Webworker canvas performance terrible - javascript

I'm trying to use webworkers to render parts of the frames for an animated mandelbrot zoomer, since there is a lot of calculating involved, and since this can be easily split up in blocks this should be an ideal situation for parallel processing.
But no matter what I try I do not get any performance in return for the extra cpu the workers use. Compared to a non worker version, in Chrome my benchmark is somewhat slower, in Firefox it is much slower.
My guess is that transferring the image data to the webworkers is incredibly expensive, I tried just receiving raw data and using that to render frames but the result is much the same. I don't think this is the ideal way to send and receive imagedata to the workers (in fact I only need to receive it, but I have not been able to create a buffer inside the workers that can be used for the canvas directly). So at it stands sending any serious amount of data creates a real bottleneck.
Dear stackoverflow, please help me answer these two questions: What am I doing wrong here, and what can be improved?
A demo can be found here for workers, and for reference a non worker version on jsfiddle.
Code is as follows:
"use strict";
/*global $*/
$(function() {
var mandelbrot = new Mandelbrot();
});
var Mandelbrot = function() {
// set some values
this.width = 500;
this.height = 500;
this.x_center = -1.407566731001088;
this.y_center = 2.741525895538953e-10;
this.iterations = 250;
this.escape = 4,
this.zoom = 10;
this.count = 0;
this.worker_size = 10;
this.received = 0;
this.refresh = true;
//let's go - create canvas, image data and workers
this.init();
//start animation loop
this.animate();
};
Mandelbrot.prototype = {
init: function() {
var self = this;
//create main canvas and append it to div
var container = $("#content");
this.canvas = document.createElement("canvas");
this.canvas.width = this.width;
this.canvas.height = this.height;
container.append(this.canvas);
//create imagedata
this.context = this.canvas.getContext("2d");
this.image = this.context.getImageData(0, 0, this.width, this.height);
this.data = new Int32Array(this.image.data.buffer);
//create imagedata for webworkers
this.worker_data = this.context.getImageData(0, 0, this.width, this.height / this.worker_size);
//create webworkers drop them in array
this.pool = [];
for (var i = 0; i < this.worker_size; i++) {
this.pool[i] = new Worker("js/worker.js");
this.pool[i].idle = true;
this.pool[i].id = i;
//on webworker finished
this.pool[i].onmessage = function(e) {
self.context.putImageData(e.data, 0, self.height / self.worker_size * e.target.id);
self.received++;
};
}
},
iterate: function() {
for (var i = 0; i < this.pool.length; i++) {
this.pool[i].postMessage({
image: this.worker_data,
id: this.pool[i].id,
worker_size: this.worker_size,
width: this.width,
height: this.height,
x_center: this.x_center,
y_center: this.y_center,
iterations: this.iterations,
escape: this.escape,
zoom: this.zoom
});
}
},
animate: function() {
requestAnimationFrame(this.animate.bind(this));
//poor man's benchmark over 250 frames
if (this.count === 0) {
console.time("timer");
}
if (this.count === 250) {
console.timeEnd("timer");
}
//refresh at init, then refresh when all webworkers are done and reset
if (this.received === this.worker_size | this.refresh) {
this.received = 0;
this.refresh = false;
this.count++;
this.zoom *= 0.95;
this.iterate();
}
}
};
and worker.js:
self.onmessage = function(e) {
"use strict";
var x_step = e.data.zoom / e.data.width;
var y_step = e.data.zoom / e.data.height;
var y_start = e.data.height / e.data.worker_size * e.data.id;
var y_end = e.data.height / e.data.worker_size;
var data = new Int32Array(e.data.image.data.buffer);
for (var y = 0; y < y_end; y++) {
var iy = e.data.y_center - e.data.zoom / 2 + (y + y_start) * y_step;
for (var x = 0; x < e.data.width; x++) {
var rx = e.data.x_center - e.data.zoom / 2 + x * x_step;
var zx = rx;
var zy = iy;
var zx2 = 0;
var zy2 = 0;
for (var i = 0; zx2 + zy2 < e.data.escape && i < e.data.iterations; ++i) {
zx2 = zx * zx;
zy2 = zy * zy;
zy = (zx + zx) * zy + iy;
zx = zx2 - zy2 + rx;
}
data[y * e.data.width + x] = (255 << 24) | (i << 16) | (i << 8) | i;
}
}
self.postMessage(e.data.image);
};

The problem is that you are iterating over every pixel in the parent picture. If you restrict the iteration to the smaller of the two images, things will be much faster. Also, if you tile the drawing, each tile could be handled in a separate web worker, thus increasing the palletization of each section of the image. I wrote this: http://robertleeplummerjr.github.io/CanvasWorker/ which does exactly what you want.

I actually tried the same thing on this experiment, this is a displacement filter:
http://www.soundstep.com/blog/experiments/displacement-js/heart/
http://www.soundstep.com/blog/2012/04/25/javascript-displacement-mapping/
I created a worker in the filter and I compute the pixel together before posting them back to the main app. Basically iterating on all the pixels inside a worker.
Before the worker, I have in a loop 4 getImageData, this can't be done in the worker. It takes around 15% CPU on chrome no matter what.
So, overall I get 70% CPU without the worker, and I get 90% CPU with the worker.
I suppose the actions that cannot be done in the worker, such as "getImageData" AND "putImageData", plus the fact of having the worker itself, takes more CPU than not having a worker.
It would probably be better if we were able to send other types of data so we could do the getImageData and putImageData inside the worker.
Not sure there's another way sending and receiving bytes to treat and reconstruct the canvas content.
http://typedarray.org/concurrency-in-javascript/

Related

How to obtain a gaussian filter in javascript

python has fspecial('gaussian', f_wid, sigma) to make gaussian easy(link). Does Javascript has similar utils?
Yes this is possible with Javascript, but it isn't as easy as it is with Python.
Since you are looking for a javascript solution, it is good to know that HTML5 Canvas element has some built-in filters. An example snippet of Canvas blur looks like this:
const canvas = document.querySelector("canvas");
const ctx = canvas.getContext("2d");
const texture = document.querySelector("img");
texture.onload = function(){
canvas.width = this.width;
canvas.height = this.height;
ctx.filter = 'blur(10px)';
ctx.drawImage(this, 0, 0);
}
Other approaches that work well for the web are CSS and SVG filters and they are compatible with Canvas as well. However, they are not well designed for cases where our code runs in Web Worker.
The makeGaussKernel function creates a one dimensional array with the appropriate filter size and coefficients.
function makeGaussKernel(sigma){
const GAUSSKERN = 6.0;
var dim = parseInt(Math.max(3.0, GAUSSKERN * sigma));
var sqrtSigmaPi2 = Math.sqrt(Math.PI*2.0)*sigma;
var s2 = 2.0 * sigma * sigma;
var sum = 0.0;
var kernel = new Float32Array(dim - !(dim & 1)); // Make it odd number
const half = parseInt(kernel.length / 2);
for (var j = 0, i = -half; j < kernel.length; i++, j++)
{
kernel[j] = Math.exp(-(i*i)/(s2)) / sqrtSigmaPi2;
sum += kernel[j];
}
// Normalize the gaussian kernel to prevent image darkening/brightening
for (var i = 0; i < dim; i++) {
kernel[i] /= sum;
}
return kernel;
}
Source Fiveko

Chrome Performance Issue With Bitmap Area Sampling Effect (JavaScript)

I am writing an HTML5 game using the engine Phaser, in which I am implementing what are essentially live backgrounds, backgrounds that respond to the movements of the game objects. The first I am working with is a water ripple effect that uses area sampling on the bitmapData object. I thought I had a performance issue in my code, but it turns out that Firefox runs it like a dream. Chrome runs a little slower to begin with and slows to less than 10 FPS when my game objects go too close to the top or bottom of the screen. (I am at a loss for why that makes a difference.)
This thread suggests that Chrome has poor image processing performance and suggests to break large image data up into smaller pieces. I don't know if this is possible in my case, because this is not simply an image displaying on the screen but an effect based on pixels next to each other that refreshes each frame. Even if it is possible, I think Chrome would end up having to do the same amount of work or more to get the four individual bitmaps to interact with each other as if they were one.
I've been doing performance tests in Chrome for a few hours, and the issue is definitely that it is getting caught up on the method that actually creates the effect by reading pixels from a source imageData and writing them to another location in a target imageData (the ws.displace(x,y) method below).
function waterStage(canvas) {
var ws = new Object();
ws.dampFactor = 16;
ws.magFactor = 150;
ws.dispFactor = 0.5;
ws.lumFactor = 1;
ws.width = canvas.width;
ws.height = canvas.height;
// Initialize height data caches
ws.pMaps = [];
var map1 = new Array(ws.width+2);
var map2 = new Array(ws.width+2);
for (x=0; x < map1.length; x++) {
map1[x] = new Array(ws.height+2);
map2[x] = new Array(ws.height+2);
}
for (x=0; x < map1.length; x++) {
for (y=0; y < map1[x].length; y++) {
map1[x][y] = 0;
map2[x][y] = 0;
}
}
ws.pMaps.push(map1, map2);
ws.stageInit = function(canvas) {
canvas.fill(100,100,100);
canvas.ctx.strokeStyle = "#000000";
canvas.ctx.lineWidth = 2;
canvas.ctx.moveTo(0,0);
for (y=0; y < ws.height; y+=10) {
canvas.ctx.beginPath();
canvas.ctx.moveTo(0,y);
canvas.ctx.lineTo(ws.width,y);
canvas.ctx.closePath();
canvas.ctx.stroke();
}
ws.sourceData = canvas.ctx.getImageData(0, 0, ws.width, ws.height);
ws.targetData = canvas.ctx.getImageData(0, 0, ws.width, ws.height);
}
ws.setWave = function(pnt) {
ws.pMaps[0][pnt.x-1][pnt.y-1] = ws.magFactor//*pnt.magnitude;
}
ws.resolveWaves = function(x,y) {
// Calculate the net result of the wave heights
ws.pMaps[1][x][y] = ((ws.pMaps[0][x-1][y]+ws.pMaps[0][x+1][y]+ws.pMaps[0][x][y-1]+ws.pMaps[0][x][y+1]) / 2)
-ws.pMaps[1][x][y];
ws.pMaps[1][x][y] -= (ws.pMaps[1][x][y]/ws.dampFactor);
}
ws.displace = function(x,y) {
var displace = Math.floor(ws.pMaps[1][x][y]*ws.dispFactor);
var xCorrect = x-1, yCorrect = y-1;
var targetIndex = (xCorrect + yCorrect * ws.width)*4;
if (displace == 0) {
ws.targetData.data[targetIndex] = ws.sourceData.data[targetIndex];
ws.targetData.data[targetIndex+1] = ws.sourceData.data[targetIndex+1];
ws.targetData.data[targetIndex+2] = ws.sourceData.data[targetIndex+2];
}
else {
if (displace < 0) {
displace += 1;
}
var sourceX = displace+xCorrect;
var sourceY = displace+yCorrect;
var sourceIndex = (sourceX + sourceY * ws.width)*4;
//var lum = ws.pMaps[1][x][y]*ws.lumFactor;
ws.targetData.data[targetIndex] = ws.sourceData.data[sourceIndex];//+lum;
ws.targetData.data[targetIndex+1] = ws.sourceData.data[sourceIndex+1];//+lum;
ws.targetData.data[targetIndex+2] = ws.sourceData.data[sourceIndex+2];//+lum;
}
}
ws.stageRefresh = function(moves, canvas) {
canvas.clear();
for (j=0; j < moves.length; j++) {
ws.setWave(moves[j]);
}
for (x=1; x <= ws.width; x++) {
if (ws.pMaps[1][x][0] != 0 || ws.pMaps[0][x][0] != 0) {
alert("TOP ROW ANOMALY");
}
for (y=1; y <= ws.height; y++) {
ws.resolveWaves(x,y);
ws.displace(x,y);
}
}
ws.pMaps.sort(function(a,b) { return 1 });
//ws.pMaps[0] = ws.pMaps[1];
//ws.pMaps[1] = temp;
canvas.ctx.putImageData(ws.targetData, 0, 0);
}
return ws;
}
canvas is the bitmapData that is given as the texture for the background (not an HTML5 canvas; sorry if that's confusing). ws.stageRefresh(moves,canvas) is called on every frame update.
Before I try to make the split-into-four-bitmaps solution work, does anyone have any guidance for other ways to improve the performance of this effect on Chrome?

Generate the Dominant Colors for an RGB image with XMLHttpRequest

A Note For Readers: This is a long question, but it needs a background to understand the question asked.
The color quantization technique is commonly used to get the dominant colors of an image.
One of the well-known libraries that do color quantization is Leptonica through the Modified Median Cut Quantization (MMCQ) and octree quantization (OQ)
Github's Color-thief by #lokesh is a very simple implementation in JavaScript of the MMCQ algorithm:
var colorThief = new ColorThief();
colorThief.getColor(sourceImage);
Technically, the image on a <img/> HTML element is backed on a <canvas/> element:
var CanvasImage = function (image) {
this.canvas = document.createElement('canvas');
this.context = this.canvas.getContext('2d');
document.body.appendChild(this.canvas);
this.width = this.canvas.width = image.width;
this.height = this.canvas.height = image.height;
this.context.drawImage(image, 0, 0, this.width, this.height);
};
And that is the problem with TVML, as we will see later on.
Another implementation I recently came to know was linked on this article Using imagemagick, awk and kmeans to find dominant colors in images that links to Using python to generate awesome linux desktop themes.
The author posted an article about Using python and k-means to find the dominant colors in images that was used there (sorry for all those links, but I'm following back my History...).
The author was super productive, and added a JavaScript version too that I'm posting here: Using JavaScript and k-means to find the dominant colors in images
In this case, we are generating the dominant colors of an image, not using the MMCQ (or OQ) algorithm, but K-Means.
The problem is that the image must be a as well:
<canvas id="canvas" style="display: none;" width="200" height="200"></canvas>
and then
function analyze(img_elem) {
var ctx = document.getElementById('canvas').getContext('2d')
, img = new Image();
img.onload = function() {
var results = document.getElementById('results');
results.innerHTML = 'Waiting...';
var colors = process_image(img, ctx)
, p1 = document.getElementById('c1')
, p2 = document.getElementById('c2')
, p3 = document.getElementById('c3');
p1.style.backgroundColor = colors[0];
p2.style.backgroundColor = colors[1];
p3.style.backgroundColor = colors[2];
results.innerHTML = 'Done';
}
img.src = img_elem.src;
}
This is because the Canvas has a getContext() method, that expose 2D image drawing APIs - see An introduction to the Canvas 2D API
This context ctx is passed to the image processing function
function process_image(img, ctx) {
var points = [];
ctx.drawImage(img, 0, 0, 200, 200);
data = ctx.getImageData(0, 0, 200, 200).data;
for (var i = 0, l = data.length; i < l; i += 4) {
var r = data[i]
, g = data[i+1]
, b = data[i+2];
points.push([r, g, b]);
}
var results = kmeans(points, 3, 1)
, hex = [];
for (var i = 0; i < results.length; i++) {
hex.push(rgbToHex(results[i][0]));
}
return hex;
}
So you can draw an image on the Canvas through the Context and get image data:
ctx.drawImage(img, 0, 0, 200, 200);
data = ctx.getImageData(0, 0, 200, 200).data;
Another nice solution is in CoffeeScript, ColorTunes, but this is using a as well:
ColorTunes.getColorMap = function(canvas, sx, sy, w, h, nc) {
var index, indexBase, pdata, pixels, x, y, _i, _j, _ref, _ref1;
if (nc == null) {
nc = 8;
}
pdata = canvas.getContext("2d").getImageData(sx, sy, w, h).data;
pixels = [];
for (y = _i = sy, _ref = sy + h; _i < _ref; y = _i += 1) {
indexBase = y * w * 4;
for (x = _j = sx, _ref1 = sx + w; _j < _ref1; x = _j += 1) {
index = indexBase + (x * 4);
pixels.push([pdata[index], pdata[index + 1], pdata[index + 2]]);
}
}
return (new MMCQ).quantize(pixels, nc);
};
But, wait, we have no <canvas/> element in TVML!
Of course, there are native solutions like Objective-C ColorCube, DominantColor - this is using K-means
and the very nice and reusable ColorArt by #AaronBrethorst from CocoaControls.
Despite the fact that this could be used in a TVML application through a native to JavaScriptCore bridge - see How to bridge TVML/JavaScriptCore to UIKit/Objective-C (Swift)?
my aim is to make this work completely in TVJS and TVML.
The simplest MMCQ JavaScript implementation does not need a Canvas: see Basic Javascript port of the MMCQ (modified median cut quantization) by Nick Rabinowitz, but needs the RGB array of the image:
var cmap = MMCQ.quantize(pixelArray, colorCount);
that is taken from the HTML <canvas/> and that is the reason for it!
function createPalette(sourceImage, colorCount) {
// Create custom CanvasImage object
var image = new CanvasImage(sourceImage),
imageData = image.getImageData(),
pixels = imageData.data,
pixelCount = image.getPixelCount();
// Store the RGB values in an array format suitable for quantize function
var pixelArray = [];
for (var i = 0, offset, r, g, b, a; i < pixelCount; i++) {
offset = i * 4;
r = pixels[offset + 0];
g = pixels[offset + 1];
b = pixels[offset + 2];
a = pixels[offset + 3];
// If pixel is mostly opaque and not white
if (a >= 125) {
if (!(r > 250 && g > 250 && b > 250)) {
pixelArray.push([r, g, b]);
}
}
}
// Send array to quantize function which clusters values
// using median cut algorithm
var cmap = MMCQ.quantize(pixelArray, colorCount);
var palette = cmap.palette();
// Clean up
image.removeCanvas();
return palette;
}
[QUESTION]
How to generate the dominant colors of a RGB image without using the HTML5 <canvas/>, but in pure JavaScript from an image's ByteArray fetched with XMLHttpRequest?
[UPDATE]
I have posted this question to Color-Thief github repo, adapting the RGB array calculations to the latest codebase.
The solution I have tried was this
ColorThief.prototype.getPaletteNoCanvas = function(sourceImageURL, colorCount, quality, done) {
var xhr = new XMLHttpRequest();
xhr.open('GET', sourceImageURL, true);
xhr.responseType = 'arraybuffer';
xhr.onload = function(e) {
if (this.status == 200) {
var uInt8Array = new Uint8Array(this.response);
var i = uInt8Array.length;
var biStr = new Array(i);
while (i--)
{ biStr[i] = String.fromCharCode(uInt8Array[i]);
}
if (typeof colorCount === 'undefined') {
colorCount = 10;
}
if (typeof quality === 'undefined' || quality < 1) {
quality = 10;
}
var pixels = uInt8Array;
var pixelCount = 152 * 152 * 4 // this should be width*height*4
// Store the RGB values in an array format suitable for quantize function
var pixelArray = [];
for (var i = 0, offset, r, g, b, a; i < pixelCount; i = i + quality) {
offset = i * 4;
r = pixels[offset + 0];
g = pixels[offset + 1];
b = pixels[offset + 2];
a = pixels[offset + 3];
// If pixel is mostly opaque and not white
if (a >= 125) {
if (!(r > 250 && g > 250 && b > 250)) {
pixelArray.push([r, g, b]);
}
}
}
// Send array to quantize function which clusters values
// using median cut algorithm
var cmap = MMCQ.quantize(pixelArray, colorCount);
var palette = cmap? cmap.palette() : null;
done.apply(this,[ palette ])
} // 200
};
xhr.send();
}
but it does not gives back the right RGB colors array.
[UPDATE]
Thanks to all the suggestions I got it working. Now a full example is available on Github,
The canvas element is being used as a convenient way to decode the image into an RGBA array. You can also use pure JavaScript libraries to do the image decoding.
jpgjs is a JPEG decoder and pngjs is a PNG decoder. It looks like the JPEG decoder will work with TVJS as is. The PNG decoder, however, looks like it's made to work in a Node or web browser environment, so you might have to tweak that one a bit.

Consistent FPS in frame by frame video with <canvas>

I'm trying to display precisely enough a video that I can stop on or jump to a specific frame. For now my approach is to display a video frame by frame on a canvas (I do have the list of images to display, I don't have to extract them from the video). The speed doesn't really matter as long as it's consistent and around 30fps. Compatibility somewhat matters (we can ignore IE≤8).
So first off, I'm pre-loading all the images:
var all_images_loaded = {};
var all_images_src = ["Continuity_0001.png","Continuity_0002.png", ..., "Continuity_0161.png"];
function init() {
for (var i = all_images_src.length - 1; i >= 0; i--) {
var objImage = new Image();
objImage.onload = imagesLoaded;
objImage.src = 'Continuity/'+all_images_src[i];
all_images_loaded[all_images_src[i]] = objImage;
}
}
var loaded_count = 0;
function imagesLoaded () {
console.log(loaded_count + " / " + all_images_src.length);
if(++loaded_count === all_images_src.length) startvid();
}
init();
and once that's done, the function startvid() is called.
Then the first solution I came up with was to draw on requestAnimationFrame() after a setTimeout (to tame the fps):
var canvas = document.getElementsByTagName('canvas')[0];
var ctx = canvas.getContext("2d");
var video_pointer = 0;
function startvid () {
video_pointer++;
if(all_images_src[video_pointer]){
window.requestAnimationFrame((function (video_pointer) {
ctx.drawImage(all_images_loaded[all_images_src[video_pointer]], 0, 0);
}).bind(undefined, video_pointer))
setTimeout(startvid, 33);
}
}
but that felt somewhat slow and irregular...
So second solution is to use 2 canvases and draw on the one being hidden and then switch it to visible with the proper timing:
var canvas = document.getElementsByTagName('canvas');
var ctx = [canvas[0].getContext("2d"), canvas[1].getContext("2d")];
var curr_can_is_0 = true;
var video_pointer = 0;
function startvid () {
video_pointer++;
curr_can_is_0 = !curr_can_is_0;
if(all_images_src[video_pointer]){
ctx[curr_can_is_0?1:0].drawImage(all_images_loaded[all_images_src[video_pointer]], 0, 0);
window.requestAnimationFrame((function (curr_can_is_0, video_pointer) {
ctx[curr_can_is_0?0:1].canvas.style.visibility = "visible";
ctx[curr_can_is_0?1:0].canvas.style.visibility = "hidden";
}).bind(undefined, curr_can_is_0, video_pointer));
setTimeout(startvid, 33);
}
}
but that too feels slow and irregular...
Yet, Google Chrome (which I'm developing on) seems to have plenty of idle time:
So what can I do?
The Problem:
Your main issue is setTimeout and setInterval are not guaranteed to fire at exactly the delay specified, but at some point after the delay.
From the MDN article on setTimeout (emphasis added by me).
delay is the number of milliseconds (thousandths of a second) that the function call should be delayed by. If omitted, it defaults to 0. The actual delay may be longer; see Notes below.
Here are the relevant notes from MDN mentioned above.
Historically browsers implement setTimeout() "clamping": successive setTimeout() calls with delay smaller than the "minimum delay" limit are forced to use at least the minimum delay. The minimum delay, DOM_MIN_TIMEOUT_VALUE, is 4 ms (stored in a preference in Firefox: dom.min_timeout_value), with a DOM_CLAMP_TIMEOUT_NESTING_LEVEL of 5.
In fact, 4ms is specified by the HTML5 spec and is consistent across browsers released in 2010 and onward. Prior to (Firefox 5.0 / Thunderbird 5.0 / SeaMonkey 2.2), the minimum timeout value for nested timeouts was 10 ms.
In addition to "clamping", the timeout can also fire later when the page (or the OS/browser itself) is busy with other tasks.
The Solution:
You would be better off using just requestAnimationFrame, and inside the callback using the timestamp arguments passed to the callback to compute the delta time into the video, and drawing the necessary frame from the list. See working example below. As a bonus, I've even included code to prevent re-drawing the same frame twice.
Working Example:
var start_time = null;
var frame_rate = 30;
var canvas = document.getElementById('video');
var ctx = canvas.getContext('2d');
var all_images_loaded = {};
var all_images_src = (function(frames, fps){//Generate some placeholder images.
var a = [];
var zfill = function(s, l) {
s = '' + s;
while (s.length < l) {
s = '0' + s;
}
return s;
}
for(var i = 0; i < frames; i++) {
a[i] = 'http://placehold.it/480x270&text=' + zfill(Math.floor(i / fps), 2) + '+:+' + zfill(i % fps, 2)
}
return a;
})(161, frame_rate);
var video_duration = (all_images_src.length / frame_rate) * 1000;
function init() {
for (var i = all_images_src.length - 1; i >= 0; i--) {
var objImage = new Image();
objImage.onload = imagesLoaded;
//objImage.src = 'Continuity/'+all_images_src[i];
objImage.src = all_images_src[i];
all_images_loaded[all_images_src[i]] = objImage;
}
}
var loaded_count = 0;
function imagesLoaded () {
//console.log(loaded_count + " / " + all_images_src.length);
if (++loaded_count === all_images_src.length) {
startvid();
}
}
function startvid() {
requestAnimationFrame(draw);
}
var last_frame = null;
function draw(timestamp) {
//Set the start time on the first call.
if (!start_time) {
start_time = timestamp;
}
//Find the current time in the video.
var current_time = (timestamp - start_time);
//Check that it is less than the end of the video.
if (current_time < video_duration) {
//Find the delta of the video completed.
var delta = current_time / video_duration;
//Find the frame for that delta.
var current_frame = Math.floor(all_images_src.length * delta);
//Only draw this frame if it is different from the last one.
if (current_frame !== last_frame) {
ctx.drawImage(all_images_loaded[all_images_src[current_frame]], 0, 0);
last_frame = current_frame;
}
//Continue the animation loop.
requestAnimationFrame(draw);
}
}
init();
<canvas id="video" width="480" height="270"></canvas>

Create a waveform of the full track with Web Audio API

Realtime moving Waveform
I'm currently playing with Web Audio API and made a spectrum using canvas.
function animate(){
var a=new Uint8Array(analyser.frequencyBinCount),
y=new Uint8Array(analyser.frequencyBinCount),b,c,d;
analyser.getByteTimeDomainData(y);
analyser.getByteFrequencyData(a);
b=c=a.length;
d=w/c;
ctx.clearRect(0,0,w,h);
while(b--){
var bh=a[b]+1;
ctx.fillStyle='hsla('+(b/c*240)+','+(y[b]/255*100|0)+'%,50%,1)';
ctx.fillRect(1*b,h-bh,1,bh);
ctx.fillRect(1*b,y[b],1,1);
}
animation=webkitRequestAnimationFrame(animate);
}
Mini question: is there a way to not write 2 times new Uint8Array(analyser.frequencyBinCount)?
DEMO
add a MP3/MP4 file and wait. (tested in Chrome)
http://jsfiddle.net/pc76H/2/
But there are many problems. I can't find a proper documentation of the various audio filters.
Also, if you look at the spectrum you will notice that after 70% or the range there is no data. What does that mean? that maybe from 16k hz to 20k hz is no sound? I would apply a text to the canvas to show the various HZ. but where??
I found out that the returned data is a power of 32 in length with a max of 2048
and the height is always 256.
BUT the real question is ... I want to create a moving waveform like in traktor.
I already did that some time ago with PHP it converts the file to low bitrate than extracts the data and coverts that to a image. i found the script somewhere...but I don't remember where...
note: needs LAME
<?php
$a=$_GET["f"];
if(file_exists($a)){
if(file_exists($a.".png")){
header("Content-Type: image/png");
echo file_get_contents($a.".png");
}else{
$b=3000;$c=300;define("d",3);
ini_set("max_execution_time","30000");
function n($g,$h){
$g=hexdec(bin2hex($g));
$h=hexdec(bin2hex($h));
return($g+($h*256));
};
$k=substr(md5(time()),0,10);
copy(realpath($a),"/var/www/".$k."_o.mp3");
exec("lame /var/www/{$k}_o.mp3 -f -m m -b 16 --resample 8 /var/www/{$k}.mp3 && lame --decode /var/www/{$k}.mp3 /var/www/{$k}.wav");
//system("lame {$k}_o.mp3 -f -m m -b 16 --resample 8 {$k}.mp3 && lame --decode {$k}.mp3 {$k}.wav");
#unlink("/var/www/{$k}_o.mp3");
#unlink("/var/www/{$k}.mp3");
$l="/var/www/{$k}.wav";
$m=fopen($l,"r");
$n[]=fread($m,4);
$n[]=bin2hex(fread($m,4));
$n[]=fread($m,4);
$n[]=fread($m,4);
$n[]=bin2hex(fread($m,4));
$n[]=bin2hex(fread($m,2));
$n[]=bin2hex(fread($m,2));
$n[]=bin2hex(fread($m,4));
$n[]=bin2hex(fread($m,4));
$n[]=bin2hex(fread($m,2));
$n[]=bin2hex(fread($m,2));
$n[]=fread($m,4);
$n[]=bin2hex(fread($m,4));
$o=hexdec(substr($n[10],0,2));
$p=$o/8;
$q=hexdec(substr($n[6],0,2));
if($q==2){$r=40;}else{$r=80;};
while(!feof($m)){
$t=array();
for($i=0;$i<$p;$i++){
$t[$i]=fgetc($m);
};
switch($p){
case 1:$s[]=n($t[0],$t[1]);break;
case 2:if(ord($t[1])&128){$u=0;}else{$u=128;};$u=chr((ord($t[1])&127)+$u);$s[]= floor(n($t[0],$u)/256);break;
};
fread($m,$r);
};
fclose($m);
unlink("/var/www/{$k}.wav");
$x=imagecreatetruecolor(sizeof($s)/d,$c);
imagealphablending($x,false);
imagesavealpha($x,true);
$y=imagecolorallocatealpha($x,255,255,255,127);
imagefilledrectangle($x,0,0,sizeof($s)/d,$c,$y);
for($d=0;$d<sizeof($s);$d+=d){
$v=(int)($s[$d]/255*$c);
imageline($x,$d/d,0+($c-$v),$d/d,$c-($c-$v),imagecolorallocate($x,255,0,255));
};
$z=imagecreatetruecolor($b,$c);
imagealphablending($z,false);
imagesavealpha($z,true);
imagefilledrectangle($z,0,0,$b,$c,$y);
imagecopyresampled($z,$x,0,0,0,0,$b,$c,sizeof($s)/d,$c);
imagepng($z,realpath($a).".png");
header("Content-Type: image/png");
imagepng($z);
imagedestroy($z);
};
}else{
echo $a;
};
?>
The script works... but you are limited to a max image size of 4k pixels.
so you have not a nice waveform if it should rappresent only some milliseconds.
What do i need to store/create a realtime waveform like the traktors app or this php script? btw the traktor has also a colored waveform(the php script not).
EDIT
I rewrote your script that it fits my idea... it's relatively fast.
As you can see inside the function createArray i push the various lines into an object with the key as x coordinate.
I'm simply taking the the highest number.
here is where we could play with the colors.
var ajaxB,AC,B,LC,op,x,y,ARRAY={},W=1024,H=256;
var aMax=Math.max.apply.bind(Math.max, Math);
function error(a){
console.log(a);
};
function createDrawing(){
console.log('drawingArray');
var C=document.createElement('canvas');
C.width=W;
C.height=H;
document.body.appendChild(C);
var context=C.getContext('2d');
context.save();
context.strokeStyle='#121';
context.globalCompositeOperation='lighter';
L2=W*1;
while(L2--){
context.beginPath();
context.moveTo(L2,0);
context.lineTo(L2+1,ARRAY[L2]);
context.stroke();
}
context.restore();
};
function createArray(a){
console.log('creatingArray');
B=a;
LC=B.getChannelData(0);// Float32Array describing left channel
L=LC.length;
op=W/L;
for(var i=0;i<L;i++){
x=W*i/L|0;
y=LC[i]*H/2;
if(ARRAY[x]){
ARRAY[x].push(y)
}else{
!ARRAY[x-1]||(ARRAY[x-1]=aMax(ARRAY[x-1]));
// the above line contains an array of values
// which could be converted to a color
// or just simply create a gradient
// based on avg max min (frequency???) whatever
ARRAY[x]=[y]
}
};
createDrawing();
};
function decode(){
console.log('decodingMusic');
AC=new webkitAudioContext
AC.decodeAudioData(this.response,createArray,error);
};
function loadMusic(url){
console.log('loadingMusic');
ajaxB=new XMLHttpRequest;
ajaxB.open('GET',url);
ajaxB.responseType='arraybuffer';
ajaxB.onload=decode;
ajaxB.send();
}
loadMusic('AudioOrVideo.mp4');
Ok, so what i would do is to load the sound with an XMLHttpRequest, then decode it using webaudio, then display it 'carefully' to have the colors you are searching for.
I just made a quick version, copy-pasting from various of my projects, it is quite working, as you might see with this picture :
The issue is that it is slow as hell. To have (more) decent speed, you'll have to do some computation to reduce the number of lines to draw on the canvas, because at 441000 Hz, you very quickly get too many lines to draw.
// AUDIO CONTEXT
window.AudioContext = window.AudioContext || window.webkitAudioContext ;
if (!AudioContext) alert('This site cannot be run in your Browser. Try a recent Chrome or Firefox. ');
var audioContext = new AudioContext();
var currentBuffer = null;
// CANVAS
var canvasWidth = 512, canvasHeight = 120 ;
var newCanvas = createCanvas (canvasWidth, canvasHeight);
var context = null;
window.onload = appendCanvas;
function appendCanvas() { document.body.appendChild(newCanvas);
context = newCanvas.getContext('2d'); }
// MUSIC LOADER + DECODE
function loadMusic(url) {
var req = new XMLHttpRequest();
req.open( "GET", url, true );
req.responseType = "arraybuffer";
req.onreadystatechange = function (e) {
if (req.readyState == 4) {
if(req.status == 200)
audioContext.decodeAudioData(req.response,
function(buffer) {
currentBuffer = buffer;
displayBuffer(buffer);
}, onDecodeError);
else
alert('error during the load.Wrong url or cross origin issue');
}
} ;
req.send();
}
function onDecodeError() { alert('error while decoding your file.'); }
// MUSIC DISPLAY
function displayBuffer(buff /* is an AudioBuffer */) {
var leftChannel = buff.getChannelData(0); // Float32Array describing left channel
var lineOpacity = canvasWidth / leftChannel.length ;
context.save();
context.fillStyle = '#222' ;
context.fillRect(0,0,canvasWidth,canvasHeight );
context.strokeStyle = '#121';
context.globalCompositeOperation = 'lighter';
context.translate(0,canvasHeight / 2);
context.globalAlpha = 0.06 ; // lineOpacity ;
for (var i=0; i< leftChannel.length; i++) {
// on which line do we get ?
var x = Math.floor ( canvasWidth * i / leftChannel.length ) ;
var y = leftChannel[i] * canvasHeight / 2 ;
context.beginPath();
context.moveTo( x , 0 );
context.lineTo( x+1, y );
context.stroke();
}
context.restore();
console.log('done');
}
function createCanvas ( w, h ) {
var newCanvas = document.createElement('canvas');
newCanvas.width = w; newCanvas.height = h;
return newCanvas;
};
loadMusic('could_be_better.mp3');
Edit : The issue here is that we have too much data to draw. Take a 3 minutes mp3, you'll have 3*60*44100 = about 8.000.000 line to draw. On a display that has, say, 1024 px resolution, that makes 8.000 lines per pixel...
In the code above, the canvas is doing the 'resampling', by drawing lines with low-opacity and in 'ligther' composition mode (e.g. pixel's r,g,b will add-up).
To speed-up things, you have to re-sample by yourself, but to get some colors, it's not just a down-sampling, you'll have to handle a set (within a performance array most probably) of 'buckets', one for each horizontal pixel (so, say 1024), and in every bucket you compute the cumulated sound pressure, the variance, min, max and then, at display time, you decide how you will render that with colors.
For instance :
values between 0 positiveMin are very clear. (any sample is below that point).
values between positiveMin and positiveAverage - variance are darker,
values between positiveAverage - variance and positiveAverage + variance are darker,
and values between positiveAverage+variance and positiveMax lighter .
(same for negative values)
That makes 5 colors for each bucket, and it's still quite some work, for you to code and for the browser to compute.
I don't know if the performance could get decent with this, but i fear the statistics accuracy and the color coding of the software you mention can't be reached on a browser (obviously not in real-time), and that you'll have to make some compromises.
Edit 2 :
I tried to get some colors out of stats but it quite failed. My guess, now, is that the guys at tracktor also change color depending on frequency.... quite some work here....
Anyway, just for the record, the code for an average / mean variation follows.
(variance was too low, i had to use mean variation).
// MUSIC DISPLAY
function displayBuffer2(buff /* is an AudioBuffer */) {
var leftChannel = buff.getChannelData(0); // Float32Array describing left channel
// we 'resample' with cumul, count, variance
// Offset 0 : PositiveCumul 1: PositiveCount 2: PositiveVariance
// 3 : NegativeCumul 4: NegativeCount 5: NegativeVariance
// that makes 6 data per bucket
var resampled = new Float64Array(canvasWidth * 6 );
var i=0, j=0, buckIndex = 0;
var min=1e3, max=-1e3;
var thisValue=0, res=0;
var sampleCount = leftChannel.length;
// first pass for mean
for (i=0; i<sampleCount; i++) {
// in which bucket do we fall ?
buckIndex = 0 | ( canvasWidth * i / sampleCount );
buckIndex *= 6;
// positive or negative ?
thisValue = leftChannel[i];
if (thisValue>0) {
resampled[buckIndex ] += thisValue;
resampled[buckIndex + 1] +=1;
} else if (thisValue<0) {
resampled[buckIndex + 3] += thisValue;
resampled[buckIndex + 4] +=1;
}
if (thisValue<min) min=thisValue;
if (thisValue>max) max = thisValue;
}
// compute mean now
for (i=0, j=0; i<canvasWidth; i++, j+=6) {
if (resampled[j+1] != 0) {
resampled[j] /= resampled[j+1]; ;
}
if (resampled[j+4]!= 0) {
resampled[j+3] /= resampled[j+4];
}
}
// second pass for mean variation ( variance is too low)
for (i=0; i<leftChannel.length; i++) {
// in which bucket do we fall ?
buckIndex = 0 | (canvasWidth * i / leftChannel.length );
buckIndex *= 6;
// positive or negative ?
thisValue = leftChannel[i];
if (thisValue>0) {
resampled[buckIndex + 2] += Math.abs( resampled[buckIndex] - thisValue );
} else if (thisValue<0) {
resampled[buckIndex + 5] += Math.abs( resampled[buckIndex + 3] - thisValue );
}
}
// compute mean variation/variance now
for (i=0, j=0; i<canvasWidth; i++, j+=6) {
if (resampled[j+1]) resampled[j+2] /= resampled[j+1];
if (resampled[j+4]) resampled[j+5] /= resampled[j+4];
}
context.save();
context.fillStyle = '#000' ;
context.fillRect(0,0,canvasWidth,canvasHeight );
context.translate(0.5,canvasHeight / 2);
context.scale(1, 200);
for (var i=0; i< canvasWidth; i++) {
j=i*6;
// draw from positiveAvg - variance to negativeAvg - variance
context.strokeStyle = '#F00';
context.beginPath();
context.moveTo( i , (resampled[j] - resampled[j+2] ));
context.lineTo( i , (resampled[j +3] + resampled[j+5] ) );
context.stroke();
// draw from positiveAvg - variance to positiveAvg + variance
context.strokeStyle = '#FFF';
context.beginPath();
context.moveTo( i , (resampled[j] - resampled[j+2] ));
context.lineTo( i , (resampled[j] + resampled[j+2] ) );
context.stroke();
// draw from negativeAvg + variance to negativeAvg - variance
// context.strokeStyle = '#FFF';
context.beginPath();
context.moveTo( i , (resampled[j+3] + resampled[j+5] ));
context.lineTo( i , (resampled[j+3] - resampled[j+5] ) );
context.stroke();
}
context.restore();
console.log('done 231 iyi');
}
Based on the top answer, I have controlled that by reducing number of lines want to draw and little canvas function call placement. see following code for your reference.
// AUDIO CONTEXT
window.AudioContext = (window.AudioContext ||
window.webkitAudioContext ||
window.mozAudioContext ||
window.oAudioContext ||
window.msAudioContext);
if (!AudioContext) alert('This site cannot be run in your Browser. Try a recent Chrome or Firefox. ');
var audioContext = new AudioContext();
var currentBuffer = null;
// CANVAS
var canvasWidth = window.innerWidth, canvasHeight = 120 ;
var newCanvas = createCanvas (canvasWidth, canvasHeight);
var context = null;
window.onload = appendCanvas;
function appendCanvas() { document.body.appendChild(newCanvas);
context = newCanvas.getContext('2d'); }
// MUSIC LOADER + DECODE
function loadMusic(url) {
var req = new XMLHttpRequest();
req.open( "GET", url, true );
req.responseType = "arraybuffer";
req.onreadystatechange = function (e) {
if (req.readyState == 4) {
if(req.status == 200)
audioContext.decodeAudioData(req.response,
function(buffer) {
currentBuffer = buffer;
displayBuffer(buffer);
}, onDecodeError);
else
alert('error during the load.Wrong url or cross origin issue');
}
} ;
req.send();
}
function onDecodeError() { alert('error while decoding your file.'); }
// MUSIC DISPLAY
function displayBuffer(buff /* is an AudioBuffer */) {
var drawLines = 500;
var leftChannel = buff.getChannelData(0); // Float32Array describing left channel
var lineOpacity = canvasWidth / leftChannel.length ;
context.save();
context.fillStyle = '#080808' ;
context.fillRect(0,0,canvasWidth,canvasHeight );
context.strokeStyle = '#46a0ba';
context.globalCompositeOperation = 'lighter';
context.translate(0,canvasHeight / 2);
//context.globalAlpha = 0.6 ; // lineOpacity ;
context.lineWidth=1;
var totallength = leftChannel.length;
var eachBlock = Math.floor(totallength / drawLines);
var lineGap = (canvasWidth/drawLines);
context.beginPath();
for(var i=0;i<=drawLines;i++){
var audioBuffKey = Math.floor(eachBlock * i);
var x = i*lineGap;
var y = leftChannel[audioBuffKey] * canvasHeight / 2;
context.moveTo( x, y );
context.lineTo( x, (y*-1) );
}
context.stroke();
context.restore();
}
function createCanvas ( w, h ) {
var newCanvas = document.createElement('canvas');
newCanvas.width = w; newCanvas.height = h;
return newCanvas;
};
loadMusic('could_be_better.mp3');
this is a bit old, sorry to bump, but it's the only post about displaying a full waveform with the Web Audio Api and I'd like to share what method i used.
This method is not perfect but it only goes through the displayed audio and it only goes over it once. it also succeeds in displaying an actual waveform for short files or big zoom :
and a convincing loudness chart for bigger files dezoomed :
here is what it's like at middle zoom, kind of pleasant too:
notice that both zooms use the same algorythm.
I still struggle about scales (the zoomed waveform is bigger than the dezoomed one (though not so bigger than displayed on the images)
this algorythm i find is quite efficient (i can change zoom on 4mn music and it redraws flawlessly every 0.1s)
function drawWaveform (audioBuffer, canvas, pos = 0.5, zoom = 1) {
const canvasCtx = canvas.getContext("2d")
const width = canvas.clientWidth
const height = canvas.clientHeight
canvasCtx.clearRect(0, 0, width, height)
canvasCtx.fillStyle = "rgb(255, 0, 0)"
// calculate displayed part of audio
// and slice audio buffer to only process that part
const bufferLength = audioBuffer.length
const zoomLength = bufferLength / zoom
const start = Math.max(0, bufferLength * pos - zoomLength / 2)
const end = Math.min(bufferLength, start + zoomLength)
const rawAudioData = audioBuffer.getChannelData(0).slice(start, end)
// process chunks corresponding to 1 pixel width
const chunkSize = Math.max(1, Math.floor(rawAudioData.length / width))
const values = []
for (let x = 0; x < width; x++) {
const start = x*chunkSize
const end = start + chunkSize
const chunk = rawAudioData.slice(start, end)
// calculate the total positive and negative area
let positive = 0
let negative = 0
chunk.forEach(val =>
val > 0 && (positive += val) || val < 0 && (negative += val)
)
// make it mean (this part makes dezommed audio smaller, needs improvement)
negative /= chunk.length
positive /= chunk.length
// calculate amplitude of the wave
chunkAmp = -(negative - positive)
// draw the bar corresponding to this pixel
canvasCtx.fillRect(
x,
height / 2 - positive * height,
1,
Math.max(1, chunkAmp * height)
)
}
}
To use it :
async function decodeAndDisplayAudio (audioData) {
const source = audioCtx.createBufferSource()
source.buffer = await audioCtx.decodeAudioData(audioData)
drawWaveform(source.buffer, canvas, 0.5, 1)
// change position (0//start -> 0.5//middle -> 1//end)
// and zoom (0.5//full -> 400//zoomed) as you wish
}
// audioData comes raw from the file (server send it in my case)
decodeAndDisplayAudio(audioData)

Categories