I'm new to WebGL, I've worked with OpenGL in Java before. I've been trying to code a simple function that draws an image on a specific location with a specific size and rotation. But after searching on the internet for a while, my code still isn't working.
Currently, I've succeeded in drawing an image, but that image is not close to being in the correct location, having the correct size and rotation. I've lost my overview over what code does and needs what because I've used code from a number of different tutorials since I didn't found one tutorial that had all my specifications.
I know that the image loading part works for sure. I just need help with making a function that
sets up the vertex and fragment shader (for drawing width a texture)
translates, resizes and rotates it into the correct location, size and rotations
and draws it
Could someone help me with that?
You should probably read up on WebGL especially about matrices.
In any case here's "drawImage" from the canvas 2d API re-written in WebGL with the full transform stack.
In other words in Canvas2D you could do this
ctx.save();
ctx.translate(x, y);
ctx.rotate(angle);
ctx.scale(w, h);
ctx.drawImage(img, x, y);
ctx.restore();
Below you can do this
save();
translate(x, y);
rotate(angle);
scale(w, h);
drawImage(targetWidth, targetHeight, tex, texWidth, texHeight, x, y);
restore();
var m4 = twgl.m4;
var gl = document.getElementById("c").getContext('webgl');
var programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);
// a unit quad
var arrays = {
position: {
numComponents: 2,
data: [
0, 0,
1, 0,
0, 1,
0, 1,
1, 0,
1, 1,
],
},
};
var bufferInfo = twgl.createBufferInfoFromArrays(gl, arrays);
// Let's use a 2d canvas for a texture just so we don't have to download anything
var ctx = document.createElement("canvas").getContext("2d");
var w = 128;
var h = 64;
ctx.canvas.width = w;
ctx.canvas.height = h;
ctx.fillStyle = "blue";
ctx.fillRect(0, 0, w, h);
ctx.fillStyle = "green";
ctx.fillRect(w / 8, h / 8, w / 8 * 6, h / 8 * 6);
ctx.fillStyle = "red";
ctx.fillRect(w / 4, h / 4, w / 2, h / 2);
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.font = "20px sans-serif";
ctx.fillStyle = "yellow";
ctx.fillText("texture", w / 2, h / 2);
var tex = twgl.createTexture(gl, { src: ctx.canvas });
// Implement a matrix stack like Canvas2d
var matrixStack = [ m4.identity() ];
function render(time) {
var t = time * 0.001;
var texWidth = w;
var texHeight = h;
var targetWidth = gl.canvas.width;
var targetHeight = gl.canvas.height;
save();
translate(
(Math.sin(t * 0.9) * 0.5 + 0.5) * targetWidth,
(Math.sin(t * 0.8) * 0.5 + 0.5) * targetHeight);
rotate(t * 0.7);
scale(
Math.sin(t * 0.7) * 0.5 + 1,
Math.sin(t * 0.6) * 0.5 + 1);
// scale and rotate from center of image
translate(texWidth * -0.5, texHeight * -0.5);
drawImage(
targetWidth, targetHeight,
tex, texWidth, texHeight,
0, 0);
restore();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
function getCurrentMatrix() {
return matrixStack[matrixStack.length - 1];
}
function save() {
matrixStack.push(m4.copy(getCurrentMatrix()));
}
function restore() {
matrixStack.pop();
if (!matrixStack.length) {
matrixStack.push(m4.identity());
}
}
function translate(x, y) {
var m = getCurrentMatrix();
m4.translate(m, [x, y, 0], m);
}
function scale(x, y) {
var m = getCurrentMatrix();
m4.scale(m, [x, y, 1], m);
}
function rotate(radians) {
var m = getCurrentMatrix();
m4.rotateZ(m, radians, m);
}
// we pass in texWidth and texHeight because unlike images
// we can't look up the width and height of a texture
// we pass in targetWidth and targetHeight to tell it
// the size of the thing we're drawing too. We could look
// up the size of the canvas with gl.canvas.width and
// gl.canvas.height but maybe we want to draw to a framebuffer
// etc.. so might as well pass those in.
// srcX, srcY, srcWidth, srcHeight are in pixels
// computed from texWidth and texHeight
// dstX, dstY, dstWidth, dstHeight are in pixels
// computed from targetWidth and targetHeight
function drawImage(
targetWidth, targetHeight,
tex, texWidth, texHeight,
srcX, srcY, srcWidth, srcHeight,
dstX, dstY, dstWidth, dstHeight
) {
// handle case where only x, y are passed in
// as in ctx.drawIimage(img, x, y);
if (srcWidth === undefined) {
srcWidth = texWidth;
srcHeight = texHeight;
}
// handle case where only x, y, width, height are passed in
// as in ctx.drawIimage(img, x, y, width, height);
if (dstX === undefined) {
dstX = srcX;
dstY = srcY;
dstWidth = srcWidth;
dstHeight = srcHeight;
}
var mat = m4.identity();
var tmat = m4.identity();
var uniforms = {
matrix: mat,
textureMatrix: tmat,
texture: tex,
};
// these adjust the unit quad to generate texture coordinates
// to select part of the src texture
// NOTE: no check is done that srcX + srcWidth go outside of the
// texture or are in range in any way. Same for srcY + srcHeight
m4.translate(tmat, [srcX / texWidth, srcY / texHeight, 0], tmat);
m4.scale(tmat, [srcWidth / texWidth, srcHeight / texHeight, 1], tmat);
// these convert from pixels to clip space
m4.ortho(0, targetWidth, targetHeight, 0, -1, 1, mat);
// Add in global matrix
m4.multiply(mat, getCurrentMatrix(), mat);
// these move and scale the unit quad into the size we want
// in the target as pixels
m4.translate(mat, [dstX, dstY, 0], mat);
m4.scale(mat, [dstWidth, dstHeight, 1], mat);
gl.useProgram(programInfo.program);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
twgl.setUniforms(programInfo, uniforms);
twgl.drawBufferInfo(gl, bufferInfo);
}
html, body, canvas {
margin: 0; width: 100%; height:100%; display: block;
}
<script src="https://twgljs.org/dist/4.x/twgl-full.js"></script>
<script id="vs" type="not-js">
// we will always pass a 0 to 1 unit quad
// and then use matrices to manipulate it
attribute vec4 position;
uniform mat4 matrix;
uniform mat4 textureMatrix;
varying vec2 texcoord;
void main () {
gl_Position = matrix * position;
texcoord = (textureMatrix * position).xy;
}
</script>
<script id="fs" type="not-js">
precision mediump float;
varying vec2 texcoord;
uniform sampler2D texture;
void main() {
gl_FragColor = texture2D(texture, texcoord);
}
</script>
<canvas id="c"></canvas>
and here's an article describing how it works
I am rendering a texture using WebGL ,However ,the way I am rendering is I am rendering few lines of data and then moving those lines to right and again drawing another set of lines.
For example : I have a image of 640*480 ,which contains 640*480*4 pixels of RGBA, however I am only filling the alpha values and it is a GrayScale medical Dicom image.
Now ,the issue that I am facing is it is rendering the texture with jerks ,the image rendering is not happening smoothly.
For example, This is what happens :
There are 640 lines of data to be rendered.
So, I took a arraybuffer of 640*480*4 and then , Suppose first line came to client via websocket from server to render ,then I will fill the indexes as 3, 640+3, 640*2+3, 640*3+3 and so on until 640*480+3. Then when the second line is received ,I will move first line to second line like 3->7, 640+3->640+7, ......640*480+3->640*480+7. And then the newly received line will be rendered to 3, 640+3, 640*2+3, 640*3+3 and this will continue until the 640th line of image data.
Here's the code that I have done.
Code:
var renderLineData = function (imageAttr) {
var data = imageAttr.data;
var LINES_PER_CHUNK = imageAttr.lines;
var alpha = 4;
if(imageAttr.newImage) {
newBuffer = new ArrayBuffer(imageAttr.width * imageAttr.height * alpha);dataTypedArray = new Uint8Array(newBuffer);
// provide texture coordinates for the rectangle.
provideTextureCoordsForRect();
setParams();
// Upload the image into the texture.
// look up uniform locations
uploadImageToTexture(gl.getUniformLocation(program, 'u_matrix'));
} else {
for (var z = imageAttr.index; z > 0; z--) {
for (i = 0 ; i < LINES_PER_CHUNK; i++) {
for (j = 0 ; j < imageAttr.height; j++) {
dataTypedArray[i * alpha + imageAttr.width*alpha * j + 3 + LINES_PER_CHUNK * alpha * z] = dataTypedArray[i * alpha + imageAttr.width*alpha * j + 3 + LINES_PER_CHUNK * alpha * (z-1)];
}
}
}
}
for (i = 0, k = imageAttr.height*LINES_PER_CHUNK; i < LINES_PER_CHUNK; i++) {
for (j = 0 ; j < imageAttr.height; j++) {
dataTypedArray[i * alpha + imageAttr.width*4 * j + 3] = data[k - imageAttr.height + j];
}
k = k - imageAttr.height;
}
imageAttrTemp = imageAttr;
renderImgSlowly(gl, imageAttr, dataTypedArray);
};
function renderImgSlowly (gl, image, dataTypedArray) {
gl.clear(gl.COLOR_BUFFER_BIT || gl.DEPTH_BUFFER_BIT);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, dataTypedArray);
//Draw the rectangle.
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
First off, nothing you are doing is likely to be the speed issue. A 640x320 image is not that large and the amount of processing you're doing in JavaScript is unlikely to be the bottleneck.
On top of that WebGL will have no trouble drawing a single quad which is all you're drawing. Nor will it have a problem uploading a 640x480 texture.
The bottleneck is the network. Sending chunks over the network is slow.
On the other hand, if you want to optimize, why are you shifting the data around in JavaScript? Just put it in the correct place in the texture to start with with gl.texSubImage2D. If you only want to draw the part that has had data put in it then adjust the texture coordinates to select that part of the texture
Also, why are you using RGBA if you only need one channel? Use LUMINANCE.
if (imageAttr.newImage) {
destColumn = imageAttr.width;
gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, imageAttr.width, imageAttr.height, 0,
gl.LUMINANCE, gl.UNSIGNED_BYTE, null);
}
destColumn -= imageAttr.lines;
// should check it destColumn does not go negative!
gl.texSubImage2D(gl.TEXTURE_2D, 0, destColumn, 0, imageAttr.lines, imageAttr.height,
gl.LUMINANCE, gl.UNSIGNED_BYTE, imageAttr.data);
var srcX = destColumn;
var srcY = 0;
var srcWidth = imageAttr.width - destColumn;
var srcHeight = imageAttr.height;
var dstX = destColumn * gl.canvas.width / imageAttr.width;
var dstY = 0;
var dstWidth = srcWidth * gl.canvas.width / imageAttr.width;
var dstHeight = srcHeight;
var texWidth = imageAttr.width;
var texHeight = imageAttr.height;
var targetWidth = gl.canvas.width;
var targetHeight = gl.canvas.height;
drawImageInWebGL(
tex, texWidth, texHeight,
srcX, srcY, srcWidth, srcHeight,
dstX, dstY, dstWidth, dstHeight,
targetWidth, targetHeight);
}
Here's an example
var m4 = twgl.m4;
var gl = document.getElementById("c").getContext("webgl");
// compiles shader, links and looks up locations
var programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);
// a unit quad
var arrays = {
position: {
numComponents: 2,
data: [
0, 0,
1, 0,
0, 1,
0, 1,
1, 0,
1, 1,
],
},
};
// calls gl.createBuffer, gl.bindBuffer, gl.bufferData for each array
var bufferInfo = twgl.createBufferInfoFromArrays(gl, arrays);
// we're only using 1 texture so just make and bind it now
var tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
var destColumn = 0;
// We're using 1 byte wide texture pieces so we need to
// set UNPACK_ALIGNMENT to 1 as it defaults to 4
gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
simulateSendingAnImageNColumnsAtATime(1, 1, addLinesToImageAndDraw);
function addLinesToImageAndDraw(imageAttr) {
if (imageAttr.newImage) {
destColumn = imageAttr.width;
gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, imageAttr.width, imageAttr.height, 0,
gl.LUMINANCE, gl.UNSIGNED_BYTE, null);
}
destColumn -= imageAttr.lines;
// should check it destColumn does not go negative!
gl.texSubImage2D(gl.TEXTURE_2D, 0, destColumn, 0, imageAttr.lines, imageAttr.height,
gl.LUMINANCE, gl.UNSIGNED_BYTE, imageAttr.data);
var srcX = destColumn;
var srcY = 0;
var srcWidth = imageAttr.width - destColumn;
var srcHeight = imageAttr.height;
var dstX = destColumn * gl.canvas.width / imageAttr.width;
var dstY = 0;
var dstWidth = srcWidth * gl.canvas.width / imageAttr.width;
var dstHeight = gl.canvas.height;
var texWidth = imageAttr.width;
var texHeight = imageAttr.height;
var targetWidth = gl.canvas.width;
var targetHeight = gl.canvas.height;
drawImage(
tex, texWidth, texHeight,
srcX, srcY, srcWidth, srcHeight,
dstX, dstY, dstWidth, dstHeight,
targetWidth, targetHeight);
}
// we pass in texWidth and texHeight because unlike images
// we can't look up the width and height of a texture
// we pass in targetWidth and targetHeight to tell it
// the size of the thing we're drawing too. We could look
// up the size of the canvas with gl.canvas.width and
// gl.canvas.height but maybe we want to draw to a framebuffer
// etc.. so might as well pass those in.
// srcX, srcY, srcWidth, srcHeight are in pixels
// computed from texWidth and texHeight
// dstX, dstY, dstWidth, dstHeight are in pixels
// computed from targetWidth and targetHeight
function drawImage(
tex, texWidth, texHeight,
srcX, srcY, srcWidth, srcHeight,
dstX, dstY, dstWidth, dstHeight,
targetWidth, targetHeight) {
var mat = m4.identity();
var tmat = m4.identity();
var uniforms = {
matrix: mat,
textureMatrix: tmat,
texture: tex,
};
// these adjust the unit quad to generate texture coordinates
// to select part of the src texture
// NOTE: no check is done that srcX + srcWidth go outside of the
// texture or are in range in any way. Same for srcY + srcHeight
m4.translate(tmat, [srcX / texWidth, srcY / texHeight, 0], tmat);
m4.scale(tmat, [srcWidth / texWidth, srcHeight / texHeight, 1], tmat);
// these convert from pixels to clip space
m4.ortho(0, targetWidth, targetHeight, 0, -1, 1, mat)
// these move and scale the unit quad into the size we want
// in the target as pixels
m4.translate(mat, [dstX, dstY, 0], mat);
m4.scale(mat, [dstWidth, dstHeight, 1], mat);
gl.useProgram(programInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
// calls gl.uniformXXX, gl.activeTexture, gl.bindTexture
twgl.setUniforms(programInfo, uniforms);
// calls gl.drawArray or gl.drawElements
twgl.drawBufferInfo(gl, gl.TRIANGLES, bufferInfo);
}
// =====================================================================
// Everything below this line represents stuff from the server.
// so it's irrelevant to the answer
//
function simulateSendingAnImageNColumnsAtATime(minColumnsPerChunk, maxColumnsPerChunk, callback) {
var imageData = createImageToSend(640, 480);
// cut data into columns at start because this work would be done on
// the server
var columns = [];
var x = 0;
while (x < imageData.width) {
// how many columns are left?
var maxWidth = imageData.width - x;
// how many columns should we send
var columnWidth = Math.min(maxWidth, rand(minColumnsPerChunk, maxColumnsPerChunk + 1));
var data = createImageChunk(imageData, imageData.width - x - columnWidth, 0, columnWidth, imageData.height);
columns.push({
newImage: x === 0,
lines: columnWidth,
width: imageData.width,
height: imageData.height,
data: data,
});
x += columnWidth;
}
var columnNdx = 0;
sendNextColumn();
function sendNextColumn() {
if (columnNdx < columns.length) {
callback(columns[columnNdx++]);
if (columnNdx < columns.length) {
// should we make this random to siumlate network speed
var timeToNextChunkMS = 17;
setTimeout(sendNextColumn, timeToNextChunkMS);
}
}
}
}
function createImageChunk(imageData, x, y, width, height) {
var data = new Uint8Array(width * height);
for (var yy = 0; yy < height; ++yy) {
for (var xx = 0; xx < width; ++xx) {
var srcOffset = ((yy + y) * imageData.width + xx + x) * 4;
var dstOffset = yy * width + xx;
// compute gray scale
var gray = Math.max(imageData.data[srcOffset], imageData.data[srcOffset + 1], imageData.data[srcOffset + 2]);
data[dstOffset] = gray;
}
}
return data;
}
function rand(min, max) {
return Math.floor(Math.random() * max - min) + min;
}
function createImageToSend(width, height) {
// create a texture using a canvas so we don't have to download one
var ctx = document.createElement("canvas").getContext("2d");
ctx.width = width;
ctx.height = height;
ctx.fillStyle = "#222";
ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.lineWidth = 20;
["#AAA", "#888", "#666"].forEach(function(color, ndx, array) {
ctx.strokeStyle = color;
ctx.beginPath();
ctx.arc((ndx + 1) / (array.length + 1) * ctx.canvas.width, ctx.canvas.height / 2,
ctx.canvas.height * 0.4, 0, Math.PI * 2, false);
ctx.stroke();
});
ctx.fillStyle = "white";
ctx.font = "40px sans-serif";
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.fillText("Some Image", ctx.canvas.width / 2, ctx.canvas.height / 2);
return ctx.getImageData(0, 0, ctx.canvas.width, ctx.canvas.height);
}
canvas { border: 1px solid black; }
<script src="https://twgljs.org/dist/twgl-full.min.js"></script>
<script id="vs" type="not-js">
// we will always pass a 0 to 1 unit quad
// and then use matrices to manipulate it
attribute vec4 position;
uniform mat4 matrix;
uniform mat4 textureMatrix;
varying vec2 texcoord;
void main () {
gl_Position = matrix * position;
texcoord = (textureMatrix * position).xy;
}
</script>
<script id="fs" type="not-js">
precision mediump float;
varying vec2 texcoord;
uniform sampler2D texture;
void main() {
gl_FragColor = texture2D(texture, texcoord);
}
</script>
<canvas id="c" width="640" height="480"></canvas>
NOTE: This will not be smooth because it is using setTimeout to simulate receiving network data but that's exactly what you're likely seeing.
Here's a sample that rotates the image independently of updating the texture. You can see it runs perfectly smooth. The slowness is not WebGL, the slowness is networking (as simulated by setTimeout)
var m4 = twgl.m4;
var gl = document.getElementById("c").getContext("webgl");
// compiles shader, links and looks up locations
var programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);
// a unit quad
var arrays = {
position: {
numComponents: 2,
data: [
0, 0,
1, 0,
0, 1,
0, 1,
1, 0,
1, 1,
],
},
};
// calls gl.createBuffer, gl.bindBuffer, gl.bufferData for each array
var bufferInfo = twgl.createBufferInfoFromArrays(gl, arrays);
// we're only using 1 texture so just make and bind it now
var tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
var destColumn = 0;
var imageWidth;
var imageHeight;
// We're using 1 byte wide texture pieces so we need to
// set UNPACK_ALIGNMENT to 1 as it defaults to 4
gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
simulateSendingAnImageNColumnsAtATime(1, 1, addLinesToImageAndDraw);
function addLinesToImageAndDraw(imageAttr) {
if (imageAttr.newImage) {
destColumn = imageAttr.width;
imageWidth = imageAttr.width;
imageHeight = imageAttr.height;
gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, imageAttr.width, imageAttr.height, 0,
gl.LUMINANCE, gl.UNSIGNED_BYTE, null);
}
destColumn -= imageAttr.lines;
// should check it destColumn does not go negative!
gl.texSubImage2D(gl.TEXTURE_2D, 0, destColumn, 0, imageAttr.lines, imageAttr.height,
gl.LUMINANCE, gl.UNSIGNED_BYTE, imageAttr.data);
}
function render(time) {
if (imageWidth) {
var srcX = destColumn;
var srcY = 0;
var srcWidth = imageWidth - destColumn;
var srcHeight = imageHeight;
var dstX = destColumn * gl.canvas.width / imageWidth;
var dstY = 0;
var dstWidth = srcWidth * gl.canvas.width / imageWidth;
var dstHeight = gl.canvas.height;
var texWidth = imageWidth;
var texHeight = imageHeight;
var targetWidth = gl.canvas.width;
var targetHeight = gl.canvas.height;
drawImageWithRotation(
time * 0.001,
tex, texWidth, texHeight,
srcX, srcY, srcWidth, srcHeight,
dstX, dstY, dstWidth, dstHeight,
targetWidth, targetHeight);
}
requestAnimationFrame(render);
}
requestAnimationFrame(render);
// we pass in texWidth and texHeight because unlike images
// we can't look up the width and height of a texture
// we pass in targetWidth and targetHeight to tell it
// the size of the thing we're drawing too. We could look
// up the size of the canvas with gl.canvas.width and
// gl.canvas.height but maybe we want to draw to a framebuffer
// etc.. so might as well pass those in.
// srcX, srcY, srcWidth, srcHeight are in pixels
// computed from texWidth and texHeight
// dstX, dstY, dstWidth, dstHeight are in pixels
// computed from targetWidth and targetHeight
function drawImageWithRotation(
rotation,
tex, texWidth, texHeight,
srcX, srcY, srcWidth, srcHeight,
dstX, dstY, dstWidth, dstHeight,
targetWidth, targetHeight) {
var mat = m4.identity();
var tmat = m4.identity();
var uniforms = {
matrix: mat,
textureMatrix: tmat,
texture: tex,
};
// these adjust the unit quad to generate texture coordinates
// to select part of the src texture
// NOTE: no check is done that srcX + srcWidth go outside of the
// texture or are in range in any way. Same for srcY + srcHeight
m4.translate(tmat, [srcX / texWidth, srcY / texHeight, 0], tmat);
m4.scale(tmat, [srcWidth / texWidth, srcHeight / texHeight, 1], tmat);
// convert from pixels to clipspace
m4.ortho(0, targetWidth, targetHeight, 0, -1, 1, mat);
// rotate around center of canvas
m4.translate(mat, [targetWidth / 2, targetHeight / 2, 0], mat);
m4.rotateZ(mat, rotation, mat);
m4.translate(mat, [-targetWidth / 2, -targetHeight / 2, 0], mat);
// these move and scale the unit quad into the size we want
// in the target as pixels
m4.translate(mat, [dstX, dstY, 0], mat);
m4.scale(mat, [dstWidth, dstHeight, 1], mat);
gl.useProgram(programInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
// calls gl.uniformXXX, gl.activeTexture, gl.bindTexture
twgl.setUniforms(programInfo, uniforms);
// calls gl.drawArray or gl.drawElements
twgl.drawBufferInfo(gl, bufferInfo);
}
// =====================================================================
// Everything below this line represents stuff from the server.
// so it's irrelevant to the answer
//
function simulateSendingAnImageNColumnsAtATime(minColumnsPerChunk, maxColumnsPerChunk, callback) {
var imageData = createImageToSend(640, 480);
// cut data into columns at start because this work would be done on
// the server
var columns = [];
var x = 0;
while (x < imageData.width) {
// how many columns are left?
var maxWidth = imageData.width - x;
// how many columns should we send
var columnWidth = Math.min(maxWidth, rand(minColumnsPerChunk, maxColumnsPerChunk + 1));
var data = createImageChunk(imageData, imageData.width - x - columnWidth, 0, columnWidth, imageData.height);
columns.push({
newImage: x === 0,
lines: columnWidth,
width: imageData.width,
height: imageData.height,
data: data,
});
x += columnWidth;
}
var columnNdx = 0;
sendNextColumn();
function sendNextColumn() {
if (columnNdx < columns.length) {
callback(columns[columnNdx++]);
if (columnNdx < columns.length) {
// should we make this random to siumlate network speed
var timeToNextChunkMS = 17;
setTimeout(sendNextColumn, timeToNextChunkMS);
}
}
}
}
function createImageChunk(imageData, x, y, width, height) {
var data = new Uint8Array(width * height);
for (var yy = 0; yy < height; ++yy) {
for (var xx = 0; xx < width; ++xx) {
var srcOffset = ((yy + y) * imageData.width + xx + x) * 4;
var dstOffset = yy * width + xx;
// compute gray scale
var gray = Math.max(imageData.data[srcOffset], imageData.data[srcOffset + 1], imageData.data[srcOffset + 2]);
data[dstOffset] = gray;
}
}
return data;
}
function rand(min, max) {
return Math.floor(Math.random() * max - min) + min;
}
function createImageToSend(width, height) {
// create a texture using a canvas so we don't have to download one
var ctx = document.createElement("canvas").getContext("2d");
ctx.width = width;
ctx.height = height;
ctx.fillStyle = "#222";
ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.lineWidth = 20;
["#AAA", "#888", "#666"].forEach(function(color, ndx, array) {
ctx.strokeStyle = color;
ctx.beginPath();
ctx.arc((ndx + 1) / (array.length + 1) * ctx.canvas.width, ctx.canvas.height / 2,
ctx.canvas.height * 0.4, 0, Math.PI * 2, false);
ctx.stroke();
});
ctx.fillStyle = "white";
ctx.font = "40px sans-serif";
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.fillText("Some Image", ctx.canvas.width / 2, ctx.canvas.height / 2);
return ctx.getImageData(0, 0, ctx.canvas.width, ctx.canvas.height);
}
canvas { border: 1px solid black; }
<script src="https://twgljs.org/dist/3.x/twgl-full.min.js"></script>
<script id="vs" type="not-js">
// we will always pass a 0 to 1 unit quad
// and then use matrices to manipulate it
attribute vec4 position;
uniform mat4 matrix;
uniform mat4 textureMatrix;
varying vec2 texcoord;
void main () {
gl_Position = matrix * position;
texcoord = (textureMatrix * position).xy;
}
</script>
<script id="fs" type="not-js">
precision mediump float;
varying vec2 texcoord;
uniform sampler2D texture;
void main() {
gl_FragColor = texture2D(texture, texcoord);
}
</script>
<canvas id="c" width="640" height="480"></canvas>
gl.texImage2D is slow and there is not much that can be done to improve on this. The reason is that texImage2D involves a state change and requires that the GPU halt all rendering and then fetch the data from CPU RAM. Depending on the hardware the interface between the main board and GPU can be very slow (in comparison to RAM access speed)
You also add to the problem with the resolution of the image. All images on the GPU have sizes that are powers of 2 (32,64,128,256,512,1024...) independently for height and width. Sending an image that is 640 by 480 does not fit this rule. To accommodate the bad size the GPU will allocate an image that is W 1024 by H 512 pixels and will thus have to re dimension the image data to fit the internal dimensions (Fast as it they are this is not something they are good at). Depending on hardware this will cause an additional slowdown on top of the already slow data transfer.
You may get a slight improvement if you make your data buffer equal to the powers of two rule (POT) (1024, 512).
Your best option is to avoid the transfer until the entire image has loaded then do it just once.
If you really need it live then I suggest you divide the image into smaller images and send the smaller separate images. For example for the POT image size 1024 by 512 could be divided into 128 by 64 images or 1024 by 8 resulting in 64 smaller images. Only send the small images as they become available and on the GPU reassemble the images as one during render. This will give you an almost 64 times improvement on the time it takes to send the image to the GPU.
Apart from that there is not much else that can be done. GPUs are good at rendering, GPUs suck when it comes to mainboard IO, avoid this at all costs (during rendering) to get the most out of the graphics hardware.
How do you blend two arrays of pixel data to create one image? with the option of using different blending modes?
Pixastic is a special framework for advanced use of canvas, here are blending examples: http://www.pixastic.com/lib/docs/actions/blend/
If you would like do this alone, you can extract pixel data from 2 images, blend it with a mathematical equation, and put into a canvas. Here is information how to get and put pixel data from/to canvas:
http://ajaxian.com/archives/canvas-image-data-optimization-tip
Update:
Simple example with alpha blending of 2 images in proportion 50-50.
(Images borrowed from http://www.pixastic.com/sample/Butterfly.jpg and http://www.pixastic.com/sample/Flower.jpg )
<img src="Butterfly.jpg" id="img1">
<img src="Flower.jpg" id="img2">
<p>Blended image<br><canvas id="canvas"></canvas></p>
<script>
window.onload = function () {
var img1 = document.getElementById('img1');
var img2 = document.getElementById('img2');
var canvas = document.getElementById("canvas");
var context = canvas.getContext("2d");
var width = img1.width;
var height = img1.height;
canvas.width = width;
canvas.height = height;
var pixels = 4 * width * height;
context.drawImage(img1, 0, 0);
var image1 = context.getImageData(0, 0, width, height);
var imageData1 = image1.data;
context.drawImage(img2, 0, 0);
var image2 = context.getImageData(0, 0, width, height);
var imageData2 = image2.data;
while (pixels--) {
imageData1[pixels] = imageData1[pixels] * 0.5 + imageData2[pixels] * 0.5;
}
image1.data = imageData1;
context.putImageData(image1, 0, 0);
};
</script>
I have created a separate, lightweight, open-source library for perform Photoshop-style blend modes from one HTML Canvas context to another: context-blender. Here's the sample usage:
// Might be an 'offscreen' canvas
var over = someCanvas.getContext('2d');
var under = anotherCanvas.getContext('2d');
over.blendOnto( under, 'screen', {destX:30,destY:15} );
See the README for more information.
I am tasked with recreating this java applet using JavaScript (must be tablet friendly, and work in all modern browsers > IE8).
I am creating images using: var image1 = new Image(); and then setting source: img.src = "some path";
So, from pepkin88 I see that the following function will blend two images by combining their pixel array data, overriding previous data from the first image with the new blended data, and finally putting the new data on the canvas resulting in a blended image:
window.onload = function () {
var img1 = document.getElementById('img1');
var img2 = document.getElementById('img2');
var canvas = document.getElementById("canvas");
var context = canvas.getContext("2d");
var width = img1.width;
var height = img1.height;
canvas.width = width;
canvas.height = height;
var pixels = 4 * width * height;
context.drawImage(img1, 0, 0);
var image1 = context.getImageData(0, 0, width, height);
var imageData1 = image1.data;
context.drawImage(img2, 0, 0);
var image2 = context.getImageData(0, 0, width, height);
var imageData2 = image2.data;
while (pixels--) {
imageData1[pixels] = imageData1[pixels] * 0.5 + imageData2[pixels] * 0.5;
}
image1.data = imageData1;
context.putImageData(image1, 0, 0); };
HOWEVER, if you viewed the java applet that I'm responsible for recreating, you see that blending happens in real-time continuously as you drag the image around with the pointer the images are constantly blending based on their overlapped regions..
SO, I'm looking to modify the code to account for this, and I continually have the x, y, positions of images drawn (based on top left corner), and the w, h of all images stays static:
the following snippets don't include everything I'm doing, just what I sense is important for you to know
//Rectangle Class from Java converted to JS
function Rectangle(x, y, width, height, src) {
this.x = x;
this.y = y;
this.w = width;
this.h = height;
this.img = new Image();
this.img.src = src;
}
//Stores instance in rect array
rect[0] = new Rectangle(1, (height - 111)/2, 150, 105, "images/mMain.png");
//Draw method that's called
Rectangle.prototype.draw = function(ctx) {
//this.checkBound();
ctx.drawImage(this.img, this.x, this.y, this.w, this.h);
prepareMix(this.img, this.x, this.y, this.w, this.h);
}
So, I'm working on a prepareMix function that receives image info and uses it to get and store image data:
function prepareMix(src, x, y, w, h) {
pixels = 4 * w * h;
var image = mtx.getImageData(x, y, w, h);
var imgData = image.data;
}
Made a list of what to do:
Sense the overlapping
Get and Store the overlapping image data
Mix the overlapping region data arrays
Replace the overlapping image data with the blended data
Put the new data on the canvas
1. Sense the Overlapping:
Plan: Store image positions and compare positions data to know whether or not overlapping is occurring.
IF overlapping is TRUE, which two images is it true for? Distinguish these images that're overlapping from other images so that methods can be called on them.
js, css, html, and images in zip here BOX