I am trying to render two objects, with two separate gl.drawArrays call. I want to have transparent parts of the objects not be visible. Also I want to render one object on top of another so the first drawn object is not visible where it overlaps the second one.
I use this setup on my render loop:
gl.clearColor(0, 0, 0, 1);
// https://stackoverflow.com/questions/18439897/webgl-fragment-shader-opacity
gl.blendFunc(gl.SRC_ALPHA, gl.ONE);
gl.enable(gl.BLEND);
gl.disable(gl.DEPTH_TEST);
I am not sure what the blend functions do but I use them so the transparency is enabled. But that causes two objects to blend and create a yellow colour. (One object is red, other is green). I want to have red if i draw red last, and vice versa, while having transparency enabled.
const fShaderSource2 = `#version 300 es
precision mediump float;
out vec4 outColor;
void main() {
outColor = vec4(0.0, 1.0, 0.0, 1.0);
}
`;
const fShaderSource = `#version 300 es
precision mediump float;
out vec4 outColor;
uniform sampler2D u_texture;
void main() {
outColor = texture(u_texture, vec2(0.0));
}
`;
const vShaderSource = `#version 300 es
precision mediump float;
in vec2 a_position;
void main() {
gl_Position = vec4(a_position, 0, 1);
}
`;
main(document.getElementById('app'));
function main(element) {
const canvas = document.createElement('canvas'),
gl = canvas.getContext('webgl2');
element.append(canvas);
const displayWidth = canvas.clientWidth,
displayHeight = canvas.clientHeight;
canvas.width = displayWidth;
canvas.height = displayHeight;
let graphics = new Graphics({width: displayWidth, height: displayHeight}, gl);
new Loop(() => {
graphics.render();
}).start();
}
function Graphics(state, gl) {
const { width, height } = state;
gl.clearColor(0, 0, 0, 0);
gl.blendFunc(gl.SRC_ALPHA, gl.ONE);
gl.enable(gl.BLEND);
gl.disable(gl.DEPTH_TEST);
let minibatch = [];
const redText = makeGlQuad(gl, fShaderSource, canvasTexture());
const greenText = makeGlQuad(gl, fShaderSource2);
this.render = () => {
minibatch.push(redText);
minibatch.push(greenText);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clear(gl.COLOR_BUFFER_BIT);
minibatch.forEach(({
program,
resUniformLocation,
vao,
glTexture
}) => {
gl.useProgram(program);
gl.uniform2f(resUniformLocation, gl.canvas.width, gl.canvas.height);
if (glTexture) {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, glTexture);
}
gl.bindVertexArray(vao);
gl.drawArrays(gl.TRIANGLES, 0, 6);
});
minibatch = [];
};
}
function makeGlQuad(gl, fShaderSource, texture) {
let vShader = createShader(gl, gl.VERTEX_SHADER, vShaderSource);
let fShader = createShader(gl, gl.FRAGMENT_SHADER, fShaderSource);
let program = createProgram(gl, vShader, fShader);
let posAttrLocation = gl.getAttribLocation(program, "a_position");
let posBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, posBuffer);
/*
(-1, 1).( 1, 1)
.
(-1,-1).( 1,-1)
*/
let positions = [
-1, 1,
-1, -1,
1, -1,
-1, 1,
1,-1,
1, 1
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
let vao = gl.createVertexArray();
gl.bindVertexArray(vao);
gl.enableVertexAttribArray(posAttrLocation);
let size = 2,
type = gl.FLOAT,
normalize = false,
stride = 0,
offset = 0;
gl.vertexAttribPointer(posAttrLocation,
size,
type,
normalize,
stride,
offset);
let glTexture;
if (texture) {
glTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, glTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, texture);
//gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
}
let resUniformLocation = gl.getUniformLocation(program, "u_resolution");
let texUniformLocation = gl.getUniformLocation(program, "u_texture");
return {
program,
resUniformLocation,
vao,
glTexture
}
}
function canvasTexture() {
return withCanvasTexture(256, 256, (w, h, canvas, ctx) => {
ctx.fillStyle = 'red';
ctx.fillRect(0, 0, w, h);
ctx.font = '50pt Comic Sans';
ctx.fillStyle = 'white';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
ctx.fillText('label', w / 2, 50);
return canvas;
});
function withCanvasTexture(width, height, f) {
var canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
f(width, height, canvas, canvas.getContext('2d'));
const texture = canvas;
//document.body.append(canvas);
return texture;
}
}
function createShader(gl, type, source) {
let shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
let success = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
if (success) {
return shader;
}
console.error(gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
};
function createProgram(gl, vShader, fShader) {
let program = gl.createProgram();
gl.attachShader(program, vShader);
gl.attachShader(program, fShader);
gl.linkProgram(program);
let success = gl.getProgramParameter(program, gl.LINK_STATUS);
if (success) {
return program;
}
console.error(gl.getProgramInfoLog(program));
gl.deleteProgram(program);
return null;
}
// Loop Library
function Loop(fn) {
const perf = window.performance !== undefined ? window.performance : Date;
const now = () => perf.now();
const raf = window.requestAnimationFrame;
let running = false,
lastUpdate = now(),
frame = 0;
this.start = () => {
if (running) {
return this;
}
running = true;
lastUpdate = now();
frame = raf(tick);
return this;
};
this.stop = () => {
running = false;
if (frame != 0) {
raf.cancel(frame);
}
frame = 0;
return this;
};
const tick = () => {
frame = raf(tick);
const time = now();
const dt = time - lastUpdate;
fn(dt);
lastUpdate = time;
};
}
#app canvas {
position: fixed;
top: 50%;
bottom: 0;
left: 50%;
right: 0;
width: 100vmin;
height: 70vmin;
transform: translate(-50%, -25%);
image-rendering: optimizeSpeed;
cursor: none;
margin: auto;
}
<div id="app">
</div>
You can see here:
minibatch.push(redText);
minibatch.push(greenText);
I render red first and then green second but I get yellow instead.
The issue is that you use the wrong blend function (blendFunc). Blending defines a function which combines the fragment color outputs with the current colors in the color buffers. The fist parameter is the factor for the fragment color output and the 2nd parameter the factor for the color in the color buffer. The colors are summed, because the default blend equation (blendEquation()) is FUNC_ADD.
So the blend function
gl.blendFunc(gl.SRC_ALPHA, gl.ONE);
can be expressed by the formula
destColor = srcColor * srcAlpha + destColor * 1
where destColor is the current color in the framebuffer, srcColor is the color which is set to fragment (outColor).
This causes that the color in the current framebuffer is kept (multiplied by 1). The new color is multiplied by the alpha channel and add to the color in the framebuffer. If the color in the framebuffer is red (1, 0, 0) and the new color is green (0, 1, 0), then the result is yellow (if the alpha channel is 1):
(0, 1, 0) * 1 + (1, 0, 0) * 1 == (1, 1, 0)
Use the blend function :
gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
which cause that the color in the frambuffer and the new color are "mixed", dependent on the alpha channel of the new color:
destColor = srcColor * srcAlpha + destColor * (1-srcAlpha)
The general principle of Alpha blending is similar as in OpenGL (since WebGL (2.0) conforms closely to the OpenGL ES (3.0)), so further information can be found at the OpenGL wiki page Blending.
Related
I'm trying to replace all RGB pixels with the value of 0 to 1 (out of the max value of 255).
Here is my code on stackblitz.
You can see that after I'm reassigning the buffer with the new pixels some of the pixles are back to 0. On Firefox it's actually works with this image:
But won't work with the full size image:
It's like the browser won't allow a certain contrast of something like that.
Sharing my code here as well:
const getBase64FromFile = async (file: File): Promise<string> => {
return new Promise((resolve: Function, reject: Function) => {
let reader = new FileReader();
reader.addEventListener(
'load',
(arg) => {
resolve(reader.result);
},
false
);
reader.readAsDataURL(file);
});
};
// Returns the amount of pixels with RGB 0 value
const howManyZeros = async (src: string): Promise<number> => {
return new Promise((resolve: Function, reject: Function) => {
const image = new Image();
image.onload = () => {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
canvas.width = image.naturalWidth;
canvas.height = image.naturalHeight;
ctx.drawImage(image, 0, 0);
const data = ctx.getImageData(
0,
0,
image.naturalWidth,
image.naturalHeight
).data;
let zeros = 0;
for (var i = 0; i < data.length; i += 4) {
if (data[i] === 0) zeros++;
if (data[i + 1] === 0) zeros++;
if (data[i + 2] === 0) zeros++;
}
resolve(zeros);
};
image.src = src;
});
};
const onFinish = async (src: string) => {
document.querySelector(
'p#after'
).textContent = `nunber of zeros after: ${await howManyZeros(src)}`;
(document.querySelector('img#after-img') as HTMLImageElement).src = src;
const a = document.querySelector('a');
a.setAttribute('href', src);
a.setAttribute('download', 'image.png');
a.style.display = '';
};
const onFileChange = async (e: Event | any) => {
const image = new Image();
image.onload = async () => {
const canvas = document.createElement('canvas');
canvas.width = image.naturalWidth;
canvas.height = image.naturalHeight;
const ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0);
let data = ctx.getImageData(
0,
0,
image.naturalWidth,
image.naturalHeight
).data;
let buffer = new Uint8ClampedArray(
image.naturalWidth * image.naturalHeight * 4
);
// Iterate over all the pixels and increase all RGB 0 values to 1
for (var i = 0; i < data.length; i += 4) {
if (data[i] === 0) buffer[i] = 1;
else buffer[i] = data[i];
if (data[i + 1] === 0) buffer[i + 1] = 1;
else buffer[i + 1] = data[i + 1];
if (data[i + 2] === 0) buffer[i + 2] = 1;
else buffer[i + 2] = data[i + 2];
buffer[i + 3] = data[i + 3];
}
const iData = ctx.createImageData(image.naturalWidth, image.naturalHeight);
iData.data.set(buffer);
ctx.putImageData(iData, 0, 0);
onFinish(canvas.toDataURL('image/png', 1));
};
let src = await getBase64FromFile(e.target.files[0]);
document.querySelector(
'p#before'
).textContent = `nunber of zeros before: ${await howManyZeros(src)}`;
(document.querySelector('img#before-img') as HTMLImageElement).src = src;
image.src = src;
};
const input: HTMLInputElement = document.querySelector('input');
input.addEventListener('change', onFileChange, false);
Appreciate any help with this and praying that's not a browser issue but something with my code.
Due to canvas spec, which doesn't guarantee that the pixels stay the same as you set them, you can't use browser built in image manipulation functions.
Due to the lossy nature of converting between color spaces and converting to and from premultiplied alpha color values, pixels that have just been set using putImageData(), and are not completely opaque, might be returned to an equivalent getImageData() as different values.
In your case pixel values with high transparency get turned to 0 again.
This doesn't happen in WebGL context with the premultipliedAlpha context attribute set to false, but the solution involves a lot of code.
The following code is based on the example from WebGL2 Fundamentals:
const vertexShaderSource = `#version 300 es
// an attribute is an input (in) to a vertex shader.
// It will receive data from a buffer
in vec2 a_position;
in vec2 a_texCoord;
// Used to pass in the resolution of the canvas
uniform vec2 u_resolution;
// Used to pass the texture coordinates to the fragment shader
out vec2 v_texCoord;
// all shaders have a main function
void main() {
// convert the position from pixels to 0.0 to 1.0
vec2 zeroToOne = a_position / u_resolution;
// convert from 0->1 to 0->2
vec2 zeroToTwo = zeroToOne * 2.0;
// convert from 0->2 to -1->+1 (clipspace)
vec2 clipSpace = zeroToTwo - 1.0;
gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
// pass the texCoord to the fragment shader
// The GPU will interpolate this value between points.
v_texCoord = a_texCoord;
}`;
const fragmentShaderSource = `#version 300 es
// fragment shaders don't have a default precision so we need
// to pick one. highp is a good default. It means "high precision"
precision highp float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
in vec2 v_texCoord;
// we need to declare an output for the fragment shader
out vec4 outColor;
void main() {
vec4 inColor = texture(u_image, v_texCoord);
outColor = vec4(
inColor.r != 0.0 ? inColor.r : 1.0/255.0,
inColor.g != 0.0 ? inColor.g : 1.0/255.0,
inColor.b != 0.0 ? inColor.b : 1.0/255.0,
inColor.a
);
}`;
function readFile(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = function() {
resolve(this.result);
}
reader.onerror = reject;
reader.readAsArrayBuffer(file);
});
}
function loadImage(url) {
return new Promise(async(resolve, reject) => {
const image = new Image();
image.onload = function() {
resolve(this);
}
image.onerror = reject;
image.src = url;
})
}
function canvasToBlob(canvas) {
return new Promise((resolve, reject) => {
canvas.toBlob(blob => blob ? resolve(blob) : reject(canvas), "image/png");
});
}
function howManyZeros(gl) {
gl.drawBuffers([gl.COLOR_ATTACHMENT0]);
let data = new Uint8Array(gl.canvas.width * gl.canvas.height * 4);
gl.readPixels(0, 0, gl.canvas.width, gl.canvas.height, gl.RGBA, gl.UNSIGNED_BYTE, data);
let zeros = 0;
for (let i=0; i<data.length; i++) {
if (i % 4 == 3) continue; // ignore alpha
if (data[i] == 0) zeros++;
}
return zeros;
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1,
x2, y1,
x1, y2,
x1, y2,
x2, y1,
x2, y2,
]), gl.STATIC_DRAW);
}
function render(image) {
const canvas = document.createElement("canvas");
canvas.width = image.width;
canvas.height = image.height;
const gl = canvas.getContext("webgl2", {
premultipliedAlpha: false
});
if (!gl) {
console.error("No WebGL2");
return;
}
canvas.gl = gl;
// setup GLSL program
const program = webglUtils.createProgramFromSources(gl, [vertexShaderSource, fragmentShaderSource]);
// look up where the vertex data needs to go.
const positionAttributeLocation = gl.getAttribLocation(program, "a_position");
const texCoordAttributeLocation = gl.getAttribLocation(program, "a_texCoord");
// lookup uniforms
const resolutionLocation = gl.getUniformLocation(program, "u_resolution");
const imageLocation = gl.getUniformLocation(program, "u_image");
// Create a vertex array object (attribute state)
const vao = gl.createVertexArray();
// and make it the one we're currently working with
gl.bindVertexArray(vao);
// Create a buffer and put a single pixel space rectangle in
// it (2 triangles)
const positionBuffer = gl.createBuffer();
// Turn on the attribute
gl.enableVertexAttribArray(positionAttributeLocation);
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Tell the attribute how to get data out of positionBuffer (ARRAY_BUFFER)
let size = 2; // 2 components per iteration
let type = gl.FLOAT; // the data is 32bit floats
let normalize = false; // don't normalize the data
let stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
let offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
positionAttributeLocation, size, type, normalize, stride, offset);
// provide texture coordinates for the rectangle.
const texCoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]), gl.STATIC_DRAW);
// Turn on the attribute
gl.enableVertexAttribArray(texCoordAttributeLocation);
// Tell the attribute how to get data out of texCoordBuffer (ARRAY_BUFFER)
size = 2; // 2 components per iteration
type = gl.FLOAT; // the data is 32bit floats
normalize = false; // don't normalize the data
stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
texCoordAttributeLocation, size, type, normalize, stride, offset);
// Create a texture.
const texture = gl.createTexture();
// make unit 0 the active texture uint
// (ie, the unit all other texture commands will affect
gl.activeTexture(gl.TEXTURE0 + 0);
// Bind it to texture unit 0' 2D bind point
gl.bindTexture(gl.TEXTURE_2D, texture);
// Set the parameters so we don't need mips and so we're not filtering
// and we don't repeat
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
// Upload the image into the texture.
const mipLevel = 0; // the largest mip
const internalFormat = gl.RGBA; // format we want in the texture
const srcFormat = gl.RGBA; // format of data we are supplying
const srcType = gl.UNSIGNED_BYTE; // type of data we are supplying
gl.texImage2D(gl.TEXTURE_2D,
mipLevel,
internalFormat,
srcFormat,
srcType,
image);
// Tell WebGL how to convert from clip space to pixels
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Clear the canvas
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
// Tell it to use our program (pair of shaders)
gl.useProgram(program);
// Bind the attribute/buffer set we want.
gl.bindVertexArray(vao);
// Pass in the canvas resolution so we can convert from
// pixels to clipspace in the shader
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
// Tell the shader to get the texture from texture unit 0
gl.uniform1i(imageLocation, 0);
// Bind the position buffer so gl.bufferData that will be called
// in setRectangle puts data in the position buffer
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Set a rectangle the same size as the image.
setRectangle(gl, 0, 0, image.width, image.height);
// Draw the rectangle.
const primitiveType = gl.TRIANGLES;
offset = 0;
const count = 6;
gl.drawArrays(primitiveType, offset, count);
return canvas;
}
async function onFileChange(e) {
const png_data = await readFile(e.target.files[0]);
const png_blob = new Blob([png_data], {
type: 'image/png'
});
const png_url = URL.createObjectURL(png_blob);
document.querySelector("#before-img").src = png_url;
const image = await loadImage(png_url);
let canvas = render(image);
document.querySelector("#after").textContent = `nunber of zeros after: ${howManyZeros(canvas.gl)}`
const new_png_blob = await canvasToBlob(canvas);
const new_png_url = URL.createObjectURL(new_png_blob);
const dl_link = document.querySelector("a");
dl_link.href = new_png_url;
dl_link.style.display = "";
document.querySelector("#after-img").src = new_png_url;
};
const input = document.querySelector('input[type=file]');
input.addEventListener('change', onFileChange, false);
<input type="file" />
<div>
<a style="display: none" download="image.png">download</a>
</div>
<div>
<p id="before"></p>
<img id="before-img" src="" />
</div>
<div>
<p id="after"></p>
<img id="after-img" src="" />
</div>
<script src="https://webgl2fundamentals.org/webgl/resources/webgl-utils.js"></script>
Alternatively you could do it with a 3rd party image manupulation library.
Here's an example using the UPNG.js library:
function readFile(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = function() {
resolve(this.result);
}
reader.onerror = reject;
reader.readAsArrayBuffer(file);
});
}
async function onFileChange(e) {
const png_data = await readFile(e.target.files[0]);
const png_blob = new Blob([png_data], {
type: 'image/png'
});
const png_url = URL.createObjectURL(png_blob);
document.querySelector("#before-img").src = png_url;
const png = UPNG.decode(png_data);
const png_frames = UPNG.toRGBA8(png);
const png_frame = new Uint8Array(png_frames[0]);
for (var i = 0; i < png_frame.length; i += 4) {
png_frame[i + 0] = png_frame[i + 0] == 0 ? 1 : png_frame[i + 0];
png_frame[i + 1] = png_frame[i + 1] == 0 ? 1 : png_frame[i + 1];
png_frame[i + 2] = png_frame[i + 2] == 0 ? 1 : png_frame[i + 2];
//png_frame[i+3] = 255; // remove transparency
}
const new_png_data = UPNG.encode([png_frame.buffer], png.width, png.height, 0);
const new_png_blob = new Blob([new_png_data], {
type: 'image/png'
});
const new_png_url = URL.createObjectURL(new_png_blob);
/*const new_png_url = "data:image/png;base64," + btoa(String.fromCharCode.apply(null, new Uint8Array(new_png_data)));*/
const dl_link = document.querySelector("a");
dl_link.href = new_png_url;
dl_link.style.display = "";
document.querySelector("#after-img").src = new_png_url;
};
const input = document.querySelector('input[type=file]');
input.addEventListener('change', onFileChange, false);
<input type="file" />
<div>
<a style="display: none" download="image.png">download</a>
</div>
<div>
<p id="before"></p>
<img id="before-img" src="" />
</div>
<div>
<p id="after"></p>
<img id="after-img" src="" />
</div>
<script type="module">
import UPNG from "https://cdn.skypack.dev/upng-js#2.1.0"; window.UPNG = UPNG;
</script>
I've converted a RGBA image to greyscale image using webgl.
When reading the pixel using gl.readPixels() with gl.RGBA format, getting the values for each pixel as YYYA because an RGBA pixel is converted to YYYA and assigned to gl_FragColor. I want only 1 byte Y component for each pixel instead of 4 bytes.
Tried reading the pixels with gl.RED format(instead of gl.RGBA)
gl.readPixels(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight, gl.RED, gl.UNSIGNED_BYTE, pixels);
but getting the following error on Chrome and getting only zeroes.
WebGL: INVALID_ENUM: readPixels: invalid format
Is it possible to make gl_FragColor to output 1 byte per pixel in LUMINANCE mode, instead of RGBA, but the input texture has to be RGBA?
If the format of gl rendering cannot be changed, is it possible to read only the first byte of each 4 bytes pixel, when calling gl.readPixels()?
Note:
3. I've already done copy the gl.readPixels() output to another array by jumping every 4 bytes. But I want to avoid this copy as it takes more time.
4. Also, I need the solution to be a mobile browser(ios safari and android chrome) compatible.
function webGL() {
var gTexture;
var gFramebuffer;
var srcCanvas = null;
var programs = {};
var program;
var pixels;
this.convertRGBA2Gray = function(inCanvas, inArray) {
// Y component from YCbCr
const shaderSourceRGB2GRAY = `
precision mediump float;
uniform sampler2D u_image;
uniform vec2 u_textureSize;
vec4 scale = vec4(0.299, 0.587, 0.114, 0.0);
void main() {
vec4 color = texture2D(u_image, gl_FragCoord.xy / u_textureSize);
gl_FragColor = vec4(vec3(dot(color,scale)), color.a);
}`;
if (srcCanvas === null) {
console.log('Setting up webgl');
srcCanvas = inCanvas;
_initialize(srcCanvas.width, srcCanvas.height);
program = _createProgram("rgb2grey", shaderSourceRGB2GRAY);
}
pixels = inArray;
_run(program);
}
///////////////////////////////////////
// Private functions
var _getWebGLContext = function(canvas) {
try {
return (canvas.getContext("webgl", {premultipliedAlpha: false, preserveDrawingBuffer: true}) || canvas.getContext("experimental-webgl", {premultipliedAlpha: false, preserveDrawingBuffer: true}));
}
catch(e) {
console.log("ERROR: %o", e);
}
return null;
}
var gl = _getWebGLContext(document.createElement('canvas'));
var _initialize = function(width, height) {
var canvas = gl.canvas;
canvas.width = width;
canvas.height = height;
if (this.originalImageTexture) {
return;
}
this.originalImageTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, this.originalImageTexture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, gTexture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, canvas.width, canvas.height, 0,
gl.RGBA, gl.UNSIGNED_BYTE, null);
gFramebuffer = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, gFramebuffer);
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
-1.0, 1.0,
1.0, 1.0,
1.0, -1.0,
-1.0, 1.0,
1.0, -1.0,
-1.0, -1.0
]), gl.STATIC_DRAW);
gl.framebufferTexture2D(
gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, gTexture, 0);
gl.bindTexture(gl.TEXTURE_2D, this.originalImageTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, srcCanvas);
}
var _createProgram = function(name, fragmentSource, vertexSource) {
shaderProgram = programs[name];
if (shaderProgram){
console.log('Reusing program');
gl.useProgram(shaderProgram);
return shaderProgram;
}
function createShader(type, source){
var shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
return shader;
}
var vertexShader, fragmentShader;
if (!vertexSource){
vertexShader = createShader(gl.VERTEX_SHADER, `attribute vec2 a_position;
void main() { gl_Position = vec4(a_position, 0.0, 1.0); }`
);
} else {
vertexShader = createShader(gl.VERTEX_SHADER, vertexSource);
}
fragmentShader = createShader(gl.FRAGMENT_SHADER, fragmentSource);
shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
gl.useProgram(shaderProgram);
return shaderProgram;
}
var _render = function(gl, program){
var positionLocation = gl.getAttribLocation(program, "a_position");
var u_imageLoc = gl.getUniformLocation(program, "u_image");
var textureSizeLocation = gl.getUniformLocation(program, "u_textureSize");
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
gl.enableVertexAttribArray(positionLocation);
var width = gl.canvas.width,
height = gl.canvas.height;
gl.bindFramebuffer(gl.FRAMEBUFFER, gFramebuffer);
gl.uniform2f(textureSizeLocation, width, height);
gl.uniform1i(u_imageLoc, 0);
gl.viewport(0, 0, width, height);
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
var _run = function(program){
let t0 = performance.now();
_render(gl, program);
gl.bindTexture(gl.TEXTURE_2D, gTexture);
let t1 = performance.now();
// gl.readPixels(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
gl.readPixels(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight, gl.RED, gl.UNSIGNED_BYTE, pixels);
let t2 = performance.now();
console.log('_render dur = ' + Number((t1-t0).toFixed(3)) + ' ms');
console.log('_run dur = ' + Number((t2-t0).toFixed(3)) + ' ms');
}
};
<div>
<canvas id="source"></canvas>
</div>
<script src="webgl.js" type="text/javascript"></script>
<script>
window.addEventListener('load', function(e) {
var srcImg = new Image();
srcImg.crossOrigin = "anonymous";
srcImg.src = "https://i.picsum.photos/id/17/480/480.jpg";
srcImg.width = 480;
srcImg.height = 480;
srcImg.onload = function(){
// image has been loaded
let srcCanvas = document.getElementById("source");
srcCanvas.width = srcImg.width;
srcCanvas.height = srcImg.height;
let ctx = srcCanvas.getContext('2d');
ctx.drawImage(srcImg, 0, 0, srcImg.width, srcImg.height);
var webgl = new webGL();
let pixels = new Uint8Array(srcCanvas.width * srcCanvas.height * 4);
webgl.convertRGBA2Gray(srcCanvas, pixels);
var outData = ctx.createImageData(srcCanvas.width, srcCanvas.height);
console.log('\n');
for (let k = 0; k < 12; ++k) {
console.log(pixels[k] + ', ');
}
console.log('\n');
// Luminance plot
for (let i = 0, j = 0; i < (srcCanvas.width * srcCanvas.height * 4); i+=4, ++j ) {
outData.data[i] = outData.data[i+1] = outData.data[i+2] = pixels[j];
outData.data[i+3] = 255;
}
// RGB plot
// for ( let i = 0; i < (srcCanvas.width * srcCanvas.height * 4); ++i ) {
// outData.data[i] = pixels[i];
// }
srcCanvas.getContext('2d').putImageData(outData, 0, 0);
};
}, true);
</script>
Is it possible to make gl_FragColor to output 1 byte per pixel in LUMINANCE mode, instead of RGBA, but the input texture has to be RGBA?
Not portably. The spec for WebGL1 says rendering to a texture only has to be supported for gl.RGBA / gl.UNSIGNED_BYTE. All other formats are optional.
If the format of gl rendering cannot be changed, is it possible to read only the first byte of each 4 bytes pixel, when calling gl.readPixels()?
No, The spec section 4.3.1 says only gl.RGBA, gl.UNSIGNED_BYTE is supported. All other formats are optional and up to the implementation. This is the same on WebGL2. Even if you make a R8 texture (red only, 8 bits) it's up to the implemenation if if you can read it as gl.RED/gl.UNSIGNED_BYTE.
See Webgl1 and Webgl2
I want to rotate and translate a 2d shape made with signed distance functions.
The docs say this is the method:
vec3 opTx( in vec3 p, in transform t, in sdf3d primitive )
{
return primitive( invert(t)*p );
}
It looks to me like primitive is some kind of function (or a struct) I can call, Is there a way to pass functions like that (or how does this make sense)?
Firstly I don't know what transform and sdf3d types are, and what is the invert function. Secondly how do I apply this to 2d?
const fShaderSource = `#version 300 es
precision mediump float;
uniform vec2 u_resolution;
out vec4 outColor;
float sdLine( in vec2 p, in vec2 a, in vec2 b )
{
vec2 pa = p-a, ba = b-a;
float h = clamp( dot(pa,ba)/dot(ba,ba), 0.0, 1.0 );
return length( pa - ba*h );
}
vec2 screenToWorld(vec2 screen) {
vec2 result = 2.0 * (screen/u_resolution.xy - 0.5);
result.x *= u_resolution.x/u_resolution.y;
return result;
}
void main() {
vec2 p = screenToWorld(gl_FragCoord.xy);
float sd = sdLine(p, vec2(0.0), vec2(0.0, 0.5));
vec3 col = vec3(0.0);
col += 1.0 - smoothstep(0.0, 0.04, abs(sd));
outColor = vec4(col, 1.0);
}
`;
const vShaderSource = `#version 300 es
precision mediump float;
in vec2 a_position;
uniform vec2 u_resolution;
void main() {
gl_Position = vec4(a_position, 0, 1);
}
`;
main(document.getElementById('app'));
function main(element) {
const canvas = document.createElement('canvas'),
gl = canvas.getContext('webgl2');
element.append(canvas);
const displayWidth = canvas.clientWidth,
displayHeight = canvas.clientHeight;
canvas.width = displayWidth;
canvas.height = displayHeight;
let graphics = new Graphics({width: displayWidth, height: displayHeight}, gl);
new Loop(() => {
graphics.render();
}).start();
}
function Graphics(state, gl) {
const { width, height } = state;
let vShader = createShader(gl, gl.VERTEX_SHADER, vShaderSource);
let fShader = createShader(gl, gl.FRAGMENT_SHADER, fShaderSource);
let program = createProgram(gl, vShader, fShader);
let posAttrLocation = gl.getAttribLocation(program, "a_position");
let posBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, posBuffer);
/*
(-1, 1).( 1, 1)
.
(-1,-1).( 1,-1)
*/
let positions = [
-1, 1,
-1, -1,
1, -1,
-1, 1,
1,-1,
1, 1
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
let vao = gl.createVertexArray();
gl.bindVertexArray(vao);
gl.enableVertexAttribArray(posAttrLocation);
let size = 2,
type = gl.FLOAT,
normalize = false,
stride = 0,
offset = 0;
gl.vertexAttribPointer(posAttrLocation,
size,
type,
normalize,
stride,
offset);
let resUniformLocation = gl.getUniformLocation(program, "u_resolution");
gl.clearColor(0, 0, 0, 0);
this.render = () => {
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(program);
gl.uniform2f(resUniformLocation, gl.canvas.width, gl.canvas.height);
gl.bindVertexArray(vao);
gl.drawArrays(gl.TRIANGLES, 0, 6);
};
}
function createShader(gl, type, source) {
let shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
let success = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
if (success) {
return shader;
}
console.error(gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
};
function createProgram(gl, vShader, fShader) {
let program = gl.createProgram();
gl.attachShader(program, vShader);
gl.attachShader(program, fShader);
gl.linkProgram(program);
let success = gl.getProgramParameter(program, gl.LINK_STATUS);
if (success) {
return program;
}
console.error(gl.getProgramInfoLog(program));
gl.deleteProgram(program);
return null;
}
// Loop Library
function Loop(fn) {
const perf = window.performance !== undefined ? window.performance : Date;
const now = () => perf.now();
const raf = window.requestAnimationFrame;
let running = false,
lastUpdate = now(),
frame = 0;
this.start = () => {
if (running) {
return this;
}
running = true;
lastUpdate = now();
frame = raf(tick);
return this;
};
this.stop = () => {
running = false;
if (frame != 0) {
raf.cancel(frame);
}
frame = 0;
return this;
};
const tick = () => {
frame = raf(tick);
const time = now();
const dt = time - lastUpdate;
fn(dt);
lastUpdate = time;
};
}
#app canvas {
position: fixed;
top: 50%;
bottom: 0;
left: 50%;
right: 0;
width: 100vmin;
height: 70vmin;
transform: translate(-50%, -25%);
image-rendering: optimizeSpeed;
cursor: none;
margin: auto;
}
<div id="app">
</div>
GLSL does not allow you to pass functions as parameters. The snippet you linked is more of a macro, where you are supposed to manually inline the primitive.
Just above the code you copy-pasted, the definition of transform is stated:
This code bellow assumes that transform encodes only a rotation and a translation (as a 3x4 matrix for example, or as a quaternion and a vector), and that it does not contain any scaling factors in it.
To work in 2D, you use 3x3 matrices, where the upper 2x2 matrix encodes a rotation and the two first columns of the bottom row encode a translation.
Putting it all together: (replace the mainImage function of https://www.shadertoy.com/view/MldcD7 with this)
void mainImage( out vec4 fragColor, in vec2 fragCoord )
{
vec2 p = (2.0*fragCoord-iResolution.xy)/iResolution.y;
p *= 1.1;
// iFrame is a uniform that shadertoy provides: the current frame number
float angle = float(iFrame) / 60.0;
// Rotation part: rotate by `angle`, or once every 60 fps.
// Translation part: Move across the screen left to right, every 60 fps.
mat3 transform = mat3(
cos(angle), sin(angle), 0.0,
-sin(angle),cos(angle), 0.0,
(float(iFrame % 60)/60.0 - 0.5) * 2.0, 0.0, 1.0
);
vec2 tri = vec2(0.3,-1.1); // width, height
// Here, we first apply the inverse transform to our input, then pass the resulting point to our primitive, here sdTriangleIsosceles
float d = sdTriangleIsosceles( tri, (inverse(transform) * vec3(p, 1.0)).xy );
vec3 col = vec3(1.0) - sign(d)*vec3(0.1,0.4,0.7);
col *= 1.0 - exp(-2.0*abs(d));
col *= 0.8 + 0.2*cos(140.0*d);
col = mix( col, vec3(1.0), 1.0-smoothstep(0.0,0.02,abs(d)) );
fragColor = vec4(col*1.2,1.0);
}
I render a grid texture. I want to manipulate uv coordinates(vQuadCoord) in fragment shader to make the black hole effect, that is make the gaps between lines go further as it approaches to the center. also with a circular effect
I think this would be possible, since if I do vQuadCoord = vQuadCoord * vQuadCoord it achieves a similar effect but in the corners.
const fShaderSource = `#version 300 es
precision mediump float;
out vec4 outColor;
uniform sampler2D u_texture;
in vec2 vQuadCoord;
void main() {
outColor = texture(u_texture, vQuadCoord);
}
`;
const vShaderSource = `#version 300 es
precision mediump float;
in vec2 a_position;
out vec2 vQuadCoord;
void main() {
vQuadCoord = (a_position + 1.0) / 2.0;
gl_Position = vec4(a_position, 0, 1);
}
`;
main(document.getElementById('app'));
function main(element) {
const canvas = document.createElement('canvas'),
gl = canvas.getContext('webgl2');
element.append(canvas);
const displayWidth = canvas.clientWidth,
displayHeight = canvas.clientHeight;
canvas.width = displayWidth;
canvas.height = displayHeight;
let graphics = new Graphics({width: displayWidth, height: displayHeight}, gl);
new Loop(() => {
graphics.render();
}).start();
}
function Graphics(state, gl) {
const { width, height } = state;
gl.clearColor(0, 0, 0, 0);
gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
//gl.blendFunc(gl.SRC_ALPHA, gl.ONE);
gl.enable(gl.BLEND);
gl.disable(gl.DEPTH_TEST);
let minibatch = [];
const redText = makeGlQuad(gl, fShaderSource, canvasTexture());
this.render = () => {
minibatch.push(redText);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clear(gl.COLOR_BUFFER_BIT);
minibatch.forEach(({
program,
resUniformLocation,
vao,
glTexture
}) => {
gl.useProgram(program);
gl.uniform2f(resUniformLocation, gl.canvas.width, gl.canvas.height);
if (glTexture) {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, glTexture);
}
gl.bindVertexArray(vao);
gl.drawArrays(gl.TRIANGLES, 0, 6);
});
minibatch = [];
};
}
function makeGlQuad(gl, fShaderSource, texture) {
let vShader = createShader(gl, gl.VERTEX_SHADER, vShaderSource);
let fShader = createShader(gl, gl.FRAGMENT_SHADER, fShaderSource);
let program = createProgram(gl, vShader, fShader);
let posAttrLocation = gl.getAttribLocation(program, "a_position");
let posBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, posBuffer);
let left = -1,
right = 1,
down = -1,
up = 1;
/*
(-1, 1).( 1, 1)
.
(-1,-1).( 1,-1)
*/
let positions = [
left, down,
left, up,
right, down,
left, up,
right, down,
right, up
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
let vao = gl.createVertexArray();
gl.bindVertexArray(vao);
gl.enableVertexAttribArray(posAttrLocation);
let size = 2,
type = gl.FLOAT,
normalize = false,
stride = 0,
offset = 0;
gl.vertexAttribPointer(posAttrLocation,
size,
type,
normalize,
stride,
offset);
let glTexture;
if (texture) {
glTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, glTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, texture);
//gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
}
let resUniformLocation = gl.getUniformLocation(program, "u_resolution");
let texUniformLocation = gl.getUniformLocation(program, "u_texture");
return {
program,
resUniformLocation,
vao,
glTexture
}
}
function canvasTexture() {
return withCanvasTexture(256, 256, (w, h, canvas, ctx) => {
const gap = w * 0.07;
ctx.fillStyle = 'green';
ctx.fillRect(0, 0, 10, 10);
ctx.strokeStyle = 'red';
ctx.lineWidth = 1;
ctx.beginPath();
for (let i = 0; i < w; i+= gap) {
ctx.moveTo(i, 0);
ctx.lineTo(i, h);
}
for (let i = 0; i < h; i+= gap) {
ctx.moveTo(0, i);
ctx.lineTo(w, i);
}
ctx.stroke();
return canvas;
});
function withCanvasTexture(width, height, f) {
var canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
f(width, height, canvas, canvas.getContext('2d'));
const texture = canvas;
document.body.append(canvas);
return texture;
}
}
function createShader(gl, type, source) {
let shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
let success = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
if (success) {
return shader;
}
console.error(gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
};
function createProgram(gl, vShader, fShader) {
let program = gl.createProgram();
gl.attachShader(program, vShader);
gl.attachShader(program, fShader);
gl.linkProgram(program);
let success = gl.getProgramParameter(program, gl.LINK_STATUS);
if (success) {
return program;
}
console.error(gl.getProgramInfoLog(program));
gl.deleteProgram(program);
return null;
}
// Loop Library
function Loop(fn) {
const perf = window.performance !== undefined ? window.performance : Date;
const now = () => perf.now();
const raf = window.requestAnimationFrame;
let running = false,
lastUpdate = now(),
frame = 0;
this.start = () => {
if (running) {
return this;
}
running = true;
lastUpdate = now();
frame = raf(tick);
return this;
};
this.stop = () => {
running = false;
if (frame != 0) {
raf.cancel(frame);
}
frame = 0;
return this;
};
const tick = () => {
frame = raf(tick);
const time = now();
const dt = time - lastUpdate;
fn(dt);
lastUpdate = time;
};
}
#app canvas {
background: #ccc;
position: fixed;
top: 50%;
bottom: 0;
left: 50%;
right: 0;
width: 100vmin;
height: 70vmin;
transform: translate(-50%, -25%);
image-rendering: optimizeSpeed;
cursor: none;
margin: auto;
}
<div id="app">
</div>
[...] in fragment shader to make the black hole effect, that is make the gaps between lines go further as it approaches to the center.
You've to do something like (1.0 - (1.0 - abs(x)) * (1.0 - abs(x))). x is a coordinate, where (0,0) is in the center of the texture.
Convert the texture coordinates from the range [0, 1] to the range [-1, 1]:
vec2 p = vQuadCoord * 2.0 - 1.0;
Calculate the "black hole effect" coordinate:
p = sign(p) * (1.0 - (1.0 - abs(p)) * (1.0 - abs(p)));
Convert back from the range [-1, 1] to [0, 1]:
vec2 uv = p * 0.5 + 0.5;
For a circular effect you've to multiply the normalized direction vector by a factor which depends on the square distance to the center or distance to the border:
p = normalize(p) * length(p) * length(p);
or
p = normalize(p) * (1.0 - (1.0 - length(p)) * (1.0 - length(p)))
Fragment shader:
precision mediump float;
out vec4 outColor;
uniform sampler2D u_texture;
in vec2 vQuadCoord;
void main() {
vec2 p = vQuadCoord * 2.0 - 1.0;
//p = sign(p) * (1.0 - (1.0 - abs(p)) * (1.0 - abs(p)));
//p = normalize(p) * (1.0 - (1.0 - length(p)) * (1.0 - length(p)));
p = normalize(p) * length(p) * length(p);
vec2 uv = p * 0.5 + 0.5;
outColor = texture(u_texture, uv);
}
To y-flip the texture you can set the UNPACK_FLIP_Y_WEBGL flag. See WebGL 2.0, 5.14.8 Texture objects:
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, texture);
I'm attempting to add video as texture in WebGL. The project is simple and only involves one object. The code I have works fine in Firefox but claims there is no video when loaded in Chrome. The ERROR seems to be that the video is not starting in Chrome for some reason even though I have a function to start the video:
function startVideo() {
videoElement.play();
intervalID = setInterval(draw, 15);
}
as well as
videoElement.preload = "auto";
tied to my video objectin JS.
Additionally, I also have
<video id="video" src="Firefox.ogv" autoplay muted>
Your browser doesn't appear to support the <code><video></code> element.
</video>
just to be sure, but it is not loadng the texture. The error is being referenced in this function for teximage2D:
function updateTexture() {
gl.bindTexture(gl.TEXTURE_2D, modelTexture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoElement);
}
Subsequently, I also get the error:
RENDER WARNING: texture bound to texture unit 0 is not renderable. It maybe non-power-of-2 and have incompatible texture filtering.
I am using this tutorial combined with my own implemenation :
https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/Tutorial/Animating_textures_in_WebGL
My JS and HTML are located here:
https://github.com/TacoB0t/CSC43/tree/prog/prog5
Everything mostly takes place in the init() and draw() function of the JS
HTML:
<html>
<head>
<script type="text/javascript" src="webgl-utils.js"></script>
<script type="text/javascript" src="webgl-debug.js"></script>
<script type="text/javascript" src="cuon-utils.js"></script>
<script type="text/javascript" src="cuon-matrix.js"></script>
<script type="text/javascript" src="prog5.js"></script>
<script type="text/javascript" src="chest.js"></script>
<script type="text/javascript" src="cube.js"></script>
</head>
<body onload="init()">
<script id="vertexShader" type="x-shader/x-vertex">
precision mediump float;
uniform mat4 modelMatrix;
uniform mat4 viewMatrix;
uniform mat4 projectionMatrix;
uniform vec4 lightPosition;
attribute vec4 vertexPosition;
attribute vec3 vertexNormal;
attribute vec2 vertexTexCoord;
varying vec3 fragmentNormal;
varying vec3 fragmentLight;
varying vec3 fragmentView;
varying vec4 fragmentPosition;
varying vec2 fragmentTexCoord;
void main() {
mat4 modelViewMatrix = viewMatrix * modelMatrix;
vec4 p = modelViewMatrix * vertexPosition;
vec4 q = viewMatrix * lightPosition;
fragmentPosition = vertexPosition;
fragmentNormal = normalize(mat3(modelViewMatrix) * vertexNormal);
fragmentLight = normalize(vec3(q - p));
fragmentView = normalize(vec3(-p));
fragmentTexCoord = vertexTexCoord;
gl_Position = projectionMatrix * modelViewMatrix * vertexPosition;
}
</script>
<script id="lightingFragmentShader" type="x-shader/x-fragment">
precision mediump float;
varying vec3 fragmentNormal;
varying vec3 fragmentLight;
varying vec3 fragmentView;
varying vec4 fragmentPosition;
varying vec2 fragmentTexCoord;
uniform sampler2D modelTexture;
uniform vec3 modelColor;
uniform vec3 lightColor;
void main() {
vec3 n = normalize(fragmentNormal);
vec3 l = normalize(fragmentLight);
vec3 v = normalize(fragmentView);
vec3 h = normalize(l + v);
vec4 modelColor = texture2D(modelTexture, fragmentTexCoord);
float d = max(dot(l,n) , 0.0);
float s = pow(max(dot(h, n), 0.0), 10.0);
vec3 fragmentColor = vec3(modelColor) * lightColor * d + lightColor * s;
gl_FragColor = vec4(fragmentColor, 1.0);
}
</script>
<center>
<canvas id="webgl" width="500px" height="500px">
This content requires WebGL
</canvas>
<font face ="Arial">
<br>
Light Source Position
<br>
X-AXIS<input id="x-light" type="range" min="-5.0" max="5.0" value="0" step="0.1" oninput="refresh()">
<br>
Y-AXIS <input id="y-light" type="range" min="-5.0" max="5.0" value="0" step="0.1" oninput="refresh()">
<br>
Z-AXIS<input id="z-light" type="range" min="-5.0" max="5.0" value="0" step="0.1" oninput="refresh()">
</font>
</center>
<video id="video" src="Firefox.ogv" autoplay muted style="display: none;">
Your browser doesn't appear to support the <code><video></code> element.
</video>
</body>
</html>
JS:
var gl;
var canvas;
var dragging = false;
var texShader;
var chestModel;
var xValue = 0;
var yValue = 0;
var zValue = 0;
var modelRotationX = 0;
var modelRotationY = 0;
var lastClientX;
var lastClientY;
var videoElement;
var modelTexture;
var copyVideo;
//refresh function used to request animation frame after moving slider in HTML
function refresh(){
xValue = document.getElementById("x-light").value;
yValue = document.getElementById("y-light").value;
zValue = document.getElementById("z-light").value;
requestAnimationFrame(draw);
}
//define 'flatten' function to flatten tables to single array
function flatten(a) {
return a.reduce(function (b, v) { b.push.apply(b, v); return b }, [])
}
//create tumble interaction functions to click and drag cube
function onmousedown(event){
dragging = true;
lastClientX = event.clientX;
lastClientY = event.clientY;
}
function onmouseup(event){
dragging = false;
}
/*using clientX and clientY derived from click event, use to create modelX and Y
rotation before passing to model matrices rotation transformations*/
function onmousemove(event){
//console.log(event.clientX, event.clientY);
if (dragging){
var dX = event.clientX - lastClientX;
var dY = event.clientY - lastClientY;
modelRotationY = modelRotationY + dX;
modelRotationX = modelRotationX + dY;
if (modelRotationX > 90.0){
modelRotationX = 90.0;
}
if (modelRotationX < -90.0){
modelRotationX = -90.0;
}
requestAnimationFrame(draw);
}
lastClientX = event.clientX;
lastClientY = event.clientY;
}
function startVideo() {
videoElement.play();
intervalID = setInterval(draw, 15);
}
function videoDone() {
clearInterval(intervalID);
}
//define Shader object constructor function
function Shader(vertexId, fragmentId){
this.program = createProgram(gl, document.getElementById( vertexId).text,
document.getElementById(fragmentId).text);
this.modelMatrixLocation = gl.getUniformLocation(this.program, 'modelMatrix');
this.viewMatrixLocation = gl.getUniformLocation(this.program, 'viewMatrix');
this.projectionMatrixLocation = gl.getUniformLocation(this.program, 'projectionMatrix');
this.vertexPositionLocation = gl.getAttribLocation(this.program, 'vertexPosition');
this.lightPositionLocation = gl.getUniformLocation(this.program, 'lightPosition');
this.modelColorLocation = gl.getUniformLocation(this.program, 'modelColor');
this.lightColorLocation = gl.getUniformLocation(this.program, 'lightColor');
this.vertexNormalLocation = gl.getAttribLocation(this.program, 'vertexNormal');
this.vertexTexCoordLocation = gl.getAttribLocation(this.program, 'vertexTexCoord');
gl.enableVertexAttribArray(this.vertexPositionLocation);
gl.enableVertexAttribArray(this.vertexNormalLocation);
gl.enableVertexAttribArray(this.vertexTexCoordLocation);
}
//define use() method for Shader objects
Shader.prototype.use = function(projectionMatrix, modelMatrix, viewMatrix){
gl.useProgram(this.program);
gl.uniformMatrix4fv(this.modelMatrixLocation, false, modelMatrix.elements);
gl.uniformMatrix4fv(this.viewMatrixLocation, false, viewMatrix.elements);
gl.uniformMatrix4fv(this.projectionMatrixLocation, false, projectionMatrix.elements);
gl.uniform4f(this.lightPositionLocation, xValue, yValue, zValue, 0.0);
gl.uniform3f(this.modelColorLocation, 0.6, 0.3, 0.2);
gl.uniform3f(this.lightColorLocation, 1.0, 1.0, 1.0);
}
//define Model object constructor function
function Model(positions, triangles, normals, texCoords){
//initialize buffer objects
this.positionBuffer = gl.createBuffer();
this.triangleBuffer = gl.createBuffer();
this.normalsBuffer = gl.createBuffer();
this.texCoordBuffer = gl.createBuffer();
//copy vertex data from array in CPU onto GPU
this.positionArray = new Float32Array(flatten(positions));
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, this.positionArray, gl.STATIC_DRAW);
//copy triangle data from array in CPU onto GPU
this.triangleArray = new Uint16Array(flatten(triangles));
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.triangleBuffer);
gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, this.triangleArray, gl.STATIC_DRAW);
this.normalsArray = new Float32Array(flatten(normals));
gl.bindBuffer(gl.ARRAY_BUFFER, this.normalsBuffer);
gl.bufferData(gl.ARRAY_BUFFER, this.normalsArray, gl.STATIC_DRAW);
this.textCoordArray = new Float32Array(flatten(texCoords));
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, this.textCoordArray, gl.STATIC_DRAW);
}
//define draw() method for Model objects to bind barray buffers
Model.prototype.draw = function(shader){
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
gl.vertexAttribPointer(shader.vertexPositionLocation, 3, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this.normalsBuffer);
gl.vertexAttribPointer(shader.vertexNormalLocation, 3, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
gl.vertexAttribPointer(shader.vertexTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.triangleBuffer);
gl.drawElements(gl.TRIANGLES, this.triangleArray.length, gl.UNSIGNED_SHORT, 0);
}
//initizlize texture object
function loadTexture(image, texture){
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
//requestAnimationFrame(draw);
}
function init(){
//initialize GL context
canvas = document.getElementById('webgl');
gl = getWebGLContext(canvas, false);
canvas.onmousedown = onmousedown;
canvas.onmouseup = onmouseup;
canvas.onmousemove = onmousemove;
//instantiate shader objects for each defined shader
texShader = new Shader('vertexShader', 'lightingFragmentShader');
//instantiate model objects for each model
chestModel = new Model(chest.positions, chest.triangles, chest.normals, chest.texCoords);
videoElement = document.getElementById("video");
modelTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, modelTexture);
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE,
new Uint8Array([255, 255, 255, 255]));
videoElement.addEventListener("canplaythrough", startVideo, true);
videoElement.addEventListener("playing", function (){ copyVideo = true; }, true);
videoElement.addEventListener("ended", videoDone, true);
/* videoElement.onload = function() {
loadTexture(videoElement, modelTexture);
}*/
loadTexture(videoElement, modelTexture);
videoElement.crossOrigin = "anonymous";
videoElement.src = "Firefox.ogv";
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.enable(gl.DEPTH_TEST);
//request animation frame
requestAnimationFrame(draw);
}
function draw(){
if (copyVideo) {
updateTexture();
}
//compose matrices for transformations
var viewMatrix = new Matrix4();
var projectionMatrix = new Matrix4();
viewMatrix.translate(0.0, 0.0, -1.8);
projectionMatrix.perspective(90, 1, 1, 10);
//set color and refresh rendering for canvas
gl.clearColor(0.5, 0.5, 0.5, 1.0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
/*instantiate model matrices for each respective model
and draw models with applied shader*/
var chestModelMatrix = new Matrix4();
chestModelMatrix.rotate(modelRotationX, 1, 0, 0 );
chestModelMatrix.rotate(modelRotationY, 0, 1, 0 );
chestModelMatrix.translate(0.0, 0.0, 0.0, 0.0 );
//set uniform locations and apply shader to designated model
texShader.use(projectionMatrix, chestModelMatrix, viewMatrix);
chestModel.draw(texShader);
}
function updateTexture() {
gl.bindTexture(gl.TEXTURE_2D, modelTexture);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoElement);
}
I'm not an HTML video expert but this works for me. You need to wait for the playing event before you start copying the video to a texture
var copyVideo; // if true we can call gl.texImage2D
var video = document.createElement("video");
video.src = someVideoUrl;
video.addEventListener("playing", function() {
copyVideo = true;
}, true);
video.addEventListener("ended", function() {
video.currentTime = 0;
video.play();
}, true);
video.play();
Working example:
"use strict";
var m4 = twgl.m4;
var gl = document.getElementById("c").getContext("webgl");
var programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);
var bufferInfo = twgl.primitives.createCubeBufferInfo(gl, 2);
var camera = m4.identity();
var view = m4.identity();
var viewProjection = m4.identity();
var texture = twgl.createTexture(gl, {
src: [0, 0, 255],
format: gl.RGB,
min: gl.LINEAR,
wrap: gl.CLAMP_TO_EDGE,
});
var uniforms = {
u_texture: texture,
u_worldViewProjection: m4.identity(),
};
var copyVideo;
var video = document.createElement("video");
video.src = "https://webglsamples.org/color-adjust/sample-video.mp4";
video.crossOrigin = "*";
video.volume = 0; // sample video has bad audio
video.addEventListener("playing", function() {
copyVideo = true;
}, true);
video.addEventListener("ended", function() {
video.currentTime = 0;
video.play();
}, true);
video.addEventListener("error", function() {
console.log("could not play video");
}, true);
video.play();
function render(time) {
time *= 0.001;
twgl.resizeCanvasToDisplaySize(gl.canvas);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.enable(gl.DEPTH_TEST);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
var radius = 5;
var orbitSpeed = time;
var projection = m4.perspective(30 * Math.PI / 180, gl.canvas.clientWidth / gl.canvas.clientHeight, 0.5, 100);
var eye = [Math.cos(orbitSpeed) * radius, 3, Math.sin(orbitSpeed) * radius];
var target = [0, 0, 0];
var up = [0, 1, 0];
if (copyVideo) {
gl.bindTexture(gl.TEXTURE_2D, texture)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, video);
}
m4.lookAt(eye, target, up, camera);
m4.inverse(camera, view);
m4.multiply(projection, view, viewProjection);
m4.copy(viewProjection, uniforms.u_worldViewProjection);
gl.useProgram(programInfo.program);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo)
twgl.setUniforms(programInfo, uniforms);
twgl.drawBufferInfo(gl, bufferInfo);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
body { margin: 0; }
#c { width: 100vw; height: 100vh; display: block; }
<script id="vs" type="notjs">
uniform mat4 u_worldViewProjection;
attribute vec4 position;
attribute vec2 texcoord;
varying vec2 v_texCoord;
void main() {
v_texCoord = texcoord;
gl_Position = u_worldViewProjection * position;
}
</script>
<script id="fs" type="notjs">
precision mediump float;
varying vec2 v_texCoord;
uniform sampler2D u_texture;
void main() {
gl_FragColor = texture2D(u_texture, v_texCoord);
}
</script>
<script src="https://twgljs.org/dist/2.x/twgl-full.min.js"></script>
<canvas id="c"></canvas>