I want to capture a frame from video every 5 seconds.
This is my JavaScript code:
video.addEventListener('loadeddata', function() {
var duration = video.duration;
var i = 0;
var interval = setInterval(function() {
video.currentTime = i;
generateThumbnail(i);
i = i+5;
if (i > duration) clearInterval(interval);
}, 300);
});
function generateThumbnail(i) {
//generate thumbnail URL data
var context = thecanvas.getContext('2d');
context.drawImage(video, 0, 0, 220, 150);
var dataURL = thecanvas.toDataURL();
//create img
var img = document.createElement('img');
img.setAttribute('src', dataURL);
//append img in container div
document.getElementById('thumbnailContainer').appendChild(img);
}
The problem I have is the 1st two images generated are the same and the duration-5 second image is not generated. I found out that the thumbnail is generated before the video frame of the specific time is displayed in < video> tag.
For example, when video.currentTime = 5, image of frame 0s is generated. Then the video frame jump to time 5s. So when video.currentTime = 10, image of frame 5s is generated.
Cause
The problem is that seeking video (by setting it's currentTime) is asynchronous.
You need to listen to the seeked event or else it will risk take the actual current frame which is likely your old value.
As it is asynchronous you must not use the setInterval() as it is asynchronous too and you will not be able to properly synchronize when the next frame is seeked to. There is no need to use setInterval() as we will utilize the seeked event instead which will keep everything is sync.
Solution
By re-writing the code a little you can use the seeked event to go through the video to capture the correct frame as this event ensures us that we are actually at the frame we requested by setting the currentTime property.
Example
// global or parent scope of handlers
var video = document.getElementById("video"); // added for clarity: this is needed
var i = 0;
video.addEventListener('loadeddata', function() {
this.currentTime = i;
});
Add this event handler to the party:
video.addEventListener('seeked', function() {
// now video has seeked and current frames will show
// at the time as we expect
generateThumbnail(i);
// when frame is captured, increase here by 5 seconds
i += 5;
// if we are not past end, seek to next interval
if (i <= this.duration) {
// this will trigger another seeked event
this.currentTime = i;
}
else {
// Done!, next action
}
});
If you'd like to extract all frames from a video, see this answer. The example below assumes that you want to extract a frame every 5 seconds, as OP requested.
This answer requires WebCodecs which is supported in Chrome and Edge as of writing.
<canvas id="canvasEl"></canvas>
<script type="module">
import getVideoFrames from "https://deno.land/x/get_video_frames#v0.0.8/mod.js"
let ctx = canvasEl.getContext("2d");
// `getVideoFrames` requires a video URL as input.
// If you have a file/blob instead of a videoUrl, turn it into a URL like this:
let videoUrl = URL.createObjectURL(fileOrBlob);
const saveFrameEverySeconds = 5;
let elapsedSinceLastSavedFrame = 0;
await getVideoFrames({
videoUrl,
onFrame(frame) { // `frame` is a VideoFrame object:
elapsedSinceLastSavedFrame += frame.duration / 1e6; // frame.duration is in microseconds, so we convert to seconds
if(elapsedSinceLastSavedFrame > saveFrameEverySeconds) {
ctx.drawImage(frame, 0, 0, canvasEl.width, canvasEl.height);
elapsedSinceLastSavedFrame = 0;
}
frame.close();
},
onConfig(config) {
canvasEl.width = config.codedWidth;
canvasEl.height = config.codedHeight;
},
});
URL.revokeObjectURL(fileOrBlob); // revoke URL to prevent memory leak
</script>
Demo: https://jsbin.com/qovapeziqi/edit?html,output
Github: https://github.com/josephrocca/getVideoFrames.js
Related
How I can make it work accurately so that each picture is exactly shown 50ms after the last one? (And so that the calculated time between seeing the picture and clicking is accurate?)
Background
I want to have a slideshow of images. The images are stored in the images directory and their names are sequential. The slideshow should start after the user clicks on the play button. And the user will click on the stop button a little after the 100th picture. And the code will show the user how many milliseconds after seeing the 100th picture they have clicked on the stop button. The problem is that when I run the code it doesn't work so accurate. It has some lag on some pictures. I was wondering
Here is my code:
<!DOCTYPE html>
<html>
<head>
<script>
var player;
var timestamp;
function preloadImage()
{
var i = 1
while(true){
var img = new Image();
img.src="images/" + i;
i = i + 1;
if (img.height == 0) { break; }
}
}
function next(){
var fullPath=document.getElementById("image").src;
var filename = fullPath.split("/").pop();
var n=parseInt(filename, 10) + 1;
document.getElementById("image").src = "images/" + n;
if (document.getElementById("image").height == 0) { clearInterval(player); }
if (n == 100) { timestamp = new Date().getTime(); }
}
function start(){
clearInterval(player);
document.getElementById("image").src = "images/1";
player = setInterval(function(){ next(); }, 50)
}
function stop(){
clearInterval(player);
alert("You clicked after " + (new Date().getTime() - timestamp) + "ms.")
}
preloadImage()
</script>
</head>
<body>
<img src="images/0" id="image"><br/>
<button type="button" onclick='start()'>start</button>
<button type="button" onclick='stop()'>stop</button>
</body>
</html>
As I stated in my comment, I believe your preloadImage() is not working as you expect. Try running the stack snippet below as a demonstration, and possibly make sure your cache is cleared:
function badPreloadImage () {
var img = new Image();
img.onload = function () {
// check it asynchronously
console.log('good', this.height);
};
img.src = 'https://www.w3schools.com/w3css/img_lights.jpg';
// don't check height synchronously
console.log('bad', img.height);
}
badPreloadImage();
To preload all your images properly, you must do so asynchronously:
function preloadImage (done, i) {
if (!i) { i = 1; }
var img = new Image();
img.onloadend = function () {
if (this.height == 0) { return done(); }
preloadImage(done, i + 1);
};
img.src = "images/" + i;
}
// usage
preloadImage(function () { console.log('images loaded'); });
Other concerns
You should consider using performance.now() instead of new Date().getTime(), as it uses a more precise time resolution (currently 20us, as pointed out in this comment).
You might also consider storing references to each image as an array of Image objects, rather than loading each frame via specifying the src property of an HTMLImageElement, so that the browser can just load the data from memory, rather than loading from cache on the hard drive, or even making a new HTTP request when caching is not enabled.
Addressing each of these issues will allow you to measure timing more precisely by using the proper API and eliminating lag spikes on the DOM thread due to inefficient animation.
I have the following two pieces of code (awful but I have no idea what I'm doing):
var stage = new createjs.Stage("canvas");
createjs.Ticker.on("tick", tick);
// Simple loading for demo purposes.
var image = document.createElement("img");
image.src = "http://dossierindustries.co/wp-content/uploads/2017/07/DossierIndustries_Cactus-e1499205396119.png";
var _obstacle = new createjs.Bitmap(image);
setInterval(clone, 1000);
function clone() {
var bmp = _obstacle.clone();
bmp.x= Math.floor((Math.random() * 1920) + 1);
bmp.y = Math.floor((Math.random() * 1080) + 1);
stage.addChild(bmp);
}
function tick(event) {
stage.update(event);
}
<script>
$j=jQuery.noConflict();
jQuery(document).ready(function($){
var interval = 1;
setInterval(function(){
if(interval == 3){
$('canvas').show();
interval = 1;
}
interval = interval+1;
console.log(interval);
},1000);
$(document).bind('mousemove keypress', function() {
$('canvas').hide();
interval = 1;
});
});
<script src="https://code.createjs.com/easeljs-0.8.2.min.js"></script>
<canvas id="canvas" width="1920" height="1080"></canvas>
Basically what I'm hoping to achieve is that when a user is inactive for x amount of time the full page (no matter on size) slowly fills with the repeated image. When anything happens they all clear and it begins again after the set amount of inactivity.
The code above relies on an external resource which I'd like to avoid and needs to work on Wordpress.
Site is viewable at dossierindustries.co
Rather than interpret your code, I made a quick demo showing how I might approach this.
The big difference is that drawing new images over time is going to add up (they have to get rendered every frame), so this approach uses a cached container with one child, and each tick it just adds more to the cache (similar to the "updateCache" demo in GitHub.
Here is the fiddle.
http://jsfiddle.net/dcs5zebm/
Key pieces:
// Move the contents each tick, and update the cache
shape.x = Math.random() * stage.canvas.width;
shape.y = Math.random() * stage.canvas.height;
container.updateCache("source-over");
// Only do it when idle
function tick(event) {
if (idle) { addImage(); }
stage.update(event);
}
// Use a timeout to determine when idle. Clear it when the mouse moves.
var idle = false;
document.body.addEventListener("mousemove", resetIdle);
function resetIdle() {
clearTimeout(this.timeout);
container.visible = false;
idle = false;
this.timeout = setTimeout(goIdle, TIMEOUT);
}
resetIdle();
function goIdle() {
idle = true;
container.cache(0, 0, stage.canvas.width, stage.canvas.height);
container.visible = true;
}
Caching the container means this runs the same speed forever (no overhead), but you still have control over the rest of the stage (instead of just turning off auto-clear). If you have more complicated requirements, you can get fancier -- but this basically does what you want I think.
I'm building a cross-platform web app where audio is generated on-the-fly on the server and live streamed to a browser client, probably via the HTML5 audio element. On the browser, I'll have Javascript-driven animations that must precisely sync with the played audio. "Precise" means that the audio and animation must be within a second of each other, and hopefully within 250ms (think lip-syncing). For various reasons, I can't do the audio and animation on the server and live-stream the resulting video.
Ideally, there would be little or no latency between the audio generation on the server and the audio playback on the browser, but my understanding is that latency will be difficult to control and probably in the 3-7 second range (browser-, environment-, network- and phase-of-the-moon-dependent). I can handle that, though, if I can precisely measure the actual latency on-the-fly so that my browser Javascript knows when to present the proper animated frame.
So, I need to precisely measure the latency between my handing audio to the streaming server (Icecast?), and the audio coming out of the speakers on the computer hosting the speaker. Some blue-sky possibilities:
Add metadata to the audio stream, and parse it from the playing audio (I understand this isn't possible using the standard audio element)
Add brief periods of pure silence to the audio, and then detect them on the browser (can audio elements yield the actual audio samples?)
Query the server and the browser as to the various buffer depths
Decode the streamed audio in Javascript and then grab the metadata
Any thoughts as to how I could do this?
Utilize timeupdate event of <audio> element, which is fired three to four times per second, to perform precise animations during streaming of media by checking .currentTime of <audio> element. Where animations or transitions can be started or stopped up to several times per second.
If available at browser, you can use fetch() to request audio resource, at .then() return response.body.getReader() which returns a ReadableStream of the resource; create a new MediaSource object, set <audio> or new Audio() .src to objectURL of the MediaSource; append first stream chunks at .read() chained .then() to sourceBuffer of MediaSource with .mode set to "sequence"; append remainder of chunks to sourceBuffer at sourceBuffer updateend events.
If fetch() response.body.getReader() is not available at browser, you can still use timeupdate or progress event of <audio> element to check .currentTime, start or stop animations or transitions at required second of streaming media playback.
Use canplay event of <audio> element to play media when stream has accumulated adequate buffers at MediaSource to proceed with playback.
You can use an object with properties set to numbers corresponding to .currentTime of <audio> where animation should occur, and values set to css property of element which should be animated to perform precise animations.
At javascript below, animations occur at every twenty second period, beginning at 0, and at every sixty seconds until the media playback has concluded.
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
<title></title>
<style>
body {
width: 90vw;
height: 90vh;
background: #000;
transition: background 1s;
}
span {
font-family: Georgia;
font-size: 36px;
opacity: 0;
}
</style>
</head>
<body>
<audio controls></audio>
<br>
<span></span>
<script type="text/javascript">
window.onload = function() {
var url = "/path/to/audio";
// given 240 seconds total duration of audio
// 240/12 = 20
// properties correspond to `<audio>` `.currentTime`,
// values correspond to color to set at element
var colors = {
0: "red",
20: "blue",
40: "green",
60: "yellow",
80: "orange",
100: "purple",
120: "violet",
140: "brown",
160: "tan",
180: "gold",
200: "sienna",
220: "skyblue"
};
var body = document.querySelector("body");
var mediaSource = new MediaSource;
var audio = document.querySelector("audio");
var span = document.querySelector("span");
var color = window.getComputedStyle(body)
.getPropertyValue("background-color");
//console.log(mediaSource.readyState); // closed
var mimecodec = "audio/mpeg";
audio.oncanplay = function() {
this.play();
}
audio.ontimeupdate = function() {
// 240/12 = 20
var curr = Math.round(this.currentTime);
if (colors.hasOwnProperty(curr)) {
// set `color` to `colors[curr]`
color = colors[curr]
}
// animate `<span>` every 60 seconds
if (curr % 60 === 0 && span.innerHTML === "") {
var t = curr / 60;
span.innerHTML = t + " minute" + (t === 1 ? "" : "s")
+ " of " + Math.round(this.duration) / 60
+ " minutes of audio";
span.animate([{
opacity: 0
}, {
opacity: 1
}, {
opacity: 0
}], {
duration: 2500,
iterations: 1
})
.onfinish = function() {
span.innerHTML = ""
}
}
// change `background-color` of `body` every 20 seconds
body.style.backgroundColor = color;
console.log("current time:", curr
, "current background color:", color
, "duration:", this.duration);
}
// set `<audio>` `.src` to `mediaSource`
audio.src = URL.createObjectURL(mediaSource);
mediaSource.addEventListener("sourceopen", sourceOpen);
function sourceOpen(event) {
// if the media type is supported by `mediaSource`
// fetch resource, begin stream read,
// append stream to `sourceBuffer`
if (MediaSource.isTypeSupported(mimecodec)) {
var sourceBuffer = mediaSource.addSourceBuffer(mimecodec);
// set `sourceBuffer` `.mode` to `"sequence"`
sourceBuffer.mode = "sequence";
fetch(url)
// return `ReadableStream` of `response`
.then(response => response.body.getReader())
.then(reader => {
var processStream = (data) => {
if (data.done) {
return;
}
// append chunk of stream to `sourceBuffer`
sourceBuffer.appendBuffer(data.value);
}
// at `sourceBuffer` `updateend` call `reader.read()`,
// to read next chunk of stream, append chunk to
// `sourceBuffer`
sourceBuffer.addEventListener("updateend", function() {
reader.read().then(processStream);
});
// start processing stream
reader.read().then(processStream);
// do stuff `reader` is closed,
// read of stream is complete
return reader.closed.then(() => {
// signal end of stream to `mediaSource`
mediaSource.endOfStream();
return mediaSource.readyState;
})
})
// do stuff when `reader.closed`, `mediaSource` stream ended
.then(msg => console.log(msg))
}
// if `mimecodec` is not supported by `MediaSource`
else {
alert(mimecodec + " not supported");
}
};
}
</script>
</body>
</html>
plnkr http://plnkr.co/edit/fIm1Qp?p=preview
There no way for you to measure latency directly, but any AudioElement generate events like 'playing' if it just played (fired quite often), or 'stalled' if stoped streaming, or 'waiting' if data is loading. So what you can do, is to manipulate your video based on this events.
So play while stalled or waiting is fired, then continue playing video if playing fired again.
But I advice you check other events that might affect your flow (error for example would be important for you).
https://developer.mozilla.org/en-US/docs/Web/API/HTMLAudioElement
What i would try is first create a timestamp with performance.now, process the data, and record it in a blob with the new web recorder api.
The web recorder will ask user access to his audio card, this can be a problem for your app, but it look like mandatory to get the real latency.
As soon this done, there is many way to measure the actual latency between the generation and the actual rendering. Basically, a sound event.
For further reference and example:
Recorder demo
https://github.com/mdn/web-dictaphone/
https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder_API/Using_the_MediaRecorder_API
I'm having an issue while using canvas in a background page to create data URLs for desktop notifications' images.
I want to use the "image" notifications which require a 3:2 ratio to display properly. The images I want to use (from hulu.com) are a different ratio, so I decided to use the canvas element to create the corresponding data URL off of these images so that the ratio is correct. It kind of works in theory, but…
…I'm having issues if I'm creating more than one canvas/notification in the background page. One image is created properly, but the rest comes out empty.
Confusingly, opening the same background page in a new tab (i.e. exact same code) makes everything works just fine: all the notifications are created with the images loaded from hulu.com. Also, just changing the dimensions from 360x240 to 300x200 makes it work. Finally, though they're similar computers with the same Chrome version (34.0.1847.116), it works without modification at work while it doesn't on my own laptop.
I made a test extension available at the bottom of this post. Basically, it only has a generated background page. The code for that page is this:
var images = ["http://ib2.huluim.com/video/60376901?size=290x160&img=1",
"http://ib2.huluim.com/video/60366793?size=290x160&img=1",
"http://ib4.huluim.com/video/60372951?size=290x160&img=1",
"http://ib1.huluim.com/video/60365336?size=290x160&img=1",
"http://ib3.huluim.com/video/60376290?size=290x160&img=1",
"http://ib4.huluim.com/video/60377231?size=290x160&img=1",
"http://ib4.huluim.com/video/60312203?size=290x160&img=1",
"http://ib1.huluim.com/video/60376972?size=290x160&img=1",
"http://ib4.huluim.com/video/60376971?size=290x160&img=1",
"http://ib1.huluim.com/video/60376616?size=290x160&img=1"];
for (var i = 0; i < 10; i++) {
getDataURL(i);
}
/*
* Gets the data URL for an image URL
*/
function getDataURL(i) {
var img = new Image();
img.onload = function() {
var canvas = document.createElement('canvas');
canvas.width = 360;
canvas.height = 240;
var ctx = canvas.getContext('2d');
ctx.drawImage(this, 0, 0);
ctx.fillStyle = "rgb(200,0,0)";
ctx.fillRect (10, 10, 55, 50);
var dataURL = canvas.toDataURL('image/png');
chrome.notifications.create('', {
type: 'image',
iconUrl: 'logo_128x128.png',
title: String(i),
message: 'message',
imageUrl: dataURL
}, function(id) {});
}
//img.src = chrome.extension.getURL('logo_128x128.png');;
img.src = images[i];
}
The commented out line for img.src = ... is a test where it loads a local file instead of a remote one. In that case, all the images are created.
The red rectangle added to the canvas is to show that it's not just the remote image that is an issue: the whole resulting canvas is empty, without any red rectangle.
If you download and add the test extension below, you should get 10 notifications but only one with an image.
Then, to open the background page in a new tab, you can inspect the background page, type this in the console:
chrome.extension.getURL('_generated_background_page.html')
and right-click the URL, and click "Open in a new Tab" (or window). Once open you should get 10 notifications that look fine.
Any idea of what is going on? I haven't been able to find any kind of limitations for background pages relevant to that. Any help would be appreciated, because this has been driving me crazy!
Files available here: https://www.dropbox.com/s/ejbh6wq0qixb7a8/canvastest.zip
edit: based on #GameAlchemist's comment, I also tried the following: same getDataURL method, but the loop wrapped inside an onload for the logo:
function loop() {
for (var i = 0; i < 10; i++) {
getDataURL(i);
}
}
var logo = new Image();
logo.onload = function () {
loop();
}
logo.src = chrome.extension.getURL('logo_128x128.png');
Remember that the create() method is asynchronous and you should use a callback with. The callback can invoke next image fetching.
I would suggest doing this in two steps:
Load all the images first
Process the image queue
The reason is that you can utilize the asynchronous image loading better this way instead of chaining the callbacks which would force you to load one and one image.
For example:
Image loader
var urls = ["http://ib2.huluim.com/video/60376901?size=290x160&img=1",
"http://ib2.huluim.com/video/60366793?size=290x160&img=1",
"http://ib4.huluim.com/video/60372951?size=290x160&img=1",
"http://ib1.huluim.com/video/60365336?size=290x160&img=1",
"http://ib3.huluim.com/video/60376290?size=290x160&img=1",
"http://ib4.huluim.com/video/60377231?size=290x160&img=1",
"http://ib4.huluim.com/video/60312203?size=290x160&img=1",
"http://ib1.huluim.com/video/60376972?size=290x160&img=1",
"http://ib4.huluim.com/video/60376971?size=290x160&img=1",
"http://ib1.huluim.com/video/60376616?size=290x160&img=1"];
var images = [], // store image objects
count = urls.length; // for loader
for (var i = 0; i < urls.length; i++) {
var img = new Image; // create image
img.onload = loader; // share loader handler
img.src = urls[i]; // start loading
images.push(img); // push image object in array
}
function loader() {
count--;
if (count === 0) process(); // all loaded, start processing
}
//TODO need error handling here as well
Fiddle with concept code for loader
Processing
Now the processing can be isolated from the loading:
function process() {
// share a single canvas (use clearRect() later if needed)
var canvas = document.createElement('canvas'),
ctx = canvas.getContext('2d'),
current = 0;
canvas.width = 360;
canvas.height = 240;
createImage(); // invoke processing for first image
function createImage() {
ctx.drawImage(images[current], 0, 0); // draw current image
ctx.fillStyle = "rgb(200,0,0)";
ctx.fillRect (10, 10, 55, 50);
chrome.notifications.create('', {
type : 'image',
iconUrl : 'logo_128x128.png',
title : String(i),
message : 'message',
imageUrl: canvas.toDataURL() // png is default
},
function(id) { // use callback
current++; // next in queue
if (current < images.length) {
createImage(); // call again if more images
}
else {
done(); // we're done -> continue to done()
}
});
}
}
Disclaimer: I don't have a test environment to test Chrome extensions so typos/errors may be present.
Hope this helps!
I am trying to generate a group of thumbnails in the browser out of a HTML5 video using canvas with this code:
var fps = video_model.getFps(); //frames per second, comes from another script
var start = shot.getStart(); //start time of capture, comes from another script
var end = shot.getEnd(); //end time of capture, comes from another script
for(var i = start; i <= end; i += 50){ //capture every 50 frames
video.get(0).currentTime = i / fps;
var capture = $(document.createElement("canvas"))
.attr({
id: video.get(0).currentTime + "sec",
width: video.get(0).videoWidth,
height: video.get(0).videoHeight
})
var ctx = capture.get(0).getContext("2d");
ctx.drawImage(video.get(0), 0, 0, video.get(0).videoWidth, video.get(0).videoHeight);
$("body").append(capture, " ");
}
The the amount of captures is correct, but the problem is that in Chrome all the canvases appear black and in Firefox they always show the same image.
Maybe the problem is that the loop is too fast to let the canvases be painted, but I read that .drawImage() is asynchronous, therefore, in theory, it should let the canvases be painted before jumping to the next line.
Any ideas on how to solve this issue?
Thanks.
After hours of fighting with this I finally came up with a solution based on the "seeked" event. For this to work, the video must be completely loaded:
The code goes like this:
var fps = video_model.getFps(); //screenshot data, comes from another script
var start = shot.getStart();
var end = shot.getEnd();
video.get(0).currentTime = start/fps; //make the video jump to the start
video.on("seeked", function(){ //when the time is seeked, capture screenshot
setTimeout( //the trick is in giving the canvas a little time to be created and painted, 500ms should be enough
function(){
if( video.get(0).currentTime <= end/fps ){
var capture = $(document.createElement("canvas")) //create canvas element on the fly
.attr({
id: video.get(0).currentTime + "sec",
width: video.get(0).videoWidth,
height: video.get(0).videoHeight
})
.appendTo("body");
var ctx = capture.get(0).getContext("2d"); //paint canvas
ctx.drawImage(video.get(0), 0, 0, video.get(0).videoWidth, video.get(0).videoHeight);
if(video.get(0).currentTime + 50/fps > end/fps){
video.off("seeked"); //if last screenshot was captured, unbind
}else{
video.get(0).currentTime += 50/fps; //capture every 50 frames
}
}
}
, 500); //timeout of 500ms
});
This has worked for me in Chrome and Firefox, I've read that the seeked event can be buggy in some version of particular browsers.
Hope this can be useful to anybody. If anyone comes up with a cleaner, better solution, it would be nice to see it.