I would like to stream different user-selected videos to my front-end. For this I am using NodeJS and Express. The source of the -element in which the video should be displayed is 'http://localhost:4201/video'.
The code I am using to stream the video looks like this:
async function loadLocalVideo(_, filePath) {
if (!filePath) {
console.log('No file selected');
return;
} else {
fs.access(filePath, (err) => {
if (err) {
console.log(`File does not exist at path ${filePath}`);
return;
}
});
}
expressApp.get('/video', function (req, res) {
const path = filePath;
const stat = fs.statSync(path);
const fileSize = stat.size;
const range = req.headers.range;
if (range) {
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
if (start >= fileSize) {
res.status(416).send(
'Requested range not satisfiable\n' + start + ' >= ' + fileSize
);
return;
}
const chunksize = end - start + 1;
const file = fs.createReadStream(path, { start, end });
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4',
};
res.writeHead(206, head);
file.pipe(res);
} else {
const head = {
'Content-Length': fileSize,
'Content-Type': 'video/mp4',
};
res.writeHead(200, head);
fs.createReadStream(path).pipe(res);
}
});
}
However, when I want to stream a different video and call the same function again but with a different filePath-param the same video keeps playing. How can I stream another video and display it in the -element?
I think you are saying that you are using the same file name and path but want the backend server to change the actual contents that are streamed to the client.
This approach may confuse the caching downstream of your server - i.e. the network or browser cache may not recognise that the file has changed and it may simply serve the cached versions.
Some CDN's will allow you add edge functionality to intercept a request and then decide which of a number of options you want to return, or you could disable caching but I suspect these approaches might be overly complex.
If all you want is to be able to display different videos in a video element on your web page, it may be easier to simply change the source attribute on the video on the page itself and then call load() on the video element on the page.
Related
In this code I can use local videos to stream but how do I use cloudinary to serve my videos. I have tried to replace the filepath to cloudinary video url but it didn't work. CAn anyone please help me out on this part?
const http=require('http');
const fs=require("fs");
const path=require("path");
const server=http.createServer((req, res)=>{
// return res.end(req.url+req.method);
if(req.method==='GET' && req.url==="/"){
/*we will send a index.html page when
user visits "/" endpoint*/
/*index.html will have video component
that displays the video*/
fs.createReadStream(path.resolve(
"index.html")).pipe(res);
return;
}
//if video content is requesting
if(req.method==='GET' && req.url==="/video"){
const filepath = path.resolve("video1.mp4");
const stat = fs.statSync(filepath)
const fileSize = stat.size
const range = req.headers.range
/*when we seek the video it will put
range header to the request*/
/*if range header exists send some
part of video*/
if (range) {
//range format is "bytes=start-end",
const parts =
range.replace(/bytes=/, "").split("-");
console.log(parts);
const start = parseInt(parts[0], 10)
/*in some cases end may not exists, if its
not exists make it end of file*/
console.log(start);
const end =
parts[1] ?parseInt(parts[1], 10) :fileSize - 1
console.log(end);
//chunk size is what the part of video we are sending.
const chunksize = (end - start) + 1
/*we can provide offset values as options to
the fs.createReadStream to read part of content*/
const file = fs.createReadStream(filepath, {start, end})
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4',
}
/*we should set status code as 206 which is
for partial content*/
// because video is continuosly fetched part by part
res.writeHead(206, head);
file.pipe(res);
}else{
//if not send the video from start.
/* anyway html5 video player play content
when sufficient frames available*/
// It doesn't wait for the entire video to load.
const head = {
'Content-Length': fileSize,
'Content-Type': 'video/mp4',
}
res.writeHead(200, head);
fs.createReadStream(path).pipe(res);
}
}
/*if anything other than handler routes then send
400 status code, is for bad request*/
else{
res.writeHead(400);
res.end("bad request");
}
})
/*check if system has environment variable
for the port, otherwise defaults to 3000*/
const PORT = process.env.PORT || 3000;
//start the server
server.listen(PORT, () => {
console.log(`server listening on port:${PORT}`);
})
I am trying to speed up the upload. So I tried with different solution, with both BackEnd and Front-End. Those are,
1) I uploaded the tar file (already compressed one)
2) I tried chunk upload (sequentially), if the response is success next API will get triggered. In the back-end side, in the same file the content will get appended.
3) I tried chunk upload but in parallel, at a single time I make the 50 request to upload the chunk content (I know, at a time browser handle only 6 requests). From the backend side, we are storing all the chunk file separately, after receiving the final request, appending all those chunks in to the single file.
But observed is, I am not seeing the much difference with all these cases.
Following is my service file
export class largeGeneUpload {
chromosomeFile: any;
options: any;
chunkSize = 1200000;
activeConnections = 0;
threadsQuantity = 50;
totalChunkCount = 0;
chunksPosition = 0;
failedChunks = [];
sendNext() {
if (this.activeConnections >= this.threadsQuantity) {
return;
}
if (this.chunksPosition === this.totalChunkCount) {
console.log('all chunks are done');
return;
}
const i = this.chunksPosition;
const url = 'gene/human';
const chunkIndex = i;
const start = chunkIndex * this.chunkSize;
const end = Math.min(start + this.chunkSize, this.chromosomeFile.size);
const currentchunkSize = this.chunkSize * i;
const chunkData = this.chromosomeFile.webkitSlice ? this.chromosomeFile.webkitSlice(start, end) : this.chromosomeFile.slice(start, end);
const fd = new FormData();
const binar = new File([chunkData], this.chromosomeFile.upload.filename);
console.log(binar);
fd.append('file', binar);
fd.append('dzuuid', this.chromosomeFile.upload.uuid);
fd.append('dzchunkindex', chunkIndex.toString());
fd.append('dztotalfilesize', this.chromosomeFile.upload.total);
fd.append('dzchunksize', this.chunkSize.toString());
fd.append('dztotalchunkcount', this.chromosomeFile.upload.totalChunkCount);
fd.append('isCancel', 'false');
fd.append('dzchunkbyteoffset', currentchunkSize.toString());
this.chunksPosition += 1;
this.activeConnections += 1;
this.apiDataService.uploadChunk(url, fd)
.then(() => {
this.activeConnections -= 1;
this.sendNext();
})
.catch((error) => {
this.activeConnections -= 1;
console.log('error here');
// chunksQueue.push(chunkId);
});
this.sendNext();
}
uploadChunk(resrc: string, item) {
return new Promise((resolve, reject) => {
this._http.post(this.baseApiUrl + resrc, item, {
headers: this.headers,
withCredentials: true
}).subscribe(r => {
console.log(r);
resolve();
}, err => {
console.log('err', err);
reject();
});
});
}
But the thing is, If I upload the same file in google drive it is not taking much time.
Let's consider, I have 700 MB file, to upload it in google drive it took 3 mins. But the same 700 MB file to upload with my Angular code with our back-end server it took 7 mins to finish it.
How do I improve the performance of file upload.?
forgive me ,
it seems silly answer but this depend on your hosting infrastructure
A lot of variables can cause this, but by your story it has nothing to do with your front-end code. Making it into chunks is not going to help, because browsers have their own optimized algorithm to upload files. The most likely culprit is your backend server or the connection from your client to the server.
You say that google drive is fast, but you should also know that google has a very widespread global infrastructure with top of the line cloud servers. If you are using, for example, a 2 euro per month fixed place hosting provider, you cannot expect the same processing and network power as google.
I'm trying to stream videos from an Amazon S3 Bucket.
Streaming works fine if I call my REST endpoint only once. But if I want to stream the video from multiple browsers at the same time, I get the following error:
TimeoutError: Connection timed out after 120000ms
My code so far:
var express = require("express");
var fs = require("fs");
var app = express();
var path = require("path");
var AWS = require("aws-sdk");
app.get("/video", function(req, res, next) {
res.set({
"Accept-Ranges": "bytes",
"Content-Type": "video/mp4",
"Content-Length": 41811600,
"Cache-Control": "max-age=31536000"
});
var stream;
if (req.headers.range) {
const size = 41811600;
const parts = req.headers.range.replace(/bytes=/, "").split("-");
const start = parseInt(parts[0], 10);
const end = parts[1] ? parseInt(parts[1], 10) : size - 1;
const length = end - start + 1;
res.set({
"Content-Range": `bytes ${start}-${end}/${size}`,
"Content-Length": length
});
stream = getReadStream(start, end);
} else {
stream = getReadStream(undefined, undefined);
}
res.writeHead(res.getHeader("Content-Range") ? 206 : 200);
return stream.pipe(res);
});
function getReadStream(startByte, endByte) {
const params = {
Bucket: "s3-fancy-test-bucket",
Key: path.join(
"SOME_KEY"
),
Range: "bytes=" + (startByte || 0) + "-" + (endByte || "")
};
const stream = getS3Instance()
.getObject(params)
.createReadStream();
stream.on("error", err => {
console.error(`Unable to get File Stream for ${params.Key} - ${err}`);
});
return stream;
}
function getS3Instance() {
AWS.config.update({
accessKeyId: "ACCESS_KEY_ID",
secretAccessKey: "SECRET",
signatureVersion: "v4",
region: "eu-central-1"
});
return new AWS.S3({ apiVersion: "2006-03-01" });
}
app.listen(process.env.PORT || 3000);
I'm not quite sure whether this is a problem with aws-sdk-js or a general problem with streams...
What am I doing wrong?
Is anyone experiencing the same problems?
In the meantime I've found out, that the behavior described above heavily depends on how the video is created. I'm using ffmpeg to convert videos to mp4. Converting videos with the faststart flag makes a big difference as far as the amount of range requests is concerned. Once I had the faststart flag in place there were a lot less range requests from the browser in order to load the video.
As there are fewer requests now, express / aws-sdk seem to be able to deal with it. There are no timeout any longer.
We are trying exactly the same thing only the reason we are doing it is because we want to restrict access to the videos through our backend before streaming the video to the user. So serving videos directly from s3 is not an option. I realized that video is streamed just fine but getObject() times out 120000 ms(aws default timeout duration) even after the stream ends successfully. There seems to be an active issue regarding this on aws-sdk repository that is still not resolved.
https://github.com/aws/aws-sdk-js/issues/2087
i am creating a streaming module that will stream any file on my server to the user
for this i have the following node code:
.get(function (req, res) {
if (req.query.resourceToken) {
var decoded = jwt.decode(req.query.resourceToken, require('../secret')());
var mediaObject = decoded.mediaObject;
var file = path.resolve(mediaObject.targetDir, mediaObject.file.originalname);
fs.stat(file, function (err, stats) {
if (err) {
if (err.code === 'ENOENT') {
// 404 Error if file not found
return res.sendStatus(404);
}
res.end(err);
}
var range = req.headers.range;
if (!range) {
// 416 Wrong range
return res.sendStatus(416);
}
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
var total = stats.size;
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": mediaObject.file.mimetype
});
var stream = fs.createReadStream(file, {start: start, end: end})
.on("open", function () {
stream.pipe(res);
}).on("error", function (err) {
res.end(err);
});
});
}
else {
res.status(500).send('Missing resource token!');
}
});
Now to download a simple document i have the follow html tag:
<a target="_self" href="http://localhost:433/resource?resourceToken=token" class="btn btn-default"> <i class="fa fa-download"> </i></a>
Now when i press the button it correctly calls the method however the header range is undefined.
which means it sends the error code 416 (as shown in the above code).
Can anyone tell me why this is happening and what im missing?
Range requests serve a rather specific purpose (like enabling partial downloads during audio/video scrubbing).
Your HTML suggests that you just want to present a download link for a particular resource, and the browser won't issue a range request for that (or it might, but only after the initial request).
So your code should allow for the Range header to not exist.
Tl;Dr - The Question:
What is the right way to handle streaming a video file to an html5 video player with Node.js so that the video controls continue to work?
I think it has to do with the way that the headers are handled. Anyway, here's the background information. The code is a little lengthy, however, it's pretty straightforward.
Streaming small video files to HTML5 video with Node is easy
I learned how to stream small video files to an HTML5 video player very easily. With this setup, the controls work without any work on my part, and the video streams flawlessly. A working copy of the fully working code with sample video is here, for download on Google Docs.
Client:
<html>
<title>Welcome</title>
<body>
<video controls>
<source src="movie.mp4" type="video/mp4"/>
<source src="movie.webm" type="video/webm"/>
<source src="movie.ogg" type="video/ogg"/>
<!-- fallback -->
Your browser does not support the <code>video</code> element.
</video>
</body>
</html>
Server:
// Declare Vars & Read Files
var fs = require('fs'),
http = require('http'),
url = require('url'),
path = require('path');
var movie_webm, movie_mp4, movie_ogg;
// ... [snip] ... (Read index page)
fs.readFile(path.resolve(__dirname,"movie.mp4"), function (err, data) {
if (err) {
throw err;
}
movie_mp4 = data;
});
// ... [snip] ... (Read two other formats for the video)
// Serve & Stream Video
http.createServer(function (req, res) {
// ... [snip] ... (Serve client files)
var total;
if (reqResource == "/movie.mp4") {
total = movie_mp4.length;
}
// ... [snip] ... handle two other formats for the video
var range = req.headers.range;
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
if (reqResource == "/movie.mp4") {
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
res.end(movie_mp4.slice(start, end + 1), "binary");
}
// ... [snip] ... handle two other formats for the video
}).listen(8888);
But this method is limited to files < 1GB in size.
Streaming (any size) video files with fs.createReadStream
By utilizing fs.createReadStream(), the server can read the file in a stream rather than reading it all into memory at once. This sounds like the right way to do things, and the syntax is extremely simple:
Server Snippet:
movieStream = fs.createReadStream(pathToFile);
movieStream.on('open', function () {
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
// This just pipes the read stream to the response object (which goes
//to the client)
movieStream.pipe(res);
});
movieStream.on('error', function (err) {
res.end(err);
});
This streams the video just fine! But the video controls no longer work.
The Accept Ranges header (the bit in writeHead()) is required for the HTML5 video controls to work.
I think instead of just blindly send the full file, you should first check the Accept Ranges header in the REQUEST, then read in and send just that bit. fs.createReadStream support start, and end option for that.
So I tried an example and it works. The code is not pretty but it is easy to understand. First we process the range header to get the start/end position. Then we use fs.stat to get the size of the file without reading the whole file into memory. Finally, use fs.createReadStream to send the requested part to the client.
var fs = require("fs"),
http = require("http"),
url = require("url"),
path = require("path");
http.createServer(function (req, res) {
if (req.url != "/movie.mp4") {
res.writeHead(200, { "Content-Type": "text/html" });
res.end('<video src="http://localhost:8888/movie.mp4" controls></video>');
} else {
var file = path.resolve(__dirname,"movie.mp4");
fs.stat(file, function(err, stats) {
if (err) {
if (err.code === 'ENOENT') {
// 404 Error if file not found
return res.sendStatus(404);
}
res.end(err);
}
var range = req.headers.range;
if (!range) {
// 416 Wrong range
return res.sendStatus(416);
}
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
var total = stats.size;
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
var stream = fs.createReadStream(file, { start: start, end: end })
.on("open", function() {
stream.pipe(res);
}).on("error", function(err) {
res.end(err);
});
});
}
}).listen(8888);
The accepted answer to this question is awesome and should remain the accepted answer. However I ran into an issue with the code where the read stream was not always being ended/closed. Part of the solution was to send autoClose: true along with start:start, end:end in the second createReadStream arg.
The other part of the solution was to limit the max chunksize being sent in the response. The other answer set end like so:
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
...which has the effect of sending the rest of the file from the requested start position through its last byte, no matter how many bytes that may be. However the client browser has the option to only read a portion of that stream, and will, if it doesn't need all of the bytes yet. This will cause the stream read to get blocked until the browser decides it's time to get more data (for example a user action like seek/scrub, or just by playing the stream).
I needed this stream to be closed because I was displaying the <video> element on a page that allowed the user to delete the video file. However the file was not being removed from the filesystem until the client (or server) closed the connection, because that is the only way the stream was getting ended/closed.
My solution was just to set a maxChunk configuration variable, set it to 1MB, and never pipe a read a stream of more than 1MB at a time to the response.
// same code as accepted answer
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
// poor hack to send smaller chunks to the browser
var maxChunk = 1024 * 1024; // 1MB at a time
if (chunksize > maxChunk) {
end = start + maxChunk - 1;
chunksize = (end - start) + 1;
}
This has the effect of making sure that the read stream is ended/closed after each request, and not kept alive by the browser.
I also wrote a separate StackOverflow question and answer covering this issue.
Firstly create app.js file in the directory you want to publish.
var http = require('http');
var fs = require('fs');
var mime = require('mime');
http.createServer(function(req,res){
if (req.url != '/app.js') {
var url = __dirname + req.url;
fs.stat(url,function(err,stat){
if (err) {
res.writeHead(404,{'Content-Type':'text/html'});
res.end('Your requested URI('+req.url+') wasn\'t found on our server');
} else {
var type = mime.getType(url);
var fileSize = stat.size;
var range = req.headers.range;
if (range) {
var parts = range.replace(/bytes=/, "").split("-");
var start = parseInt(parts[0], 10);
var end = parts[1] ? parseInt(parts[1], 10) : fileSize-1;
var chunksize = (end-start)+1;
var file = fs.createReadStream(url, {start, end});
var head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': type
}
res.writeHead(206, head);
file.pipe(res);
} else {
var head = {
'Content-Length': fileSize,
'Content-Type': type
}
res.writeHead(200, head);
fs.createReadStream(url).pipe(res);
}
}
});
} else {
res.writeHead(403,{'Content-Type':'text/html'});
res.end('Sorry, access to that file is Forbidden');
}
}).listen(8080);
Simply run node app.js and your server shall be running on port 8080. Besides video it can stream all kinds of files.