i'm having a serious issue with an app i'm building with node.js and express.js.
the app converts videos to mp3. when the video is small upto 5min length everything work as expected, the http server respond with a download button to the client.
but when the video is too big the server prematurely closes connection, and because i'm using http protocol, the client retry the request and this time receives ERR_EMPTY_RESPONSE after a certain amount of time of waiting.
app.post('/', function(req, res) {
var obj_dlConvert = apart_dl_cv.dlConvert(req.body.yt_url,140,apart_dl_cv.generateDir()); //the function that download from youtube and convert
var lien = obj_dlConvert.link;
var dossier = obj_dlConvert.dossier;
var video_stream = obj_dlConvert.streame;
obj_dlConvert.processus.on('end', () =>{
fs.rename(path.join(__dirname,'uploads',dossier,dossier+'.mp3'), path.join(__dirname,'uploads',dossier,'video.mp3'), function(err) {
if (err) {
res.render('dlpage.hbs',{
renameError: true
});
}else res.render('dlpage.hbs',{
dossier: dossier,
fullLink: lien
});
});
}
}
req.on("close", function() {
obj_dlConvert.processus.kill();
obj_dlConvert.processus.on('error', () => {
if (fs.existsSync(path.join(__dirname,'uploads',dossier))){
fse.removeSync(path.join(__dirname,'uploads',dossier));
}
});
});
});
Serving video is not a one time deal. There is a hand-shake between the browser and server. The server needs to be able to provide the 'next' chunk when asked by the browser. Following may by used as an inspiration:
var fs = require("fs"),
http = require("http"),
url = require("url");
exports.serveVideo = function(req, res, file) {
var range = req.headers.range;
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
fs.stat(file, function(err, stats) {
var total = stats.size;
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
var stream = fs.createReadStream(file, { start: start, end: end })
.on("open", function() {
stream.pipe(res);
}).on("error", function(err) {
res.end(err);
});
});
}
Related
I am trying to parse a JSON string upon loading the page but I get the following error in the web dev tools: GET http://ipaddress/CulturalEvents/calWrapper 404 not found (Note: ipaddress is the address for our IIS web server). When I click on the error I get the following error: Failed to load resource: the server responded with a status of 404 not found.
Here is my index.js
var titles = new Array();
var descriptions = new Array();
var count = 0;
// Function to cycle through events on display
function changeText() {
$('#evtName').html(titles[count]);
$('#evtDesc').html(descriptions[count]);
if (count < titles.length - 1) {
count++;
} else {
count = 0;
}
}
$(document).ready(function () {
$.ajax({
url:'/CulturalEvents/calWrapper',
type: 'GET',
dataType: 'json',
success: function(calJSON){
let eventCheck = 0;
var today = new Date();
var yyyy = today.getFullYear();
var dd = String(today.getDate()).padStart(2, '0');
var mm = String(today.getMonth() + 1).padStart(2, '0');
today = yyyy + mm + dd;
console.log(today);
for (let i = 0; i < calJSON.length; i++){
if (calJSON[i].startDT == today){
eventCheck = 1;
} else {
eventCheck = 0;
}
if (eventCheck == 1){
titles.push(calJSON[i].summary);
if (calJSON[i].description == ""){
descriptions.push("No description.");
} else{
descriptions.push(calJSON[i].description)
}
} else {
titles.push("No events today.");
descriptions.push("If you know of an event that is not displayed feel free to contact the Diversity Equity and Inclusion committee.")
}
}
}
});
// Rotate through events
changeText();
setInterval(changeText, 10000);
});
It can't find my ajax url '/CulturalEvents/calWrapper'. Note I can run this locally and look for the endpoint /calWrapper and it works perfectly fine, but when I run it on the IIS server it stops working.
Here is my app.js as well:
// C library API
const ffi = require('ffi');
// Express app
const express = require('express');
const app = express();
const path = require('path')
const port = process.env.PORT
const fs = require('fs');
app.use(express.static('public'));
// Send HTML
app.get('/CulturalEvents/', function(req, res){
res.sendFile(path.join(__dirname + '/public/index.html'));
});
// Send style
app.get('/CulturalEvents/style.css', function(req, res) {
res.sendFile(path.join(__dirname + '/public/style.css'));
});
// send JavaScript
app.get('/CulturalEvents/index.js', function (req, res) {
res.readFile(path.join(__dirname + '/public/index.js'), 'utf8', function(err, contents){
res.send(contents);
});
});
// Wrapper function for c library
let wrapper = ffi.Library('./bin/libcalWrapper', {
'calWrapper': [ 'string', [ 'string' ] ]
});
app.get('/CulturalEvents/calWrapper', function (req, res) {
var tempStr = JSON.parse(wrapper.calWrapper(__dirname + "/multiculturalcalendar2021.ics"));
res.send(tempStr);
});
app.listen(port, () => {
console.log(__dirname + '/public/index.js');
});
Also the directory structure is as follows:
CulturalEvents/
public/
index.js
index.html
style.css
app.js
package.json
web.confi
I'm trying to download a file (+200mb) from an url (that requires to be logged in) using the request module from nodejs but when it finishes the download the server starts to slow down until it crashes or gets really slow.
here's my current code (it downloads the whole file but my server crashes eventually):
//Required modules
var http = require('http'),
url = require("url"),
fs = require('fs'),
request = require('request'),
path = require("path"),
events = require("events"),
j = request.jar(),
request = request.defaults({ jar : j });
// make te request login in with cookies
console.log("downloading file :)");
request({
url:"http://example.com/",
method:"POST",
form:{u: "username",p: "password"}
},
function(error,response,body){
setTimeout(function(){
request
.get('http://example.com/test.ashx?file=15')
.on('error', function(err) {
console.log(err);
})
.pipe(fs.createWriteStream("/var/www/filesDir/CustomName.zip"));
console.log(body);
},1000)
}
);
I've tried applying another solution from this answer but for some reason the file is not being downloaded properly, it only shows "Download progress: 0 bytes" all the time maybe it's something related with the login access.
here I put the other code I'm trying to implement from the last sentence:
var http = require('http');
var fs = require('fs');
var url = require("url");
var request = require('request');
var path = require("path");
var events = require("events");
var j = request.jar();
var request = request.defaults({ jar : j });
request({
url:"http://example.com/",
method:"POST",
form:{u:"username",p:"password"}
}, function(error,response,body){
var downloadfile = "http://example.com/test.ashx?file=15";
var host = url.parse(downloadfile).hostname;
var filename = "1977.zip";
var req = http.request({port: 80, host: host, method: 'GET'});
console.log("Downloading file: " + filename);
console.log("Before download request");
req.end();
dlprogress = 0;
setInterval(function () {
console.log("Download progress: " + dlprogress + " bytes");
}, 1000);
req.addListener('response', function (response) {
var downloadfile = fs.createWriteStream(filename, {'flags': 'a'});
console.log("File size " + filename + ": " + response.headers['content-length'] + " bytes.");
response.addListener('data', function (chunk) {
dlprogress += chunk.length;
downloadfile.write(chunk, encoding='binary');
});
response.addListener("end", function() {
downloadfile.end();
console.log("Finished downloading " + filename);
});
});
}
);
It doesn't matter which way you decide to help me with.
I ended up doing it like this, I've tested the code multiple times and the server didn't crash anymore:
var request = require('request');
var filed = require('filed');
var j = request.jar();
var request = request.defaults({ jar : j });
// make the request and login
request({
url: "http://example.com/login",
method:"POST",
// 'u' and 'p' are the field names on the form
form:{u:"username",p:"password"}
}, function(error,response,body){
setTimeout(function(){
var downloadURL = 'http://example.com/download/file.zip';
var downloadPath = "/path/to/download/localNameForFile.zip";
var downloadFile = filed(downloadPath);
var r = request(downloadURL).pipe(downloadFile);
r.on('data', function(data) {
console.log('binary data received');
});
downloadFile.on('end', function () {
console.log(downloadPath, 'file downloaded to path');
});
downloadFile.on('error', function (err) {
console.log(err, 'error downloading file');
});
},3000)
}
);
I am currently working on an auction script using node.js and socket.io.But site was developed by using PHP & MySQL. Here I'm using node.js + socket.io for auction bidding process only. The site will have 500-1000 logged in users viewing a single page during the auction. Only one item will be on auction and it will be sold at one day once.
I will be broadcasting(emitting) a countdown timer to all of the users from the server to the client. On the server side I will be using setInterval(),recursive setTimeout() of 1 second to countdown to the auction end time. Apart from this the only other message being sent across will be the current bid being passed from a single client to the server then broadcast to all. This way to do be a reliable? And will it be able to handle the usage on the server?. Here I've tested with 500 users means in browsers getting hanging the timer.
Server.js
var cluster = require('cluster');
var app = require('express')();
//var http = require('http');
var https = require('https');
var socket = require('socket.io');
var redis = require('redis');
var redisAdapter = require('socket.io-redis');
var request = require('request');
var fs = require('fs');
var options = {
key: fs.readFileSync('keys/e1317_0f2c9_71565598d419e37e376ccef5c2827113.key'),
cert: fs.readFileSync('certs/e1317_0f2c9_1468152279_2dc46c1f2cc135a.crt'),
ca: fs.readFileSync('cabundles/90490a5c829d2aca24f22b5820864c6e_1935558000.cabundle')
};
//var server = http.createServer( app );
var server = https.createServer(options, app);
var io = socket.listen(server);
var port = process.env.PORT || 8080;
var workers = process.env.WORKERS || require('os').cpus().length;
var redisUrl = process.env.REDISTOGO_URL || 'redis://127.0.0.1:6379';
var redisOptions = require('parse-redis-url')(redis).parse(redisUrl);
var pub = redis.createClient(redisOptions.port, redisOptions.host, {
detect_buffers: true,
return_buffers: true,
auth_pass: redisOptions.password
});
var sub = redis.createClient(redisOptions.port, redisOptions.host, {
detect_buffers: true,
return_buffers: true,
auth_pass: redisOptions.password
});
io.adapter(redisAdapter({
pubClient: pub,
subClient: sub
}));
console.log('Redis adapter started with url: ' + redisUrl);
io.sockets.on('connection', function(client) {
//console.log('first');
client.on('nauction', function(data) {
io.sockets.emit('nauction', data);
});
});
io.on('connection', function(socket) {
//console.log('in');
console.log('connected client count:' + io.sockets.sockets.length);
var recursive = function() {
//console.log("It has been one second!");
if (io.sockets.sockets.length > 0) {
request('https://www.example.com/file.php', function(error, response, body) {
if (!error && response.statusCode == 200) {
data = JSON.parse(body);
socket.volatile.emit('auction_data', {
'auction_data': data
});
//console.log(data);
} else {
//console.log('else');
console.log(error);
}
});
} //else{
//console.log('No clients connected now');
//}
setTimeout(recursive, 1000);
}
recursive();
socket.on("disconnect", function() {
console.log('clear interval')
//clearInterval(interval);
clearTimeout(recursive);
});
});
if (cluster.isMaster) {
console.log('start cluster with %s workers', workers - 1);
workers--;
for (var i = 0; i < workers; ++i) {
var worker = cluster.fork();
console.log('worker %s started.', worker.process.pid);
}
cluster.on('death', function(worker) {
console.log('worker %s died. restart...', worker.process.pid);
});
} else {
start();
}
function start() {
server.listen(port, function() {
console.log('listening on *:' + port);
});
}
Client.js
socket.on('auction_data', function(auction_details) {
//console.log(auction_details);
$.each(auction_details, function(keys, values) {
//countdwon formation
var tm, days, hrs, mins, secs;
days = value.auction_data.time.days;
if (value.auction_data.time.hours < 10) {
hrs = ("0" + value.auction_data.time.hours);
} else {
hrs = value.auction_data.time.hours;
}
if (value.auction_data.time.mins < 10) {
mins = ("0" + value.auction_data.time.mins);
} else {
mins = value.auction_data.time.mins;
}
if (value.auction_data.time.secs < 10) {
secs = ("0" + value.auction_data.time.secs);
} else {
secs = value.auction_data.time.secs;
}
if (days == 0) {
tm = '' + hrs + '' + '' + mins + '' + '' + secs + '';
} else {
tm = '' + days + '' + '' + hrs + '' + '' + mins + '' + '' + secs + '';
}
$('#auction_' + value.auction_data.product_id + " .countdown").html(tm);
});
});
I'm waiting for your answers to fix the browser hanging problem.
First Question: Is This way to do be a reliable?
Sending the time every Second to EVERY client is not necessary. Simply send them the time at their first visit and use a local timer (at their local page) to reduce the time every second.
You also need to check for server-time on every bid (more secure).
If this is not "secure" enough for you, send the time with the changing bid. You only have to send the actual Bid, when it changed (using Broadcast) or when the user joins the site (just send it to him).
Second Question: And will it be able to handle the usage on the server?
Yes and No.
If your Sever is good enough (every 5$ server with endless traffic would fit),
you should not get in trouble. Only, if your script is very very bad and seeded with Memory Leaks.
Now a few tips:
Never trust the user input - parse it before you use it!
Recalculate everything you get from the client on the Server.
Send the Client only what he needs. He does not need information about stuff that he does not use.
If this was the answer you hoped for, please select the green arrow on the left.
If not, write a comment here and I will give more tips.
I want to send a video file to the client and display the video with .createObjectURL().
Node server.js:
var fs = require("fs"),
http = require("http");
http.createServer(function (req, res) {
if (req.url == "/") {
res.writeHead(200, { "Content-Type": "text/html" });
res.end('<video id="video" src="" autoplay controls loop width="200px" height="200px" muted></video>' +
'<script src="https://code.jquery.com/jquery-2.1.4.min.js"></script>' +
'<script src="blobvideo.js"></script>');
}
else if (req.url == "/blobvideo.js") {
res.writeHead(200, { "Content-Type": "application/javascript" });
fs.readFile('./blobvideo.js', function(err, data) {
res.end(data.toString());
});
}
else if (req.url == "/video") {
fs.readFile('video.mp4', function(err, data) {
res.end(data);
});
}
}).listen(3000);
Client blobvideo.js:
$.ajax( "/video" ).done(function(data) {
var ab = new ArrayBuffer(data.length);
var view = new Uint8Array(ab);
for(var i = 0; i < data.length; ++i) {
view[i] = data[i];
}
blob = new Blob([ab], { type: "video/mp4" });
document.getElementById("video").src = (window.URL || window.webkitURL).createObjectURL(blob);
});
In this code, the video is sent all in one piece, and the video doesn't play. My questions:
How can I fix this to play the video?
How can I change it to stream the file rather than wait for the entire video to download?
Edit for Clarification
I want to use Blob and .createObjectURL() on the client because I am trying to build a peer-to-peer video implementation of the WebRTC RTCPeerConnection, so that static video data can be sent from the client to another client without sending it through the server.
This is pure nodejs javascript which streams video/audio without outside library dependencies and requires NO client code ... launch it using :
node this_code.js
then point your client browser at
http://localhost:8888
Huge benefit is the browser client video rendering UI widgets just work - (ability to jump to some random media location, etc.)
var http = require('http'),
fs = require('fs'),
util = require('util');
// put any audio or video file here
var path = "/path/to/audio/or/video/file/local/to/server/cool.mp4";
var port = 8888;
var host = "localhost";
http.createServer(function (req, res) {
var stat = fs.statSync(path);
var total = stat.size;
if (req.headers.range) {
// meaning client (browser) has moved the forward/back slider
// which has sent this request back to this server logic ... cool
var range = req.headers.range;
var parts = range.replace(/bytes=/, "").split("-");
var partialstart = parts[0];
var partialend = parts[1];
var start = parseInt(partialstart, 10);
var end = partialend ? parseInt(partialend, 10) : total-1;
var chunksize = (end-start)+1;
console.log('RANGE: ' + start + ' - ' + end + ' = ' + chunksize);
var file = fs.createReadStream(path, {start: start, end: end});
res.writeHead(206, { 'Content-Range': 'bytes ' + start + '-' + end + '/' + total, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, 'Content-Type': 'video/mp4' });
file.pipe(res);
} else {
console.log('ALL: ' + total);
res.writeHead(200, { 'Content-Length': total, 'Content-Type': 'video/mp4' });
fs.createReadStream(path).pipe(res);
}
}).listen(port, host);
console.log("Server running at http://" + host + ":" + port + "/");
enjoy,
I would just serve video.mp4 as a static asset and set the src of the video element to the URL for that video.
Here is a simple TCP-Server stress test. As long as we send only one message per client everything works as expected. But when two messages per client are send,
the server suddenly stops without an exception or error.
So is this a bug or a feature?
var net = require("net");
var async = require("async");
var cluster = require("cluster");
// `ulimit -n` tells us that we can open max. 1024 files per process.
// Creating a socket means opening a file so we are limited.
var CLIENTS = 1000;
// Here is the weird part:
// - sending one message per client works fine
// - sending multiple message per client sucks
var MESSAGES = 2;
var TOTAL = CLIENTS * MESSAGES;
var PORT = 1234;
var HOST = "127.0.0.1";
if (cluster.isMaster) {
var count = 0;
var start = new Date;
var server = net.createServer(function(socket) {
socket.on("data", function(data) {
var t;
count++;
console.log("server received " + count + " messages");
socket.write(data, function(err) { if (err) console.error(err); });
if (count === TOTAL) {
t = (new Date) - start;
console.log("server received and sent " + count + " messages within " + t + "ms");
}
});
});
server.listen(PORT, HOST, function() { cluster.fork(); });
} else {
var run = function(i) {
var c = net.connect({ port: PORT, host: HOST }, function() {
var tasks = (function() {
var results = [];
for (var x = 1; x <= MESSAGES; ++x) {
results.push((function(x) {
return function(next) { c.write("Hello server!", next); };
})(x));
}
return results;
})();
async.series(tasks, function(err) {
if (err) { console.error(err); }
});
});
};
for (var i = 1; i <= CLIENTS; ++i) { run(i); }
}
Tested on Linux 3.11, Node.js 0.10.21
You're assuming that calling .write twice from a client triggers the data event twice on the server, not taking into account any buffering that might be going on which will coalecse multiple writes.
When the callback to .write is called, it doesn't mean the message it actually sent, it means that the message is put in some kernel buffer (which might contain more than one message when it's sent to the server).