I'm trying to download a file (+200mb) from an url (that requires to be logged in) using the request module from nodejs but when it finishes the download the server starts to slow down until it crashes or gets really slow.
here's my current code (it downloads the whole file but my server crashes eventually):
//Required modules
var http = require('http'),
url = require("url"),
fs = require('fs'),
request = require('request'),
path = require("path"),
events = require("events"),
j = request.jar(),
request = request.defaults({ jar : j });
// make te request login in with cookies
console.log("downloading file :)");
request({
url:"http://example.com/",
method:"POST",
form:{u: "username",p: "password"}
},
function(error,response,body){
setTimeout(function(){
request
.get('http://example.com/test.ashx?file=15')
.on('error', function(err) {
console.log(err);
})
.pipe(fs.createWriteStream("/var/www/filesDir/CustomName.zip"));
console.log(body);
},1000)
}
);
I've tried applying another solution from this answer but for some reason the file is not being downloaded properly, it only shows "Download progress: 0 bytes" all the time maybe it's something related with the login access.
here I put the other code I'm trying to implement from the last sentence:
var http = require('http');
var fs = require('fs');
var url = require("url");
var request = require('request');
var path = require("path");
var events = require("events");
var j = request.jar();
var request = request.defaults({ jar : j });
request({
url:"http://example.com/",
method:"POST",
form:{u:"username",p:"password"}
}, function(error,response,body){
var downloadfile = "http://example.com/test.ashx?file=15";
var host = url.parse(downloadfile).hostname;
var filename = "1977.zip";
var req = http.request({port: 80, host: host, method: 'GET'});
console.log("Downloading file: " + filename);
console.log("Before download request");
req.end();
dlprogress = 0;
setInterval(function () {
console.log("Download progress: " + dlprogress + " bytes");
}, 1000);
req.addListener('response', function (response) {
var downloadfile = fs.createWriteStream(filename, {'flags': 'a'});
console.log("File size " + filename + ": " + response.headers['content-length'] + " bytes.");
response.addListener('data', function (chunk) {
dlprogress += chunk.length;
downloadfile.write(chunk, encoding='binary');
});
response.addListener("end", function() {
downloadfile.end();
console.log("Finished downloading " + filename);
});
});
}
);
It doesn't matter which way you decide to help me with.
I ended up doing it like this, I've tested the code multiple times and the server didn't crash anymore:
var request = require('request');
var filed = require('filed');
var j = request.jar();
var request = request.defaults({ jar : j });
// make the request and login
request({
url: "http://example.com/login",
method:"POST",
// 'u' and 'p' are the field names on the form
form:{u:"username",p:"password"}
}, function(error,response,body){
setTimeout(function(){
var downloadURL = 'http://example.com/download/file.zip';
var downloadPath = "/path/to/download/localNameForFile.zip";
var downloadFile = filed(downloadPath);
var r = request(downloadURL).pipe(downloadFile);
r.on('data', function(data) {
console.log('binary data received');
});
downloadFile.on('end', function () {
console.log(downloadPath, 'file downloaded to path');
});
downloadFile.on('error', function (err) {
console.log(err, 'error downloading file');
});
},3000)
}
);
Related
I am trying to parse a JSON string upon loading the page but I get the following error in the web dev tools: GET http://ipaddress/CulturalEvents/calWrapper 404 not found (Note: ipaddress is the address for our IIS web server). When I click on the error I get the following error: Failed to load resource: the server responded with a status of 404 not found.
Here is my index.js
var titles = new Array();
var descriptions = new Array();
var count = 0;
// Function to cycle through events on display
function changeText() {
$('#evtName').html(titles[count]);
$('#evtDesc').html(descriptions[count]);
if (count < titles.length - 1) {
count++;
} else {
count = 0;
}
}
$(document).ready(function () {
$.ajax({
url:'/CulturalEvents/calWrapper',
type: 'GET',
dataType: 'json',
success: function(calJSON){
let eventCheck = 0;
var today = new Date();
var yyyy = today.getFullYear();
var dd = String(today.getDate()).padStart(2, '0');
var mm = String(today.getMonth() + 1).padStart(2, '0');
today = yyyy + mm + dd;
console.log(today);
for (let i = 0; i < calJSON.length; i++){
if (calJSON[i].startDT == today){
eventCheck = 1;
} else {
eventCheck = 0;
}
if (eventCheck == 1){
titles.push(calJSON[i].summary);
if (calJSON[i].description == ""){
descriptions.push("No description.");
} else{
descriptions.push(calJSON[i].description)
}
} else {
titles.push("No events today.");
descriptions.push("If you know of an event that is not displayed feel free to contact the Diversity Equity and Inclusion committee.")
}
}
}
});
// Rotate through events
changeText();
setInterval(changeText, 10000);
});
It can't find my ajax url '/CulturalEvents/calWrapper'. Note I can run this locally and look for the endpoint /calWrapper and it works perfectly fine, but when I run it on the IIS server it stops working.
Here is my app.js as well:
// C library API
const ffi = require('ffi');
// Express app
const express = require('express');
const app = express();
const path = require('path')
const port = process.env.PORT
const fs = require('fs');
app.use(express.static('public'));
// Send HTML
app.get('/CulturalEvents/', function(req, res){
res.sendFile(path.join(__dirname + '/public/index.html'));
});
// Send style
app.get('/CulturalEvents/style.css', function(req, res) {
res.sendFile(path.join(__dirname + '/public/style.css'));
});
// send JavaScript
app.get('/CulturalEvents/index.js', function (req, res) {
res.readFile(path.join(__dirname + '/public/index.js'), 'utf8', function(err, contents){
res.send(contents);
});
});
// Wrapper function for c library
let wrapper = ffi.Library('./bin/libcalWrapper', {
'calWrapper': [ 'string', [ 'string' ] ]
});
app.get('/CulturalEvents/calWrapper', function (req, res) {
var tempStr = JSON.parse(wrapper.calWrapper(__dirname + "/multiculturalcalendar2021.ics"));
res.send(tempStr);
});
app.listen(port, () => {
console.log(__dirname + '/public/index.js');
});
Also the directory structure is as follows:
CulturalEvents/
public/
index.js
index.html
style.css
app.js
package.json
web.confi
I want to download multiple files from the web using this code:
var fs = require('fs');
var http = require('http');
var request = require('request');
var file;
for(var i = 1; i <= 5; i++) {
//CHECK IF REMOTE FILE EXISTS
request('http://webaddress.com/filename' + i + '.jar', function (err, resp) {
//IF EXISTS DO
if (resp.statusCode == 200) {
//DOWNLOAD DATA AND CREATE A NEW .JAR FILE
file = fs.createWriteStream('D:\\filename' + i + '.jar');
http.get('http://webaddress.com/filename' + i + '.jar', function(response) {
response.pipe(file);
file.on('finish', function() {
file.close();
});
});
}
//FILE DOES NOT EXIST
});
}
The result I want is: multiple files downloaded with filenames filename1-5.jar. The result I am getting is just 1 file with filename filename5.jar (or the last value of the i var in the loop). What am I doing wrong?
Like #Ionut said your requests are async so you need to wait for it
let fs = require('fs');
let request = require('request');
let download = (uri, filename) => {
return new Promise ((resolve, reject) => {
request.head(uri, function(err, res) {
if (res.statusCode === 200) {
request(uri).pipe(fs.createWriteStream(filename)).on('close', resolve);
} else {
reject(res.statusCode);
}
});
});
};
let promises = [];
for(let i = 1; i <= 5; i++) {
promises.push(download('http://webaddress.com/filename' + i + '.jar', 'D:\\filename' + i + '.jar'));
}
Promise.all(promises).then(() => {
process.exit(0);
});
Your request is asynchronous and it will execute only after your loop finishes hence the 5 from the filename. A solution for this is to threat your code separately by creating a new function and call it inside the loop:
var fs = require('fs');
var http = require('http');
var request = require('request');
var file;
function customRequest(i){
//CHECK IF REMOTE FILE EXISTS
return request('http://webaddress.com/filename' + i + '.jar', function(err, resp) {
//IF EXISTS DO
if (resp.statusCode == 200) {
//DOWNLOAD DATA AND CREATE A NEW .JAR FILE
file = fs.createWriteStream('D:\\filename' + i + '.jar');
http.get('http://webaddress.com/filename' + i + '.jar', function(response) {
response.pipe(file);
file.on('finish', function() {
file.close();
});
});
}
//FILE DOES NOT EXIST
});
}
for (var i = 1; i <= 5; i++) {
customRequest(i)
}
i'm having a serious issue with an app i'm building with node.js and express.js.
the app converts videos to mp3. when the video is small upto 5min length everything work as expected, the http server respond with a download button to the client.
but when the video is too big the server prematurely closes connection, and because i'm using http protocol, the client retry the request and this time receives ERR_EMPTY_RESPONSE after a certain amount of time of waiting.
app.post('/', function(req, res) {
var obj_dlConvert = apart_dl_cv.dlConvert(req.body.yt_url,140,apart_dl_cv.generateDir()); //the function that download from youtube and convert
var lien = obj_dlConvert.link;
var dossier = obj_dlConvert.dossier;
var video_stream = obj_dlConvert.streame;
obj_dlConvert.processus.on('end', () =>{
fs.rename(path.join(__dirname,'uploads',dossier,dossier+'.mp3'), path.join(__dirname,'uploads',dossier,'video.mp3'), function(err) {
if (err) {
res.render('dlpage.hbs',{
renameError: true
});
}else res.render('dlpage.hbs',{
dossier: dossier,
fullLink: lien
});
});
}
}
req.on("close", function() {
obj_dlConvert.processus.kill();
obj_dlConvert.processus.on('error', () => {
if (fs.existsSync(path.join(__dirname,'uploads',dossier))){
fse.removeSync(path.join(__dirname,'uploads',dossier));
}
});
});
});
Serving video is not a one time deal. There is a hand-shake between the browser and server. The server needs to be able to provide the 'next' chunk when asked by the browser. Following may by used as an inspiration:
var fs = require("fs"),
http = require("http"),
url = require("url");
exports.serveVideo = function(req, res, file) {
var range = req.headers.range;
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
fs.stat(file, function(err, stats) {
var total = stats.size;
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
var stream = fs.createReadStream(file, { start: start, end: end })
.on("open", function() {
stream.pipe(res);
}).on("error", function(err) {
res.end(err);
});
});
}
I want to send a video file to the client and display the video with .createObjectURL().
Node server.js:
var fs = require("fs"),
http = require("http");
http.createServer(function (req, res) {
if (req.url == "/") {
res.writeHead(200, { "Content-Type": "text/html" });
res.end('<video id="video" src="" autoplay controls loop width="200px" height="200px" muted></video>' +
'<script src="https://code.jquery.com/jquery-2.1.4.min.js"></script>' +
'<script src="blobvideo.js"></script>');
}
else if (req.url == "/blobvideo.js") {
res.writeHead(200, { "Content-Type": "application/javascript" });
fs.readFile('./blobvideo.js', function(err, data) {
res.end(data.toString());
});
}
else if (req.url == "/video") {
fs.readFile('video.mp4', function(err, data) {
res.end(data);
});
}
}).listen(3000);
Client blobvideo.js:
$.ajax( "/video" ).done(function(data) {
var ab = new ArrayBuffer(data.length);
var view = new Uint8Array(ab);
for(var i = 0; i < data.length; ++i) {
view[i] = data[i];
}
blob = new Blob([ab], { type: "video/mp4" });
document.getElementById("video").src = (window.URL || window.webkitURL).createObjectURL(blob);
});
In this code, the video is sent all in one piece, and the video doesn't play. My questions:
How can I fix this to play the video?
How can I change it to stream the file rather than wait for the entire video to download?
Edit for Clarification
I want to use Blob and .createObjectURL() on the client because I am trying to build a peer-to-peer video implementation of the WebRTC RTCPeerConnection, so that static video data can be sent from the client to another client without sending it through the server.
This is pure nodejs javascript which streams video/audio without outside library dependencies and requires NO client code ... launch it using :
node this_code.js
then point your client browser at
http://localhost:8888
Huge benefit is the browser client video rendering UI widgets just work - (ability to jump to some random media location, etc.)
var http = require('http'),
fs = require('fs'),
util = require('util');
// put any audio or video file here
var path = "/path/to/audio/or/video/file/local/to/server/cool.mp4";
var port = 8888;
var host = "localhost";
http.createServer(function (req, res) {
var stat = fs.statSync(path);
var total = stat.size;
if (req.headers.range) {
// meaning client (browser) has moved the forward/back slider
// which has sent this request back to this server logic ... cool
var range = req.headers.range;
var parts = range.replace(/bytes=/, "").split("-");
var partialstart = parts[0];
var partialend = parts[1];
var start = parseInt(partialstart, 10);
var end = partialend ? parseInt(partialend, 10) : total-1;
var chunksize = (end-start)+1;
console.log('RANGE: ' + start + ' - ' + end + ' = ' + chunksize);
var file = fs.createReadStream(path, {start: start, end: end});
res.writeHead(206, { 'Content-Range': 'bytes ' + start + '-' + end + '/' + total, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, 'Content-Type': 'video/mp4' });
file.pipe(res);
} else {
console.log('ALL: ' + total);
res.writeHead(200, { 'Content-Length': total, 'Content-Type': 'video/mp4' });
fs.createReadStream(path).pipe(res);
}
}).listen(port, host);
console.log("Server running at http://" + host + ":" + port + "/");
enjoy,
I would just serve video.mp4 as a static asset and set the src of the video element to the URL for that video.
I've tried to search through stackoverflow for a similar question but most people are asking about the client-side of the NTLMv2 protocol.
I'm implementing a proxy that is performing the server-side of the protocol to authenticate users connecting to the proxy.
I've coded a lot of the protocol but I'm now stuck because the documentation that should take me further is difficult to understand.
This is the best documentation I've found so far: http://www.innovation.ch/personal/ronald/ntlm.html, but how to deal with the LM and NT responses is oblivious to me.
The proxy is located on an application server. The domain server is a different machine.
Example code for the node proxy:
var http = require('http')
, request = require('request')
, ProxyAuth = require('./proxyAuth');
function handlerProxy(req, res) {
ProxyAuth.authorize(req, res);
var options = {
url: req.url,
method: req.method,
headers: req.headers
}
req.pipe(request(options)).pipe(res)
}
var server = http.createServer(handlerProxy);
server.listen(3000, function(){
console.log('Express server listening on port ' + 3000);
});
ProxyAuth.js code:
ProxyAuth = {
parseType3Msg: function(buf) {
var lmlen = buf.readUInt16LE(12);
var lmoff = buf.readUInt16LE(16);
var ntlen = buf.readUInt16LE(20);
var ntoff = buf.readUInt16LE(24);
var dlen = buf.readUInt16LE(28);
var doff = buf.readUInt16LE(32);
var ulen = buf.readUInt16LE(36);
var uoff = buf.readUInt16LE(40);
var hlen = buf.readUInt16LE(44);
var hoff = buf.readUInt16LE(48);
var domain = buf.slice(doff, doff+dlen).toString('utf8');
var user = buf.slice(uoff, uoff+ulen).toString('utf8');
var host = buf.slice(hoff, hoff+hlen).toString('utf8');
var lmresp = buf.slice(lmoff, lmoff+lmlen).toString('utf8');
var ntresp = buf.slice(ntoff, ntoff+ntlen).toString('utf8');
console.log(user, lmresp, ntresp);
/* NOW WHAT DO I DO? */
},
authorize: function(req, res) {
var auth = req.headers['authorization'];
if (!auth) {
res.writeHead(401, {
'WWW-Authenticate': 'NTLM',
});
res.end('<html><body>Proxy Authentication Required</body></html>');
}
else if(auth) {
var header = auth.split(' ');
var buf = new Buffer(header[1], 'base64');
var msg = buf.toString('utf8');
console.log("Decoded", msg);
if (header[0] == "NTLM") {
if (msg.substring(0,8) != "NTLMSSP\x00") {
res.writeHead(401, {
'WWW-Authenticate': 'NTLM',
});
res.end('<html><body>Header not recognized</body></html>');
}
// Type 1 message
if (msg[8] == "\x01") {
console.log(buf.toString('hex'));
var challenge = require('crypto').randomBytes(8);
var type2msg = "NTLMSSP\x00"+
"\x02\x00\x00\x00"+ // 8 message type
"\x00\x00\x00\x00"+ // 12 target name len/alloc
"\x00\x00\x00\x00"+ // 16 target name offset
"\x01\x82\x00\x00"+ // 20 flags
challenge.toString('utf8')+ // 24 challenge
"\x00\x00\x00\x00\x00\x00\x00\x00"+ // 32 context
"\x00\x00\x00\x00\x00\x00\x00\x00"; // 40 target info len/alloc/offset
type2msg = new Buffer(type2msg).toString('base64');
res.writeHead(401, {
'WWW-Authenticate': 'NTLM '+type2msg.trim(),
});
res.end();
}
else if (msg[8] == "\x03") {
console.log(buf.toString('hex'));
ProxyAuth.parseType3Msg(buf);
/* NOW WHAT DO I DO? */
}
}
else if (header[0] == "Basic") {
}
}
}
};
module.exports = ProxyAuth;
The /* NOW WHAT DO I DO? */ comment specifies where I am stuck.
I hope I put enough information there, but let me know if anything else is needed.