Node.js - spawn gzip process - javascript

I'm trying to gzip some data using Node.js...
Specifically, I have data in 'buf' and I want to write a gzipped form of this to 'stream'.
Here is my code:
c1.on('data',function(buf){
var gzip = spawn('gzip', ['-' + (compressionRate-0),'-c', '-']);
gzip.stdin.write(buf);
gzip.stdout.on('data',function(data){
console.log(data);
stream.write(data,'binary');
});
});
The trouble is, it simply won't work! I'm not sure of the exact syntax for spawning processes and piping data to them.
Any help greatly appreciated.
Many thanks in advance,
Edit: here is the original working code where I got the idea from. The project is at: https://github.com/indutny/node.gzip
Can anyone work out how to do this spawning in node.js cos I'm totally stuck!
var spawn = require('child_process').spawn,
Buffer = require('buffer').Buffer;
module.exports = function (data) {
var rate = 8,
enc = 'utf8',
isBuffer = Buffer.isBuffer(data),
args = Array.prototype.slice.call(arguments, 1),
callback;
if (!isBuffer && typeof args[0] === 'string') {
enc = args.shift();
}
if (typeof args[0] === 'number') {
rate = args.shift() - 0;
}
callback = args[0];
var gzip = spawn('gzip', ['-' + (rate - 0), '-c', '-']);
var promise = new
process.EventEmitter,
output = [],
output_len = 0;
// No need to use buffer if no
callback was provided
if (callback) {
gzip.stdout.on('data', function (data) {
output.push(data);
output_len += data.length;
});
gzip.on('exit', function (code) {
var buf = new Buffer(output_len);
for (var a = 0, p = 0; p < output_len; p += output[a++].length) {
output[a].copy(buf, p, 0);
}
callback(code, buf);
});
}
// Promise events
gzip.stdout.on('data', function (data) {
promise.emit('data', data);
});
gzip.on('exit', function (code) {
promise.emit('end');
});
if (isBuffer) {
gzip.stdin.encoding = 'binary';
gzip.stdin.end(data.length ? data : '');
} else {
gzip.stdin.end(data ? data.toString() : '', enc);
}
// Return EventEmitter, so node.gzip can be used for streaming
// (thx #indexzero for that tip)
return promise;
};

Why don't you simply use the gzip node library that you are "inspired from" instead of copying the code?
var gzip = require('gzip');
c1.on('data' function(buf){
gzip(buf, function(err, data){
stream.write(data, 'binary');
}
}
Should work using the library. To install it simply type npm install gzip in your terminal.

Do you need to call the 'end' method on gzip.stdin? I.e.:
gzip.stdin.write(buf);
gzip.stdout.on('data',function(data){
console.log(data);
stream.write(data,'binary');
});
gzip.stdin.end();

Related

Unzipping encrypted compressed report from Amazon Selling Partner

I am using the getReport operation to fetch the documentId, which later I use to download the report document itself which is encrypted and compressed.
The code looks like this:
const documentData = await this.sellingPartner.callAPI({
operation: "getReportDocument",
endpoint: "reports",
path: { reportDocumentId: reportData.reportDocumentId }
})
const request = https.get(documentData.url, function(res) {
const data = [];
res.on("data", chunk => data.push(chunk));
res.on("end", () => {
const key = new Buffer.from(documentData.encryptionDetails.key, 'base64')
const initializationVector = new Buffer.from(documentData.encryptionDetails.initializationVector, 'base64')
const input = Buffer.concat(data)
let result;
try {
result = aes.decryptText(
aes.CIPHERS.AES_256,
key,
initializationVector,
input
)
} catch (e) {
console.log(e)
}
console.log(">>>>")
console.log(result)
zlib.gunzip(result, (err, unzipped) => {
debugger
});
});
}
The current error I am getting is from zlib:
Error: incorrect header check
at Zlib.zlibOnError [as onerror] (node:zlib:189:17)
I am getting the same even if I pass the unencrypted value directly to zlib.
There is a Sample Java code example in the docs, but I cannot understand very well where they do the decryption: before unzipping or after?
In any case, what is the right way to solve this: unzip and decrypt or decrypt and unzip? The former does not work at all, the latter almost works but fails at the unzipping part.
How can I solve the unzip problem?
Short Answer:
Decrypt first and then decompress if compression type is specified.
Longer answer:
After a little research I stumbled upon this example (sadly written in python not in javascript) but it covers the steps to do in more detail.
https://github.com/amzn/selling-partner-api-docs/issues/186#issuecomment-756108550
It contains both compressed or not compressed cases.
Specifically for compressed it looks like this:
import gzip
import requests
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
def ase_cbc_decryptor(key, iv, encryption):
cipher = Cipher(algorithms.AES(base64.b64decode(key)), modes.CBC(base64.b64decode(iv)))
decryptor = cipher.decryptor()
decrypted_text = decryptor.update(encryption)
unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder()
unpaded_text = unpadder.update(decrypted_text)
return unpaded_text + unpadder.finalize()
def get_report_document_content(self, key, iv, url, compression_type=None):
resp = requests.get(url=url)
resp_content = resp.content
decrypted_content = ase_cbc_decryptor(key=key, iv=iv, encryption=resp_content)
if compression_type == 'GZIP':
decrypted_content = gzip.decompress(decrypted_content)
code = 'utf-8'
if 'cp1252' in resp.headers.get('Content-Type', '').lower():
code = 'Cp1252'
return decrypted_content.decode(code)
P.S. keep in mind you need to use AES in CBC mode
Short example here:
https://gist.github.com/manuks/5cef1e536ef791e97b39
var keyhex = "8479768f48481eeb9c8304ce0a58481eeb9c8304ce0a5e3cb5e3cb58479768f4"; //length 32
var blockSize = 16;
function encryptAES(input) {
try {
var iv = require('crypto').randomBytes(16);
//console.info('iv',iv);
var data = new Buffer(input).toString('binary');
//console.info('data',data);
key = new Buffer(keyhex, "hex");
//console.info(key);
var cipher = require('crypto').createCipheriv('aes-256-cbc', key, iv);
// UPDATE: crypto changed in v0.10
// https://github.com/joyent/node/wiki/Api-changes-between-v0.8-and-v0.10
var nodev = process.version.match(/^v(\d+)\.(\d+)/);
var encrypted;
if( nodev[1] === '0' && parseInt(nodev[2]) < 10) {
encrypted = cipher.update(data, 'binary') + cipher.final('binary');
} else {
encrypted = cipher.update(data, 'utf8', 'binary') + cipher.final('binary');
}
var encoded = new Buffer(iv, 'binary').toString('hex') + new Buffer(encrypted, 'binary').toString('hex');
return encoded;
} catch (ex) {
// handle error
// most likely, entropy sources are drained
console.error(ex);
}
}
function decryptAES(encoded) {
var combined = new Buffer(encoded, 'hex');
key = new Buffer(keyhex, "hex");
// Create iv
var iv = new Buffer(16);
combined.copy(iv, 0, 0, 16);
edata = combined.slice(16).toString('binary');
// Decipher encrypted data
var decipher = require('crypto').createDecipheriv('aes-256-cbc', key, iv);
// UPDATE: crypto changed in v0.10
// https://github.com/joyent/node/wiki/Api-changes-between-v0.8-and-v0.10
var nodev = process.version.match(/^v(\d+)\.(\d+)/);
var decrypted, plaintext;
if( nodev[1] === '0' && parseInt(nodev[2]) < 10) {
decrypted = decipher.update(edata, 'binary') + decipher.final('binary');
plaintext = new Buffer(decrypted, 'binary').toString('utf8');
} else {
plaintext = (decipher.update(edata, 'binary', 'utf8') + decipher.final('utf8'));
}
return plaintext;
}
var input="testing";
var encrypted = encryptAES(input);
console.info('encrypted:', encrypted);
var decryped = decryptAES(encrypted);
console.info('decryped:',decryped);

Calculate MD5 hash of a large file using javascript

How do you upload a 500mb file and get a MD5 hash with CryptoJS?
Here is my code:
$('#upload-file').change(function(){
var reader = new FileReader();
reader.addEventListener('load',function () {
var hash = CryptoJS.MD5(CryptoJS.enc.Latin1.parse(this.result));
window.md5 = hash.toString(CryptoJS.enc.Hex);
});
reader.readAsBinaryString(this.files[0]);
});
If the file is under 200mb, it works. Anything bigger, this.result is an empty "".
I've tried:
filereader api on big files
javascript FileReader - parsing long file in chunks
and almost got this to work , but console is complaining about .join("")
http://dojo4.com/blog/processing-huge-files-with-an-html5-file-input
CryptoJS has a progressive api for hash digests. The rest is taken form alediaferia's answer with slight modifications.
function process() {
getMD5(
document.getElementById("my-file-input").files[0],
prog => console.log("Progress: " + prog)
).then(
res => console.log(res),
err => console.error(err)
);
}
function readChunked(file, chunkCallback, endCallback) {
var fileSize = file.size;
var chunkSize = 4 * 1024 * 1024; // 4MB
var offset = 0;
var reader = new FileReader();
reader.onload = function() {
if (reader.error) {
endCallback(reader.error || {});
return;
}
offset += reader.result.length;
// callback for handling read chunk
// TODO: handle errors
chunkCallback(reader.result, offset, fileSize);
if (offset >= fileSize) {
endCallback(null);
return;
}
readNext();
};
reader.onerror = function(err) {
endCallback(err || {});
};
function readNext() {
var fileSlice = file.slice(offset, offset + chunkSize);
reader.readAsBinaryString(fileSlice);
}
readNext();
}
function getMD5(blob, cbProgress) {
return new Promise((resolve, reject) => {
var md5 = CryptoJS.algo.MD5.create();
readChunked(blob, (chunk, offs, total) => {
md5.update(CryptoJS.enc.Latin1.parse(chunk));
if (cbProgress) {
cbProgress(offs / total);
}
}, err => {
if (err) {
reject(err);
} else {
// TODO: Handle errors
var hash = md5.finalize();
var hashHex = hash.toString(CryptoJS.enc.Hex);
resolve(hashHex);
}
});
});
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.2/components/core.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.2/components/md5.js"></script>
<input id="my-file-input" type="file">
<button onclick="process()">Process</button>
You don't need to read the whole file at once and feed it all in one go to CryptoJS routines.
You can create the hasher object, and feed chunks as you read them, and then get the final result.
Sample taken from the CryptoJS documentation
var sha256 = CryptoJS.algo.SHA256.create();
sha256.update("Message Part 1");
sha256.update("Message Part 2");
sha256.update("Message Part 3");
var hash = sha256.finalize();

node.js webtorrent collect all files by magnet link

I have not used node.js before.
Have a .txt file with list of magnet links.
Want to write a json file with list of all files contained in these links.
var WebTorrent = require('webtorrent');
var fs = require('fs');
var client = new WebTorrent();
var array = fs.readFileSync('yop.txt').toString().split("\n");
i = 0;
while (i < array.length) {
//console.log(array[i]);
var magnetURI = array[i];
n = 0;
client.add(magnetURI, function (torrent) {
torrent.files.forEach(function (file) {
//console.log( file.name)
jsonString = JSON.stringify({'book': file.name});
fs.appendFile("data.json", jsonString, function (err) {
if (err) {console.log(err);} else { n++ }
});
if (n == torrent.files.length) {i++ }
})
})
}
when run gives the following error
Sorry for such a terrible code.
var WebTorrent = require('webtorrent')
var fs = require('fs')
var stream = fs.createWriteStream("2.txt");
var client = new WebTorrent()
var array = fs.readFileSync('yop.txt').toString().split("\n");
i = 0;
function parseMagnet (uri){
var magnetURI = uri[i]
console.log(magnetURI)
client.add(magnetURI, function (torrent) {
torrent.files.forEach(function (file) {
writeStr = (uri[i]+ '\n'+ file.name+ '\n');
stream.write(writeStr);
console.log(file.name)
});
console.log('Done !')
console.log(i)
i += 1
parseMagnet(array);
client.remove(magnetURI);
})
}
parseMagnet(array)

Express render broken after save to firebase

I am writing an express app to generate a google map from geo coordinates out of photos. I am attempting to use firebase to save data about the images. The code is fully working except when I save the photo data to firebase it breaks the map rendering on the next page showing connection errors to all my local files in the console like so
So the page is rendering but the map doesn't load and nor do the images. The data I am saving to firebase is actually saving though, and If I remove the function that saves the data to firebase everything works as expected. I think it may have something to do with the way the response is being pushed but I am at a loss. In any other page where I am saving data to firebase it works fine.
Here is the code for the route that is generating the photo data and saving it to firebase:
var express = require('express');
var router = express.Router();
var util = require('util');
var fs = require('fs');
var im = require('imagemagick');
var stormpath = require('express-stormpath');
var _ = require('lodash')
var Firebase = require('firebase');
router.post("/:campaignId", stormpath.loginRequired, function(req, res, next) {
function gatherImages(files, callback) {
//accept single image upload
if (!_.isArray(files)) {
files = [files];
}
var uploads = [];
var count = 0;
files.forEach(function(file) {
fs.exists(file.path, function(exists) {
if (exists) {
var name = req.body[file.originalname];
console.log(name);
var path = file.path;
var upFile = file.name;
uploads.push({
file: upFile,
imgPath: path,
caption: name || 'no comment'
});
count++;
}
if (files.length === count) {
callback(uploads);
}
});
});
}
function getGeoLoc(path, callback) {
im.readMetadata('./' + path, function(error, metadata) {
var geoCoords = false;
if (error) throw error;
if (metadata.exif.gpsLatitude && metadata.exif.gpsLatitudeRef) {
var lat = getDegrees(metadata.exif.gpsLatitude.split(','));
var latRef = metadata.exif.gpsLatitudeRef;
if (latRef === 'S') {
lat = lat * -1;
}
var lng = getDegrees(metadata.exif.gpsLongitude.split(','));
var lngRef = metadata.exif.gpsLongitudeRef;
if (lngRef === 'W') {
lng = lng * -1;
}
var coordinate = {
lat: lat,
lng: lng
};
geoCoords = coordinate.lat + ' ' + coordinate.lng;
console.log(geoCoords);
}
callback(geoCoords);
});
}
function getDegrees(lat) {
var degrees = 0;
for (var i = 0; i < lat.length; i++) {
var cleanNum = lat[i].replace(' ', '');
var parts = cleanNum.split('/');
var coord = parseInt(parts[0]) / parseInt(parts[1]);
if (i == 1) {
coord = coord / 60;
} else if (i == 2) {
coord = coord / 3600;
}
degrees += coord;
}
return degrees.toFixed(6);
}
function processImages(uploads, callback) {
var finalImages = [];
var count = 0;
uploads.forEach(function(upload) {
var path = upload.imgPath;
getGeoLoc(path, function(geoCoords) {
upload.coords = geoCoords;
finalImages.push(upload);
count++;
if (uploads.length === count) {
callback(finalImages);
}
});
});
}
function saveImageInfo(finalImages, callback) {
var campaignId = req.param('campaignId');
var user = res.locals.user;
var count = 0;
var campaignPhotosRef = new Firebase('https://vivid-fire-567.firebaseio.com/BSB/userStore/' + user.username + '/campaigns/' + campaignId + '/photos');
finalImages.forEach(function(image) {
campaignPhotosRef.push(image, function(err) {
if (err) {
console.log(err);
} else {
count++;
if (finalImages.length === count) {
callback(finalImages);
} else {
return;
}
}
});
});
}
if (req.files) {
if (req.files.size === 0) {
return next(new Error("Why didn't you select a file?"));
}
gatherImages(req.files.imageFiles, function(uploads) {
processImages(uploads, function(finalImages) {
saveImageInfo(finalImages, function(finalImages) {
var campaignId = req.param('campaignId');
console.log(res.req.next);
res.render("uploadMapPage", {
title: "File(s) Uploaded Successfully!",
files: finalImages,
campaignId: campaignId,
scripts: ['https://maps.googleapis.com/maps/api/js?key=AIzaSyCU42Wpv6BtNO51t7xGJYnatuPqgwnwk7c', '/javascripts/getPoints.js']
});
});
});
});
}
});
module.exports = router;
This is the only file I have written trying to push multiple objects to firebase. This is my first time using Firebase and Stormpath so any help would be greatly appreciated. Also one other thing that may be helpful is the error from the terminal being output when the issue happens:
POST /uploaded/-JapMLDYzPnbtjvt001X 200 690.689 ms - 2719
/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/firebase/lib/firebase-node.js:24
?a:null}function Db(a){try{a()}catch(b){setTimeout(function(){throw b;},Math.f
^
TypeError: Property 'next' of object #<IncomingMessage> is not a function
at fn (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/response.js:899:25)
at EventEmitter.app.render (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/application.js:532:5)
at ServerResponse.res.render (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/response.js:904:7)
at /Users/jpribesh/Desktop/Code/BanditSignBoss/routes/campaigns.js:20:25
at Array.forEach (native)
at /Users/jpribesh/Desktop/Code/BanditSignBoss/routes/campaigns.js:16:18
at /Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/firebase/lib/firebase-node.js:25:533
at Db (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/firebase/lib/firebase-node.js:24:165)
at Ye (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/firebase/lib/firebase-node.js:124:216)
at Ze (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/firebase/lib/firebase-node.js:123:818)
UPDATE: It seems that the connection errors are inconsistent. Sometimes the images display just fine, sometimes only some of the images get a connection error, and other times everything including the google map script gets a connection error. This is really throwing me off no idea what the issue is. Any help or suggestions is greatly appreciated!
UPDATE 2: I changed the function saving the image data to firebase to use the firebase push function callback (to indicate completion) and added a length check on the forEach loop running to save each image's data. See updated code above. I am now getting the following error for each image that is uploaded in the terminal, but the connection errors are gone:
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (http.js:689:11)
at ServerResponse.header (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/response.js:666:10)
at ServerResponse.send (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/response.js:146:12)
at fn (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/response.js:900:10)
at View.exports.renderFile [as engine] (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/jade/lib/jade.js:325:12)
at View.render (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/view.js:93:8)
at EventEmitter.app.render (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/application.js:530:10)
at ServerResponse.res.render (/Users/jpribesh/Desktop/Code/BanditSignBoss/node_modules/express/lib/response.js:904:7)
at /Users/jpribesh/Desktop/Code/BanditSignBoss/routes/campaigns.js:20:25
at Array.forEach (native)
OK I finally figured out the issue here. I did a few things to remedy my problem. First I converted the route to use next properly to separate out each part of the route out, it processes the images, then saves, then renders. Here is the updated code from that file:
var express = require('express');
var router = express.Router();
var util = require('util');
var fs = require('fs');
var im = require('imagemagick');
var stormpath = require('express-stormpath');
var _ = require('lodash')
var Firebase = require('firebase');
function processData(req, res, next) {
function gatherImages(files, callback) {
//accept single image upload
if (!_.isArray(files)) {
files = [files];
}
var uploads = [];
var count = 0;
files.forEach(function(file) {
fs.exists(file.path, function(exists) {
if (exists) {
var name = req.body[file.originalname];
console.log(name);
var path = file.path;
var upFile = file.name;
uploads.push({
file: upFile,
imgPath: path,
caption: name || 'no comment'
});
count++;
}
if (files.length === count) {
callback(uploads);
}
});
});
}
function getGeoLoc(path, callback) {
im.readMetadata('./' + path, function(error, metadata) {
var geoCoords = false;
if (error) throw error;
if (metadata.exif.gpsLatitude && metadata.exif.gpsLatitudeRef) {
var lat = getDegrees(metadata.exif.gpsLatitude.split(','));
var latRef = metadata.exif.gpsLatitudeRef;
if (latRef === 'S') {
lat = lat * -1;
}
var lng = getDegrees(metadata.exif.gpsLongitude.split(','));
var lngRef = metadata.exif.gpsLongitudeRef;
if (lngRef === 'W') {
lng = lng * -1;
}
var coordinate = {
lat: lat,
lng: lng
};
geoCoords = coordinate.lat + ' ' + coordinate.lng;
console.log(geoCoords);
}
callback(geoCoords);
});
}
function getDegrees(lat) {
var degrees = 0;
for (var i = 0; i < lat.length; i++) {
var cleanNum = lat[i].replace(' ', '');
var parts = cleanNum.split('/');
var coord = parseInt(parts[0]) / parseInt(parts[1]);
if (i == 1) {
coord = coord / 60;
} else if (i == 2) {
coord = coord / 3600;
}
degrees += coord;
}
return degrees.toFixed(6);
}
function processImages(uploads, callback) {
var finalImages = [];
var count = 0;
uploads.forEach(function(upload) {
var path = upload.imgPath;
getGeoLoc(path, function(geoCoords) {
upload.coords = geoCoords;
finalImages.push(upload);
count++;
if (uploads.length === count) {
callback(finalImages);
}
});
});
}
if (req.files) {
if (req.files.size === 0) {
return next(new Error("Why didn't you select a file?"));
}
gatherImages(req.files.imageFiles, function(uploads) {
processImages(uploads, function(finalImages) {
req.finalImages = finalImages;
req.campaignId = req.param('campaignId');
next();
});
});
}
}
function saveImageInfo(req, res, next) {
var user = res.locals.user;
var count = 0;
var campaignPhotosRef = new Firebase('https://vivid-fire-567.firebaseio.com/BSB/userStore/' + user.username + '/campaigns/' + req.campaignId + '/photos');
var finalImages = req.finalImages;
finalImages.forEach(function(image) {
campaignPhotosRef.push(image, function(err) {
if (err) {
console.log(err);
} else {
console.log('Data saved successfully: ' + image);
count++;
if (req.finalImages.length === count) {
next();
}
}
});
});
}
router.post("/:campaignId", stormpath.loginRequired, processData, saveImageInfo, function(req, res) {
res.render("uploadMapPage", {
title: "File(s) Uploaded Successfully!",
files: req.finalImages,
campaignId: req.campaignId,
scripts: ['https://maps.googleapis.com/maps/api/js?key=AIzaSyCU42Wpv6BtNO51t7xGJYnatuPqgwnwk7c', '/javascripts/getPoints.js']
});
});
module.exports = router;
Then I realized in the tracestack I included in my question part of it was tracing back to another file I was using firebase in. I was using a call to .on() instead of using .once() when pulling my data. After reorganizing my route and changing all my calls to .on to .once for firebase data everything is now working properly. I think the real issue here was the use of .on() on my firebase calls instead of .once() as the .on() watches for events continually rather than .once which obviously only watches for it once.

Conceptual: Create a counter in an external function

I've written some javascript to successfully download hundreds of files from an external site, using wget at the core.
After downloading all of the files, I would like to do some stuff with them. The issue is, the files aren't of equal size. So, the last wget formed isn't necessarily the last file downloaded, meaning I can't really tell when the last file has completed.
I do, however, know how many files there are in total, and the number associated with each wget.
I have 3 js files, [parseproducts.js] ==> [createurl.js] ==> [downloadurl.js]
Using this information, how can I tell when all of the files have been downloaded?
I tried creating a "ticker" function in another file but the function resets itself on each instance, so it doesn't work at all!
Edit: Code added Didn't do this initially because I didn't think people would want to trawl through it! I'm new to programming/javascript/node. Please let me know if there's something that I could do better (I'm sure most of it could be more efficient!)
parseproducts.js
var fs = require('fs');
var iset = require('./ticker.js');
var createurl = require('./createurl.js');
var array = [];
filename = 'productlist.txt';
fs.readFile(filename, 'utf8', function(err, data) {
if (err) throw err;
content = data;
parseFile();
});
function parseFile() {
var stringarray = String(content).split(";");
for (var index = 0; index < stringarray.length; ++index) {
createurl(stringarray[index],index,stringarray.length);
console.log(index+'/'+stringarray.length+' sent.');
if (index === 0) {
iset(true,stringarray.length);
} else {
iset (false,stringarray.length);
}
};
};
createurl.js
function create(partnumber,iteration,total) {
var JSdownloadURL = require('./downloadurl.js');
JSdownloadURL(createurl(partnumber),partnumber,iteration,total);
function createurl(partnumber) {
var URL = ('"https://data.icecat.biz/xml_s3/xml_server3.cgi?prod_id='+partnumber+';vendor=hp;lang=en;output=productxml"');
return URL;
};
};
module.exports = create;
downloadurl.js
function downloadurl(URL,partnumber,iteration,total) {
// Dependencies
var fs = require('fs');
var url = require('url');
var http = require('http');
var exec = require('child_process').exec;
var spawn = require('child_process').spawn;
var checkfiles = require('./checkfiles.js');
// App variables
var file_url = URL;
var DOWNLOAD_DIR = './downloads/';
// We will be downloading the files to a directory, so make sure it's there
var mkdir = 'mkdir -p ' + DOWNLOAD_DIR;
var child = exec(mkdir, function(err, stdout, stderr) {
if (err) throw err;
else download_file_wget(file_url);
});
// Function to download file using wget
var download_file_wget = function(file_url) {
// compose the wget command
var wget = 'wget --http-user="MyAccount" --http-password="MyPassword" -P ' + DOWNLOAD_DIR + ' ' + file_url;
// excute wget using child_process' exec function
var child = exec(wget, function(err, stdout, stderr) {
if (err) throw err;
else console.log(iteration+'/'+total+' downloaded. '+partnumber + ' downloaded to ' + DOWNLOAD_DIR);
});
};
};
module.exports = downloadurl;
Failed attempt ticker.js
function iset(bol,total) {
if (bol === true) {
var i = 0;
} else {
var i = 1;
};
counter(i, total);
}
function counter(i,total) {
var n = n + i;
if (n === (total - 1)) {
var checkfiles = require('./checkfiles.js');
checkfiles(total);
} else {
console.log('nothing done');
};
}
module.exports = iset;
Update In response to answer
This is what my code looks like now. However, I get the error
child_process.js:945
throw errnoException(process._errno, 'spawn');
^
Error: spawn EMFILE
// Dependencies
var fs = require('fs');
var url = require('url');
var http = require('http');
var exec = require('child_process').exec;
var spawn = require('child_process').spawn;
var checkfiles = require('./checkfiles.js');
function downloadurl(URL,partnumber,iteration,total,clb) {
// App variables
var file_url = URL;
var DOWNLOAD_DIR = './downloads/';
// We will be downloading the files to a directory, so make sure it's there
var mkdir = 'mkdir -p ' + DOWNLOAD_DIR;
var child = exec(mkdir, function(err, stdout, stderr) {
if (err) throw err;
else download_file_wget(file_url);
});
var child = exec(mkdir, function(err, stdout, stderr) {
if (err) {
clb(err);
} else {
var wget = 'wget --http-user="amadman114" --http-password="Chip10" -P ' + DOWNLOAD_DIR + ' ' + file_url;
// excute wget using child_process' exec function
var child = exec(wget, function(err, stdout, stderr) {
if (err) {
clb(err);
} else {
console.log(iteration+'/'+total+' downloaded. '+partnumber + ' downloaded to ' + DOWNLOAD_DIR);
clb(null); // <-- you can pass more args here if you want, like result
// as a general convention callbacks take a form of
// callback(err, res1, res2, ...)
}
});
}
});
};
function clb() {
var LIMIT = 100,
errs = [];
for (var i = 0; i < LIMIT; i++) {
downloadurl(URL,partnumber,iternation,total, function(err) {
if (err) {
errs.push(err);
}
LIMIT--;
if (!LIMIT) {
finalize(errs);
}
});
}
}
function finalize(errs) {
// you can now check for err
//or do whatever stuff to finalize the code
}
module.exports = downloadurl;
OK, so you have this function downloadurl. What you need to do is to pass one more argument to it: the callback. And please, move requirements outside the function and don't define a function in a function unless necessary:
var fs = require('fs');
// other dependencies and constants
function downloadurl(URL,partnumber,iteration,total, clb) { // <-- new arg
// some code
var child = exec(mkdir, function(err, stdout, stderr) {
if (err) {
clb(err);
} else {
var wget = 'wget --http-user="MyAccount" --http-password="MyPassword" -P ' + DOWNLOAD_DIR + ' ' + file_url;
// excute wget using child_process' exec function
var child = exec(wget, function(err, stdout, stderr) {
if (err) {
clb(err);
} else {
console.log(iteration+'/'+total+' downloaded. '+partnumber + ' downloaded to ' + DOWNLOAD_DIR);
clb(null); // <-- you can pass more args here if you want, like result
// as a general convention callbacks take a form of
// callback(err, res1, res2, ...)
}
});
}
});
};
This look nicer, doesn't it? Now when you call that function multiple times you do:
var LIMIT = 100,
errs = [];
for (var i = 0; i < LIMIT; i++) {
downloadurl(..., function(err) {
if (err) {
errs.push(err);
}
LIMIT--;
if (!LIMIT) {
finalize(errs);
}
});
}
function finalize(errs) {
// you can now check for err
//or do whatever stuff to finalize the code
}
That's a general idea. You have to tweak it to your needs (in particular you have to modify the intermediate function to accept a callback as well). Of course there are libraries which will take care of most this for you like kriskowal's Q (Q.all) or caolan's async (async.parallel).
Not sure if I have understood the problem correctly as I don't see the code. I have worked on creating a download engine. I used to make background AJAX calls to download files. After every successful download or 'onComplete' event I used to increment one variable to keep track of downloaded files. Provdided user won't refresh the page till all the download is complete. Else the download counter can be saved in LocalStorage also.

Categories