I have a Node.Js web app with Express.Js that reads values from xml files, store values from all xml files into an array with sub-array represent the separation of per xml file. At the moment, I have the following code on the Node:
app.get('/get_software_requests', function (req, res) {
console.log("loading software requests");
requests_callback(function(all_software_requests){
console.log(all_software_requests);
});
function requests_callback(callback){
loadAllSoftwareRequests(function(all_software_requests){
callback(all_software_requests);
});
}
});
function loadAllSoftwareRequests(callback){
console.log("loading requests");
fs.readdir("/project_requests", function(error, files) {
files.forEach(filename => {
var software_request = new Array();
loadSoftwareRequestXML(filename, software_request, function(software_request){
all_software_requests.push(software_request);
callback(all_software_requests);
});
});
});
}
function loadSoftwareRequestXML(filename, software_request, callback){
var xmlparser = new xml2js.Parser();
var filepath = "/project_requests/" + filename;
fs.readFile(filepath, "utf-8", function(error, values){
xmlparser.parseString(values, function(error, xmlfile){
var xmldata = xmlfile;
date_requested = xmldata.ProjectRequest.DateRequested;
client_org = xmldata.ProjectRequest.ClientOrganization;
proposed_budget = xmldata.ProjectRequest.ProposedBudget;
contact_name = xmldata.ProjectRequest.ContactName;
delivery_date = xmldata.ProjectRequest.DeliveryDate;
requirements = xmldata.ProjectRequest.UserRequirements;
software_request.push(date_requested);
software_request.push(client_org);
callback(software_request);
});
});
}
So far, for "console.log(all_software_requests);" on the main app.get, the console outputs:
I want the Node to only return the last iteration result, like
Any help or suggestion is appreciated. Please feel free to comment. Thanks.
You can use native promises for that:
Promise.all(files.map(
filename => new Promise(ok => load(filename, request => ok(request)))
)).then(requests => callback(requests));
I mess with promise/callback style here to minify your code editions.
Better to use promises instead of callbacks in client code too.
Then it becomes just:
let loadAll = files => Promise.all(files.map(load));
let load = filename => {/*return some promise with result*/}
Without promises at all it is not too difficult:
let c = files.length; // initialize a counter
files.forEach(filename => {
var software_request = new Array();
loadSoftwareRequestXML(filename, software_request, function(software_request){
all_software_requests.push(software_request);
if(!--c) { // all async calls are finished
callback(all_software_requests);
}
});
});
You could also add my "next" method that will handle the looping for you. This has worked well for me in these types of situations.
app.get('/get_software_requests', function (req, res) {
console.log("loading software requests");
requests_callback(function(all_software_requests){
console.log(all_software_requests);
});
function requests_callback(callback){
loadAllSoftwareRequests(function(all_software_requests){
callback(all_software_requests);
});
}
});
function loadAllSoftwareRequests(callback){
console.log("loading requests");
fs.readdir("/project_requests", function(error, files) {
files.forEach(filename => {
var software_request = new Array();
loadSoftwareRequestXML(filename, software_request, function(software_request){
all_software_requests.push(software_request);
callback(all_software_requests);
});
});
});
}
function loadSoftwareRequestXML(filename, software_request, callback){
var xmlparser = new xml2js.Parser();
var filepath = "/project_requests/" + filename;
fs.readFile(filepath, "utf-8", function(error, values){
var index = 0;
var next = function () {
if (index >= values.length) {
callback(null, values);
return;
}
var value = values[index];
xmlparser.parseString(value, function(error, xmlfile){
var xmldata = xmlfile;
date_requested = xmldata.ProjectRequest.DateRequested;
client_org = xmldata.ProjectRequest.ClientOrganization;
proposed_budget = xmldata.ProjectRequest.ProposedBudget;
contact_name = xmldata.ProjectRequest.ContactName;
delivery_date = xmldata.ProjectRequest.DeliveryDate;
requirements = xmldata.ProjectRequest.UserRequirements;
software_request.push(date_requested);
software_request.push(client_org);
value.software_request = software_request;
index++;
next()
});
}
next();
});
}
Related
I'm new to nodejs.
Here is my .js file. I'm trying to expose audioData variable to other functions. audioData variable value is being empty outside the function. I see the value when I print inside the function. What could be wrong?
'use strict';
var asyncrequest = require('request');
var xml2js = require('xml2js');
var parseString = xml2js.parseString;
var audioData = [];
asyncrequest("http://example.com/feed", function(error, responsemeta, body) {
parseString(body, function(err, result){
var stories = result['rss']['channel'][0]['item'];
console.log("Total stories: " + stories.length);
stories.forEach(function(entry) {
var singleObj = {}
singleObj['title'] = entry['title'][0];
singleObj['value'] = entry['enclosure'][0].$.url;
audioData.push(singleObj);
});
});
console.dir(audioData);
});
module.exports = audioData;
console.log("Program ended");
You'll have to return a promise for the audioData, not the audioData itself! You can learn more about promises elsewhere. Happily there's a promisified version of request, request-promise, that you can use like so:
'use strict';
var rp = require('request-promise');
var xml2js = require('xml2js');
var parseString = xml2js.parseString;
var audioData = [];
var promiseForAudioData = rp('http://example.com/feed')
.then(body => {
parseString(body, function(err, result){
var stories = result['rss']['channel'][0]['item'];
console.log("Total stories: " + stories.length);
stories.forEach(function(entry) {
var singleObj = {}
singleObj['title'] = entry['title'][0];
singleObj['value'] = entry['enclosure'][0].$.url;
audioData.push(singleObj);
});
});
return audioData;
})
.catch(console.error.bind(console));
module.exports = promiseForAudioData;
console.log("Program ended");
If you don't want to use promises, you can either export inside the callback or export the request method itself.
asyncrequest("http://example.com/feed", function(error, responsemeta, body) {
parseString(body, function(err, result){
var stories = result['rss']['channel'][0]['item'];
console.log("Total stories: " + stories.length);
stories.forEach(function(entry) {
var singleObj = {}
singleObj['title'] = entry['title'][0];
singleObj['value'] = entry['enclosure'][0].$.url; audioData.push(singleObj);
});
module.exports = audioData;
});
});
// Or
exports.get = function (callback) {
return asyncrequest(/* ... */, callback);
}
// Other module
require("./module").get(function (audioData) {
/* Do something */
})
I have writen simple JS object which has function csvFileToArray. Function should return parsed CSV array.
The problem is that I don't have output from anonymous function which is passed to readFileSync.
test1 is loged to console correctly but test2 is not.
It's my first time with node.js What am I doing wrong?
var passwdArray = [];
var csv = function () {
this.csvFileToArray = function (fileName, delimiter) {
console.log("test1");
var fs = require('fs');
fs.readFileSync(fileName, 'utf8', function(err, data) {
console.log("test2");
if (err) {
throw err;
} else {
var csvLineArray = data.split("\n");
var csvArray = [];
csvArray['header'] = csvLineArray[0].split(delimiter);
csvArray['data'] = [];
for(var i = 1; i < csvLineArray.length; i++) {
csvArray['data'].push(csvLineArray[i].split(delimiter));
}
return csvArray;
}
});
}
}
var csvHandler = new csv();
var test =csvHandler.csvFileToArray('test.csv', ',');
console.log(test);
You are using readFileSync function and it's working sync. So you can not use callback inside it. DOC
So you can use it like:
var passwdArray = [];
var csv = function () {
this.csvFileToArray = function (fileName, delimiter) {
console.log("test1");
var fs = require('fs');
var data = fs.readFileSync(fileName, 'utf8');
var returnedData = doSomething(null,data);
console.log(returnedData);
}
};
function doSomething(err, data) {
console.log("test2");
if (err) {
throw err;
} else {
var csvLineArray = data.split("\n");
var csvArray = [];
csvArray['header'] = csvLineArray[0].split(delimiter);
csvArray['data'] = [];
for(var i = 1; i < csvLineArray.length; i++) {
csvArray['data'].push(csvLineArray[i].split(delimiter));
}
return csvArray;
}
};
var csvHandler = new csv();
var test =csvHandler.csvFileToArray('test.csv', ',');
console.log(test);
If you want to use it async you can use readFile function.
Are you mixing up fs.readFileSync(path[, options]) and fs.readFile(path[, options], callback)File System | Node.js v8.1.4 Documentation ?
The method you are using does not accept a callback parameter.
I have not used node.js before.
Have a .txt file with list of magnet links.
Want to write a json file with list of all files contained in these links.
var WebTorrent = require('webtorrent');
var fs = require('fs');
var client = new WebTorrent();
var array = fs.readFileSync('yop.txt').toString().split("\n");
i = 0;
while (i < array.length) {
//console.log(array[i]);
var magnetURI = array[i];
n = 0;
client.add(magnetURI, function (torrent) {
torrent.files.forEach(function (file) {
//console.log( file.name)
jsonString = JSON.stringify({'book': file.name});
fs.appendFile("data.json", jsonString, function (err) {
if (err) {console.log(err);} else { n++ }
});
if (n == torrent.files.length) {i++ }
})
})
}
when run gives the following error
Sorry for such a terrible code.
var WebTorrent = require('webtorrent')
var fs = require('fs')
var stream = fs.createWriteStream("2.txt");
var client = new WebTorrent()
var array = fs.readFileSync('yop.txt').toString().split("\n");
i = 0;
function parseMagnet (uri){
var magnetURI = uri[i]
console.log(magnetURI)
client.add(magnetURI, function (torrent) {
torrent.files.forEach(function (file) {
writeStr = (uri[i]+ '\n'+ file.name+ '\n');
stream.write(writeStr);
console.log(file.name)
});
console.log('Done !')
console.log(i)
i += 1
parseMagnet(array);
client.remove(magnetURI);
})
}
parseMagnet(array)
I have a text file which simply lists some URL's. I'm trying to take each line from the text file, and add it to an array of urls for further operation.
var fs = require('fs'),
Urls = [];
var returnURLS = function(file) {
var read = function(callback) {
fs.readFile(file, function(err, logData){
if (err) throw err;
var text = logData.toString();
var lines = text.split('\n');
lines.forEach(function(line, callback){
var url = "http://www." + line;
Urls.push(url);
});
callback();
});
};
var giveBackAnswer = function() {
console.log("1: ", Urls);
return Urls;
};
read(giveBackAnswer);
};
console.log("2: ", returnURLS('textFileWithListOfURLs.txt'));
My console output clearly shows that the file system operations have not completed by the time the program is supposed to log the results, but that the results eventually do show up.
2: the urls are undefined
1: [ 'http://www.cshellsmassage.com',
'http://www.darsanamartialarts.com',
'http://www.davidgoldbergdc.com',
'http://www.dayspaofbroward.com',.... (etc)
What is the best way to get these functions to operate synchronously?
1) Compile the Urls array through file system operations
2) Print the array to the console once it has been filled
Well, your function takes returns undefined. This is because all functions in JavaScript return undefined.
If you would like to hook on your function using callbacks, it has to take a callback itself and then you'd place your continuation in that callback:
var returnURLS = function(file, whenDone) {
var read = function(callback) {
fs.readFile(file, function(err, logData){
if (err) whenDone(err);
var text = logData.toString();
var lines = text.split('\n');
lines.forEach(function(line, callback){
var url = "http://www." + line;
Urls.push(url);
});
callback();
});
};
var giveBackAnswer = function() {
console.log("1: ", Urls);
whenDone(null, Urls);
};
read(giveBackAnswer);
};
Which would let you do:
returnURLS("textFileWithList.txt", function(err, list){
console.log("2: ", list);
});
The alternative solution using promises (bluebird) would look something like:
var fs = Promise.promisify(require("fs"));
var returnURLS = function(file) {
return fs.readFileAsync(file).then(function(logData){
var text = logData.toString();
var lines = text.split('\n');
return lines.map(function(line){
return "http://www." + line;
});
});
};
Which would let you do:
returnURLS("url.txt").then(function(data){
console.log("Got data!", data);
});
You could use fs.readFileSync in that simple case :
var returnURLS = function(file) {
var text = fs.readFileSync(file).toString();
var lines = text.split('\n');
lines.forEach(function(line, callback){
var url = "http://www." + line;
Urls.push(url);
});
return Urls;
};
That's perfectly OK when you don't need parallelism, like in this small utility program.
But the solution you'll reapply everywhere else is to be wholly asynchronous by not returning the result but passing it as argument to a callback :
var fetchURLS = function(callback) {
fs.readFile(file, function(err, logData){
if (err) throw err;
var text = logData.toString();
var lines = text.split('\n');
lines.forEach(function(line, callback){
var url = "http://www." + line;
Urls.push(url);
});
callback(Urls);
});
};
};
fetchURLS('textFileWithListOfURLs.txt', function(urls){
console.log("2: ", urls);
});
When your code grows in complexity, it becomes convenient to use promises to reduce the "callback hell".
Wrap the function with a callback
var fs = require('fs'),
Urls = [];
function doit(cb){
var returnURLS = function(file) {
var read = function(callback) {
fs.readFile(file, function(err, logData){
if (err) throw err;
var text = logData.toString();
var lines = text.split('\n');
lines.forEach(function(line, callback){
var url = "http://www." + line;
Urls.push(url);
});
callback();
});
};
var giveBackAnswer = function() {
console.log("1: ", Urls);
return Urls;
};
read(giveBackAnswer);
};
cb(returnURLS);
}
doit(function(result){
console.log("2: ", result('textFileWithListOfURLs.txt'));
});
I am just learning server-side JavaScript so please bear with any glaring mistakes I've made.
I am trying to write a file parser that operates on HTML files in a directory and returns a JSON string once all files have been parsed. I started it with a single file and it works fine. it loads the resource from Apache running on the same machine, injects jquery, does the parsing and returns my JSON.
var request = require('request'),
jsdom = require('jsdom'),
sys = require('sys'),
http = require('http');
http.createServer(function (req, res) {
request({uri:'http://localhost/tfrohe/Car3E.html'}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var window = jsdom.jsdom(body).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
// jQuery is now loaded on the jsdom window created from 'body'
var emps = {};
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step++;
var name = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = name.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2];
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
emps = JSON.stringify(emps);
//console.log(emps);
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end(emps);
});
} else {
res.writeHead(200, {"Content-Type": "text/plain"});
res.end("empty");
//console.log(response.statusCode);
}
});
}).listen(8124);
Now I am trying to extend this to using the regular file system (fs) and get all HTML files in the directory and parse them the same way and return a single combined JSON object once all files have been parsed. Here is what I have so far but it does not work.
var sys = require("sys"),
fs = require("fs"),
jsdom = require("jsdom"),
emps = {};
//path = '/home/inet/www/media/employees/';
readDirectory = function(path) {
fs.readdir(path, function(err, files) {
var htmlfiles = [];
files.forEach(function(name) {
if(name.substr(-4) === "html") {
htmlfiles.push(name);
}
});
var count = htmlfiles.length;
htmlfiles.forEach(function(filename) {
fs.readFile(path + filename, "binary", function(err, data) {
if(err) throw err;
window = jsdom.jsdom(data).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step++;
var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = empname.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2]
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
});
});
});
});
}
readDirectory('/home/inet/www/media/employees/', function() {
console.log(emps);
});
In this particular case, there are 2 html files in the directory. If i console.log(emps) during the htmlfiles.forEach() it shows me the results from the first file then the results for both files together the way I expect. how do I get emps to be returned to readDirectory so i can output it as desired?
Completed Script
After the answers below, here is the completed script with a httpServer to serve up the detail.
var sys = require('sys'),
fs = require("fs"),
http = require('http'),
jsdom = require('jsdom'),
emps = {};
var timed = setInterval(function() {
emps = {};
readDirectory('/home/inet/www/media/employees/', function(emps) {
});
}, 3600000);
readDirectory = function(path, callback) {
fs.readdir(path, function(err, files) {
var htmlfiles = [];
files.forEach(function(name) {
if(name.substr(-4) === "html") {
htmlfiles.push(name);
}
});
var count = htmlfiles.length;
htmlfiles.forEach(function(filename) {
fs.readFile(path + filename, "binary", function(err, data) {
if(err) throw err;
window = jsdom.jsdom(data).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
var imagecount = jquery("tr td img").length;
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step += 1;
var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = empname.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2]
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
count -= 1;
if (count <= 0) {
callback(JSON.stringify(emps));
}
});
});
});
});
}
var init = readDirectory('/home/inet/www/media/employees/', function(emps) {
});
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end(JSON.stringify(emps));
}).listen(8124);
That sure is a lot of code a couple of mistakes.
You're never calling the callback function you supply to readDirectory
You need to keep track of the files you have parsed, when you parsed all of them, call the callback and supply the emps
This should work:
var sys = require("sys"),
fs = require("fs"),
jsdom = require("jsdom"),
//path = '/home/inet/www/media/employees/';
// This is a nicer way
function readDirectory(path, callback) {
fs.readdir(path, function(err, files) {
// make this local
var emps = {};
var htmlfiles = [];
files.forEach(function(name) {
if(name.substr(-4) === "html") {
htmlfiles.push(name);
}
});
// Keep track of the number of files we have parsed
var count = htmlfiles.length;
var done = 0;
htmlfiles.forEach(function(filename) {
fs.readFile(path + filename, "binary", function(err, data) {
if(err) throw err;
window = jsdom.jsdom(data).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step++;
var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = empname.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2]
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
// As soon as all have finished call the callback and supply emps
done++;
if (done === count) {
callback(emps);
}
});
});
});
});
}
readDirectory('/home/inet/www/media/employees/', function(emps) {
console.log(emps);
});
You seem to be doing this a tad wrong
readDirectory('/home/inet/www/media/employees/', function() {
console.log(emps);
});
But you've defined your function as:
readDirectory = function(path) {
Where is the callback argument? Try this:
readDirectory = function(path, callback) {
then under emps[last + ",_" + first] = jquery(this).children('img').attr('src'); put
callback.call(null, emps);
Your callback function will be called however many times your loop goes on for. If you want it to return all of them at once, you'll need to get a count of how many times the loop is going to run for, count up until that number then call your callback when the emps array is full of the data you need.