Here I'm trying to make an array of functions with arguments to Async.js.
The array consists of instances of RunRequest that are supposed to be set inside the loop in MakeRequest, right before I try pass the function array to Async.
So the request in request[i] is fine when I pass it to RunRequest, but inside RunRequest function its undefined?
// Process Requests
function RunRequest(db, collection, request, requestHandler, callback) {
console.log('this happening?')
// Connect to the database
db.open(function(err, db) {
if(err) callback(err, null);
// Connect to the collection
db.collection(collection, function(err, collection) {
if (err) callback(err, null);
// Process the correct type of command
requestHandler(db, collection, request, callback);
});
});
}
function MakeRequest(request, requestHandler, collection, callback) {
var data = [];
var doneRequest = function(err, results) {
console.log('done was called')
if (err) callback(err, null);
else if(results) data = data.concat(results);
}
// Make Request Array
var requestArray = [];
for(var i = 0; i < request.length; i++) {
console.log('run request was called')
var dbConnection = new Db('KidzpaceDB', new Server(Host, Port, {auto_reconnect: true}))
requestArray.push(function() {RunRequest(dbConnection, collection, request[i], requestHandler, doneRequest)});
}
// Make all requests in Parallel then invoke callback
Async.parallel(requestArray, function(err, results) {
console.log('Step WORKS')
if(data) {
var uniqueResults = [];
for(var i = 0; i < data.length; i++) {
if( !uniqueResults[data[i]['_id']] ) {
uniqueResults[uniqueResults.length] = data[i];
uniqueResults[data[i]['_id']] = true;
}
callback (null, uniqueResults);
}
}
});
}
// Request Handlers
var FindHandler = function(db, collection, request, callback) {
console.log('FindHandler was called')
console.log('Request Query' + request);
collection.find(request.query, function(err, cursor) {
if (err) callback(err, null);
cursor.toArray(function(err, docs) {
if (err) callback(err, null);
if(docs.length <= 0) console.log("No documents match your query");
var requestResults = [];
for(var i=0; i<docs.length; i++) {
requestResults[requestResults.length] = docs[i];
}
db.close();
callback(null, requestResults);
});
});
}
This is just a shot in the dark:
I think the problem is how you call RunRequest inside MakeRequest. Inside the first for-loop you are iterating over request and use request[i] inside an anonymous function, but i changes in the next iteration and the current scope gets lost when RunRequest is actually executed.
It's hard to reproduce, but try this:
var requestArray = [];
for(var i = 0; i < request.length; i++) {
console.log('run request was called')
var dbConnection = new Db('KidzpaceDB', new Server(Host, Port, {auto_reconnect: true}))
function wrap(dbConnection, collection, request, requestHandler, doneRequest) {
return function() {
RunRequest(dbConnection, collection, request, requestHandler, doneRequest);
}
}
requestArray.push(wrap(dbConnection, collection, request[i], requestHandler, doneRequest));
}
This is a scoping issue. When the loop finishes variable i is set to request.length, so request[i] is undefined.
Wrap your code with anonymous function like that:
var requestArray = [];
for(var i = 0; i < request.length; i++) {
(function(i) {
console.log('run request was called');
var dbConnection = ...;
requestArray.push( ... );
})(i);
}
or even better ( avoids unnecessary overhead when creating anonymous functions ):
var requestArray = [];
request.forEach( function( el ) {
console.log('run request was called');
// the other code goes here, use el instead of request[i]
});
EDIT The callback is not called, because you don't define functions in arrays correctly. You will have to refactor your code a bit, so let me just show you how it should be:
requestArray.push(function(callback) { // <---- note the additional parameter here
// do some stuff, for example call db
db.open(function(err, db) {
if (err) {
callback( err );
} else {
callback( );
}
});
});
If you want to use RunRequest, then you need to pass callback as an additional parameter to RunRequest ( so use callback instead of doneRequest ).
Related
I'm new to working with promises (I'm using 'co' in node) so I'm not entirely sure what's failing with this code:
function* excelToJSON(excelFileNames) {
var jsonData = [];
for (let index = 0; index < excelFileNames.length; index++) {
parseXlsx(excelFilesNames[index], function (err, data) {
jsonData.push(data);
console.log(jsonData); //***Shows data correctly
});
}
console.log(jsonData); //***Empty array
return yield jsonData;
}
It reads the file, converts it and, at least within the loop, it shows everything correctly, but once we get out of the loop the data seems to disappear. I've also tried to return one of the values from within the loop but that doesn't work either.
EDIT:
parseXlsx is from the 'excel' module here: https://github.com/trevordixon/excel.js
I'm not entirely sure if it's async or sync, to be honest. This seems to be its code, and I know 'extractFiles' returns a promise but since it then goes through 'parseXlsx' I'm not sure what happens afterwards:
function parseXlsx(path, sheet, cb) {
if (typeof cb === 'undefined') {
cb = sheet;
sheet = '1';
}
extractFiles(path, sheet).then(function(files) {
cb(null, extractData(files));
},
function(err) {
cb(err);
});
};
EDIT2:
What I used to solve it is a combination of several answers, thanks to all of you.
function* excelToJSON(excelFileNames) {
return new Promise(function(resolve, reject) {
var jsonData = [];
if (excelFilesNames === null || excelFilesNames.length === 0) {
reject();
}
for (let index = 0; index < excelFilesNames.length; index++) {
parseXlsx(excelFilesNames[index], function(err, data) {
if (err) {
throw err;
}
jsonData.push(data);
if (jsonData.length === excelFilesNames.length) {
resolve(jsonData);
}
});
}
});
}
Use just a counter & When to return, try something like this.
function* excelToJSON(excelFileNames) {
var jsonData = [];
var count=0;
for (let index = 0; index < excelFileNames.length; index++) {
parseXlsx(excelFilesNames[index], function (err, data) {
jsonData.push(data);
console.log(jsonData); //***Shows data correctly
if(count==excelFileNames.length){
console.log(jsonData);
return yield jsonData;
}
count++
});
}
}
So what's happening here is that your code just runs through that for block, invoking parseXlsx a few times, but never actually waits for it to finish.
So this is why your empty array log comes first, and then there come the logs with the 'correct data'. Look up javascript event loop to get a better understanding of how asynchronous functions work.
What you essentially need is either a promise that you resolve when you're done, or get a callback function that you'll call when you're done.
And you'll know when you're done when your jsonData.push(data); has been called as many times as long as your excelFileNames array is.
For example:
function excelToJSON(excelFileNames) {
var deferred = Promise.defer();
var jsonData = [];
for (let index = 0; index < excelFileNames.length; index++) {
parseXlsx(excelFilesNames[index], function (err, data) {
jsonData.push(data);
console.log(jsonData); //***Shows data correctly
if (jsonData.length === excelFileNames.length) {
deferred.resolve(jsonData);
}
});
}
return deferred.promise;
}
// And use it as a promise:
var exelToJsonPromise = excelToJSON(["apples.xlsx", "pears.xlsx]);
exelToJsonPromise.then(function(jsonData){
console.log(jsonData); // Now this will have everything in it.
});
Reason:
parseXlsx is an Asynchronous call, so you won't get data back immediately.
How to fix:
do things in callback.
function* excelToJSON(excelFileNames, callback) {
var jsonData = [];
for (let index = 0; index < excelFileNames.length; index++) {
parseXlsx(excelFilesNames[index], function (err, data) {
jsonData.push(data);
console.log(jsonData); //***Shows data correctly
callback(jsonData); // do what you want with the jsonData here.
});
}
// console.log(jsonData); //***Empty array
// return yield jsonData;
}
Node.js is a asynchronous framework. What is happening in your case is that the console.log(jsonData) outside the parseXlsx is getting called before the one inside. You can try the async waterfall method like this.
var pushData = function (err, data) {
jsonData.push(data);
console.log(jsonData);
};
function* excelToJSON(excelFileNames) {
var jsonData = [];
async.waterfall([
function(){
for (let index = 0; index < excelFileNames.length; index++) {
parseXlsx(excelFilesNames[index], pushData);
}
}
], function() {
console.log(jsonData);
return yield jsonData;
});
}
You can read more about it here.
PS. It is also not a good practice to define a function inside of a loop.
I have search function once i have search String from clinet i want to loop through files and match the string from files in fs, I have problem in loop i want to get all match result and send result to client. Below trying to achieve but getting an error pasted in question. New to async library any help will be appreciated.
app.js
app.get('/serverSearch', function (req, res) {
var searchTxt = req.query.searchTxt;
dirDirectory.readDirectory(function(logFiles){
// res.json(logFiles);
if(logFiles){
searchFileService.readFile(searchTxt,logFiles,function(lines,err){
console.log('Logs',lines);
if (err)
return res.send();
res.json(lines);
})
}
});
console.log('Search text', searchTxt);
});
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var searchStr;
var result = [];
//Async Method
function readFile(str, logFiles, callback) {
async.series([
//Load user to get `userId` first
function(callback) {
searchStr = str;
for (var i = 0; i < logFiles.length; i++) {
if (logFiles[i].filename !== '.gitignore') {
fs.readFile('logs/dit/' + logFiles[i].filename, 'utf8', function(err, data) {
if (err) {
return console.log(err);
}
inspectFile(data);
});
}
callback(result);
}
},
//Load posts (won't be called before task 1's "task callback" has been called)
function() {
function inspectFile(data, callback) {
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
result.push(line);
// then log it
return line;
}
});
}
}
], function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return err;
});
};
Error
if (fn === null) throw new Error("Callback was already called.");
You should be using async.map instead of series. You are miss understanding what series does, series process request top down. You are attempting to break this chain by accessing a function within the series itself. Which is a no, no.
for example:
async.series([
function() {
let i = 0;
do {
console.log("I'm first in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
},
function() {
let i = 0;
do {
console.log("I'm next in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
}
]);
The output of this would be:
I'm next in the series: 0
I'm next in the series: 1
I'm next in the series: 2
until the callback is fired, which then tells async to move to the next function in the series array.
The output then would be:
I'm last in the series: 0
I'm last in the series: 1
I'm last in the series: 2
At no point in this series should you be accessing the function within the series after the current. So you should never be trying to cross access that.
With async.map you can actually perform on operation on each entity within your array, which is essentially what you are trying to do.
var results = [];
async.map(logFiles, function(logfile, callback) {
if (logfile.filename !== '.gitignore') {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(err, null);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(null, results);
}
});
}
}), function(error, result) {
results.map(result => {
console.log(result);
});
});
Also you should use util.inspect instead of console.log, it's much cleaner and has more options.
The documentation on this is a bit rough, but here it is. https://caolan.github.io/async/docs.html#map hope this helps!
You should use async.eachSeries method:
function readFile(str, logFiles, callback) {
async.eachSeries(array, function(item, cb){
//Process item
cb(error,item);
}, function(err){
if (err) {
console.log("Some error in one of the items");
callback(err);
} else {
console.log("All arrays items have been treated successfully");
callback(null);
}
});
}
And I would recommend to load the user and posts before using the async.eachSeries function.
I have a async function call queue.enqueue in each iteration of docs.forEach. How do I determine when is the last iteration of docs.forEach?
The following code prints the console.log message multiple times instead of on the last queue.enqueue call.
var mongojs = require('mongojs')
var db = mongojs('localhost/test')
var collection = db.collection('myCollection')
var monq = require('monq')
var client = monq('mongodb://localhost:27017/test')
var queue = client.queue('testQueue')
async.waterfall([
function(callback) {
collection.find({}, function(err, docs) {
callback(null, docs)
})
},
function(docs, callback) {
docs.forEach(function(doc, index) {
// Async call here
queue.enqueue('someTask', doc, function(err, job) {
// Determining the last iteration of docs.forEach
if(index = docs.length-1) {
console.log('this is the last async call')
callback(null)
}
}
})
}
])
I have a parse cloud code function, in this function I preform a query on some items then using a for loop I save some of those items. But the for loop continues and does not save some of the items before correctly.
Heres a general version of the code:
Parse.Cloud.define("saveCurrentDayItems", function(request, response) {
var xmlReader = require('cloud/xmlreader.js');
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
queryMenuURL.find().then(function(resultsMenuURL) {
//********************************************************************************************************
//I want the save to happen before it goes thought this for loop for the second time, and so on
for (var i = 0; i < resultsMenuURL.length; i++) {
var location = resultsMenuURL[i].get("Location");
Parse.Cloud.httpRequest({
url: url,
success: function(httpResponse) {
var xmlData = httpResponse.text;
xmlReader.read(xmlData, function (err, res){
if(err) return console.log(err);
for (var i3 = 0; i3 < res.menu.day.at(dayNumber).meal.count(); i3++) {
var meal = res.menu.day.at(dayNumber).meal.at(i3).attributes().name;
testItem.set("meal", meal);
testItem.set("location", location);
testItem.save().then(function(testItem) {
});
}
}
});
},
error: function(httpResponse) {
console.error('Request failed with response code ' + httpResponse.status);
}
});
}
});
});
I have looked at the parse docs, but I can't make sense of them, the promises section I just can't grasp.
Thanks so much for the help in advance
EDIT 2
When I have your code like this I get the error TypeError: Cannot call method 'reduce' of undefined
Parse.Cloud.define("saveCurrentDayItems23", function(request, response) {
var xmlReader = require('cloud/xmlreader.js');
//First worker function, promisify xmlReader.read();
function readResponse_async(xlmString) {
var promise = new Parse.Promise();
xmlReader.read(xlmString, function (err, res) {
if(err) {
promise.reject(err);
} else {
promise.resolve(res);
results = res;
}
});
return promise;
}
//Second worker function, perform a series of saves
function saveMeals_async(meals, location, testItem) {
return meals.reduce(function(promise, meal) {
return promise.then(function() {
testItem.set("meal", meal.attributes().name);
//the line above does not work it cannot get meal, it is undefined
testItem.set("location", location);
return testItem.save();
});
}, Parse.Promise.as());
}
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
//Master routine
queryMenuURL.find().then(function(resultsMenuURL) {
for (var i = 0; i < resultsMenuURL.length; i++) {
var url = resultsMenuURL[i].get('URL');
return resultsMenuURL.reduce(function(promise, item) {
return promise.then(function() {
return Parse.Cloud.httpRequest({
url: url,
//data: ... //some properties of item?
}).then(function(httpResponse) {
return readResponse_async(httpResponse.text).then(function() {
var TestItem = Parse.Object.extend("TestItem");
var testItem = new TestItem();
return saveMeals_async(result.menu.day.meal.counter.dish.name.text(),item.get("Location"),
testItem);
//this line above does not work, it sends only one string, not an array, so reduce cannot be called
});
});
});
}, Parse.Promise.as());
}
}).fail(function(err) {
console.error(err);
});
});
To do as the question asks ("I want the save to happen before it goes [through] this for loop for the second time, and so on"), is fairly involved - not really a beginners' problem.
It appears that you have several async operations here, viz :
queryMenuURL.find()
Parse.Cloud.httpRequest()
xmlReader.read()
testItem.save()
These operations need to work in cooperation with each other to give the desired effect.
queryMenuURL.find(), Parse.Cloud.httpRequest() and testItem.save() each appear to return a promise, while xmlReader.read() takes a node style callback, which makes things slightly awkward but not too bad.
You could write the code as one big block but you would end up with patterns within patterns. To make everything readable, you can pull out some of the code as (readabe) worker functions, leaving behind a (readable) master routine.
To convert your current outer for loop to set of sequential operations, you need the following pattern, which exploits Array.prototype.reduce() to build a .then chain, and returns a promise :
function doThings_async(arr) {
return arr.reduce(function(promise, item) {
return promise.then(function(result) {
return doSomething_async(item, result);
});
}, resolvedPromise);
}
You will see below that this pattern is also used for the inner for loop, though other possibilities exist.
Parse.Cloud.define("saveCurrentDayItems", function(request, response) {
var xmlReader = require('cloud/xmlreader.js');
//First worker function, promisify xmlReader.read();
function readResponse_async(xlmString) {
var promise = new Parse.Promise();
xmlReader.read(xlmString, function (err, res) {
if(err) {
promise.reject(err);
} else {
promise.resolve(res);
}
}
return promise;
}
//Second worker function, perform a series of saves
function saveMeals_async(meals, location, testItem) {
return meals.reduce(function(promise, meal) {
return promise.then(function() {
testItem.set("meal", meal.attributes().name);
testItem.set("location", location);
return testItem.save();
});
}, Parse.Promise.as());
}
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
//Master routine
queryMenuURL.find().then(function(resultsMenuURL) {
...
return resultsMenuURL.reduce(function(promise, item) {
return promise.then(function() {
return Parse.Cloud.httpRequest({
url: url,
//data: ... //some properties of item?
}).then(function(httpResponse) {
return readResponse_async(httpResponse).then(function() {
return saveMeals_async(res.menu.day.at(dayNumber).meal, item.get("Location"), testItem);
});
});
});
}, Parse.Promise.as());
}).fail(function(err) {
console.error(err);
});
});
saveMeals_async() could be written to perform its saves in parallel rather than in series, but it depends on what you want. For parallel saves, only saveMeals_async() would need to be rewritten, using a different pattern.
EDIT
Revised code based on the edits in the question.
Due to changes in saveMeals_async(), the arr.reduce(...) pattern is now used only once in the master routine.
Parse.Cloud.define("saveCurrentDayItems", function(request, response) {
// ***
// insert all the Date/dayNumber code here
// ***
var xmlReader = require('cloud/xmlreader.js');
//First worker function, promisify xmlReader.read();
function readResponse_async(xlmString) {
var promise = new Parse.Promise();
xmlReader.read(xlmString, function (err, res) {
if(err) {
promise.reject(err);
} else {
promise.resolve(res);
}
}
return promise;
}
//Second worker function, perform a series of saves
function saveMeals_async(meals, school, diningHallNumber, menuLocation) {
//Declare all vars up front
var i3, i4, i5, m,
TestItem = Parse.Object.extend("TestItem"),//can be reused within the loops?
promise = Parse.Promise.as();//resolved promise to start a long .then() chain
for (i3 = 0; i3 < meals.count(); i3++) {
m = meals.at(i3);
//get number of stations in day
for (i4 = 0; i4 < m.counter.count(); i4++) {
//get number of items at each station
for (i5 = 0; i5 < m.counter.at(i4).dish.count(); i5++) {
//Here a self-executing function is used to form a closure trapping `testItem`.
//Otherwise the `testItem` used in `promise.then(...)` would be the last
//remaining `testItem` created when all iterations are complete.
(function(testItem) {
testItem.set("item", m.counter.at(i4).dish.at(i5).name.text());
testItem.set("meal", m.attributes().name);
testItem.set("school", school);
testItem.set("diningHallNumber", diningHallNumber);
testItem.set("schoolMenu", menuLocation);
//build the .then() chain
promise = promise.then(function() {
return testItem.save();
});
})(new TestItem());
});
}
}
return promise;
}
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
//Master routine
queryMenuURL.find().then(function(resultsMenuURL) {
return resultsMenuURL.reduce(function(promise, menuItem) {
var url = menuItem.get('URL'),
school = menuItem.get("school"),
diningHallNumber = menuItem.get("diningHallNumber"),
menuLocation = menuItem.get("menuLocation");
return promise.then(function() {
return Parse.Cloud.httpRequest({
url: url,
//data: ... //some properties of menuItem?
}).then(function(httpResponse) {
return readResponse_async(httpResponse).then(function(res) {
if (res.menu.day.at(dayNumber).meal) {
return saveMeals_async(res.menu.day.at(dayNumber).meal, school, diningHallNumber, menuLocation);
} else {
return Parse.Promise.as();//return resolved promise to keep the promise chain going.
}
});
});
});
}, Parse.Promise.as());
}).fail(function(err) {
console.error(err);
});
});
untested so may well need debugging
I have a array of ids:
var ids = ['53asd3','53asd2','53asd5'];
Each id has a corresponding document in the mongodb.
I want to generate a object by populating data from each of them and save in some other document. Like this:
{
person: { /* data from some collection populated with first id */},
company : { /* data from some collection populated with first id */},
employee : {/* data from some collection populated with first id */}
}
WHAT I DID
var document = {}
models.persons.find({_id:'53asd3'},function(err,data){
if(!err) {
document['persons']=data;
models.company.find({_id:'53asd2'},function(err,data){
if(!err) {
document['company']=data;
models.employee.find({_id:'53asd2'},function(err,data){
if(!err) {
document['employee']=data;
document.save(function(err){ });
}
});
}
});
}
});
So I just use nested calls using callbacks and somewhat make it synchronous. Is there a chance where I can execute all these three find queries in parallel and then execute the save command? I actually want to leverage the async nature of node.js. Any suggestions?
You could build something like async.parallel yourself if you don't want to include an external library. Here's what a simple parallel function might look like. It could be a nice exercise to implement the other functions in the async library.
var parallel = function () {
var functs = arguments[0];
var callback = arguments[1];
// Since we're dealing with a sparse array when we insert the results,
// we cannot trust the `length` property of the results.
// Instead we count the results separately
var numResults = 0;
var results = [];
var getCallback = function (i) {
return function (err, res) {
if (err) {
callback(err)
}
else {
results[i] = res;
numResults += 1;
if (numResults === functs.length) {
callback(null, results);
}
}
}
}
functs.forEach(function (fn, i) {
fn(getCallback(i));
});
};
var getTest = function (timeout) {
return function (callback) {
setTimeout(function () {
callback(null, timeout);
}, timeout);
}
};
parallel([getTest(99), getTest(1000), getTest(199)], console.log.bind(console));
>> null [99, 1000, 199]
Then in your case you can do something like
var findItem = function (collection, id) {
return function (callback) {
collection.find({
_id: id
}, callback);
};
};
parallel([
findItem(models.persons, '53asd3'),
findItem(models.company, '53asd2'),
findItem(models.employee, '53dsa2')
], function (err, results) {
document.persons = results[0];
document.company = results[1];
document.employee = results[2];
document.save(function (err) {
// and so on
});
});