I'm just starting to work with Javascript and Node, and Async and callbacks concepts are not something I have under control right now.
I have to call a function for each element of a documents Array. This function will call to DB and get me an array of the document annotations. I want to get all the annotations and put them on the same array. Something similar to this:
//function in an async waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc, callback2) => {
getAnnotationsFromDocument(doc.Id, callback2);
}, function (err,annotations){
if (err){
callback(err);
}
annotationsArray = annotationsArray.concat(annotations);
callback(null, annotationsArray);
});
},
//Next waterfall function
About the getAnnotationsFromDocument function, this is a simplified structure of it:
function getAnnotationsFromDocument(docId,callback){
initDB();
var async = require('async');
async.waterfall([
function authorize(callback){
//checkAuthorization
(...)
},
function getRfpdocAnnotations(auth, metadata, callback){
//call to DB
(...)
},
function processRfpdocAnnotations(rfpDocAnnotations,metadata,callback){
(...)
callback(null,annotationsList);
}
], function (err, result) {
if(err) {
callback(err);
} else {
callback(null, result);
}
});
}
Unfortunately, I'm unable to code it properly. I'm unable to get the results from the function before exiting the async.each. Could somebody explain me how to structurate the code for this?
Debugging I've found that the function getAnnotationsFromDocument gets the data and execute the last callback(null, result); properly, but when I get to function (err,annotations){, annotations is undefined.
Ok, I think I got it:
First problem was that async.each doesn't return the results on the callback like I was expecting. Unlike waterfall, it just returns the errors. I should have payed more attention reading the documentation.
Secondly, I had to create a callback on the getAnnotationsFromDocument call to process the results.
And finally, I was not executing the call to the callback of async.each, so the execution didn't get to the async.each callback and didn't continue to the next async.waterfall function.
To be quite honest, I'm not sure it's a correct answer, but it does what I was trying to achieve.
// function part of an async.waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc,callback2) => {
getAnnotationsFromDocument(doc._id, function(err,result){
if (err){
callback2(err);
}else{
annotationsArray = annotationsArray.concat(result);
}
callback2();
})
}, (err) =>{
if( err ) {
callback(err);
} else {
callback(null,annotationsArray); //to the next waterfall function
}
});
Related
I have a async.each that iterates an array and for every element inside the array it execute a function "check" that has a request inside. This is the code but when I run it I see from the debugger that node doesn't execute the check function and it block its execution.
async.each(array,
function(v_page, callback){
if(v_page.url.length>=5){
internals.check(v_page.url,array2, function (body) {
callback();
});
}
},
function(err){
internals.calls(var1,var2,var3);
});
I tried with a normal for loop but it jump at the internals.calls function without execute the internals.check function for the async nature of node. So how can i force the execution of the check function?
This is the code of the check function:
internals.check = (url,array2,cb) => {
request(url, function(err, recordset) {
if(err){
cb(array2);
}else {
//some stuffs
cb(array2)
}
});
};
You call callback only when v_page.url.length >= 5, but you need to do that for each element:
async.each(array, function(v_page, callback) {
if(v_page.url.length >= 5) {
internals.check(v_page.url,array2, function(body) {
callback();
});
} else {
callback(); <== call callback here, when condition is not met
}
...
Another problem is that, you incorrectly call callback in internals.check. According Node.js notation, the first parameter of callback must be an error or null (async uses this notation). But in your case you call callback with array2 anyway:
internals.check = (url, array2, cb) => {
request(url, function(err, recordset) {
if (err) {
cb(err); // <== pass error here
} else {
cb(null, array2); // <== no error, so first parameter is null, the second is data
}
});
};
Code
Server.js result
My code is this
app.post('/get_page_data', function(req, res) {
function find (i,documents)
{
Item_data.find({item_name:documents[i].item_name}).exec(function (err,asd){
console.log(documents[i].item_name+": found");
console.log(asd[0].like);
}
)
}
Page_data.find().lean().exec(function (err, documents) {
var doc=documents;
async.series([
// 1st
function(done){
for(var i=0;i<documents.length;i++)
{
find(i,doc);
done()
}
},
// 2nd
function(done){
console.log("1");
console.log(doc);
console.log("2");
res.end(JSON.stringify(doc));
console.log("3");
done()
}
]);
}
)});
I'll introduce my code simply
when ajax called /get_page_data
i bring the pages data (page1,page2,page3 ...) in documents
every page has a item_name but pages data does not have the "like" data
so i find the value of "like" in the other collection(Item_data) by the same item name and put the "like" in the pages data documents
but this makes me crazy
the pages data is sent before i put the like value
so i read about the non blocking io and async blahblah...
and i found the async module. And used it.
But as you can see the pictures above, they're not working
(before trying async moudle they didn't work neither)
I don't know why res.end part excutes previously and Please tell me the solution of this situation Thank you for reading
Your find function is doing an asynchronous call, so the done() is being called before it returns any data.
You should move it inside the inner find callback.
Edit: Using async.each instead of async.series directly.
app.post('/get_page_data', function(req, res) {
function find (document, cb) {
Item_data.find({item_name:document.item_name}).exec(function (err,asd) {
console.log(document.item_name+": found");
console.log(asd[0].like);
cb();
})
}
Page_data.find().lean().exec(function (err, documents) {
var doc=documents;
async.each(documents,
// 1st
find(document, cb),
// 2nd
function() {
console.log("1");
console.log(doc);
console.log("2");
res.end(JSON.stringify(doc));
console.log("3");
});
}
)});
PS: there's also a lot of things that you should improve in that code. But that way it should work.
put the done thing inside the aysn function and call the done when the index value
app.post('/get_page_data', function(req, res) {
function find (i,documents,cb)
{
Item_data.find({item_name:documents[i].item_name}).exec(function (err,asd){
console.log(documents[i].item_name+": found");
console.log(asd[0].like);
console.log(i +"/"+ documents.length);
documents[i]["like"]=asd[0].like;
if(i==documents.length-1) cb();
//call done() when the index reached maximum
})
}
Page_data.find().lean().exec(function (err, documents) {
var doc=documents;
async.series([
// 1st
function(done){
for(var i=0;i<documents.length;i++)
{
find(i,doc,done); //put the done thing inside the function
}
},
// 2nd
function(done){
console.log("1");
console.log(doc);
console.log("2");
res.end(JSON.stringify(doc));
console.log("3");
done()
}
]);
working properly
So right now I'm trying to use Nodejs to access files in order to write them to a server and process them.
I've split it into the following steps:
Traverse directories to generate an array of all of the file paths
Put the raw text data from each of file paths in another array
Process the raw data
The first two steps are working fine, using these functions:
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function(file) {
file = path.resolve(dir, file);
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
results.push(file);
if (!--pending) done(null, results);
}
});
});
});
};
function processfilepaths(callback) {
// reading each file
for (var k in filepaths) { if (arrayHasOwnIndex(filepaths, k)) {
fs.readFile(filepaths[k], function (err, data) {
if (err) throw err;
rawdata[k] = data.toString().split(/ *[\t\r\n\v\f]+/g);
for (var j in rawdata[k]) { if (arrayHasOwnIndex(rawdata[k], j)) {
rawdata[k][j] = rawdata[k][j].split(/: *|: +/);
}}
});
}}
if (callback) callback();
}
Obviously, I want to call the function processrawdata() after all of the data has been loaded. However, using callbacks doesn't seem to work.
walk(rootdirectory, function(err, results) {
if (err) throw err;
filepaths = results.slice();
processfilepaths(processrawdata);
});
This never causes an error. Everything seems to run perfectly except that processrawdata() is always finished before processfilepaths(). What am I doing wrong?
You are having a problem with callback invocation and asynchronously calling functions. IMO I'll recommend that you use a library such as after-all to execute a callback once all your functions get executed.
Here's a example, here the function done will be called once all the functions wrapped with next are called.
var afterAll = require('after-all');
// Call `done` once all the functions
// wrapped with next() get called
next = afterAll(done);
// first execute this
setTimeout(next(function() {
console.log('Step two.');
}), 500);
// then this
setTimeout(next(function() {
console.log('Step one.');
}), 100);
function done() {
console.log("Yay we're done!");
}
I think for your problem, you can use async module for Node.js:
async.series([
function(){ ... },
function(){ ... }
]);
To answer you actual question, I need to explain how Node.js works:
Say, when you call an async operation (say mysql db query), Node.js sends "execute this query" to MySQL. Since this query will take some time (may be some milliseconds), Node.js performs the query using the MySQL async library - getting back to the event loop and doing something else there while waiting for MySQL to get back to us. Like handling that HTTP request.
So, In your case both functions are independent and executes almost in parallel.
For more information:
Async.js for use with Node.js
function processfilepaths(callback) {
// reading each file
for (var k in filepaths) { if (arrayHasOwnIndex(filepaths, k)) {
fs.readFile(filepaths[k], function (err, data) {
if (err) throw err;
rawdata[k] = data.toString().split(/ *[\t\r\n\v\f]+/g);
for (var j in rawdata[k]) { if (arrayHasOwnIndex(rawdata[k], j)) {
rawdata[k][j] = rawdata[k][j].split(/: *|: +/);
}}
});
}}
if (callback) callback();
}
Realize that you have:
for
readfile (err, callback) {... }
if ...
Node will call each readfile asynchronously, which only sets up the event and callback, then when it is done calling each readfile, it will do the if, before the callback probably even has a chance to get invoked.
You need to use either Promises, or a promise module like async to serialize it. What you would then do looks like:
async.XXXX(filepaths, processRawData,
function (err, ...) {
// function for when all are done
if (callback) callback();
}
);
Where XXXX is one of the functions from the library like series, parallel, each, etc... The only thing you also need to know is in your process raw data, async gives you a callback to call when done. Unless you really need sequential access (I don't think you do) use parallel so that you can queue up as many i/o events as possible, it should execute faster, maybe only marginally, but it'll better leverage the hardware.
So basically I am making a database query, to get all posts with a certain id, then add them to a list, so I can return. But the list is returned, before the callback has finished.
How do I prevent it from being returned before callback has finished?
exports.getBlogEntries = function(opid) {
var list12 =[];
Entry.find({'opid' : opid}, function(err, entries) {
if(!err) {
console.log("adding");
entries.forEach( function(currentEntry){
list12.push(currentEntry);
});
}
else {
console.log("EEEERROOR");
}
//else {console.log("err");}
});
console.log(list12);
return list12;
};
ALL callback is asynchronous, so we don't have any guarantee if they will run exactly in the order we have leave them.
To fix it and make the process "synchronous" and guarantee an order executation you have two solutions:
First: make all process in nested list:
instead of this:
MyModel1.find({}, function(err, docsModel1) {
callback(err, docsModel1);
});
MyModel2.find({}, function(err, docsModel2) {
callback(err, docsModel2);
});
use this:
MyModel1.find({}, function(err, docsModel1) {
MyModel2.find({}, function(err, docsModel2) {
callback(err, docsModel1, docsModel2);
});
});
The last snippet above guarantee us that MyModel2 will be executed AFTER MyModel1 is executed.
Second: Use some framework as Async. This framework is awesome and have several helper functions to execute code in series, parallels, whatever way we want.
Example:
async.series(
{
function1 : function(callback) {
//your first code here
//...
callback(null, 'some result here');
},
function2 : function(callback) {
//your second code here (called only after the first one)
callback(null, 'another result here');
}
},
function(err, results) {
//capture the results from function1 and function2
//if function1 raise some error, function2 will not be called.
results.function1; // 'some result here'
results.function2; // 'another result here'
//do something else...
}
);
You could use sync database calls but that would work around the concept of node.js.
The proper way is to pass a callback to the function that queries the database and then call the provided callback inside the database query callback.
How do I prevent it from being returned before callback has finished?
The callback is asynchronous, and you cannot avoid that. Hence, you must not return a list.
Instead, offer a callback for when it's filled. Or return a Promise for the list. Example:
exports.getBlogEntries = function(opid, callback) {
Entry.find({'opid': opid}, callback); // yes, that's it.
// Everything else was boilerplate code
};
There is an alternate way to handle this scenario. You can use the async module and when the forEach has finished then make the return call. Please find the code snippet below for the same:
var async = requires('async');
exports.getBlogEntries = function(opid) {
var list12 =[];
Entry.find({'opid' : opid}, function(err, entries) {
if(!err) {
console.log("adding");
async.forEachSeries(entries,function(entry,returnFunction){
list12.push(entry);
},function(){
console.log(list12);
return list12;
});
}
else{
console.log("EEEERROOR");
}
});
};
I have the following code in my nodejs application:
function someFileOperation(callback) {
var files = ...;
files.forEach(function (file) {
doSomethingAsync(file, function (err, result) {
});
});
}
What is an elegant way to call the callback of someFileOperation() in case all doSomethingAsync() called their callback function and call it only once, when an error in doSomethingAsync() occurred?
For now I came up with something like this:
function someFileOperation(callback) {
var files = ...;
var noFiles = files.length;
files.forEach(function (file) {
doSomethingAsync(file, function (err, result) {
if (err) {
callback(err);
callback = function () {}; // I don't want it to be called again
} else if (--noFiles <= 0) {
callback(null, true);
}
});
});
}
But I think this is a lot of overhead for such a simple task. So I am looking for a much more elegant way or maybe a little framework for these kind of problems
Use async.map or async.foreach see here: https://github.com/caolan/async#map and https://github.com/caolan/async#forEach
the async.map method takes an array of items and performs the same async call for each item in your array in parallel. If no errors are encountered, it will call a callback with a new array of results.