Where is the async.waterfall callback defined - javascript

I am using the async library and am using the async.waterfall function whose signature is async.waterfall([function1,function2,function3],function(err,result){}). Now i understand that the results of each function is passed on to the next and the final callback function is executed in case of some error in between or when the execution is over.
What i do not understand is that each function in the array of functions passed takes a callback as an argument and the callback is executed in each function with the result of that function. But i do not understand where that callback function is defined.
Below is a sample piece of code
async.waterfall([
function getUserAvatar(callback) {
console.log("Callback function is " + callback)//print the
callback definition function
github.search.users({ q: 'airbnb' }, function(err, res) {
if (err) {
callback(err, null);
return;
}
var avatarUrl = res.items[0].avatar_url;
callback(null, avatarUrl);
});
},
function wrapAvatarInHtml(avatarUrl, callback) {
var avatarWithHtml = '<img src="' + avatarUrl + '"/>';
callback(null, avatarWithHtml);
}
], function(err, result) {
if (err) {
console.error(err);
return;
}
console.log(result);
})
When i execute the above code, the line where i do console.log("Callback function is " + callback) it prints the below definition.
function () {
if (fn === null) throw new Error("Callback was already called.");
var callFn = fn;
fn = null;
callFn.apply(this, arguments);
}
Now i do not understand where it is getting this definition. I searched the async library also for this but i could not find it.
Can anyone explain this.

If you go through the code in this link. The next() defined is the callback being called. It is implicit, you don't have to define and is because even the tasks you are passing to async waterfall is not being called by you, but by async.waterfall()

Here is the library code of async waterfall. You will get the fair idea about how it works
exports.default = function (tasks, callback) {
callback = (0, _once2.default)(callback || _noop2.default);
if (!(0, _isArray2.default)(tasks)) return callback(new Error('First argument to waterfall must be an array of functions'));
if (!tasks.length) return callback();
var taskIndex = 0;
function nextTask(args) {
var task = (0, _wrapAsync2.default)(tasks[taskIndex++]);
args.push((0, _onlyOnce2.default)(next));
task.apply(null, args);
}
function next(err /*, ...args*/) {
if (err || taskIndex === tasks.length) {
return callback.apply(null, arguments);
}
nextTask((0, _slice2.default)(arguments, 1));
}
nextTask([]);
};
For more information, please install the async library and look into the node modules folder. You will get the complete idea of async library.

Related

Calling a callback function from async.each and processing the results

I'm just starting to work with Javascript and Node, and Async and callbacks concepts are not something I have under control right now.
I have to call a function for each element of a documents Array. This function will call to DB and get me an array of the document annotations. I want to get all the annotations and put them on the same array. Something similar to this:
//function in an async waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc, callback2) => {
getAnnotationsFromDocument(doc.Id, callback2);
}, function (err,annotations){
if (err){
callback(err);
}
annotationsArray = annotationsArray.concat(annotations);
callback(null, annotationsArray);
});
},
//Next waterfall function
About the getAnnotationsFromDocument function, this is a simplified structure of it:
function getAnnotationsFromDocument(docId,callback){
initDB();
var async = require('async');
async.waterfall([
function authorize(callback){
//checkAuthorization
(...)
},
function getRfpdocAnnotations(auth, metadata, callback){
//call to DB
(...)
},
function processRfpdocAnnotations(rfpDocAnnotations,metadata,callback){
(...)
callback(null,annotationsList);
}
], function (err, result) {
if(err) {
callback(err);
} else {
callback(null, result);
}
});
}
Unfortunately, I'm unable to code it properly. I'm unable to get the results from the function before exiting the async.each. Could somebody explain me how to structurate the code for this?
Debugging I've found that the function getAnnotationsFromDocument gets the data and execute the last callback(null, result); properly, but when I get to function (err,annotations){, annotations is undefined.
Ok, I think I got it:
First problem was that async.each doesn't return the results on the callback like I was expecting. Unlike waterfall, it just returns the errors. I should have payed more attention reading the documentation.
Secondly, I had to create a callback on the getAnnotationsFromDocument call to process the results.
And finally, I was not executing the call to the callback of async.each, so the execution didn't get to the async.each callback and didn't continue to the next async.waterfall function.
To be quite honest, I'm not sure it's a correct answer, but it does what I was trying to achieve.
// function part of an async.waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc,callback2) => {
getAnnotationsFromDocument(doc._id, function(err,result){
if (err){
callback2(err);
}else{
annotationsArray = annotationsArray.concat(result);
}
callback2();
})
}, (err) =>{
if( err ) {
callback(err);
} else {
callback(null,annotationsArray); //to the next waterfall function
}
});

how to force execution of requests inside async.each with nodejs?

I have a async.each that iterates an array and for every element inside the array it execute a function "check" that has a request inside. This is the code but when I run it I see from the debugger that node doesn't execute the check function and it block its execution.
async.each(array,
function(v_page, callback){
if(v_page.url.length>=5){
internals.check(v_page.url,array2, function (body) {
callback();
});
}
},
function(err){
internals.calls(var1,var2,var3);
});
I tried with a normal for loop but it jump at the internals.calls function without execute the internals.check function for the async nature of node. So how can i force the execution of the check function?
This is the code of the check function:
internals.check = (url,array2,cb) => {
request(url, function(err, recordset) {
if(err){
cb(array2);
}else {
//some stuffs
cb(array2)
}
});
};
You call callback only when v_page.url.length >= 5, but you need to do that for each element:
async.each(array, function(v_page, callback) {
if(v_page.url.length >= 5) {
internals.check(v_page.url,array2, function(body) {
callback();
});
} else {
callback(); <== call callback here, when condition is not met
}
...
Another problem is that, you incorrectly call callback in internals.check. According Node.js notation, the first parameter of callback must be an error or null (async uses this notation). But in your case you call callback with array2 anyway:
internals.check = (url, array2, cb) => {
request(url, function(err, recordset) {
if (err) {
cb(err); // <== pass error here
} else {
cb(null, array2); // <== no error, so first parameter is null, the second is data
}
});
};

Callback problems

I am new into javascript, and currently I'm trying to learning callback to my script. This script should return reduced words in array of objects
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = []
list.forEach(function (val) {
readFile(val, function(error,data){
txtObj.push(data)
})
})
function readFile(src, cb){
fs.readFile(src,'utf8', function (error,data) {
if (error) return callback(error,null)
return mapred(data)
})
}
return callback(null,txtObj)
}
But it returns empty array. Any help would be appreciated.
Thanks!
`fs.readFile`
is an asynchronous function, before it's done and result callback is invoked, you are returning the empty txtObj array.
how to fix it ?
call return callback(null,txtObj) after fs.readFile is finished running.
and also, as you are running asynchronous function on an array of items one-by-one, it might not still work the way you want. might want to use modudles like async in nodejs
Here comes an asynchronous version using module async. synchronous file operation is strongly objected :)
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = [],
async = require('async');
async.each(list, readFile, function(err) {
callback(err,txtObj)
});
function readFile(src, cb) {
fs.readFile(src,'utf8', function (error,data) {
if (error) {
cb(error);
}
else {
txtObj.push(mapred(data));
cb(null);
}
})
}
}
EDIT : You can do this without async, but it is little bit dirty isn't it ? also its OK if you remove the self invoking function inside the forEach, i included so that you can access the val, even after the callback is done
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = [],
counter = list.length,
start = 0;
list.forEach(function (val) {
(function(val)
readFile(val, function(error,data) {
txtObj.push(data);
start++;
if(error || (start === counter)) {
callback(error,txtObj);
}
}))(val);
})
function readFile(src, cb) {
fs.readFile(src,'utf8', function (error,data) {
if (error) {
cb(error);
}
else {
txtObj.push(mapred(data));
cb(null);
}
})
}
}
The reason you are getting an empty array result is that you are performing the callback before the readFile function has a chance to populate the array. You are performing multiple asynchronous actions but not letting them to complete before continuing.
If there was only one async action, you would call callback() in the callback function of readFile, but as you need to perform multiple async actions before calling callback(), you should consider using fs.readFileSync().
Sometimes sync cannot be avoided.
function mapping(list, callback)
{
var txtObj = []
list.forEach(function(val)
{
try { txtObj.push(mapred(fs.readFileSync(val, 'utf8'))) }
catch(err) { callback(err) }
})
callback(null, txtObj)
}

Async.each final callback not firing when it is inside a deferred promise callback function

Based on my research thus far, I believe I'm using async.each correctly. The bottom function addToPlaylist calls the top function getPerformances. I'm trying to figure out why the final callback in addToPlaylist isn't called and how to fix it. Thanks!
var getPerformances = function(playlist, performances, add){
var deferred = Q.defer();
var tracks = [];
async.each(performances.performance, function(performance, cb) {
getSpotifyArtistId(performance.artist.displayName).then(function (artistId) {
if (add == true) {
getTopTracks(artistId).then(function (trackstoAdd) {
tracks.push.apply(tracks, trackstoAdd);
cb();
});
} else {
//Other stuff... only looking at the true option at this point
}
});
}, function(err){
if( err ) {
console.log(err);
deferred.resolve(tracks);
} else {
deferred.resolve(tracks);
}
});
return deferred.promise;
}
The problem is here- final callback isn't called if cb() is inside the then(). if CB is moved outside of the promise callback, finalCallback is called but of course, it is called too soon.
var AddToPlaylist = function (playlist, results) {
var allSongs = [];
var handler = function (performances, cb) {
getPerformances(playlist, performances, true)
.then(
function (trackstoAdd) { // all is well!
allSongs.push.apply(allSongs, trackstoAdd);
//console.log(allSongs) is as expected
cb();
}
,
function (err) { // something bad happened!
cb(err);
}
);
}
var finalCallback = function () {
console.log('test');
}
async.each(results, handler, finalCallback);
}

Executing asynchronous calls in a synchronous manner

I've been trying to wrap my head around this issue for the last hours but can't figure it out. I guess I still have to get used to the functional programming style ;)
I wrote a recursive function that traverses through a directory structure and does things to certain files. This functions uses the asynchronous IO methods. Now I want to perform some action when this whole traversing is done.
How would I make sure that this action is performed after all parse calls have been performed but still use the asynchronous IO functions?
var fs = require('fs'),
path = require('path');
function parse(dir) {
fs.readdir(dir, function (err, files) {
if (err) {
console.error(err);
} else {
// f = filename, p = path
var each = function (f, p) {
return function (err, stats) {
if (err) {
console.error(err);
} else {
if (stats.isDirectory()) {
parse(p);
} else if (stats.isFile()) {
// do some stuff
}
}
};
};
var i;
for (i = 0; i < files.length; i++) {
var f = files[i];
var p = path.join(dir, f);
fs.stat(p, each(f, p));
}
}
});
}
parse('.');
// do some stuff here when async parse completely finished
Look for Step module. It can chain asynchronous functions calls and pass results from one to another.
You could use async module . Its auto function is awesome . If you have function A() and function B() and function C() . Both function B() and C() depend of function A() that is using value return from function A() . using async module function you could make sure that function B and C will execute only when function A execution is completed .
Ref : https://github.com/caolan/async
async.auto({
A: functionA(){//code here },
B: ['A',functionB(){//code here }],
C: ['A',functionC(){//code here }],
D: [ 'B','C',functionD(){//code here }]
}, function (err, results) {
//results is an array that contains the results of all the function defined and executed by async module
// if there is an error executing any of the function defined in the async then error will be sent to err and as soon as err will be produced execution of other function will be terminated
}
})
});
In above example functionB and functionC will execute together once function A execution will be completed . Thus functionB and functionC will be executed simultaneously
functionB: ['A',functionB(){//code here }]
In above line we are passing value return by functionA using 'A'
and functionD will be executed only when functionB and functionC execution will be completed .
if there will be error in any function , then execution of other function will be terminated and below function will be executed .where you could write your logic of success and failure .
function (err, results) {}
On succesfull execution of all function "results" will contain the result of all the functions defined in async.auto
function (err, results) {}
Take a look at modification of your original code which does what you want without async helper libs.
var fs = require('fs'),
path = require('path');
function do_stuff(name, cb)
{
console.log(name);
cb();
}
function parse(dir, cb) {
fs.readdir(dir, function (err, files) {
if (err) {
cb(err);
} else {
// cb_n creates a closure
// which counts its invocations and calls callback on nth
var n = files.length;
var cb_n = function(callback)
{
return function() {
--n || callback();
}
}
// inside 'each' we have exactly n cb_n(cb) calls
// when all files and dirs on current level are proccessed,
// parent cb is called
// f = filename, p = path
var each = function (f, p) {
return function (err, stats) {
if (err) {
cb(err);
} else {
if (stats.isDirectory()) {
parse(p, cb_n(cb));
} else if (stats.isFile()) {
do_stuff(p+f, cb_n(cb));
// if do_stuff does not have async
// calls inself it might be easier
// to replace line above with
// do_stuff(p+f); cb_n(cb)();
}
}
};
};
var i;
for (i = 0; i < files.length; i++) {
var f = files[i];
var p = path.join(dir, f);
fs.stat(p, each(f, p));
}
}
});
}
parse('.', function()
{
// do some stuff here when async parse completely finished
console.log('done!!!');
});
Something like this would work -- basic change to your code is the loop turned into a recursive call that consumes a list until it is done. That makes it possible to add an outer callback (where you can do some processing after the parsing is done).
var fs = require('fs'),
path = require('path');
function parse(dir, cb) {
fs.readdir(dir, function (err, files) {
if (err)
cb(err);
else
handleFiles(dir, files, cb);
});
}
function handleFiles(dir, files, cb){
var file = files.shift();
if (file){
var p = path.join(dir, file);
fs.stat(p, function(err, stats){
if (err)
cb(err);
else{
if (stats.isDirectory())
parse(p, function(err){
if (err)
cb(err);
else
handleFiles(dir, files, cb);
});
else if (stats.isFile()){
console.log(p);
handleFiles(dir, files, cb);
}
}
})
} else {
cb();
}
}
parse('.', function(err){
if (err)
console.error(err);
else {
console.log('do something else');
}
});
See following solution, it uses deferred module:
var fs = require('fs')
, join = require('path').join
, promisify = require('deferred').promisify
, readdir = promisify(fs.readdir), stat = promisify(fs.stat);
function parse (dir) {
return readdir(dir).map(function (f) {
return stat(join(dir, f))(function (stats) {
if (stats.isDirectory()) {
return parse(dir);
} else {
// do some stuff
}
});
});
};
parse('.').done(function (result) {
// do some stuff here when async parse completely finished
});
I've been using syncrhonize.js with great success. There's even a pending pull request (which works quite well) to support async functions which have multiple parameters. Far better and easier to use than node-sync imho. Added bonus that it has easy-to-understand and thorough documentation, whereas node-sync does not.
Supports two different methods for wiring up the sync, a defered/await model (like what #Mariusz Nowak was suggesting) and a slimmer though not-as-granular function-target approach. The docs are pretty straightforward for each.
Recommend to use node-seq
https://github.com/substack/node-seq
installed by npm.
I'm using it, and I love it..
Look for node-sync, a simple library that allows you to call any asynchronous function in synchronous way. The main benefit is that it uses javascript-native design - Function.prototype.sync function, instead of heavy APIs which you'll need to learn. Also, asynchronous function which was called synchronously through node-sync doesn't blocks the whole process - it blocks only current thread!

Categories