We Keep Coding

Calling a callback function from async.each and processing the results

I'm just starting to work with Javascript and Node, and Async and callbacks concepts are not something I have under control right now.
I have to call a function for each element of a documents Array. This function will call to DB and get me an array of the document annotations. I want to get all the annotations and put them on the same array. Something similar to this:
//function in an async waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc, callback2) => {
getAnnotationsFromDocument(doc.Id, callback2);
}, function (err,annotations){
if (err){
callback(err);
}
annotationsArray = annotationsArray.concat(annotations);
callback(null, annotationsArray);
});
},
//Next waterfall function
About the getAnnotationsFromDocument function, this is a simplified structure of it:
function getAnnotationsFromDocument(docId,callback){
initDB();
var async = require('async');
async.waterfall([
function authorize(callback){
//checkAuthorization
(...)
},
function getRfpdocAnnotations(auth, metadata, callback){
//call to DB
(...)
},
function processRfpdocAnnotations(rfpDocAnnotations,metadata,callback){
(...)
callback(null,annotationsList);
}
], function (err, result) {
if(err) {
callback(err);
} else {
callback(null, result);
}
});
}
Unfortunately, I'm unable to code it properly. I'm unable to get the results from the function before exiting the async.each. Could somebody explain me how to structurate the code for this?
Debugging I've found that the function getAnnotationsFromDocument gets the data and execute the last callback(null, result); properly, but when I get to function (err,annotations){, annotations is undefined.

Ok, I think I got it:
First problem was that async.each doesn't return the results on the callback like I was expecting. Unlike waterfall, it just returns the errors. I should have payed more attention reading the documentation.
Secondly, I had to create a callback on the getAnnotationsFromDocument call to process the results.
And finally, I was not executing the call to the callback of async.each, so the execution didn't get to the async.each callback and didn't continue to the next async.waterfall function.
To be quite honest, I'm not sure it's a correct answer, but it does what I was trying to achieve.
// function part of an async.waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc,callback2) => {
getAnnotationsFromDocument(doc._id, function(err,result){
if (err){
callback2(err);
}else{
annotationsArray = annotationsArray.concat(result);
}
callback2();
})
}, (err) =>{
if( err ) {
callback(err);
} else {
callback(null,annotationsArray); //to the next waterfall function
}
});

How to use async for callbacks using nodejs?

I have search function once i have search String from clinet i want to loop through files and match the string from files in fs, I have problem in loop i want to get all match result and send result to client. Below trying to achieve but getting an error pasted in question. New to async library any help will be appreciated.
app.js
app.get('/serverSearch', function (req, res) {
var searchTxt = req.query.searchTxt;
dirDirectory.readDirectory(function(logFiles){
// res.json(logFiles);
if(logFiles){
searchFileService.readFile(searchTxt,logFiles,function(lines,err){
console.log('Logs',lines);
if (err)
return res.send();
res.json(lines);
})
}
});
console.log('Search text', searchTxt);
});
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var searchStr;
var result = [];
//Async Method
function readFile(str, logFiles, callback) {
async.series([
//Load user to get `userId` first
function(callback) {
searchStr = str;
for (var i = 0; i < logFiles.length; i++) {
if (logFiles[i].filename !== '.gitignore') {
fs.readFile('logs/dit/' + logFiles[i].filename, 'utf8', function(err, data) {
if (err) {
return console.log(err);
}
inspectFile(data);
});
}
callback(result);
}
},
//Load posts (won't be called before task 1's "task callback" has been called)
function() {
function inspectFile(data, callback) {
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
result.push(line);
// then log it
return line;
}
});
}
}
], function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return err;
});
};
Error
if (fn === null) throw new Error("Callback was already called.");

You should be using async.map instead of series. You are miss understanding what series does, series process request top down. You are attempting to break this chain by accessing a function within the series itself. Which is a no, no.
for example:
async.series([
function() {
let i = 0;
do {
console.log("I'm first in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
},
function() {
let i = 0;
do {
console.log("I'm next in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
}
]);
The output of this would be:
I'm next in the series: 0
I'm next in the series: 1
I'm next in the series: 2
until the callback is fired, which then tells async to move to the next function in the series array.
The output then would be:
I'm last in the series: 0
I'm last in the series: 1
I'm last in the series: 2
At no point in this series should you be accessing the function within the series after the current. So you should never be trying to cross access that.
With async.map you can actually perform on operation on each entity within your array, which is essentially what you are trying to do.
var results = [];
async.map(logFiles, function(logfile, callback) {
if (logfile.filename !== '.gitignore') {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(err, null);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(null, results);
}
});
}
}), function(error, result) {
results.map(result => {
console.log(result);
});
});
Also you should use util.inspect instead of console.log, it's much cleaner and has more options.
The documentation on this is a bit rough, but here it is. https://caolan.github.io/async/docs.html#map hope this helps!

You should use async.eachSeries method:
function readFile(str, logFiles, callback) {
async.eachSeries(array, function(item, cb){
//Process item
cb(error,item);
}, function(err){
if (err) {
console.log("Some error in one of the items");
callback(err);
} else {
console.log("All arrays items have been treated successfully");
callback(null);
}
});
}
And I would recommend to load the user and posts before using the async.eachSeries function.

What is the right way (nested functions or...)

Are both my examples the same in terms of functionality considering the fact that in error handeling I'm terminating by res.json(400, err)? Also I would like to know that in my second example the second async.each always run after the first async.each, so using results1 in the second async.each is safe? Sorry I'm new to Node and async!
Example1: where I'm using the results of each async.each in the last block as an input of the other async.each:
var results1 = {};
var results2 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
}
});
or Example2: where I have separate async.each blocks
var results1 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
}
});
var results2 = {};
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
UPDATED:
Since the second approach is not right way and it is not guaranteed that the second async.each runs after the first one the problem is: Does it mean I cannot have a simple for loop like the following example either? If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well and not a for loop, do you know how I can have this recursive functionality here?
var results1 = {};
var results2 = [];
var results3 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
// So in this case that I need to have nested function, does it mean I cannot have a simple for loop like this as well?
// If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well, do you know how I can have this recursive functionality here?
for (var input in inputs) {
inferFromUnion(inputs[input], results1);
results2.push(inputs[input]);
}
async.each(results2, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results3);
}
});
}
});
// Here just checking each object schema and if they are missing any fields from results1 we add that field with a value of null
function inferFromUnion(obj, allFields) {
Object.keys(allFields).forEach(function (key) {
if (lodash.isUndefined(obj[key])) {
if (lodash.isPlainObject(allFields[key])) {
obj[key] = {};
inferFromUnion(obj[key], allFields[key]);
} else {
obj[key] = null;
}
}
});
}

The first example is the way to go, if you want to use results of the first bunch of calls in the second bunch. The second example won't work, because the second async.each() is guaranteed to run before the callbacks bound to your asynchronous operations.
Asynchronous recursion with loops is very much possible:
(function doSomeAsyncRecursion (results) {
async.each(someItems, function (item, callback) {
// ...
}, function () {
if (results /* ... (are incomplete) */) {
doSomeAsyncRecursion(results);
} else {
// ... (results are complete now, do something with them)
}
});
})(/* initial value of results */);

These two examples are different in desing. First example will run second async after first async is successful. But second example will run second async everytime, if theres an error or not.

Returing result of asynchronous function within Async.map

I'm very new to JS and functional programming in general and am struggling to find a graceful solution to this problem. Essentially, I want to make async requests to a MongoDB server, and return the results to an async to map function. The problem I am having in that the actual function within async.map is asynchronous itself. I would like to know a graceful solution here, or at least get a pointer in the right direction! Thanks!
async.map(subQuery,
function(item){
collection.distinct("author", item, function(err, authors){
counter++;
console.log("Finished query: " + counter);
var key = item['subreddit'];
return { key: authors };
})
},
function(err, result){
if (err)
console.log(err);
else{
console.log("Preparing to write to file...");
fs.writeFile("michaAggregate.json", result, function() {
console.log("The file was saved!");
});
}
db.close();
}
);

You should process item only when the data is fetched. Just use callback That the common way of JavaScript. Like this:
var processItem = function(item){
// Do some street magic with your data to process it
// Your callback function that will be called when item is processed.
onItemProccessed();
}
async.map(subQuery,
function(item){
collection.distinct("author", item, function(err, authors){
counter++;
console.log("Finished query: " + counter);
var key = item['subreddit'];
processItem(item);
})
},
function(err, result){
if (err)
console.log(err);
else{
// That string added **ADDED**
console.log('HEEY! I done with processing all data so now I can do what I want!');
console.log("Preparing to write to file...");
fs.writeFile("michaAggregate.json", result, function() {
console.log("The file was saved!");
});
}
db.close();
}
);
ADDED
By the specification of async.map you can see:
https://github.com/caolan/async
async.map(arr, iterator, callback):
callback(err, results) - A callback which is called when all iterator functions have finished, or an error occurs. Results is an array of the transformed items from the arr. As you see that callback is exactly what you need!

Using async module to fire a callback once all files are read

I'm using caolan's 'async' module to open an array of filenames (in this case, template file names).
Per the documentation, I'm using async.forEach(),so I can fire a callback once all operations have completed.
A simple test case is:
var async = require('async')
var fs = require('fs')
file_names = ['one','two','three'] // all these files actually exist
async.forEach(file_names,
function(file_name) {
console.log(file_name)
fs.readFile(file_name, function(error, data) {
if ( error) {
console.log('oh no file missing')
return error
} else {
console.log('woo '+file_name+' found')
}
})
}, function(error) {
if ( error) {
console.log('oh no errors!')
} else {
console.log('YAAAAAAY')
}
}
)
The output is as follows:
one
two
three
woo one found
woo two found
woo three found
I.e, it seems the final callback isn't firing. What do I need to do to make the final callback fire?

The function that is being run across all items must take a callback, and pass its results to the callback. See below (I've also separated fileName to improve readability):
var async = require('async')
var fs = require('fs')
var fileNames= ['one','two','three']
// This callback was missing in the question.
var readAFile = function(fileName, callback) {
console.log(fileName)
fs.readFile(fileName, function(error, data) {
if ( error) {
console.log('oh no file missing')
return callback(error)
} else {
console.log('woo '+fileName+' found')
return callback()
}
})
}
async.forEach(fileNames, readAFile, function(error) {
if ( error) {
console.log('oh no errors!')
} else {
console.log('YAAAAAAY')
}
})
Returns:
one
two
three
woo one found
woo two found
woo three found
YAAAAAAY

This is the best way to do it in my opinion. The results param will have an array of strings containing the file data and all of the files will be read in parallel.
var async = require('async')
fs = require('fs');
async.map(['one','two','three'], function(fname,cb) {
fs.readFile(fname, {encoding:'utf8'}, cb);
}, function(err,results) {
console.log(err ? err : results);
});