We Keep Coding

How to use async for callbacks using nodejs?

I have search function once i have search String from clinet i want to loop through files and match the string from files in fs, I have problem in loop i want to get all match result and send result to client. Below trying to achieve but getting an error pasted in question. New to async library any help will be appreciated.
app.js
app.get('/serverSearch', function (req, res) {
var searchTxt = req.query.searchTxt;
dirDirectory.readDirectory(function(logFiles){
// res.json(logFiles);
if(logFiles){
searchFileService.readFile(searchTxt,logFiles,function(lines,err){
console.log('Logs',lines);
if (err)
return res.send();
res.json(lines);
})
}
});
console.log('Search text', searchTxt);
});
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var searchStr;
var result = [];
//Async Method
function readFile(str, logFiles, callback) {
async.series([
//Load user to get `userId` first
function(callback) {
searchStr = str;
for (var i = 0; i < logFiles.length; i++) {
if (logFiles[i].filename !== '.gitignore') {
fs.readFile('logs/dit/' + logFiles[i].filename, 'utf8', function(err, data) {
if (err) {
return console.log(err);
}
inspectFile(data);
});
}
callback(result);
}
},
//Load posts (won't be called before task 1's "task callback" has been called)
function() {
function inspectFile(data, callback) {
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
result.push(line);
// then log it
return line;
}
});
}
}
], function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return err;
});
};
Error
if (fn === null) throw new Error("Callback was already called.");

You should be using async.map instead of series. You are miss understanding what series does, series process request top down. You are attempting to break this chain by accessing a function within the series itself. Which is a no, no.
for example:
async.series([
function() {
let i = 0;
do {
console.log("I'm first in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
},
function() {
let i = 0;
do {
console.log("I'm next in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
}
]);
The output of this would be:
I'm next in the series: 0
I'm next in the series: 1
I'm next in the series: 2
until the callback is fired, which then tells async to move to the next function in the series array.
The output then would be:
I'm last in the series: 0
I'm last in the series: 1
I'm last in the series: 2
At no point in this series should you be accessing the function within the series after the current. So you should never be trying to cross access that.
With async.map you can actually perform on operation on each entity within your array, which is essentially what you are trying to do.
var results = [];
async.map(logFiles, function(logfile, callback) {
if (logfile.filename !== '.gitignore') {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(err, null);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(null, results);
}
});
}
}), function(error, result) {
results.map(result => {
console.log(result);
});
});
Also you should use util.inspect instead of console.log, it's much cleaner and has more options.
The documentation on this is a bit rough, but here it is. https://caolan.github.io/async/docs.html#map hope this helps!

You should use async.eachSeries method:
function readFile(str, logFiles, callback) {
async.eachSeries(array, function(item, cb){
//Process item
cb(error,item);
}, function(err){
if (err) {
console.log("Some error in one of the items");
callback(err);
} else {
console.log("All arrays items have been treated successfully");
callback(null);
}
});
}
And I would recommend to load the user and posts before using the async.eachSeries function.

Node for-loop callback required

I want to change the Tags format which I am fetching form one of the collections. Tags data contains some KC ids in an array which I am using to get KC data and insert in TagUnit to get final response format.
var newTags = Tags.map(function(TagUnit) {
for (var i = 0; i < TagUnit.kcs.length; i++) {
KCArray = [];
KC.findById(TagUnit.kcs[i], function(error, data) {
KCMap = {};
KCMap['kc_id'] = data._id;
KCMap['kc_title'] = data.title;
KCArray.push(KCMap);
if (KCArray.length == TagUnit.kcs.length) {
TagUnit.kcs = KCArray;
}
});
}
return TagUnit;
});
response.send(JSON.stringify(newTags));
But I am not getting desired result. Response is giving out Tag data in initial for instead of formatted form. I guess it is due to event looping. I will be grateful if someone can help me with this.
**Edit: ** I am using MongoDB as database and mongoose as ORM.

I'd suggest using promises to manage your async operations which is now the standard in ES6. You don't say what database you're using (it may already have a promise-based interface). If it doesn't, then we manually promisify KC.findById():
function findById(key) {
return new Promise(function(resolve, reject) {
KC.findById(key, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
Then, assuming you can do all these find operations in parallel, you can use Promise.all() to both keep track of when they are all done and to order them for you.
var allPromises = Tags.map(function(TagUnit) {
var promises = TagUnit.kcs.map(function(key) {
return findById(key).then(function(data) {
// make resolved value be this object
return {kc_id: data._id, kc_title: data.title};
});
});
// this returns a promise that resolves with an array of the kc_id and kc_title objects
return Promise.all(promises).then(function(results) {
return {
_id: TagUnit._id,
kcs: results
};
});
});
// now see when they are all done
Promise.all(allPromises).then(function(results) {
response.send(JSON.stringify(results));
}).catch(function(err) {
// send error response here
});

You can use promises or Async module
var async = require('async');
...
function getKC (kc, callback) {
KC.findById(kc, function(err, data) {
if (err)
return callback(err);
callback(null, {kc_id: data._id, kc_title: data.title})
});
}
function getKCs (tag, callback) {
async.map(tag.kcs, getKC, callback);
}
async.map(Tags, getKCs, function(err, results){
if (err)
return console.log(err.message);
res.json(results); // or modify and send
});
P.S. Perhaps, code contains errors. I cann't test it.

sync independet callbacks result

I am searching for an elegant way to sync indepentent callbacks result invoked in unknown order.
function callback1() {
var result;
};
function callback2() {
var result;
};
//When done then call
function success(res1, res2) {
// do whatever
}
I know I can do something like:
var res = {};
var dfd = $.Deferred();
function callback1() {
var result;
res.res1 = result;
(res.res1 && res.res2) && (dfd.resolve(res));
};
function callback1() {
var result;
res.res2 = result;
(res.res1 && res.res2) && (dfd.resolve(res));
};
dfd.done(function(result){
// do whatever
});
but I would appreciate if somebody comes up with more elegant solution

If you return promises (builtin promises, not jQuery deferreds) and you don't care about order, then you can use Promise.all:
function callback1() {
return Promise.resolve(1)
}
function callback2() {
return Promise.resolve(2)
}
var ps = [callback1(), callback2()]
function add(x, y) {
return x + y
}
Promise.all(ps).then(function(result) {
return result.reduce(add)
}).then(console.log) // => 3
If you want to sequence them you can do it in such a way that you can apply a curried function that expects as many arguments as resolved promises there are by lifting it into the promise world. In other words:
function apply(pa, pf) {
return pf.then(function(f) {
return pa.then(f)
})
}
function lift(f, ps) {
return ps.reduce(function(pa, pb) {
return apply(pb, pa)
}, Promise.resolve(f))
}
function add(x) {
return function(y) {
return x + y
}
}
lift(add, ps).then(console.log) //=> 3
You can also sequence them in such a way that you don't need a curried function, by collecting the results in an array first then reducing it:
function sequence(ps) {
return ps.reduceRight(function(pa, pb) {
return pa.then(function(a) {
return pb.then(function(b) {
return [b].concat(a)
})
})
}, Promise.resolve([]))
}
function add(x, y) {
return x + y
}
// This looks similar to the Promise.all approach
// but they run in order
sequence(ps).then(function(result) {
return result.reduce(add)
}).then(console.log) // => 3

There are libraries that do this, such as the async library, but here's a "from scratch" solution. I'm also avoiding promises to avoid overwhelming you, but you should really read about them as they are the most elegant solution, albeit complicated for first timers.
function runInParallel(jobs, done) {
// Store all our results in an array.
var results = [];
// If one job fails, set this to true and use it to
// ignore all job results that follow.
var failureOccurred = false;
// Iterate over each of our registered jobs.
jobs.forEach(function (runJob, index) {
// Create a jobDone callback to pass to the job.
var jobDone = function (err, result) {
// If another job failed previously, abort processing
// this job's result. We no longer care.
if (failureOccurred) return;
// If this job passed in an error, set failure to true
// and pass the error to the final done callback.
if (err) {
failureOccurred = true;
done(err);
return;
}
// If we made it this far then push the job result into
// the results array at the same position as the job in
// the jobs array.
results[index] = result;
// If the results array contains as many results as the
// jobs array had jobs then we have finished processing
// them all. Invoke our done callback with an array of
// all results.
if (results.length === jobs.length) {
done(null, results);
}
};
// Begin the job and pass in our jobDone callback.
runJob(jobDone);
});
}
This will call all of your job functions in the array, passing in a jobDone callback the job should call when finished. If any job passes an error in then the function will immediately invoke the result callback with the error and ignore everything else. If the jobs succeed then you'll end up with an array of job results in the same positions as the jobs were in the jobs array. Simply modify your job functions to accept the jobDone callback.
var jobs = [
function job1(done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
},
function job2(done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
}
];
runInParallel(jobs, function (err, results) {
if (err) {
console.error(err);
return;
}
// results[0] = jobs[0] result
// results[1] = jobs[1] result
// etc...
});
Instead of an array of jobs you could modify this code to accept an object with property names. Then instead of assigning the results to the same position as the jobs in the jobs array you could assign the results to an object using the same property names.
Example (without comments this time):
function runInParallel(jobs, done) {
var results = {};
var failureOccurred = false;
Object.keys(jobs).forEach(function (jobName) {
var jobDone = function (err, result) {
if (failureOccurred) return;
if (err) {
failureOccurred = true;
done(err);
return;
}
results[jobName] = result;
if (results.length === jobs.length) {
done(null, results);
}
};
jobs[jobName](jobDone);
});
}
Then you can consume it like this:
var jobs = {
job1: function (done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
},
job2: function (done) {
try {
var result;
done(null, result);
} catch (err) {
done(err);
}
}
};
runInParallel(jobs, function (err, results) {
if (err) {
console.error(err);
return;
}
// results.job1 = job1 result
// results.job2 = job2 result
// etc...
});
The parallel function in the async library does almost exactly what we've done above. It even accepts an array of jobs or an object of named jobs like we did :)

Assuming your tasks (callback1() and callback2()) are synchronous, you might choose to write a reusable generalisation of the code in the question, in the form of a function that returns a function, trapping a couple of private vars in a closure :
function resultAggregator(n, fn) {
var results = {},
count = 0;
return function(id, res) {
count++;
results[id] = res;
if (count == n) {
fn(results);
}
}
}
So after calling resultAccumulator(), you have a function that can kept in-scope of other functions or passed to other parts of your code base. It makes no assumptions about ids or the nature of the results, except that they are synchronously derived. It will fire its callback when n results have been delivered.
var myResults = resultAggregator(2, function(results) {
// do whatever;
});
//The following commands may be in different parts of your code base
myResults('id1', synchTask1());
...
myResults('id2', synchTask2());
...
myResults('id3', synchTask3());
//The second tasks to deliver its data (ostensibly `synchTask1()` and `synchTask2()`, but not necessarily) will trigger the callback.
Demo
This is just one way to perform result aggregation. You might do something different depending on the exact scenario. Here's a slightly different formulation which records the order in which the results arrived :
Demo
Whatever you write, Deferreds/Promises are not necessary for the aggregation of synchronously derived data.
However, if any one task is, or may be, asynchronous then you may need a promise aggregator, eg jQuery.when() or Promise.all(), somewhere in the pattern.

What is the right way (nested functions or...)

Are both my examples the same in terms of functionality considering the fact that in error handeling I'm terminating by res.json(400, err)? Also I would like to know that in my second example the second async.each always run after the first async.each, so using results1 in the second async.each is safe? Sorry I'm new to Node and async!
Example1: where I'm using the results of each async.each in the last block as an input of the other async.each:
var results1 = {};
var results2 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
}
});
or Example2: where I have separate async.each blocks
var results1 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
}
});
var results2 = {};
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
UPDATED:
Since the second approach is not right way and it is not guaranteed that the second async.each runs after the first one the problem is: Does it mean I cannot have a simple for loop like the following example either? If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well and not a for loop, do you know how I can have this recursive functionality here?
var results1 = {};
var results2 = [];
var results3 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
// So in this case that I need to have nested function, does it mean I cannot have a simple for loop like this as well?
// If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well, do you know how I can have this recursive functionality here?
for (var input in inputs) {
inferFromUnion(inputs[input], results1);
results2.push(inputs[input]);
}
async.each(results2, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results3);
}
});
}
});
// Here just checking each object schema and if they are missing any fields from results1 we add that field with a value of null
function inferFromUnion(obj, allFields) {
Object.keys(allFields).forEach(function (key) {
if (lodash.isUndefined(obj[key])) {
if (lodash.isPlainObject(allFields[key])) {
obj[key] = {};
inferFromUnion(obj[key], allFields[key]);
} else {
obj[key] = null;
}
}
});
}

The first example is the way to go, if you want to use results of the first bunch of calls in the second bunch. The second example won't work, because the second async.each() is guaranteed to run before the callbacks bound to your asynchronous operations.
Asynchronous recursion with loops is very much possible:
(function doSomeAsyncRecursion (results) {
async.each(someItems, function (item, callback) {
// ...
}, function () {
if (results /* ... (are incomplete) */) {
doSomeAsyncRecursion(results);
} else {
// ... (results are complete now, do something with them)
}
});
})(/* initial value of results */);

These two examples are different in desing. First example will run second async after first async is successful. But second example will run second async everytime, if theres an error or not.

Async parallel bug?

Here I'm trying to make an array of functions with arguments to Async.js.
The array consists of instances of RunRequest that are supposed to be set inside the loop in MakeRequest, right before I try pass the function array to Async.
So the request in request[i] is fine when I pass it to RunRequest, but inside RunRequest function its undefined?
// Process Requests
function RunRequest(db, collection, request, requestHandler, callback) {
console.log('this happening?')
// Connect to the database
db.open(function(err, db) {
if(err) callback(err, null);
// Connect to the collection
db.collection(collection, function(err, collection) {
if (err) callback(err, null);
// Process the correct type of command
requestHandler(db, collection, request, callback);
});
});
}
function MakeRequest(request, requestHandler, collection, callback) {
var data = [];
var doneRequest = function(err, results) {
console.log('done was called')
if (err) callback(err, null);
else if(results) data = data.concat(results);
}
// Make Request Array
var requestArray = [];
for(var i = 0; i < request.length; i++) {
console.log('run request was called')
var dbConnection = new Db('KidzpaceDB', new Server(Host, Port, {auto_reconnect: true}))
requestArray.push(function() {RunRequest(dbConnection, collection, request[i], requestHandler, doneRequest)});
}
// Make all requests in Parallel then invoke callback
Async.parallel(requestArray, function(err, results) {
console.log('Step WORKS')
if(data) {
var uniqueResults = [];
for(var i = 0; i < data.length; i++) {
if( !uniqueResults[data[i]['_id']] ) {
uniqueResults[uniqueResults.length] = data[i];
uniqueResults[data[i]['_id']] = true;
}
callback (null, uniqueResults);
}
}
});
}
// Request Handlers
var FindHandler = function(db, collection, request, callback) {
console.log('FindHandler was called')
console.log('Request Query' + request);
collection.find(request.query, function(err, cursor) {
if (err) callback(err, null);
cursor.toArray(function(err, docs) {
if (err) callback(err, null);
if(docs.length <= 0) console.log("No documents match your query");
var requestResults = [];
for(var i=0; i<docs.length; i++) {
requestResults[requestResults.length] = docs[i];
}
db.close();
callback(null, requestResults);
});
});
}

This is just a shot in the dark:
I think the problem is how you call RunRequest inside MakeRequest. Inside the first for-loop you are iterating over request and use request[i] inside an anonymous function, but i changes in the next iteration and the current scope gets lost when RunRequest is actually executed.
It's hard to reproduce, but try this:
var requestArray = [];
for(var i = 0; i < request.length; i++) {
console.log('run request was called')
var dbConnection = new Db('KidzpaceDB', new Server(Host, Port, {auto_reconnect: true}))
function wrap(dbConnection, collection, request, requestHandler, doneRequest) {
return function() {
RunRequest(dbConnection, collection, request, requestHandler, doneRequest);
}
}
requestArray.push(wrap(dbConnection, collection, request[i], requestHandler, doneRequest));
}

This is a scoping issue. When the loop finishes variable i is set to request.length, so request[i] is undefined.
Wrap your code with anonymous function like that:
var requestArray = [];
for(var i = 0; i < request.length; i++) {
(function(i) {
console.log('run request was called');
var dbConnection = ...;
requestArray.push( ... );
})(i);
}
or even better ( avoids unnecessary overhead when creating anonymous functions ):
var requestArray = [];
request.forEach( function( el ) {
console.log('run request was called');
// the other code goes here, use el instead of request[i]
});
EDIT The callback is not called, because you don't define functions in arrays correctly. You will have to refactor your code a bit, so let me just show you how it should be:
requestArray.push(function(callback) { // <---- note the additional parameter here
// do some stuff, for example call db
db.open(function(err, db) {
if (err) {
callback( err );
} else {
callback( );
}
});
});
If you want to use RunRequest, then you need to pass callback as an additional parameter to RunRequest ( so use callback instead of doneRequest ).