We Keep Coding

Calling a callback function from async.each and processing the results

I'm just starting to work with Javascript and Node, and Async and callbacks concepts are not something I have under control right now.
I have to call a function for each element of a documents Array. This function will call to DB and get me an array of the document annotations. I want to get all the annotations and put them on the same array. Something similar to this:
//function in an async waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc, callback2) => {
getAnnotationsFromDocument(doc.Id, callback2);
}, function (err,annotations){
if (err){
callback(err);
}
annotationsArray = annotationsArray.concat(annotations);
callback(null, annotationsArray);
});
},
//Next waterfall function
About the getAnnotationsFromDocument function, this is a simplified structure of it:
function getAnnotationsFromDocument(docId,callback){
initDB();
var async = require('async');
async.waterfall([
function authorize(callback){
//checkAuthorization
(...)
},
function getRfpdocAnnotations(auth, metadata, callback){
//call to DB
(...)
},
function processRfpdocAnnotations(rfpDocAnnotations,metadata,callback){
(...)
callback(null,annotationsList);
}
], function (err, result) {
if(err) {
callback(err);
} else {
callback(null, result);
}
});
}
Unfortunately, I'm unable to code it properly. I'm unable to get the results from the function before exiting the async.each. Could somebody explain me how to structurate the code for this?
Debugging I've found that the function getAnnotationsFromDocument gets the data and execute the last callback(null, result); properly, but when I get to function (err,annotations){, annotations is undefined.

Ok, I think I got it:
First problem was that async.each doesn't return the results on the callback like I was expecting. Unlike waterfall, it just returns the errors. I should have payed more attention reading the documentation.
Secondly, I had to create a callback on the getAnnotationsFromDocument call to process the results.
And finally, I was not executing the call to the callback of async.each, so the execution didn't get to the async.each callback and didn't continue to the next async.waterfall function.
To be quite honest, I'm not sure it's a correct answer, but it does what I was trying to achieve.
// function part of an async.waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc,callback2) => {
getAnnotationsFromDocument(doc._id, function(err,result){
if (err){
callback2(err);
}else{
annotationsArray = annotationsArray.concat(result);
}
callback2();
})
}, (err) =>{
if( err ) {
callback(err);
} else {
callback(null,annotationsArray); //to the next waterfall function
}
});

Returning results with callbacks

I'm trying to query a database, then make an array of objects from the results, turn them into a JSON object.
I'm not used to Javascript's asynchronous nature and I'm really confused about how to implement something that needs to work synchronously. I know I probably need to use callbacks, but after looking at many tutorials I'm only left more confused.
This is the code without callbacks:
var foreignTable = (tablename,idArr)=>{
var dataArray = [];
//call a query for each of the ids
var objectToAdd;
for(var id of idArr){
objectToAdd = queryForeignTable(tablename,id);
dataArray.push(objectToAdd);
}
return dataArray;
connection.end();
};
var queryForeignTable = (tablename,id)=>{
connection.query("SELECT * FROM "+tablename+" WHERE id="+id, function (error, results, fields) {
if(error)throw error;
var objectToAddToArray={};
//Go through each field in a result and make the object
for(packet of fields){
var label = packet.name;
objectToAddToArray[label] = results[0][label];
}
return objectToAddToArray;
});
};
var arrayOfDrivers = foreignTable("driver",[1,2]);
outputJson["drive"]=arrayOfDrivers;
console.log(outputJson); // { drive: [ undefined, undefined ] }
I attempted foreignTable(tablename, idArr, callback) with the callback calling queryForeignTable with no luck.
Can someone explain how I can get this piece of code working with callbacks?

A callback function is a function passed into another function as an argument, which is then invoked inside the outer function to complete some kind of routine or action.
from MDN: Callback function
Callbacks are a way of telling a function what to do next, as in "after you're done, run this function".
For example:
first = function (callback) {
console.log("First!");
callback();
}
second = function () {
console.log("Second!");
}
first(second);
Will produce:
First!
Second!
You can also use anonymous function to product the same result:
first(function() {
console.log("Second!")
});
Regarding the specific example from your question, you are correct that you need to use callbacks a bit differently. Rather than using a return statement in each of your two functions, you'll need to use a callback. connection.query is asynchronously coming back with your results as results, but you can't return them to your queryForeignTable function. Instead, give queryForeignTable a callback function to run. The same idea goes for your foreignTable function.
I'm not connected to your database, obviously, so I stubbed a DB connection and simplified what you're trying to do, but it should look something like this:
// Stubbed DB connection
var connection = {};
connection.query = (id, cb) => {
var results = [
{
id: id,
name: 'Name of ' + id,
},
];
cb(null, results);
};
var foreignTable = (ids, cb) => {
var data = [];
for (var i = 0; i < ids.length; i++) {
queryForeignTable(ids[i], (error, obj) => {
data.push(obj);
if (i == ids.length - 1) {
cb(null, data);
}
});
}
};
var queryForeignTable = (id, cb) => {
connection.query(id, (error, results) => {
if (error) {
cb(error, null);
}
cb(null, results[0]);
});
};
foreignTable([1, 2], (error, data) => {
if (error) {
console.error(error);
}
console.log(data);
});
That produces:
[ { id: 1, name: 'Name of 1' }, { id: 2, name: 'Name of 2' } ]
In essence, when you have an urge to return some value(s) from a function in an asynchronous way, give the function a callback parameter, then invoke that callback with your return values.
You can run the code here: https://repl.it/K0YI/3

When you have an asynchronous call, like connection.query(statement, callback), then whatever you want to do with the results of that call needs to be done within the callback.
Bear in mind that async functions don't return the final value that you generally want (usually they return undefined). Instead of using a return value, you pass a callback as a way of saying, "when you're finished, carry on and do this with the results", aka. continuation-passing style.
One of the challenges in your code is that you're issuing separate queries for each ID, and then aggregating the results into an array of responses. This means you'll need to check when all the queries have completed, and only then proceed to display the final results.
Here's your example, re-written, commented, and simplified. Hopefully this helps to explain how the control flow works.
// Read row IDs 1 and 2 from the "driver" table.
readTable("driver", [1, 2], displayData);
// Print the results.
function displayData (arrayOfDrivers) {
console.log(arrayOfDrivers);
}
// Read all rows matching IDs in `idArray` from `tableName`,
// put results into an array, and finally invoke `callback`.
function readTable (tablename, idArray, callback) {
var resultsArray = [];
// Queue up all the async queries.
for (var id of idArray){
queryTable(table, id, handleResponse);
}
// A query finished, so handle the result.
function handleResponse (error, results, fields) {
if (error) {
throw error;
}
// Add the query result to array of results.
resultsArray.push(results[0]);
// Check if all queries are done.
if (resultsArray.length === idArray.length) {
// Invoke the callback with the resultsArray.
// The callback is in fact the `displayData` function.
callback(resultsArray);
}
}
}
// Execute a query, using the `cb` callback to handle the response.
function queryForeignTable (tablename, id, cb) {
var query = "SELECT * FROM " + tablename + " WHERE id=" + id;
connection.query(query, cb);
}
Note that the handleResponse function is defined within the scope of the readTable function, so it can access the variables in readTables scope, such as resultsArray and callback.
Hope that helps.

Async.eachSeries executing in parallel

I'm trying to understand async library from node.js and apply it, but it doesn't work as expected. Called are made in parallels when I was expecting them to be made in series (i.e. one after another)
Here is my code
var users = [{_id:'1',username:'user1'},{_id:'2',username:'user2'}];
async.eachSeries(users,function function1(user,callbackEach){
var username = user.username;
var incomes = [{source:'sourceA',provider:providerA},{source:'sourceB',provider:providerB},{source:'sourceC',provider:providerC}];
async.eachSeries(incomes,function function2(income,callbackSmallEach){
var source = income.source;
income.provider.getEarnings(user._id,username,yesterday,function callbackFromGetEarnings(err,result){
if (err){
// error
} else {
income.earnings = {day : new Number(result)};
income.provider.getMonthEarnings(user._id,username,yesterday,function callbackFromGetMonthEarnings(err,monthTotal){
if (err){
// error
} else {
income.earnings.month = new Number(monthTotal);
callbackSmallEach();
}
});
}
});
},
function sendEmails(err){
if (err) {
// error
} else {
// send email
}
});
console.log("Just before calling callbackEach()");
callbackEach();
});
getEarnings and getMonthEarnings use an external provider, so some time can occur until callback functions callbackFromGetEarnings and callbackFromGetMonthEarnings are called.
My problem is that I dont want both calls to getEarnings to be executed in parallel. I want function getEarnings for user2 to be called only after getEarnings for user1 has returned (and corresponding callbacks have been made).
As you can see I have tried to make the calls for user1 and 2 in a serie, with async.eachSeries but calls are made in parallel. When I execute, the log Just before calling callbackEach() is always executed before the callback function callbackFromGetEarnings is called...
I hope this is clear enough.

The problem is where you are calling your callback. Example:
async.eachSeries(something, function(item, callback) {
async.eachSeries(item.somethingElse, function(subitem, callback2) {
//do something
return callback2();
}, function() {
//when all item.somethingElse is done, call the upper callback
return callback();
})
}, function() {
console.log('done');
})
This way, for each item in something, you will execute in series all the item.somethingElse, before going to the next item.

Javascript: Utilising data from an async callback function (MongoClient)

I've spent so much time trying to find the answer to this on here and come up with nothing. Hoping someone can enlighten me..
I have code which is making an async call to a database and returning data in a callback function (in my case I'm using MongoClient, which returns a Promise). However, I can't work out how to use the resulting data to actually set function-level variables - whenever I try to do it the resulting value that I log is either undefined or a pending Promise object.
There's lots of posts on this subject but I can't find any methods that work when I try to apply them. Any and all help gratefully received!
function lookupOneDbEntry(key, value) {
var responseData = "initial data"
// search for the entry in the database
db.collection("my_collection").findOne({key: value}, function(err, result) {
if (err) {
//if database throws an error
responseData = "db error";
}
else {
// if the entry is found, return the data
responseData = result;
}
});
return responseData;
}
EDIT: I am aware of other posts on this (like this one here and, while exhaustive documentation is useful to an extent, I;m having trouble using this information practically in a real-life implementation like the one above. Hence my question here.

Async calls happens outside of the call stack that you are on. you can't return it onto the current stack.
So we use the promises to hook into the results of our call.
function lookupOneDbEntry(key, value) {
return new Promise(function (resolve, reject) {
// search for the entry in the database
db.collection("my_collection").findOne({key: value}, function(err, result) {
if (err) {
//if database throws an error
reject(err);
}
else {
// if the entry is found, return the data
resolve(result);
}
});
});
}
lockupOneDbEntry('myKey', 'myValue').then(function (result) {
console.log('result', result);
}, function (err) {
console.log("error!", err);
});

After a long while of experimenting I've finally managed to do it - I didn't need any fancy callbacks or additional Promises in the end, I just removed the optional callback in the database request and instead processed the returned promise separately.
function lookupOneDbEntry(key, value) {
var responseData = "initial data";
var solution = db.collection("accounting_module").findOne({key: value});
solution.then(function (result) {
// validation of result here
responseData = result;
});
return responseData;
}

NodeJS async callback. How to return the list, from a async callback?

So basically I am making a database query, to get all posts with a certain id, then add them to a list, so I can return. But the list is returned, before the callback has finished.
How do I prevent it from being returned before callback has finished?
exports.getBlogEntries = function(opid) {
var list12 =[];
Entry.find({'opid' : opid}, function(err, entries) {
if(!err) {
console.log("adding");
entries.forEach( function(currentEntry){
list12.push(currentEntry);
});
}
else {
console.log("EEEERROOR");
}
//else {console.log("err");}
});
console.log(list12);
return list12;
};

ALL callback is asynchronous, so we don't have any guarantee if they will run exactly in the order we have leave them.
To fix it and make the process "synchronous" and guarantee an order executation you have two solutions:
First: make all process in nested list:
instead of this:
MyModel1.find({}, function(err, docsModel1) {
callback(err, docsModel1);
});
MyModel2.find({}, function(err, docsModel2) {
callback(err, docsModel2);
});
use this:
MyModel1.find({}, function(err, docsModel1) {
MyModel2.find({}, function(err, docsModel2) {
callback(err, docsModel1, docsModel2);
});
});
The last snippet above guarantee us that MyModel2 will be executed AFTER MyModel1 is executed.
Second: Use some framework as Async. This framework is awesome and have several helper functions to execute code in series, parallels, whatever way we want.
Example:
async.series(
{
function1 : function(callback) {
//your first code here
//...
callback(null, 'some result here');
},
function2 : function(callback) {
//your second code here (called only after the first one)
callback(null, 'another result here');
}
},
function(err, results) {
//capture the results from function1 and function2
//if function1 raise some error, function2 will not be called.
results.function1; // 'some result here'
results.function2; // 'another result here'
//do something else...
}
);

You could use sync database calls but that would work around the concept of node.js.
The proper way is to pass a callback to the function that queries the database and then call the provided callback inside the database query callback.

How do I prevent it from being returned before callback has finished?
The callback is asynchronous, and you cannot avoid that. Hence, you must not return a list.
Instead, offer a callback for when it's filled. Or return a Promise for the list. Example:
exports.getBlogEntries = function(opid, callback) {
Entry.find({'opid': opid}, callback); // yes, that's it.
// Everything else was boilerplate code
};

There is an alternate way to handle this scenario. You can use the async module and when the forEach has finished then make the return call. Please find the code snippet below for the same:
var async = requires('async');
exports.getBlogEntries = function(opid) {
var list12 =[];
Entry.find({'opid' : opid}, function(err, entries) {
if(!err) {
console.log("adding");
async.forEachSeries(entries,function(entry,returnFunction){
list12.push(entry);
},function(){
console.log(list12);
return list12;
});
}
else{
console.log("EEEERROOR");
}
});
};