We Keep Coding

NodeJS + ExpressJS: How to wait for forEach to finish with SQL queries inside

I'm trying to wait for a forEach to finish, and the forEach loop has two nested requests inside.
I need to wait untill the forEach finish beacuse I fill an array with the queries results and then, when the forEach is finish, then call another function, but I cannot do it well because sometimes, the array is fully filled, but othertimes the array is incomplete.
Here is my code:
readAllClientsAndInvoices: function(request, response) {
let clientsInvoices = [];
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) {
console.log(err);
} else {
clients.forEach(function (client, idx, array) {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
clientsInvoices.push(pair);
}
});
}
if (idx === array.length - 1) {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices, function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
}
});
});
}
});
}
This is how I do it now, but I need to wait untill the array is fully filled with all the clients and its invoices and then call to createclientPDFReportWOCommentsV2 function but I don't know how to do it.
Thanks everyone

You can try to use a map instead of forEach in order to accept a return value from every call of the callback function, that return value will have to be a Promise, resolving after particular call has been completed. Since I don't see any particular error handling in your example I just made it so that in case of error Promise resolves undefined which is filtered afterwards in the createClientPDFReportWOCommentsV2 call.
function readAllClientsAndInvoices(request, response) {
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) return console.log(err);
Promise.all(clients.map(client => {
return new Promise(resolve => {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
resolve();
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
resolve();
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
resolve(pair);
}
});
}
});
});
})).then(clientsInvoices => {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices.filter(Boolean), function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
});
});
}

To solve these problems i would use Async/Await https://javascript.info/async-await. Make sure all the methods you're calling on DAOClients returns a Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
For example
function readAllClientesById() {
return new Promise((resolve, reject) => {
// Wait for some data to get fetched from SQL
// and call resolve instead of callback function
resolve(data)
// Or of there was an error
reject(err)
})
}
This is natively supported in the latest versions of Node.js.
Example of Async/Await if promises is implemented:
async function readAllClientsAndInvoices(req, res) {
try {
const clientInvoices = []
const clients = await DAOClients.readAllClientesById(req.session.id)
for (const client of clients) {
const clientData = await DAOClients.readClientDataById(client.id)
const clientInvoices = await DAOClients.readAllclientInvoices(clientData.id)
clientInvoices.push({
clientData: data,
invoicesList: invoices
})
}
// This code won't be executed until the for loop is completed
await DAOClients.createClientPDFReportWOCommentsV2(clientInvoices)
} catch (err) {
return res.status(err.code).send(err)
}
res.redirect(307, '/client/searchClient');
}
I haven't tested the code, it's just an example of how I approach these type of problems.

How to break Async series on SIGTERM?

Assume I have the following scenario -
async.series(
[
function (cbi) {
students.getAll('student', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
students.deleteAll('student', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
teachers.getAll('teacher', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
teachers.deleteAll('teacher', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
};
]
);
And I want a graceful cleanup when SIGTERM is sent. That is a clean up of all the students or all teachers whichever is in progress when the signal was sent should complete and the next one should not begin.
function (cbi) {
students.getAll('student', function (err, response) {
if (err || GLOBAL_VAR_SIGTERM === true) {
logger.error(err);
}
cbi(err, response);
});
}
I was thinking that I should set a global variable to keep track of the SIGTERM signal.
process.on('SIGTERM', function onSigterm () {
GLOBAL_VAR_SIGTERM = true;
}
Is there any better way to break an async series to break a SIGTERM signal?

As #adamrights points out in his answer, the main problem in your code is you didn't call cbi(err, response) with a truthy err 1st param, which is essential to stop async.series continuing to next task in the queue.
Now your code should work, but you have a repeating pattern in your code that goes:
function (cbi) {
students.getAll('student', function (err, response) {
// these 3 lines appear in every callback function
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
if (err) logger.error(err);
cbi(err, response);
// end of repeat pattern
});
}
Your callback passed to every async task always do the same 3-liner thing. We know the DRY rule, it's always a good idea to extract the repeating pattern into another function to reuse it as much as possible.
So instead of repeatedly declaring anonymous functions, you should declare a factory function.
function callbackFactory(cbi) {
return function(err, response) {
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
if (err) logger.error(err);
cbi(err, response);
}
}
// use arrow function to write more concise code
async.series(
[
cbi => students.getAll('student', callbackFactory(cbi)),
cbi => students.deleteAll('student', callbackFactory(cbi)),
cbi => teachers.getAll('teacher', callbackFactory(cbi)),
cbi => teachers.deleteAll('teacher', callbackFactory(cbi)),
]
);
Advance Topic: use decorator to handle cross cutting concern
Let's explore this topic a bit more. Obviously, abort early on receiving SIGTERM is a cross cutting concern that should be separated from business logic. Suppose your business logic varies from task to task:
async.series(
[
cbi => students.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateStudentCount(response.data.length) // <- extra work
cbi(err, response);
}),
cbi => teachers.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateTeacherCount(response.data.length) // <- different extra work
cbi(err, response);
})
]
);
Because the callback is varying, it can be difficult to be extracted into a factory function like before. From this perspective, we'd better inject the abort-early behavior to each task, keep it easy to write normal business logic.
This is where decorator pattern comes handy. But global variable isn't the best tool to implement it, we'll use event listener.
The basic interface of the decorator looks like:
// `task` will be things like `cbi => students.getAll('student', ... )`
function decorateTaskAbortEarly(task) {
return (originalCbi) => {
...
task(originalCbi)
}
}
Below is our implementation checklist:
we're going to call originalCbi if we receive SIGTERM
but when we don't receive SIGTERM, the originalCbi is still callable inside callback of any async task like normal
if originalCbi is ever called once, we should unsubscribe from SIGTERM to prevent memory leak
The implementation:
function decorateTaskAbortEarly(task) {
return (originalCbi) => {
// subscribe to `SIGTERM`
var listener = () => originalCbi(new Error("SIGTERM: Aborting remaining tasks"));
process.once('SIGTERM', listener);
var wrappedCbi = (err, response) => {
// unsubscribe if `cbi` is called once
process.off('SIGTERM', listener);
return originalCbi(err, response);
};
// pass `cbi` through to `task`
task(wrappedCbi);
}
}
// Usage:
async.series(
[
cbi => students.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateStudentCount(response.data.length)
cbi(err, response);
}),
cbi => teachers.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateTeacherCount(response.data.length)
cbi(err, response);
})
].map(decorateTaskAbortEarly) // <--- nice API
);

If you want to respond to the SIGTERM event from within the async.series() then you are correct, the easiest way would be to track with a global variable.
But you need to set the first parameter (error-first callback) in the cbi(err, response) function to true in order to break the series.
so:
if (err || GLOBAL_VAR_SIGTERM === true) {
logger.error(err);
}
should be more like:
if (err) logger.error(err);
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
// You could just do err = true
// But best practice is to use an Error instance.
Then because cbi(err, response) will be called with err value equal totrue the remaining tasks will not be run.

I liked the other answers. This is another way to achieve the same. I am using my own example:
var async = require('async');
var ifAsync = require('if-async')
var GLOBAL_VAR_SIGTERM = false;
async.series({
one: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('one');
callback(null, 1);
}, 1000);
}),
two: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('two');
callback(null, 2);
}, 1000);
}),
three: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('three');
callback(null, 3);
}, 1000);
}),
four: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('four');
callback(null, 4);
}, 1000);
}),
}, function (err, results) {
if (err) {
//Handle the error in some way. Here we simply throw it
//Other options: pass it on to an outer callback, log it etc.
throw err;
}
console.log('Results are ' + JSON.stringify(results));
});
process.on('SIGTERM', function onSigterm () {
console.log('SIGTERM caught');
GLOBAL_VAR_SIGTERM = true;
});
function notsigterm(callback) {
if (!GLOBAL_VAR_SIGTERM) return callback(null, true)
else return callback(null, false)
}
I am using a package called ifAsync which allows you to use a predicate notsigterm to decide if a callback should be invoked or not. If notsigterm returns true then the callback will be invoked else it will be skipped. This is similar answer to others but somehow I find this cleaner. Let me know if you have questions.

Cannot access db result in Nodejs, it always returns null

var robject=[];
async.waterfall([
function (callback) {
for(var i in serial){
Router.find({},{r_serial_no:serial[i]},function (err,routerData) {
robject = robject.concat(routerData);
});
}
console.log('Robject= '+robject); //THIS RETURNS NULL
callback(null, robject);
},
function (blogs, callback) {
res.render('index', {dispatched_data:dispatched_data });
callback(null, 'Ended..' );
}
], function (err, result) {
console.log(result);
});
this is my waterfall model, here i need to access the robject from schema.find method to outside of that method. but it always return null..
how to access that??

You have the syntax error:
for(var i in serial){
Router.find({},{r_serial_no: i},function (err,routerData) {
robject = robject.concat(routerData);
});
}
the "for" loop defines "i" as next item in the array each iteration

The problem I see here is in for...in loop. Your callback will be fired even if your process i.e. Router.find is not completed. You can try below code, It might help.
Unlike your serial object please create a array called serials.
var robject=[];
async.waterfall([
function (callback) {
async.each(serials,
function(serial, localCb){
Router.find({},{r_serial_no:serial},function (err,routerData) {
robject = robject.concat(routerData);
localCb()
});
},
function(err){
console.log('Robject= '+robject);
callback(null, robject);
}
);
},
function (blogs, callback) {
res.render('index', {dispatched_data:dispatched_data });
callback(null, 'Ended..' );
}
], function (err, result) {
console.log(result);
});

Callback function getting called before initial function finishes

I am having issues with callback functions - I have a main function which has two callback functions inside. This is the main function
socket.on('play next video', function(data) {
removeVideo(cue[0], getCueFromDb(function() {
io.sockets.emit('next video');
}));
});
My removeVideo function looks like this:
function removeVideo(id, callback) {
Video.find({'id' : id}).remove(function(err, data) {
if (err)
console.log(err)
console.log("Removed video", id)
});
if (callback)
callback();
else
return
}
and the getCueFromDb function looks like this
function getCueFromDb(callback) {
Video.find({}).exec(function(err, videos) {
if (err) {
console.log(err)
}
if (videos.length) {
cue.length = 0 // empty array
videos.forEach(function(video) {
cue.push(video.id) // push all the videos from db into cue array
});
io.sockets.emit('send cue', {cue: cue});
}
else {
console.log("No more videos in database!")
}
if (callback)
callback();
else
return
});
}
However the functions aren't getting called in the correct order - am I doing something wrong?

Your callback needs to be inside your find.remove()
function removeVideo(id, callback) {
Video.find({'id' : id}).remove(function(err, data) {
if (err)
console.log(err)
console.log("Removed video", id)
if (callback)
callback();
else
return
});

You have to change removeVideo so callback we be called only after delete.
That's the code:
function removeVideo(id, callback) {
Video.find({'id' : id}).remove(function(err, data) {
if (err)
console.log(err)
console.log("Removed video", id)
if (callback)
callback();
else
return
}
});
So callback will be called only after vide is really removed.

Calling a parameterized callback function within a mongoose async callback function becomes 'undefined'

I am having a weird problem with calling a callback inside another callback from mongoose.
Setup : MEAN Stack.
myFunc = function (cb) {
var projection = {
'_id': 0,
'var1': 1,
'var2': 1
}
var order = {
'var1': 1
}
User.find({})
.select(projection).sort(order)
.exec(function(err, docs){
if(err){
console.log(err);
cb(err,docs);
} else {
console.log(docs);
cb(err,docs);
}
});
};
going to the lines where cb(err,docs) will result in
"ReferenceError: cb is not defined"
the weird part is
I have functions with even deeper nested callbacks that can invoke the "cb" normaly.
myFunc = function(cb){
model1.count({var1:'test'}, function (err, count) {
if(count) {
model2.findOne({dat1:'hoho'}, function (err, doc){
if (err) {
console.error(err);
cb(err,doc);
} else {
cb(err,doc);
}
});
} else {
cb({message: "No items found"}, null);
}
})
}
The code above will be invoked like so...
function init(something){
myfunc(function(err, doc) {
if (err){
console.log(err.message);
} else {
//do something about doc
}
});
}

ugh, it seems that the calling function did not properly follow the rules.
it called
myFunc(json, function(err,doc){
//do something
})
wrong param count...