We Keep Coding

NodeJS + ExpressJS: How to wait for forEach to finish with SQL queries inside

I'm trying to wait for a forEach to finish, and the forEach loop has two nested requests inside.
I need to wait untill the forEach finish beacuse I fill an array with the queries results and then, when the forEach is finish, then call another function, but I cannot do it well because sometimes, the array is fully filled, but othertimes the array is incomplete.
Here is my code:
readAllClientsAndInvoices: function(request, response) {
let clientsInvoices = [];
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) {
console.log(err);
} else {
clients.forEach(function (client, idx, array) {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
clientsInvoices.push(pair);
}
});
}
if (idx === array.length - 1) {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices, function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
}
});
});
}
});
}
This is how I do it now, but I need to wait untill the array is fully filled with all the clients and its invoices and then call to createclientPDFReportWOCommentsV2 function but I don't know how to do it.
Thanks everyone

You can try to use a map instead of forEach in order to accept a return value from every call of the callback function, that return value will have to be a Promise, resolving after particular call has been completed. Since I don't see any particular error handling in your example I just made it so that in case of error Promise resolves undefined which is filtered afterwards in the createClientPDFReportWOCommentsV2 call.
function readAllClientsAndInvoices(request, response) {
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) return console.log(err);
Promise.all(clients.map(client => {
return new Promise(resolve => {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
resolve();
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
resolve();
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
resolve(pair);
}
});
}
});
});
})).then(clientsInvoices => {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices.filter(Boolean), function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
});
});
}

To solve these problems i would use Async/Await https://javascript.info/async-await. Make sure all the methods you're calling on DAOClients returns a Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
For example
function readAllClientesById() {
return new Promise((resolve, reject) => {
// Wait for some data to get fetched from SQL
// and call resolve instead of callback function
resolve(data)
// Or of there was an error
reject(err)
})
}
This is natively supported in the latest versions of Node.js.
Example of Async/Await if promises is implemented:
async function readAllClientsAndInvoices(req, res) {
try {
const clientInvoices = []
const clients = await DAOClients.readAllClientesById(req.session.id)
for (const client of clients) {
const clientData = await DAOClients.readClientDataById(client.id)
const clientInvoices = await DAOClients.readAllclientInvoices(clientData.id)
clientInvoices.push({
clientData: data,
invoicesList: invoices
})
}
// This code won't be executed until the for loop is completed
await DAOClients.createClientPDFReportWOCommentsV2(clientInvoices)
} catch (err) {
return res.status(err.code).send(err)
}
res.redirect(307, '/client/searchClient');
}
I haven't tested the code, it's just an example of how I approach these type of problems.

Using javascript promises with Mongoose queries

Normally if I was going to run multiple mongoose queries I would use the built in promise to chain them all together. In this case, the user chooses which schemas to search. This could be one of them or both. The following example uses the post data to define which schemas to search and if one is false, it should continue through the promise chain. Right now the final promise is being called before the queries.
Example in my express controller:
app.post('/custom-search', function (req, res) {
var single = false
var multi = false
if(req.body.single){
var single = true
}
if(req.body.multi){
var multi = true
}
var promise = new Promise(function (resolve, reject) {
if(multi){
multiSchema.find({}, function (err, result) {
if(!err){
console.log(result);
resolve()
}
})
}else{
resolve()
}
}).then(function (value) {
if(single){
singleSchema.find({}, function (err, result) {
if(!err){
console.log(result);
resolve()
}
})
}else{
resolve()
}
}).then(function (value) {
console.log("done");
})
})
});
output:
>done
>[singleResults]
>[multiResults]
done should be printing last so that is the first problem.

Like we discussed, few things had to be clean up. First by actually using and returning the promise for it work properly, Second, creating a mini-promise within your first .then() to resolve and reject your single conditional statement. And third, handling/catching promises.
I wrote a pseudo version of your code to illustrate my point of view, hopefully it may be of a good use.
app.get('/custom-search', function (req, res) {
// Manipulating values to test responses
var single = false;
var multi = true;
var promise = new Promise(function (resolve, reject) {
if (multi) {
setTimeout(function () {
resolve('MULTI RESOLVED!');
}, 3000);
} else {
reject('MULTI REJECTED!');
}
})
.then(function (value) {
new Promise(function (resolve, reject) {
if (single) {
setTimeout(function () {
resolve('SINGLE RESOLVED!');
}, 3000);
} else {
reject('SINGLE REJECTED!');
}
})
.catch(function (error) {
console.log('SINGLE ERROR!', error);
})
.then(function (value) {
console.log('SINGLE DONE', value);
});
})
.catch(function (error) {
console.log('MULTI ERROR!', error);
});
return promise;
});

Node.js - How to chain Promise in The right way

I recently moved from callback functions to promises in node.js. I want to preform async query to the DB (psql) in the most elegant way. I was wondering if the following code is the right way to do it or if I can chain for example the promises in a way of first().then(second).then(third).
function queryAll(req, res) {
searchQuery()
.then((rows) => {
console.log(rows);
totalQuery()
.then((total) => {
console.log(total);
});
});
res.json({"rows": rows, "total": total});
}
function searchQuery() {
return new Promise(function(resolve, reject) {
var rowData = { rows: {} };
pool.query('select age, sex from workers;', values, function(err, result) {
if(err) {
return console.error('error running query', err);
reject(err);
}
rowData.rows = result.rows;
resolve(rowData);
});
});
}
function totalQuery() {
return new Promise(function(reject, resolve) {
var totalData = { totals: {} };
pool.query('select sex, scores from workers group by sex;', values, function(err, result) {
if(err) {
return console.error('error running query', err);
reject(err);
}
totalData.totals = result.rows;
resolve(totalData);
});
});
}

var rowData = { rows: {} };
var totalData = { totals: {} };
First of all, these make no sense stored in variables since there's nothing else on the object. Just resolve with the rows directly instead.
return console.error('error running query', err);
Also, don't just console.log your errors. then accepts a second callback that executes on thrown errors or rejected promises. Throw this message in an error or reject with it instead. Also, I would leave logging to the consumer.
function queryAll(req, res) {
searchQuery()
.then((search) => {
console.log(rows);
totalQuery()
.then((total) => {
console.log(total);
});
});
res.json({"rows": rows, "total": total});
}
rows and total don't exist anywhere. Plus, by the time res.json executes, rows and total (assuming they come from inside the callbacks) won't exist yet since the whole sequence is async. You'll get undefined values as results.
I see little point in running searchQuery and totalQuery in sequence as they're not dependent on each other. It's better to run them parallel instead. Use Promise.all for that.
function queryAll(req, res) {
Promise.all([
searchQuery(),
totalQuery()
]).then(values => {
const rows = values[0];
const total = values[1];
res.json({"rows": rows, "total": total});
}, function(e){
// something went wrong
});
}
function searchQuery() {
return new Promise(function(resolve, reject) {
pool.query('select age, sex from workers;', values, function(err, result) {
if(err) reject(err);
else resolve(result.rows);
});
});
}
function totalQuery() {
return new Promise(function(reject, resolve) {
pool.query('select sex, scores from workers group by sex;', values, function(err, result) {
if(err) reject(err);
else resolve(result.rows);
});
});
}

You have a few issues in the code:
You return before executing reject()
There is an undefined rows variable (mismatch with search)
res.json is executed before the results are in.
The promises resolve to objects like { rows: rows }, but the main function seems to expect the plain numbers, not the objects. So let the promises just resolve to numeric values.
The second SQL is ambiguous since the second field is not aggregated and does not appear in the group by clause either. Assuming you want to sum the scores, use sum().
The second query is only launched after the first one has returned results, but this can be done in parallel
You have very similar code repeated. Try to reuse code and make the SQL statement an argument.
Here is how I would suggest to do it:
function queryAll(req, res) {
return Promise.all([searchQuery(), totalQuery()]).then(([rows, total]) => {
console.log('rows', rows);
console.log('total', total);
// Make sure to only access the promised values in the `then` callback
res.json({rows, total});
});
}
function searchQuery() {
return promiseQuery('select age, sex from workers;');
}
function totalQuery() {
// When you group, make sure to aggregate:
return promiseQuery('select sex, sum(scores) as scores from workers group by sex;');
}
function promiseQuery(sql) { // reusable for other SQL queries
return new Promise(function(resolve, reject) {
pool.query(sql, values, function(err, result) {
if(err) {
// Do not return before calling reject!
console.error('error running query', err);
reject(err);
return;
}
// No need for a variable or object, just resolve with the number of rows
resolve(result.rows);
});
});
}

The most elegant solution would be via pg-promise:
function queryAll(req, res) {
db.task(t => {
return t.batch([
t.any('SELECT age, sex FROM workers', values),
t.any('SELECT sex, scores FROM workers GROUP BY sex', values)
]);
})
.then(data => {
res.json({rows: data[0], total: data[1]});
})
.catch(error => {
// handle error
});
}
And that's everything. You don't have to reinvent promise patterns for working with the database, they are all part of the library already.
And if your queries have a dependency, see: How to get results from multiple queries at once.
Or if you prefer ES6 generators:
function queryAll(req, res) {
db.task(function* (t) {
let rows = yield t.any('SELECT age, sex FROM workers', values);
let total = yield t.any('SELECT sex, scores FROM workers GROUP BY sex', values);
return {rows, total};
})
.then(data => {
res.json(data);
})
.catch(error => {
// handle error
});
}
And with the ES7's await/async it would be almost the same.

First of all there are some errors in your code, you have to place the reject before the return, otherwise it will be never called, and create a dangling promise:
function searchQuery() {
return new Promise(function(resolve, reject) {
var rowData = {
rows: {}
};
pool.query('select age, sex from workers;', values, function(err, result) {
if (err) {
reject(err);
console.error('error running query', err);
} else {
rowData.rows = result.rows;
resolve(rowData);
}
});
});
}
Beside that you should not nest the Promises when ever possible.
So it should be:
function queryAll(req, res) {
var result = {};
searchQuery()
.then((search) => {
console.log(search);
result.rows = search;
return totalQuery();
})
.then((total) => {
result.total = total;
console.log(total);
});
}
The res.json has to be called in the then part of the Promise:
function queryAll(req, res) {
var result = {};
searchQuery()
.then((search) => {
console.log(search);
result.rows = search;
return totalQuery();
})
.then((total) => {
result.total = total;
console.log(total);
})
.then(() => {
res.json({
"rows": result.rows,
"total": result.total
});
});
}
If your queryAll is called as e.g. middleware of express, then you should handle the catch case within queryAll:
function queryAll(req, res) {
var result = {};
searchQuery()
.then((search) => {
console.log(search);
result.rows = search;
return totalQuery();
})
.then((total) => {
result.total = total;
console.log(total);
})
.then(() => {
res.json({
"rows": result.rows,
"total": result.total
});
})
.catch( err => {
res.status(500).json({error: 'some error'})
});
}
For postgress I would suggest to use pg-promise instead of using a callback style library and wrapping it into promises yourself.
You could simplify the code if you use a library like bluebird:
const bPromise = require('bluebird')
function queryAll(req, res) {
bPromise.all([
searchQuery(),
totalQuery()
])
.spread((rows, total) => {
res.json({
"rows": rows,
"total": total
});
})
.catch(err => {
res.status(500).json({
error: 'some error'
})
});
}

With nsynjs your logic may be coded as simple as this:
var resp = {
rows: dbQuery(nsynjsCtx, conn, 'select age, sex from workers', values1).data,
total: dbQuery(nsynjsCtx, conn, 'select sex, scores from workers group by sex', values2).data
};
Please see example of multiple sequential queries here: https://github.com/amaksr/nsynjs/tree/master/examples/node-mysql

Promise chains and anonymous promise returns

Here I have a chain of promises that works fine. All the *.destroy's are promises that return promises:
function callDBDestroy() {
var db;
DB_Categories.destroy().then(function () {
return DB_Equipment.destroy();
}).catch(function (err) {
showMsg("Error in callDBDestroy: " + err);
}).then(function () {
return DB_Certificates.destroy();
}).catch(function (err) {
showMsg("Error in callDBDestroy: " + err);
}).then(function () {
return DB_Locations.destroy();
}).catch(function (err) {
showMsg("Error in callDBDestroy: " + err);
});
}
But I want to add an if statement into each one to check to see if the PouchDB database exists (which it doesn't if the DB_* is null).
If it exists, I want to destroy it then return (and these all return promises).
If it doesn't exist, I want to return an anonymous promise which returns nothing as none of the promises have any data I am concerned with.
In the example, I added in some sample code to do the if statement and I was wondering what I would put in the null instance that would pass a promise (resolve) value.
function callDBDestroy() {
var db;
DB_Categories.destroy().then(function () {
if( DB_Equipment != null) {
return DB_Equipment.destroy();
}
else {
Anonymous empty promise - something like:
new Promise().resolve();
}
}).then(function () {
return DB_Certificates.destroy();
}).then(function () {
return DB_Locations.destroy();
}).catch(function (err) {
showMsg("Error in callDBDestroy: " + err);
});
}
Thanks,
Tom

It looks like you are just wondering how to manually resolve/reject a Promise. If that is the case you can just call Promise.resolve(optionalValue) or Promise.reject(optionalValue) if you want to go to the catch handler:
function callDBDestroy() {
var db;
DB_Categories.destroy()
.then(function () {
if( DB_Equipment != null) {
return DB_Equipment.destroy();
} else {
return Promise.resolve();
}
}).then(function () {
return DB_Certificates.destroy();
}).then(function () {
return DB_Locations.destroy();
}).catch(function (err) {
showMsg("Error in callDBDestroy: " + err);
});
}

You could wrap it:
function withDb(db, handler) {
return function onFulfilled(value) {
if(db === null) throw new Error("DB Null");
return handler(value);
});
}
Which would let you do:
function callDBDestroy() {
var db;
var w = withDb(db); // or whatever instance
DB_Categories.destroy().then(w(function () {
// do stuff
}))); // .then(w( to chain calls here.
...
}

I want to return an anonymous promise which returns nothing as none of the promises have any data I am concerned with. Something like:
new Promise().resolve();
You are looking for Promise.resolve(undefined). Though you can omit the undefined, that's implicit.
….then(function () {
if (DB_Equipment != null) {
return DB_Equipment.destroy();
} else {
return Promise.resolve(undefined);
}
}).…
And you don't even have to return a promise from a then callback, simply returning undefined (or not returning) will have the same effect.
….then(function () {
if (DB_Equipment != null) {
return DB_Equipment.destroy();
}
}).…
In your case, I'd recommend a wrapper function:
function destroyDatabase(db, name = "db") {
if (db != null)
return db.destroy().catch(err => {
showMsg(`Error in destroying ${name}: ${err}`);
});
else
return Promise.resolve();
}
function callDBDestroy() {
return destroyDatabase(DB_Categories, "categories")
.then(() => destroyDatabase(DB_Certificates, "certificates"))
.then(() => destroyDatabase(DB_Locations, "locations"))
}
// or even in parallel:
function callDBDestroy() {
return Promise.all([
destroyDatabase(DB_Categories, "categories"),
destroyDatabase(DB_Certificates, "certificates"),
destroyDatabase(DB_Locations, "locations")
]);
}

How about using an Array, since you do the very same task, and only the DB changes:
//serial
function callDBDestroy() {
var databases = [
DB_Categories,
DB_Equipment,
DB_Certificates,
DB_Locations
];
function errorMessage(err){ showMsg("Error in callDBDestroy: " + err) };
databases.reduce(
(prev, db) => db == null?
prev:
prev.then(() => db.destroy().catch(errorMessage)),
Promise.resolve()
)
}
//parallel
function callDBDestroy() {
var databases = [
DB_Categories,
DB_Equipment,
DB_Certificates,
DB_Locations
];
function errorMessage(err){ showMsg("Error in callDBDestroy: " + err) };
databases.forEach( db => db && db.destroy().catch(errorMessage) );
}
I've added a serial and a paralell version.

It seems that you can DRY this out and replace a lot of redundant code by using an array of databases and then just loop through the array:
function callDbDestroy();
var dbs = [DB_Categories, DB_Equipment, DB_Certificates, DB_Locations];
// chain all the destroys together
return dbs.reduce((p, db) => {
return p.then(() => {
if (db) {
return db.destroy().catch(err => {
showMsg("Error in callDBDestroy: " + err);
});
}
});
}, Promise.resolve());
}
You do not have to return a promise from a .then() handler. If you just have no return value, then it's just like doing return undefined which just means that no value will be passed to the next .then() handler, but the promise chain will continue just fine. Conceptually, it works the same as return Promise.resolve(), but there's no need to make an extra promise there.
Since you aren't passing a value from one .then() to the next in the chain, you have nothing to pass there so you can just not return anything if there's no db value to call destroy on.
FYI, using .reduce() to loop through an array is with the return p.then(...) structure is a common design pattern for sequencing async operations on an array.
FYI, using the Bluebird promise library (which has some useful helpers), this could be done like this:
function callDbDestroy();
var dbs = [DB_Categories, DB_Equipment, DB_Certificates, DB_Locations];
return Promise.mapSeries(dbs, db => {
if (db) {
return db.destroy().catch(err => {
showMsg("Error in callDBDestroy: " + err);
});
}
});
}
For more info on why the Bluebird (or other promise libraries) are still useful even with ES6, see Are there still reasons to use promise libraries like Q or BlueBird now that we have ES6 promises?
Since it appears that these databases might all be independent, I'm wondering why you are forcing them to be executed in sequence. If they don't have to be forced into sequence, then you could do this:
function callDbDestroy();
var dbs = [DB_Categories, DB_Equipment, DB_Certificates, DB_Locations];
return Promise.all(dbs.map(db => {
if (db) {
return db.destroy().catch(err => {
showMsg("Error in callDBDestroy: " + err);
});
}
}));
}
Since this runs the operations in parallel, it has the opportunity for faster end-to-end execution time vs. strict serialization.

Issue with promises in a for loop

I'm confronted to a situation which drives me a bit mad.
So The situation is as below :
module.exports = {
generation: function (req, res) {
// Let's firstly fetch all the products from the productTmp Table
function fetchProductsTmp (){
ProductsTmp.find().then(function (products) {
return Promise.all(products.map (function (row){
Service.importProcess(row);
}));
});
}
fetchProductsTmp();
}
Here I simply call my model ProductsTmp to fetch my datas and iterate through my rows calling importProcess.
importProcess :
importProcess: function (product) {
async.series([
function (callback) {
return SousFamille.findOne({name: product.sous_famille}).then(function (sf) {
console.log('1');
if (!sf) {
return SousFamille.create({name: product.sous_famille}).then(function (_sf) {
console.log('2');
callback(null, _sf.sf_id);
});
} else {
callback(null, sf.sf_id);
}
});
},
function (callback){
console.log('3');
},
], function(err, results){
if(err) return res.send({message: "Error"});
});
}
So I got with my console log :
1
1
1
2
3
2
3
2
3
What I want to Obtain is 1 2 3 1 2 3 1 2 3 So that each function wait for the promise to finish before calling the next one.

In the generation function of the first section, replace
return Promise.all(products.map (function (row){
Service.importProcess(row);
}));
with
var results = [],
pushResult = id => results.push(id);
return products.reduce(function(prev, row){//Go through all the products
//Take the previous promise, and schedule next call to Service.importProcess to be
//made after the previous promise has been resolved
return prev.then(function(){
return Service.importProcess(row).then(pushResult);
});
}, Promise.resolve())
.then(() => results);
You also need to return a promise from importProcess for this to work. Just ditch the whole async.series thingy and do something like
return new Promise(function(resolve, reject){
...
resolve(sf.sf_id); //instead of the callback(null, sf.sf_id)
...
});
Update: This forces the calls to Service.importProcess to be sequential instead of concurrent, which does affect the overall performance of calls to generation. But I guess you have more solid reasons to do so than sequential console.logs.

Sorry, can't help the urge to do it in ES6, basically things can be reduced to single line, like Bergi said, async is redundant( using Bluebird Promise Library):
importProcess: product =>
SousFamille.findOne({name: product.sous_famille})
.then(sf => sf? sf.sf_id : SousFamille.create({name: product.sous_famille}).then(_sf => _sf.sf_id))
// the other module
module.exports = {
generation: (req, res) => ProductsTmp.find()
.then(products => Promise.mapSeries(products, Service.importProcess.bind(Service)) )
.then(ids => res.send({ids}))
.catch(error => res.send({message: 'Error'}))
}
also like noppa said, your problem is the missing return in Service.importProcess(row), same code in ES5:
module.exports = {
generation: function (req, res) {
ProductsTmp.find()
.then(function (products) {
return Promise.mapSeries(products, Service.importProcess.bind(Service)) );
}).then(function(ids){
res.send({ids: ids});
}).catch(function(error){
res.send({message: 'Error'});
})
}
importProcess: function (product) {
return SousFamille.findOne({name: product.sous_famille})
.then(function (sf) {
if (sf) return sf.sf_id;
return SousFamille.create({name: product.sous_famille})
.then(function (_sf){ return _sf.sf_id});
});
}