We Keep Coding

NodeJS + ExpressJS: How to wait for forEach to finish with SQL queries inside

I'm trying to wait for a forEach to finish, and the forEach loop has two nested requests inside.
I need to wait untill the forEach finish beacuse I fill an array with the queries results and then, when the forEach is finish, then call another function, but I cannot do it well because sometimes, the array is fully filled, but othertimes the array is incomplete.
Here is my code:
readAllClientsAndInvoices: function(request, response) {
let clientsInvoices = [];
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) {
console.log(err);
} else {
clients.forEach(function (client, idx, array) {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
clientsInvoices.push(pair);
}
});
}
if (idx === array.length - 1) {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices, function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
}
});
});
}
});
}
This is how I do it now, but I need to wait untill the array is fully filled with all the clients and its invoices and then call to createclientPDFReportWOCommentsV2 function but I don't know how to do it.
Thanks everyone

You can try to use a map instead of forEach in order to accept a return value from every call of the callback function, that return value will have to be a Promise, resolving after particular call has been completed. Since I don't see any particular error handling in your example I just made it so that in case of error Promise resolves undefined which is filtered afterwards in the createClientPDFReportWOCommentsV2 call.
function readAllClientsAndInvoices(request, response) {
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) return console.log(err);
Promise.all(clients.map(client => {
return new Promise(resolve => {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
resolve();
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
resolve();
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
resolve(pair);
}
});
}
});
});
})).then(clientsInvoices => {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices.filter(Boolean), function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
});
});
}

To solve these problems i would use Async/Await https://javascript.info/async-await. Make sure all the methods you're calling on DAOClients returns a Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
For example
function readAllClientesById() {
return new Promise((resolve, reject) => {
// Wait for some data to get fetched from SQL
// and call resolve instead of callback function
resolve(data)
// Or of there was an error
reject(err)
})
}
This is natively supported in the latest versions of Node.js.
Example of Async/Await if promises is implemented:
async function readAllClientsAndInvoices(req, res) {
try {
const clientInvoices = []
const clients = await DAOClients.readAllClientesById(req.session.id)
for (const client of clients) {
const clientData = await DAOClients.readClientDataById(client.id)
const clientInvoices = await DAOClients.readAllclientInvoices(clientData.id)
clientInvoices.push({
clientData: data,
invoicesList: invoices
})
}
// This code won't be executed until the for loop is completed
await DAOClients.createClientPDFReportWOCommentsV2(clientInvoices)
} catch (err) {
return res.status(err.code).send(err)
}
res.redirect(307, '/client/searchClient');
}
I haven't tested the code, it's just an example of how I approach these type of problems.

Javascript how the better way to code nested callback?

I have 3 layer callbacks like this :
app.post('/', (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
//first
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`, function (err) {
if (err) return res.status(500).send(err);
//second
trainOutput.mv(`inputs/${req.body.caseName}/train_output.csv`, function (err) {
if (err) return res.status(500).send(err);
//third
testInput.mv(`inputs/${req.body.caseName}/test_input.csv`, function (err) {
if (err) return res.status(500).send(err);
res.send('success');
});
});
});
});
In this case, there are only 3 file uploads. In another case, I have more than 10 file uploads, and it makes 10 layer callbacks. I know it because of JavaScript asynchronous.
Is there any way, with this case, to make a beautiful code? This is because when it 10 layer callbacks, the code looks horizontally weird.
Thanks

You can use the following code to make you code look better and avoid callback hell
app.post('/', async (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
try {
var result1 = await trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`);
var result2 = await trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`);
var result2 = await testInput.mv(`inputs/${req.body.caseName}/test_input.csv`);
res.send('success');
}
catch (error) {
res.status(500).send(error);
}
});

You can make the functions return a Promise
I advice to make one function because you do the same thing 3 times. In this case I called the function 'save' but you can call it what ever you want. The first parameter is the file end the second the output filename.
function save(file, output) = return new Promise((resolve, reject) => {
file.mv(`inputs/${req.body.caseName}/${output}`, err =>
if (err) return reject(err)
resolve()
})
Promise.all([
save(req.files.trainInput, 'train_input.csv'),
save(req.files.trainInput, 'train_output.csv'),
save(req.files.trainInput, 'test_input.csv')
])
.then(_ => res.send(200))
.catch(err => res.send(400);

What version of Node you using? If async/await is available that cleans it up a bunch.
const moveCsv = (file, dest) => {
return new Promise((resolve, reject) => {
//third
file.mv(dest, function (err) {
if (err) reject(err);
resolve();
});
})
}
app.post('/', async(req, res) => {
try {
var filename = `outputs/${Date.now()}_output.json`;
const {
trainInput,
trainOutput,
testInput
} = req.files;
const prefix = `inputs/${req.body.caseName}`;
await moveCsv(trainInput, `${prefix}/train_input.csv`);
await moveCsv(trainOutput, `${prefix}/train_output.csv`);
await moveCsv(testInput, `${prefix}/test_input.csv`);
res.send('success');
} catch(err) {
res.status(500).send(err);
}
});
I'm also assuming here that your trainInput, trainOutput, testOutput weren't all meant to be req.files.trainInput.
Just be careful since the synchronous nature of the await calls are thread blocking. If that writer function takes ages you could also looking at putting those calls onto a worker thread. Won't really matter if your requests to that server endpoint are fast and non-frequent.

You can add RXJS to your project and use Observables.forkJoin()
Solution with Observables(assuming that trainInput.mv() returns Observable):
/* Without a selector */
var source = Rx.Observable.forkJoin(
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`),
trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`),
trainInput.mv(`inputs/${req.body.caseName}/test_input.csv`)
);
var subscription = source.subscribe(
function (x) {
// On success callback
console.log('Success: %s', x);
},
function (err) {
// Error callback
console.log('Error');
},
function () {
// Completed - runs always
console.log('Completed');
});
// => Success: [result_1, result_2, result_3] or Error
// => Completed

node js to return global variable set in callback function

I have 2 functions in javascript and I want to set a global variable in a callback and then return it from my main function. I know javascript is async by nature so what would be the best way to do this? I have the code below:
var results="";
function getData() {
var sql = require("mssql");
var dbConfig={
server:"server",
database: "db",
user:"user",
password: "pw"
}
var conn = new sql.Connection(dbConfig);
var req = new sql.Request(conn);
conn.connect(function (err){
if (err) {
console.log(err);
return;
}
req.query("SELECT * FROM table",resultsCallback)
conn.close();
});
return results;
}
function resultsCallback (err, recordset) {
var tableify = require('tableify');
if (err) {
console.log(err);
}
else {
var html = tableify(recordset);
html = html.replace('<table>','');
html = html.replace('</table>',');
results=html;
}
};

There is no right way to do this, because what you want to do is not right. You are trying to return a value before it was fetched. And you are using global variable, which can be dangerous for your code.
I understand why you are trying to use global variable to get the results from your callback, but it can't work.
When your code return results; executes, the SQL query has not even started, so it returns "".
Instead, use Promise:
function getData() {
return new Promise((resolve, reject) => {
var sql = require("mssql");
var dbConfig = {
server: "server",
database: "db",
user: "user",
password: "pw"
}
var conn = new sql.Connection(dbConfig);
var req = new sql.Request(conn);
conn.connect(function (err) {
if (err) {
console.log(err);
reject(err);
return;
}
req.query("SELECT * FROM table",
(err, recordset) => {
// Here we call the resolve/reject for the promise
try {
// If the results callback throws exception, it will be caught in
// the catch block
resolve(resultsCallback(err, recordset));
}
catch (e) {
reject(e);
}
}
);
conn.close();
});
})
}
function resultsCallback(err, recordset) {
var tableify = require('tableify');
if (err) {
console.log(err);
throw err;
}
else {
var html = tableify(recordset);
html = html.replace('<table>', '');
html = html.replace('</table>', '');
return html;
}
};
Global variables can be dangerous, you don't need them. Promise solves your problem. You can have as many callbacks nested, but you always return value from Promise using resolve. This is the standard modern way to do this in JavaScript.
Usage:
getData().then((data)=>{console.log("Table data:",data);})
.catch((error)=>{console.log("ERROR LOADING SQL:",error);})
Or, in an async function:
async function doSomethingWithData() {
console.log("Start loading data.");
const data = await getData();
console.log("Done loading data.");
// your results are in the data variable now
}

How to return a sdtout from a server to a client inside a string?

I'm trying to return the stdout of my method but on the client I always have undefined despite the server says it's a string with content.
I do that:
'getExistingFiles': function () {
var list = "";
return new Promise(Meteor.bindEnvironment(function(resolve) {
child = exec_tool("ls -al",
function (error, stdout, stderr) {
if (error !== null) {
console.error('exec error: ' + error);
list = "error: " + error;
resolve(list);
return list;
}else{
list = stdout;
console.log(typeof list);
console.log("LIST:------------");
console.log(list);
resolve(list);
return list;
}
});
}));
}
On my terminal I have the logs:
But on the client when I try to access the value it's undefined:
Meteor.call("getExistingFiles",function(list){
console.log(list);
console.log(typeof list);
});
So my question is, how could I send this list to the client ?
[EDIT] I tried like this to test if it was my client who was wrong but no it's on the server I think
//server
var et = Meteor.wrapAsync(exec_tool);
try {
child = et('ls -al');
console.log("LIST:------------");
console.log(child);
console.log(typeof child);
return child;
} catch (err) {
throw new Meteor.Error(err, err.stack);
}
[EDIT2] even like this it send a undefined WHY ?!
var et = Meteor.wrapAsync(exec_tool);
try {
var proc = exec_tool("ls -al")
proc.stdout.on('data', function (data) {
//do something with data
console.log(data);
list = list + data;
});
proc.stdout.on('end', function() {
console.log("end");
console.log(list);
return list;
});
} catch (err) {
throw new Meteor.Error(err, err.stack);
}

On the server (promises from the server are evaluated and then sent to the client when they're done):
getExistingFiles: function () {
return new Promise((resolve, reject) => {
child = exec_tool("ls -al",
function (error, stdout, stderr) {
if (error) {
reject(error);
} else {
resolve(stdout);
}
});
}));
}
And on the client:
Meteor.call("getExistingFiles", function(err, list) {
if(err) {
// handle your error
throw err;
}
console.log(list);
});
Promises don't have return. Callbacks from async functions usually have the function (error, result) argument list. So, the wanted result should be in the second argument. Try it like this.

Looks like this is a dupe question of this
See if this works for you. Async function using fiber/future. Let's tweak this in case you run into issues.
Server.js
//
// Load future from fibers
var Future = Npm.require("fibers/future");
// Load exec
var exec = Npm.require("child_process").exec;
Meteor.methods({
runListCommand: function () {
// This method call won't return immediately, it will wait for the
// asynchronous code to finish, so we call unblock to allow this client
// to queue other method calls (see Meteor docs)
this.unblock();
var future=new Future();
var command="cd /home/me/files/; ls *.svg";
exec(command,function(error,stdout,stderr){
if(error){
console.log(error);
throw new Meteor.Error(500,command+" failed");
}
future.return(stdout.toString());
});
return future.wait();
}
});
Client.js:
Meteor.call('runListCommand', function (err, response) {
console.log(response);
});

Node.js - How to chain Promise in The right way

I recently moved from callback functions to promises in node.js. I want to preform async query to the DB (psql) in the most elegant way. I was wondering if the following code is the right way to do it or if I can chain for example the promises in a way of first().then(second).then(third).
function queryAll(req, res) {
searchQuery()
.then((rows) => {
console.log(rows);
totalQuery()
.then((total) => {
console.log(total);
});
});
res.json({"rows": rows, "total": total});
}
function searchQuery() {
return new Promise(function(resolve, reject) {
var rowData = { rows: {} };
pool.query('select age, sex from workers;', values, function(err, result) {
if(err) {
return console.error('error running query', err);
reject(err);
}
rowData.rows = result.rows;
resolve(rowData);
});
});
}
function totalQuery() {
return new Promise(function(reject, resolve) {
var totalData = { totals: {} };
pool.query('select sex, scores from workers group by sex;', values, function(err, result) {
if(err) {
return console.error('error running query', err);
reject(err);
}
totalData.totals = result.rows;
resolve(totalData);
});
});
}

var rowData = { rows: {} };
var totalData = { totals: {} };
First of all, these make no sense stored in variables since there's nothing else on the object. Just resolve with the rows directly instead.
return console.error('error running query', err);
Also, don't just console.log your errors. then accepts a second callback that executes on thrown errors or rejected promises. Throw this message in an error or reject with it instead. Also, I would leave logging to the consumer.
function queryAll(req, res) {
searchQuery()
.then((search) => {
console.log(rows);
totalQuery()
.then((total) => {
console.log(total);
});
});
res.json({"rows": rows, "total": total});
}
rows and total don't exist anywhere. Plus, by the time res.json executes, rows and total (assuming they come from inside the callbacks) won't exist yet since the whole sequence is async. You'll get undefined values as results.
I see little point in running searchQuery and totalQuery in sequence as they're not dependent on each other. It's better to run them parallel instead. Use Promise.all for that.
function queryAll(req, res) {
Promise.all([
searchQuery(),
totalQuery()
]).then(values => {
const rows = values[0];
const total = values[1];
res.json({"rows": rows, "total": total});
}, function(e){
// something went wrong
});
}
function searchQuery() {
return new Promise(function(resolve, reject) {
pool.query('select age, sex from workers;', values, function(err, result) {
if(err) reject(err);
else resolve(result.rows);
});
});
}
function totalQuery() {
return new Promise(function(reject, resolve) {
pool.query('select sex, scores from workers group by sex;', values, function(err, result) {
if(err) reject(err);
else resolve(result.rows);
});
});
}

You have a few issues in the code:
You return before executing reject()
There is an undefined rows variable (mismatch with search)
res.json is executed before the results are in.
The promises resolve to objects like { rows: rows }, but the main function seems to expect the plain numbers, not the objects. So let the promises just resolve to numeric values.
The second SQL is ambiguous since the second field is not aggregated and does not appear in the group by clause either. Assuming you want to sum the scores, use sum().
The second query is only launched after the first one has returned results, but this can be done in parallel
You have very similar code repeated. Try to reuse code and make the SQL statement an argument.
Here is how I would suggest to do it:
function queryAll(req, res) {
return Promise.all([searchQuery(), totalQuery()]).then(([rows, total]) => {
console.log('rows', rows);
console.log('total', total);
// Make sure to only access the promised values in the `then` callback
res.json({rows, total});
});
}
function searchQuery() {
return promiseQuery('select age, sex from workers;');
}
function totalQuery() {
// When you group, make sure to aggregate:
return promiseQuery('select sex, sum(scores) as scores from workers group by sex;');
}
function promiseQuery(sql) { // reusable for other SQL queries
return new Promise(function(resolve, reject) {
pool.query(sql, values, function(err, result) {
if(err) {
// Do not return before calling reject!
console.error('error running query', err);
reject(err);
return;
}
// No need for a variable or object, just resolve with the number of rows
resolve(result.rows);
});
});
}

The most elegant solution would be via pg-promise:
function queryAll(req, res) {
db.task(t => {
return t.batch([
t.any('SELECT age, sex FROM workers', values),
t.any('SELECT sex, scores FROM workers GROUP BY sex', values)
]);
})
.then(data => {
res.json({rows: data[0], total: data[1]});
})
.catch(error => {
// handle error
});
}
And that's everything. You don't have to reinvent promise patterns for working with the database, they are all part of the library already.
And if your queries have a dependency, see: How to get results from multiple queries at once.
Or if you prefer ES6 generators:
function queryAll(req, res) {
db.task(function* (t) {
let rows = yield t.any('SELECT age, sex FROM workers', values);
let total = yield t.any('SELECT sex, scores FROM workers GROUP BY sex', values);
return {rows, total};
})
.then(data => {
res.json(data);
})
.catch(error => {
// handle error
});
}
And with the ES7's await/async it would be almost the same.

First of all there are some errors in your code, you have to place the reject before the return, otherwise it will be never called, and create a dangling promise:
function searchQuery() {
return new Promise(function(resolve, reject) {
var rowData = {
rows: {}
};
pool.query('select age, sex from workers;', values, function(err, result) {
if (err) {
reject(err);
console.error('error running query', err);
} else {
rowData.rows = result.rows;
resolve(rowData);
}
});
});
}
Beside that you should not nest the Promises when ever possible.
So it should be:
function queryAll(req, res) {
var result = {};
searchQuery()
.then((search) => {
console.log(search);
result.rows = search;
return totalQuery();
})
.then((total) => {
result.total = total;
console.log(total);
});
}
The res.json has to be called in the then part of the Promise:
function queryAll(req, res) {
var result = {};
searchQuery()
.then((search) => {
console.log(search);
result.rows = search;
return totalQuery();
})
.then((total) => {
result.total = total;
console.log(total);
})
.then(() => {
res.json({
"rows": result.rows,
"total": result.total
});
});
}
If your queryAll is called as e.g. middleware of express, then you should handle the catch case within queryAll:
function queryAll(req, res) {
var result = {};
searchQuery()
.then((search) => {
console.log(search);
result.rows = search;
return totalQuery();
})
.then((total) => {
result.total = total;
console.log(total);
})
.then(() => {
res.json({
"rows": result.rows,
"total": result.total
});
})
.catch( err => {
res.status(500).json({error: 'some error'})
});
}
For postgress I would suggest to use pg-promise instead of using a callback style library and wrapping it into promises yourself.
You could simplify the code if you use a library like bluebird:
const bPromise = require('bluebird')
function queryAll(req, res) {
bPromise.all([
searchQuery(),
totalQuery()
])
.spread((rows, total) => {
res.json({
"rows": rows,
"total": total
});
})
.catch(err => {
res.status(500).json({
error: 'some error'
})
});
}

With nsynjs your logic may be coded as simple as this:
var resp = {
rows: dbQuery(nsynjsCtx, conn, 'select age, sex from workers', values1).data,
total: dbQuery(nsynjsCtx, conn, 'select sex, scores from workers group by sex', values2).data
};
Please see example of multiple sequential queries here: https://github.com/amaksr/nsynjs/tree/master/examples/node-mysql