We Keep Coding

Node.js async taking too long to execute

I've got my array of queries
var array = [ 'UPDATE EVALUATION SET mark = "16" WHERE id_eval = "21" AND id_usr = "125"',
'UPDATE EVALUATION SET mark = "9" WHERE id_eval = "22" AND id_usr = "125"',
'UPDATE EVALUATION SET mark = "8" WHERE id_eval = "34" AND id_usr = "125"'
]
However, when I try to execute them all at once with async, my web page says Waiting for localhost... and it keeps on loading forever. What am I doing wrong?
async.forEach(array, function(query, callback) {
connection.query(query, function(err, rows, fields) {
if(err) {
return console.error(err);
}
callback();
});
}, function(err){
if(err) {
return console.log(err);
}
});

Just make sure that you return the response after the forEach callback is called:
async.forEach(array, function(query, callback) {
connection.query(query, function(err, rows, fields) {
if(err) {
console.error(err);
}
callback();
});
}, function(err){
if(err) {
console.log(err);
}
res.redirect('/next-page');
});
That way, the redirection will occur only at the end of all the queries.
Some things you should verify:
Verify you didn't call res.end() or res.redirect() or anything similar before the above code.
Verify that your DB query method actually expect only 2 arguments: query and callback, and not anything in between (e.g. the query parameters).
Verify this piece of code is actually called when you expect it. Try debugging the request all the way.
Currently there's no real error handling here. You should consider returning an HTTP error if something went wrong. This should also help you debugging this code in the future.

Usually you have a request handler like this:
const server = http.createServer((req, res) => {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
res.end('Hello World\n');
});
Your browser holds the connection until res.endwas called!
In your environment it should look somethinge like
const server = http.createServer((req, res) => {
// ...
async.forEach(array, function(query, callback){
connection.query(query, function(err, rows, fields) {
// you may do some work here but leave it *alyways* via callback!
callback(err);
});
}, function(err){
if(err){ // this may be errors from above
return console.log(err);
}
// Exit here !
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
res.end('All done.\n');
});
});
...or a bit cleaner
function doSomeDatabaseUpdates(onFinished) {
var array = [...];
async.forEach(array, function(query, callback){
connection.query(query, function(err, rows, fields) {
// you may do some work here but leave it *alyways* via callback!
callback(err);
});
},
function IGetCalledAfterAboveCallbackWereExecuted(err){
if(err){ // this may be errors from above
return console.log(err);
}
// Exit here !
onFinished();
});
}
const server = http.createServer((req, res) => {
// ... do some work
if (doSomeUpdated === true) {
doSomeDatabaseUpdates(function calledAfterUpdates() {
res.end("updates something");
});
} else {
res.end("nothing to do");
}
});

Node js: Express js asynchronous db query execution-return results got undefiend

Just started to learn express js framework ,here is my simple database query execution part its invoked with this url localhost:3000/api/test.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
return results;
});
Does it really asynchronous?? suppose another user request this url does he need to wait for the previous query execution??.
I've heard about async package ,but don't know how this is applicable in my case
UPDATE
I got proper result in console.log(); but when i return the result i got undefined error
Here is my model.js
module.exports = {
getUser:function () {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
});
}
}
From my controller.js
var model = require('../models/user.js');
module.exports = {
getData : function(req, res){
//invoke model
console.log(model.getUser());
}
}

Node is non-blocking and will serve this request as and when it's called.
If another user hits this endpoint then it will execute again regardless if the first query has completed or not (unless the SQL has locked the table, in which case all consecutive connections/queries will wait and may timeout because of it). This happens on a connection basis.
You should make sure to check your SQL server (MySQL?) configs here to make sure there are enough max_connections to be able to cope with whatever load you are expecting.
Remember that the biggest bottleneck to an application is usually the database.
Your query above will need a callback to return the data asynchronously.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
//cb=callback function passed in to context
if (cb) cb(results);
});
Updated answer from updated question
In your model.js:
module.exports = {
getUser:function (cb) {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
if (cb) cb(results);
});
}
}
In your controller.js:
module.exports = {
getData : function(req, res){
//invoke model
model.getUser(function(results) {
console.log(results);
});
}
}

When you deal with callback, the safe and clean way to handle them is Promises. It's now standard in JavaScript and don't require any module.
And yes it is asynchronous. Behind, there'll be network access and dialogs with the database server. Only when they're done chatting will the callback be called.
module.exports = {
getUser: function () {
// wrap asynchronously called callback in Promise
new Promise((resolve, reject) => {
db.query("SELECT * FROM user", (error, results, fields) => {
if (error) {
reject(error); // similar to "throw"
}
else {
resolve({ results, fields }); // similar to "return"
}
});
});
}
};
How do you use it:
Vanilla notation:
// similar to "try"
model.getUser()
.then((answer) => {
console.log("answer", answer);
})
// similar to "catch"
.catch((error) => {
console.log("error", error);
});
async-await notation (only available in last versions of nodejs & browsers):
// you must be in an async environement to use "await"
async function wrapper() {
try {
var answer = await model.getUser(); // wait for Promise resolution
console.log("answer", answer);
}
catch(error) {
console.log("error", error);
}
}
// async function return automatically a Promise so you can chain them easily
wrapper();

Nodejs express and promises not doing what I expect

I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.

Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.

Variable not retaining value outside of function

I am attempting to pass the data variable from the function to the app.get call.
But the global variable 'data' in my function is not retaining the value returned from the database query. I believe this to be a scoping issue. I set the value for 'data' to null at the beginning to make the variable accessible throughout the function. I am not hitting either of the error conditions.
function getCommands(query) {
var data = null;
try {
pg.connect(cString, function(err, client, done) {
// Catch any connection errors, and display them to the screen
if(err) {
return console.error('could not connect to postgres', err);
}
client.query(query, function(q_err, result) {
// Release the client back to the pool
done();
if(q_err) {
return console.error('error running query', q_err);
}
// Data object with an empty array to hold the row information
data = {"data":[]};
for (var row in result.rows)
{
//console.log(result.rows[row]);
data.data.push(result.rows[row]);
}
//Here, data, has the correct return values.
console.log(data);
});
});
}
catch( e )
{
console.log(e);
}
//Here, data, is null.
console.log(data);
return data;
}
app.get('/clients/', function(req, res) {
res.send(getCommands('SELECT clientid, clientname FROM hourglass.clients ORDER BY clientid ASC'));
});
Could someone help me determine why 'data' is not retaining value outside of the pg.connect function?

I think your problem is not that the variable is not retaining the value outside of the function, but rather that you console.log(data) is executing before the variable is set.
If you were to put console.log('step X') in your code as in the following example you'll see the sequence in which your code is executed.
function getCommands(query) {
var data = null;
console.log('STEP 1');
pg.connect(cString, function(err, client, done) {
console.log('STEP 3');
});
console.log('STEP 2');
}

function getCommands(query, callback) {
try {
pg.connect(cString, function(err, client, done) {
// Catch any connection errors, and display them to the screen
if(err) {
return console.error('could not connect to postgres', err);
}
client.query(query, function(q_err, result) {
// Release the client back to the pool
done();
if(q_err) {
return console.error('error running query', q_err);
}
// Data object with an empty array to hold the row information
var data = {"data":[]};
for (var row in result.rows)
{
data.data.push(result.rows[row]);
}
callback(data); //After data is set, the value is passed back
});
});
}
catch( e )
{
console.log(e);
}
}
app.get('/clients', function(req, res) {.....])
Using a callback function mentioned by #dystroy worked perfectly.

passing variables to another function

I'm getting "client is not defined". I can see client is not defined for the function but I'm not sure how to pass in client into the async. I don't think I'm actually returning the value of the each correct to pg_commit either or am I?
Basically I want to have an array of queries and loop over them and make them queries and then when all done commit those as a transaction.
var pg_conn_str = "postgres://postgres:5432#localhost/test2";
var pg = require ('pg');
var rollback = function (client, done) {
client.query ('ROLLBACK', function (err) {
return done (err);
});
};
var queries = ["INSERT INTO foo (bar) VALUES (4)",
"INSERT INTO foo (bar) VALUES (5)"];
pg.connect (pg_conn_str, function (err, client, done) {
if (err) throw err;
client.query ('BEGIN', function (err) {
if (err) return rollback (client, done);
process.nextTick (function() {
if (err)
console.log (err);
async.each (queries, pg_commit, function () {
client.query ('COMMIT', done);
console.log ('done');
});
}); //nextTick
}); //begin
}); //connect
function pg_commit (val) {
client.query (val, function (err) {
if (err) return rollback (client, done);
});
return (val);
}

The problem is that client variable is defined only within the callback function called by pg.connect method (as its param). But in pg_commit this name is meaningless - as this function is not defined within the scope of the callback function and doesn't have access to its name bindings.
Hence you have two solutions: either move pg_commit definition into the callback...
function (err, client, done) {
if (err) throw err;
var pg_commit = function(val) { client.query(val, function(err) { ... }};
// ...
... or leave it where it is now, but change it into a function generator:
function pg_commit_generator(client) {
return function(val) { client.query(val, function(){}); /*...*/ }
}
... then use this generator to create a function called by async.each:
async.each(queries, pg_commit_generator(client), function() {...});
Alternatively, you can just supply into async.each the anonymous function that calls pg_commit:
async.each(queries, function(val) { return pg_commit(val, client); }, ...);
... and adjust pg_commit accordingly (so it'll take client as a second param).