We Keep Coding

Node js: Express js asynchronous db query execution-return results got undefiend

Just started to learn express js framework ,here is my simple database query execution part its invoked with this url localhost:3000/api/test.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
return results;
});
Does it really asynchronous?? suppose another user request this url does he need to wait for the previous query execution??.
I've heard about async package ,but don't know how this is applicable in my case
UPDATE
I got proper result in console.log(); but when i return the result i got undefined error
Here is my model.js
module.exports = {
getUser:function () {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
});
}
}
From my controller.js
var model = require('../models/user.js');
module.exports = {
getData : function(req, res){
//invoke model
console.log(model.getUser());
}
}

Node is non-blocking and will serve this request as and when it's called.
If another user hits this endpoint then it will execute again regardless if the first query has completed or not (unless the SQL has locked the table, in which case all consecutive connections/queries will wait and may timeout because of it). This happens on a connection basis.
You should make sure to check your SQL server (MySQL?) configs here to make sure there are enough max_connections to be able to cope with whatever load you are expecting.
Remember that the biggest bottleneck to an application is usually the database.
Your query above will need a callback to return the data asynchronously.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
//cb=callback function passed in to context
if (cb) cb(results);
});
Updated answer from updated question
In your model.js:
module.exports = {
getUser:function (cb) {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
if (cb) cb(results);
});
}
}
In your controller.js:
module.exports = {
getData : function(req, res){
//invoke model
model.getUser(function(results) {
console.log(results);
});
}
}

When you deal with callback, the safe and clean way to handle them is Promises. It's now standard in JavaScript and don't require any module.
And yes it is asynchronous. Behind, there'll be network access and dialogs with the database server. Only when they're done chatting will the callback be called.
module.exports = {
getUser: function () {
// wrap asynchronously called callback in Promise
new Promise((resolve, reject) => {
db.query("SELECT * FROM user", (error, results, fields) => {
if (error) {
reject(error); // similar to "throw"
}
else {
resolve({ results, fields }); // similar to "return"
}
});
});
}
};
How do you use it:
Vanilla notation:
// similar to "try"
model.getUser()
.then((answer) => {
console.log("answer", answer);
})
// similar to "catch"
.catch((error) => {
console.log("error", error);
});
async-await notation (only available in last versions of nodejs & browsers):
// you must be in an async environement to use "await"
async function wrapper() {
try {
var answer = await model.getUser(); // wait for Promise resolution
console.log("answer", answer);
}
catch(error) {
console.log("error", error);
}
}
// async function return automatically a Promise so you can chain them easily
wrapper();

Error : Callback was already called when using pg-promise with async series

I'm having trouble understanding the output printed why executing this code :
1
2
Unhandled rejection Error: Callback was already called.
It seems like both then and catch are executed when the query is successful.
Any idea ?
Cheers
async.series([
function(callback) {
db.none(query)
.then(function () {
return callback(null, true);
})
.catch(function (err) {
return callback(err, null);
});
},
function(callback) {
db.any(query)
.then(function (data) {
console.log('1')
return callback(null, data);
})
.catch(function (err) {
console.log('2')
console.log(err);
return callback(err, null);
});
}
],
function(err, results) {
if (results && !results[1].isEmpty()) {
// do something
}
});
EDIT :
TypeError: results[1].isEmpty is not a function
It seems like the problem come from the rest of the code and was just a simple undefined function error, thanks.
But i still don't understand something : why is this error catched inside the second query instead of outside the async queries ?

I'm the author of pg-promise.
You should never use async library with pg-promise, it goes against the concept of shared/reusable connections.
Implementation with proper use of the same connection, via a task:
db.task(t => {
return t.batch([
t.none(query1),
t.any(query2)
]);
})
.then(data => {
// data[0] = null - result of the first query
// data[1] = [rows...] - result of the second query
callback(null, data); // this will work, but ill-advised
})
.catch(error => {
callback(error, null); // this will work, but ill-advised
});
See also: Chaining Queries.
However, in your case it looks like when you call the successful callback(null, data), it throws an error, which in turn results in it being caught in the following .catch section. To test this, you can change your promise handler like this:
.then(data => {
callback(null, data);
}, error => {
callback(error, null);
});
It should normally throw an error about Promise missing .catch because you threw an error while in .then and there is no corresponding .catch chained below, which you can also check through this code:
.then(data => {
callback(null, data);
}, error => {
callback(error, null);
})
.catch(error => {
// if we are here, it means our callback(null, data) threw an error
});
P.S. You really should learn to use promises properly, and avoid any callbacks altogether. I only provided an example consistent with your own, but in general, converting promises into callbacks is a very bad coding technique.

This is what happens:
callback(null, data) is called within the context of the .then();
async notices that this was the last item of the series, so it calls the final handler (still within the context of the .then());
the final handler throws an error;
because the code runs in the context of .then(), the promise implementation catches the error and calls the .catch();
this calls the callback again;
PoC:
const async = require('async');
async.series([
callback => {
Promise.resolve().then(() => {
callback(null);
}).catch(e => {
callback(e);
});
}
], err => {
throw Error();
})

Have you try to define your function externally:
function onYourFunction() {
console.log('Hi function');
}
and than do:
.then(onYourFunction) //-->(onYourFunction without parentheses )
Unfortunately i don't use pg-promise but i can advise promise
at this point i create all promises that are necessary:
function createPromise(currObj) {
return new Promise(function (resolve, reject) {
currObj.save(function (errSaving, savedObj) {
if(errSaving){
console.log("reject!");
return reject(errSaving, response);
}
console.log('currObj:' + currObj);
return resolve(savedObj);
});
});
}
and then in cascades:
var allPromiseOs = Promise.all(promise1, promise2, promise3);

Nodejs express and promises not doing what I expect

I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.

Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.

Calling another prototype method after promise completed

I have following code in my node.js file;
GameHelperAuth.prototype.GetUserViaApi = Promise.method(function (authCookie, callback) {
// get user from API
});
GameHelperAuth.prototype.GetObjectFromCache = Promise.method(function (authCookie, callback) {
// get user from Cache
});
GameHelperAuth.prototype.GetUser = function (authCookie, callback) {
// check cache
this.GetObjectFromCache()
.then(function (result) {
if (result) {
return callback(null, result);
}
else {
// not found in cache, get it from API
// **NOT WORKING HERE - undefined error**
this.GetUserViaApi(authCookie)
.then(function (apiResult) {
return callback(null, apiResult);
}).catch(function (err) {
throw err;
});
}
})
.catch(function (err) {
throw err;
});
I would like to access my instance method from another instance method once promise is completed. But it looks like it loses it's context and cannot find function anymore. (Please see where I am calling GetUserViaApi method)
Is there any way for me to reach that method without creating new instance of my class?

As far as I can see, the simplest fix here is to just declare var self = this in the first line of .GetUser() and then use self instead of this inside the .then callback.
Alternatively if you're using Node 4+ with ES6 compatibility, use an "arrow function" as the outer .then callback that inherits the lexical this instead of contextual this:
return this.GetObjectFromCache()
.then((result) => {
if (result) {
return callback(null, result);
} else {
// not found in cache, get it from API
return this.GetUserViaApi(authCookie)
.then(function (apiResult) {
return callback(null, apiResult);
}).catch(function (err) {
throw err;
});
}
})
.catch(function (err) {
throw err;
});
NB: note the addition of the return in the first line and in the else clause, necessary to ensure that the function and that branch both correctly return a promise.
FWIW, I also think you can refactor this substantially by eliminating the repeated call to return callback(...) through a chaining .then:
GameHelperAuth.prototype.GetUser = function (authCookie, callback) {
return this.GetObjectFromCache()
.then(result => result || this.GetUserViaApi(authCookie))
.then(result => callback(null, result));
}
I've removed both .catch blocks - doing .catch(function(err) { throw err }) is a no-op - AIUI the throw would make the caller end up in their own .catch block so you might just as well let the entire promise reject anyway.

Node.js how to read a file and then write the same file with two separate functions?

What I want to do is read a file and then be able to perform other operations with that information as I write the file. For example:
read file
write file and at the same time perform MD5 hash, digital signing etc.
I could use fs.readfile and fs.writefile as one operation and just copy the file from the web server to my computer, but I don't think I could still do these same operations. Anyway, skipping the in between stuff. How do I use fs.readfile and writefile to create two separate functions to copy a file? Here is what I have been working on, and yes I've read these forums extensively in search of an answer.
var fs = require('fs');
function getData(srcPath) {
fs.readFile(srcPath, 'utf8', function (err, data) {
if (err) throw err;
return data;
}
);
}
function writeData(savPath, srcPath) {
fs.writeFile (savPath, (getData(srcPath)), function(err) {
if (err) throw err;
console.log('complete');
}
);
}
//getData ('./test/test.txt');
writeData ('./test/test1.txt','./test/test.txt');
I want to be able to download files of any type and just make raw copies, with md5 hash etc attached to a JSON file. That will probably be a question for later though.

As suggested by dandavis in his comment, readFile does nothing because it is an asynchronous call. Check out this answer for additional information on what that means.
In short, an async call will never wait for the result to return. In your example, getData does not wait for readFile() to return the result you want, but will finish right away. Async calls are usually handled by passing callbacks, which is the last parameter to readFile and writeFile.
In any case, there are two ways to do this:
1.Do it asynchronously (which is the proper way):
function copyData(savPath, srcPath) {
fs.readFile(srcPath, 'utf8', function (err, data) {
if (err) throw err;
//Do your processing, MD5, send a satellite to the moon, etc.
fs.writeFile (savPath, data, function(err) {
if (err) throw err;
console.log('complete');
});
});
}
2.Do it synchronously. Your code won't have to change much, you will just need to replace readFile and writeFile by readFileSync and writeFileSync respectively. Warning: using this method is not only against best practises, but defies the very purpose of using nodejs (unless of course you have a very legitimate reason).
Edit: As per OP's request, here is one possible way to separate the two methods, e.g., using callbacks:
function getFileContent(srcPath, callback) {
fs.readFile(srcPath, 'utf8', function (err, data) {
if (err) throw err;
callback(data);
}
);
}
function copyFileContent(savPath, srcPath) {
getFileContent(srcPath, function(data) {
fs.writeFile (savPath, data, function(err) {
if (err) throw err;
console.log('complete');
});
});
}
This way, you are separating the read part (in getFileContent) from the copy part.

I had to use this recently, so I converted verybadallocs answer to promises.
function readFile (srcPath) {
return new Promise(function (resolve, reject) {
fs.readFile(srcPath, 'utf8', function (err, data) {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
function writeFile (savPath, data) {
return new Promise(function (resolve, reject) {
fs.writeFile(savPath, data, function (err) {
if (err) {
reject(err)
} else {
resolve()
}
})
})
}
Then using them is simple.
readFile('path').then(function (results) {
results += ' test manipulation'
return writeFile('path', results)
}).then(function () {
//done writing file, can do other things
})
Usage for async/await
const results = await readFile('path')
results += ' test manipulation'
await writeFile('path', results)
// done writing file, can do other things

To read and write a file with Non-blocking or Asynchronous way, you can use the advance features of es6 or higher like Promise or Async/await, but you must keep eye on Polyfills (https://javascript.info/polyfills) or if there are only a couple of read/write you can use call back Hell.
function readFiles(){
fs.readFile('./txt/start.txt', 'utf-8', (err, data1)=>{
if(err) return console.log(err);
fs.readFile(`./txt/${data1}.txt`, 'utf-8', (err, data2)=>{
if(err) return console.log(err);
fs.readFile('./txt/append.txt', 'utf-8', (err, data3)=>{
if(err) return console.log(err);
writeFile('./txt/final.txt', `${data2}\n${data3}`);
});
});
});
}
function writeFile(path, data){
fs.writeFile(path,data,'utf-8',err=>{
if(err){
console.log(err);
}
})
}
readFiles();