I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.
Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.
Related
I have created Node JS app, i am explicitly downloading more than 100000 records from database. While the request is in proceeding, i try to login in the same application from another browser it wont respond unless the previous request is completed. Any idea ? Anything to do with event loop or threads ??
So here is my logic.
I make a get request to my API in Step 1, API calls database layer in Step 2. Database is returning only 21 records, i am explicitly looping 100000*21 to make heavy rendering just to test the load on json2csv. While doing so, the other requests to server will not respond until the last processing completes.
Step 1:
router.get('/report/downloadOverdueTrainings/:criteria', function (req, res, next) {
var overDueTrainings = [];
var reportManager = new ReportManager();
var result = reportManager.getOverdueTrainings(JSON.parse(req.params.criteria));
result.then(function (result) {
var fields = ['Over Due Trainings'];
for (var i = 0; i < 100000; i++) { //Testing purpose
for (var training of result) {
overDueTrainings.push({
'Over Due Trainings': training.OverDueTrainings
})
}
}
json2csv({
data: overDueTrainings,
fields: fields
}, function (err, csv) {
if (err)
throw err;
res.setHeader('Content-disposition', 'attachment; filename=OverdueTrainings.csv');
res.set('Content-Type', 'text/csv');
res.send(csv);
});
}).catch(function (err) {
next(err);
});
});
Step 2: Database Logic
var xtrDatabaseConnection =require('./xtrDatabaseConnection').XTRDatabaseConnection;
ReportData.prototype.getOverdueTrainings = async function (params) {
var connection = new xtrDatabaseConnection();
var sequelize = connection.getSequelize();
try {
var query = "CALL report_getOverdueTrainings(:p_CourseCode,:p_Revision,:p_RevisionNo,:p_UserGroup,:p_Username,:p_Status,:p_SortColoumnNo,:p_SortColoumnDirection,:p_callType,:p_StartIndex,:p_PageSize)";
var replacements = {
p_CourseCode: params.CourseCode,
p_Revision: params.Revision,
p_RevisionNo: (params.RevisionNo == '' || params.RevisionNo == null) ? 0 : params.RevisionNo,
p_UserGroup: params.UserGroup,
p_Username: params.Username,
p_Status: params.Status,
p_SortColoumnNo: params.SortColoumnNo,
p_SortColoumnDirection: params.SortColoumnDirection,
p_callType: params.callType,
p_StartIndex: params.startIndex,
p_PageSize: params.pageSize
};
//console.log(replacements);
return await connection.executeQuerySequelize(sequelize, query, Sequelize.QueryTypes.RAW, replacements);
} catch (e) {
throw (e);
} finally {
//To close connections
sequelize.connectionManager.close().then(() => console.log('Connection Closed'));
}
}
XTRDatabaseConnection.prototype.executeQuerySequelize = function (sequelize, query, queryType, replacements) {
return sequelize.authenticate()
.then(() => {
return sequelize.query(query, {
replacements: replacements,
type: queryType
}).
then(result => result)
.catch(err => {
throw (err);
});
})
.catch(err => {
xtrUtility.logErrorsToWinstonTransports('Unable to connect to the database or some error occurred while executing your query. ', err);
throw new AppError(err, "dbError", "Unable to connect to the database or some error occurred while executing your query.");
});
}
NodeJS is single-thread, single process. As long as a javascript function is running, nothing else can run. This is by design. The event loop will only kick in again once your functions have stopped executing.
This is blocking:
for (var i = 0; i < 100000; i++) { //Testing purpose
for (var training of result) {
overDueTrainings.push({
'Over Due Trainings': training.OverDueTrainings
})
}
}
until the execution is finished.
also
var result = reportManager.getOverdueTrainings(JSON.parse(req.params.criteria));
result.then(function (result) {
/****/
return await connection.executeQuerySequelize(sequelize, query, Sequelize.QueryTypes.RAW, replacements);
Node.js is working on single thread. So you need use it for small process with async calls. When you use async/await you are blocking the callback function.
So you need to use your database functions as promise. It will work async.
.query('CALL someFunction()')
.then(function(response){
//done
}).error(function(err){
//error
});
But there is a problem about it. You need to load 21 times your result. So in this case you need to use recursive function. When you finished recursive call you can send your data to client as async.
Just started to learn express js framework ,here is my simple database query execution part its invoked with this url localhost:3000/api/test.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
return results;
});
Does it really asynchronous?? suppose another user request this url does he need to wait for the previous query execution??.
I've heard about async package ,but don't know how this is applicable in my case
UPDATE
I got proper result in console.log(); but when i return the result i got undefined error
Here is my model.js
module.exports = {
getUser:function () {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
});
}
}
From my controller.js
var model = require('../models/user.js');
module.exports = {
getData : function(req, res){
//invoke model
console.log(model.getUser());
}
}
Node is non-blocking and will serve this request as and when it's called.
If another user hits this endpoint then it will execute again regardless if the first query has completed or not (unless the SQL has locked the table, in which case all consecutive connections/queries will wait and may timeout because of it). This happens on a connection basis.
You should make sure to check your SQL server (MySQL?) configs here to make sure there are enough max_connections to be able to cope with whatever load you are expecting.
Remember that the biggest bottleneck to an application is usually the database.
Your query above will need a callback to return the data asynchronously.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
//cb=callback function passed in to context
if (cb) cb(results);
});
Updated answer from updated question
In your model.js:
module.exports = {
getUser:function (cb) {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
if (cb) cb(results);
});
}
}
In your controller.js:
module.exports = {
getData : function(req, res){
//invoke model
model.getUser(function(results) {
console.log(results);
});
}
}
When you deal with callback, the safe and clean way to handle them is Promises. It's now standard in JavaScript and don't require any module.
And yes it is asynchronous. Behind, there'll be network access and dialogs with the database server. Only when they're done chatting will the callback be called.
module.exports = {
getUser: function () {
// wrap asynchronously called callback in Promise
new Promise((resolve, reject) => {
db.query("SELECT * FROM user", (error, results, fields) => {
if (error) {
reject(error); // similar to "throw"
}
else {
resolve({ results, fields }); // similar to "return"
}
});
});
}
};
How do you use it:
Vanilla notation:
// similar to "try"
model.getUser()
.then((answer) => {
console.log("answer", answer);
})
// similar to "catch"
.catch((error) => {
console.log("error", error);
});
async-await notation (only available in last versions of nodejs & browsers):
// you must be in an async environement to use "await"
async function wrapper() {
try {
var answer = await model.getUser(); // wait for Promise resolution
console.log("answer", answer);
}
catch(error) {
console.log("error", error);
}
}
// async function return automatically a Promise so you can chain them easily
wrapper();
The concept of promises is very new for me (so far, I was working with async.each and async.waterfall)
I want to use promises but i'm stuck right now.
I want to get "tags" from my db.
I have two tables for this : One called 'tags' with every tag in it (with an ID) and another one 'user_tags' with every username saved and the ID of the tag that the user (username) created and saved into 'tags'.
I can put information in my DB but now I want to pull this out and log it out (I will display it later)
So far this is my idea :
var getUserprofile = function getUserprofile(username, callback){
pool.getConnection(function (err, connection) {
var dataUser = [];
// Error check
if (err) {
console.log(err);
}
connection.query('SELECT * FROM users_tags FULL JOIN tags ON (tags.id = users_tags.t_id) WHERE users_tags.user_id=666;', username , function (err, rows, fields) {
if (err) {
connection.release();
cb(err);
} else if (rows.length < 1) {
connection.release();
cb("We don't have any informations about this user yet");
} else {
console.log("we pull the information right now");
connection.release();
callback(null, rows[0]);
}
});
});
}
Is this a good idea ? What should I do if I want to use promises for this kind of function ?
Thanks in advance for any help !!!
I would use Bluebird. You can "promisify" existing APIs with Promise.promisify or Promise.promisifyAll.
I would do something like
var Promise = require('bluebird'),
... //other deps;
var pool = Promise.promisifyAll(pool);
function getUserprofile(username){
var connection = null;
return pool.getConnectionAsync()
.then(function (conn) {
connection = Promise.promisifyAll(conn);
return connection.queryAsync('...');
})
.then(function (results) {
if (results.length < 1) {
return "We don't have any informations about this user yet";
} else {
console.log("we pull the information right now");
return results[0];
}
})
.catch(function (err) {
console.log(err);
throw err;
})
.finally(function () {
if (connection) {
connection.release();
}
});
}
I'm trying to get the document retrieved by MongoClient findOne method (in r parameter) outside the scope of callback function. How can i achieve that?
Maybe my approach to the usage of MongoDB driver for Node.js is not appropiate.
function loadUser(name) {
var result = {};
function connection(err, db) {
assert.equal(null, err);
function callback(err, r) {
assert.equal(null, err);
db.close();
result = r; // This does not work
}
db.collection('users').findOne({'user.name':name}, callback);
}
MongoClient.connect(url, connection);
return result;
}
The way you're doing it, result will not be the correct object because it's returned before MongoDB can find it and the value assigned.
You should do something like:
function loadUser(name, cb) {
function connection(err, db) {
assert.equal(null, err);
function callback(err, r) {
assert.equal(null, err);
db.close();
cb(err, r) // user
}
db.collection('users').findOne({'user.name':name}, callback);
}
MongoClient.connect(url, connection);
return;
}
And the usage of loadUser would be:
loadUser("example", function(err, user){
console.log(user);
//Now do what you need with user
});
Also notice that if you are always searching for users, it would be better to just open the connection once, and close it once application is terminated.
Your 'result' variable lives in the scope of the loadUser function. The interaction with MongoDB will be asynchronous, so by the time your callback fires, the loadUser function will have finished and the result variable will no longer exist.
You could simply move the result variable to the global scope. You will probably want to restructure your code a little too though, so that the callback notifies whatever is waiting for the 'result' to return.
You can use EventEmitter to process asynchronous tasks, which makes data flow more maintainable especially if you have a lot of conditional tasks to perform :
"use strict";
var MongoClient = require('mongodb').MongoClient;
const EventEmitter = require('events');
class MyEmitter extends EventEmitter {}
var url = 'mongodb://localhost:27017/myproject';
const myEmitter = new MyEmitter();
// connect to database
var db = MongoClient.connect(url, function(err, dbs) {
if (err) {
console.log(err);
return;
}
console.log("Connected correctly to server");
db = dbs;
//get user when connected
myEmitter.emit('getUser');
});
// getUser event
myEmitter.on('getUser', function() {
db.collection('users').findOne({ 'user.name': "test" }, function callback(err, r) {
if (err) {
myEmitter.emit('processError', err);
} else {
myEmitter.emit('processResult', r);
}
db.close();
});
});
// process result event
myEmitter.on('processResult', function(result) {
//result received, process it here
console.log("result : " + result);
});
// error event
myEmitter.on('processError', function(err) {
//an error occured, process the error here
console.log("error : " + err);
});
I am new to sequelize and NodeJS promises in general. My app basically saves tweets from the Twitter API, but also needs to update some of the saved tweets' data in realtime, like the Retweet count or the Liked count.
But it seems like after fetching the new data, when trying to run all the update on my tweet instances, nothing happens. The promise doesn't go through.
To sum up : I find 100 saved tweets, chain on a callback that fetches their new data from Twitter, and then chain on updating every single 100 tweets with the new data. The later update doesn't go through.
var Sequelize = require('sequelize');
...
//Getting 100 tweets previously saved in DB
Sequelize.query("SELECT * FROM tweets WHERE ORDER BY id DESC LIMIT 100", { model: Model }).then(function(result) {
if(result.length == 0) {
callback(false);
} else {
var ids = [];
var realData = {};
for (var i in result) {
realData[result[i].dataValues.id_str] = result[i];
ids.push(result[i].dataValues.id_str);
}
//getting twitter data for 100 tweets previously saved in DB
twitVendor.get('statuses/lookup', {
id : ids.join(',')
}, function (err, tweets, response) {
if (typeof err == 'undefined') {
//to get a synchronous saving of all tweets
//this could be cleaned up with a Sequelize.Promise.push(...)
var i = 0;
var saving = false;
while (i < tweets.length) {
if (!saving) {
saving = true;
console.log('Updating tweet ', tweets[i].id_str);
//updating tweet with new data from twitter
Sequelize.query("UPDATE tweets SET retweet_count = "+tweets[i].retweet_count+", favorite_count = "+tweets[i].favorite_count+" WHERE id_str = '"+tweets[i].id_str+"'", {
model: Model
}).then(function(result) {
console.log('Updated tweet');
saving = false;
i++;
}).catch(function (err) {
console.log('Failed to update post ', err);
saving = false;
i++;
});
}
}
callback(true);
console.log("Updated tweets");
} else {
console.log("Failed :", err);
callback(false, err);
}
});
}
}).catch(function (err) {
console.log("Failed :", err);
callback(false, err);
})
EDIT : If you want to execute the above code, I'd recommend using this Twit to hit the Twitter API : https://github.com/ttezel/twit
To get credentials to hit the API, you will need to set up an app on Twitter though : https://apps.twitter.com/
EDIT 2 : I already tried to use transactions and pure Sequelize functions to make my queries, but the issue stood still.
Don't nest promises inside of promises. Instead, chain them by returning promises. If you are returning something that is not a promise, use Promise.resolve(value) to turn it into a promise. And certainly don't put promises inside of callbacks, or even mix them at all; instead create a promise that calls the action, and then in the callback resolve the promise.
Here's my attempt to rewrite what you're trying to do. You may need to wrap the first in a Promise.resolve to take advantage of returning the new promise:
Sequelize.query("SELECT * FROM tweets WHERE ORDER BY id DESC LIMIT 100"
, { model: Model }).then(function (results) {
if (results.length == 0) {
return Promise.reject(false); //reject to hit the catch of the promise. Change false to error.
}
var ids = [];
var realData = {};
for (var i in result) {
realData[result[i].dataValues.id_str] = result[i];
ids.push(result[i].dataValues.id_str);
}
return new Promise((resolve, reject) => {
twitVendor.get('status/lookup', {
id: ids.join(',')
}, function (err, tweets, response) {
if (err) {
reject(false); //reject to hit the catch of the promise. Change false to error message
}
resolve(tweets);
})
})
}).then(function (tweets) {
function updateTweet(tweet) {
return sequelize.query(...);
}
var updatesInParallel = tweets.map(updateTweet);
return Promise.all([updatesInParallel]);
}).then(function () {
callback(true);
}).catch(function (error) {
console.log("failed: ", error);
callback(false)
});