The following function takes an aggregate cursor containing several arrays of user Ids so the users with the corresponding ids can be modified and then saved.
The issue I am having is getting the function to only return once all the users have been modified and saved. Currently it doesn't do that and when I attempt to access the users later on (literally after this function) the modifications haven't been permeated through.
This is the current code that does the modification (might be some strange async additions as I have been attempting 10s of different options with no success):
function updateUsers(dataArray, done) {
dataArray.eachAsync(
function(data) {
let userArray = data.users;
async.forEach(userArray, function(userId, callBack){
mongoose.model('User').findById(userId, function(err, user) {
// User document modified
user.save(function(err, user){
callBack();
});
});
});
},
function(err) {
//Only come here once all the users have been modified and saved!
if (err) {
return done(err, false);
}
return done(err, true);
}
);
}
The data that is passed is:
dataArray: Aggregation cursor
Result for eachAsync e.g. data is:
{ _id: 'GroupValue',
users:
[ 5a0b2f5bcd0813adeac2435b,
5a0b2f5bcd0813adeac24357,
5a0b2f5bcd0813adeac24358 ] }
You are calling the return statement via the callback after every single iteration of the callback. Thus, you are returning userArray.length times.
If you just want to insert the data and don't want the function that's using it to know whether or not they were successful in inserting, you could just do the callback immediately before even iterating through the loop.
function updateUsers(dataArray, done) {
dataArray.eachAsync(
function(data) {
let userArray = data.users;
callBack();
async.forEach(userArray, function(userId, callBack){
mongoose.model('User').findById(userId, function(err, user) {
// User document modified
user.save(function(err, user){
//do nothing
});
});
});
},
function(err) {
//Only come here once all the users have been modified and saved!
if (err) {
return done(err, false);
}
return done(err, true);
}
);
}
Otherwise, let's say you want to only callback specifically after the last iteration. One dirty way you could do this is do a temporary variable outside of the loop, and an if statement in the loop that executes on the last iteration.
For this implementation, you must assume the function will never pass a non-zero length array of users. Otherwise, it would never run a single iteration, and not terminate. (You would want to write an if length == 0 case)
function updateUsers(dataArray, done) {
dataArray.eachAsync(
function(data) {
let userArray = data.users;
let tempVar = 0;
async.forEach(userArray, function(userId, callBack){
mongoose.model('User').findById(userId, function(err, user) {
// User document modified
user.save(function(err, user){
tempVar++;
if(tempVar == userArray.length) {
callBack();
}
});
});
});
},
function(err) {
//Only come here once all the users have been modified and saved!
if (err) {
return done(err, false);
}
return done(err, true);
}
);
}
},
function(err) {
//Only come here once all the users have been modified and saved!
if (err) {
return done(err, false);
}
return done(err, true);
}
);
}
Related
The concept of promises is very new for me (so far, I was working with async.each and async.waterfall)
I want to use promises but i'm stuck right now.
I want to get "tags" from my db.
I have two tables for this : One called 'tags' with every tag in it (with an ID) and another one 'user_tags' with every username saved and the ID of the tag that the user (username) created and saved into 'tags'.
I can put information in my DB but now I want to pull this out and log it out (I will display it later)
So far this is my idea :
var getUserprofile = function getUserprofile(username, callback){
pool.getConnection(function (err, connection) {
var dataUser = [];
// Error check
if (err) {
console.log(err);
}
connection.query('SELECT * FROM users_tags FULL JOIN tags ON (tags.id = users_tags.t_id) WHERE users_tags.user_id=666;', username , function (err, rows, fields) {
if (err) {
connection.release();
cb(err);
} else if (rows.length < 1) {
connection.release();
cb("We don't have any informations about this user yet");
} else {
console.log("we pull the information right now");
connection.release();
callback(null, rows[0]);
}
});
});
}
Is this a good idea ? What should I do if I want to use promises for this kind of function ?
Thanks in advance for any help !!!
I would use Bluebird. You can "promisify" existing APIs with Promise.promisify or Promise.promisifyAll.
I would do something like
var Promise = require('bluebird'),
... //other deps;
var pool = Promise.promisifyAll(pool);
function getUserprofile(username){
var connection = null;
return pool.getConnectionAsync()
.then(function (conn) {
connection = Promise.promisifyAll(conn);
return connection.queryAsync('...');
})
.then(function (results) {
if (results.length < 1) {
return "We don't have any informations about this user yet";
} else {
console.log("we pull the information right now");
return results[0];
}
})
.catch(function (err) {
console.log(err);
throw err;
})
.finally(function () {
if (connection) {
connection.release();
}
});
}
I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.
Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.
I am writing a NodeJS script that will run every hour through Heroku's scheduler. I am quering the Mongo instance I have (mongohq/compose) and then doing something with those results. I am working with Mongoose.js and the find() command. This returns an array of results. With those results I need to perform additional queries as well as some additional async processing (sending email, etc).
Long story short, due to node's async nature I need to wait until all the processing is complete before I call process.exit(). If I do not the script stops early and the entire result set is not processed.
The problem is that I have a christmas tree effect of calls at this point (5 nested asnyc calls).
Normally I'd solve this with the async.js library but I'm having a problem seeing this through with this many callbacks.
How can I make sure this entire process finishes before exiting the script?
Here's the code that I'm working with (note: also using lodash below as _):
Topic.find({ nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
finish();
} else {
_.forEach(topics, function (topic, callback) {
User.findById(topic.user, function (err, user) {
if (err) {
// TODO: impl logging
console.error(err);
} else {
// Create a new moment object (not moment.js, an actual moment mongoose obj)
var m = new Moment({ name: moment().format("MMM Do YY"), topic: topic});
m.save(function(err) {
if(err) {
// TODO: impl logging
console.error(err);
} else {
// Send an email via postmark
sendReminderTo(topic, user, m._id);
// Update the topic with next notification times.
// .. update some topic fields/etc
topic.save(function (err) {
if(err) {
console.error(err);
} else {
console.log("Topic updated.");
}
})
}
})
}
});
console.log("User: " + topic.user);
});
}
});
Part of what is making your code confusing is the usage of else statements. If you return your errors, you won't need the else statement and save 4 lines of indentation for every callback. That in and of itself will make things drastically more readable.
Using async:
Topic.find({nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
return finish(err);
}
async.each(topics, function(topic, topicCallback) {
async.auto({
user: function (callback) {
User.findById(topic.user, callback);
},
moment: function(callback) {
var m = new Moment({name: moment().format("MMM Do YY"), topic: topic});
m.save(callback);
},
topic: ["moment", "user", function (callback, results) {
var m = results.moment;
var user = results.user;
sendReminderTo(topic, user, m._id);
topic.save(callback);
}]
}, function(err) {
if (err) {
return topicCallback(err);
}
console.log("Topic updated.")
return topicCallback();
});
}, function(err) {
if (err) {
console.error(err);
return finish(err);
}
return finish();
});
});
The mobile app is sending the server an array of records to be saved to the database. The server is to iterate through the array of records, save each one, build an arrary of newly created ids and then return the array of ids back to the app.
The following code is saving the records correctly, but due to the asynchronous nature of javascript, when the outer function returns the inner function has not completed yet and the array of ids is still empty.
How do I get this function to return the returnIDs array AFTER it has been filled?
create: function(req, res) {
var returnIDs = [];
for(var i in req.body){
Test_session.create(req.body[i], function test_SessionCreated(err, test_Session) {
if (err) {
console.log(err);
return res.json(err);
}
returnIDs.push(test_Session.id);
});
}
return res.json({ "ids": returnIDs.toJSON}, 200);
}
I think use async.js is easier than promise. If you use async.js then do this.
create: function(req, res) {
var returnIDs = [];
async.each(req.body, function(item, callback) {
Test_session.create(item, function(err, test_Session) {
if (err) {
console.log(err);
callback(err);
}
else
callback();
});
}, function(err){
res.json(200, { "ids": returnIDs});
});
}
I am attempting to pass the data variable from the function to the app.get call.
But the global variable 'data' in my function is not retaining the value returned from the database query. I believe this to be a scoping issue. I set the value for 'data' to null at the beginning to make the variable accessible throughout the function. I am not hitting either of the error conditions.
function getCommands(query) {
var data = null;
try {
pg.connect(cString, function(err, client, done) {
// Catch any connection errors, and display them to the screen
if(err) {
return console.error('could not connect to postgres', err);
}
client.query(query, function(q_err, result) {
// Release the client back to the pool
done();
if(q_err) {
return console.error('error running query', q_err);
}
// Data object with an empty array to hold the row information
data = {"data":[]};
for (var row in result.rows)
{
//console.log(result.rows[row]);
data.data.push(result.rows[row]);
}
//Here, data, has the correct return values.
console.log(data);
});
});
}
catch( e )
{
console.log(e);
}
//Here, data, is null.
console.log(data);
return data;
}
app.get('/clients/', function(req, res) {
res.send(getCommands('SELECT clientid, clientname FROM hourglass.clients ORDER BY clientid ASC'));
});
Could someone help me determine why 'data' is not retaining value outside of the pg.connect function?
I think your problem is not that the variable is not retaining the value outside of the function, but rather that you console.log(data) is executing before the variable is set.
If you were to put console.log('step X') in your code as in the following example you'll see the sequence in which your code is executed.
function getCommands(query) {
var data = null;
console.log('STEP 1');
pg.connect(cString, function(err, client, done) {
console.log('STEP 3');
});
console.log('STEP 2');
}
function getCommands(query, callback) {
try {
pg.connect(cString, function(err, client, done) {
// Catch any connection errors, and display them to the screen
if(err) {
return console.error('could not connect to postgres', err);
}
client.query(query, function(q_err, result) {
// Release the client back to the pool
done();
if(q_err) {
return console.error('error running query', q_err);
}
// Data object with an empty array to hold the row information
var data = {"data":[]};
for (var row in result.rows)
{
data.data.push(result.rows[row]);
}
callback(data); //After data is set, the value is passed back
});
});
}
catch( e )
{
console.log(e);
}
}
app.get('/clients', function(req, res) {.....])
Using a callback function mentioned by #dystroy worked perfectly.