I am writing a NodeJS script that will run every hour through Heroku's scheduler. I am quering the Mongo instance I have (mongohq/compose) and then doing something with those results. I am working with Mongoose.js and the find() command. This returns an array of results. With those results I need to perform additional queries as well as some additional async processing (sending email, etc).
Long story short, due to node's async nature I need to wait until all the processing is complete before I call process.exit(). If I do not the script stops early and the entire result set is not processed.
The problem is that I have a christmas tree effect of calls at this point (5 nested asnyc calls).
Normally I'd solve this with the async.js library but I'm having a problem seeing this through with this many callbacks.
How can I make sure this entire process finishes before exiting the script?
Here's the code that I'm working with (note: also using lodash below as _):
Topic.find({ nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
finish();
} else {
_.forEach(topics, function (topic, callback) {
User.findById(topic.user, function (err, user) {
if (err) {
// TODO: impl logging
console.error(err);
} else {
// Create a new moment object (not moment.js, an actual moment mongoose obj)
var m = new Moment({ name: moment().format("MMM Do YY"), topic: topic});
m.save(function(err) {
if(err) {
// TODO: impl logging
console.error(err);
} else {
// Send an email via postmark
sendReminderTo(topic, user, m._id);
// Update the topic with next notification times.
// .. update some topic fields/etc
topic.save(function (err) {
if(err) {
console.error(err);
} else {
console.log("Topic updated.");
}
})
}
})
}
});
console.log("User: " + topic.user);
});
}
});
Part of what is making your code confusing is the usage of else statements. If you return your errors, you won't need the else statement and save 4 lines of indentation for every callback. That in and of itself will make things drastically more readable.
Using async:
Topic.find({nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
return finish(err);
}
async.each(topics, function(topic, topicCallback) {
async.auto({
user: function (callback) {
User.findById(topic.user, callback);
},
moment: function(callback) {
var m = new Moment({name: moment().format("MMM Do YY"), topic: topic});
m.save(callback);
},
topic: ["moment", "user", function (callback, results) {
var m = results.moment;
var user = results.user;
sendReminderTo(topic, user, m._id);
topic.save(callback);
}]
}, function(err) {
if (err) {
return topicCallback(err);
}
console.log("Topic updated.")
return topicCallback();
});
}, function(err) {
if (err) {
console.error(err);
return finish(err);
}
return finish();
});
});
Related
Ok, so I need to connect to a MySQL database through SSH and the connection works fine. I am able to execute queries with no problem. I can also get the results and print it out. The thing is, I need something simpler since I will have to send a lot of queries to this database. Below is the code for a promise which creates a connection to this database.
const SSHConnection = new Promise((resolve, reject) => {
sshClient.on('ready', () => {
sshClient.forwardOut(
forwardConfig.srcHost,
forwardConfig.srcPort,
forwardConfig.dstHost,
forwardConfig.dstPort,
(err, stream) => {
if (err) reject(err);
const updatedDbServer = {
...dbServer,
stream
};
const connection = mysql.createConnection(updatedDbServer);
connection.connect((error) => {
if (error) {
reject(error); // Return error
}
resolve(connection); // OK : return connection to database
});
});
}).connect(tunnelConfig);
});
Then I have this piece of code that gives me access to said connection and allows me to send queries. The problem is that I need the return value of my queries and be able to use it in other modules for my project. For example, export a single function to be used to send queries like sendQuery('Enter SQL here').
function sendQuery(sql) {
SSHConnection.then(
function(connection) {
connection.query(
sql,
function(err, results, fields) {
return results; // <---------- I want to return this from the function 'sendQuery()'
}
);
},
function(error) {
console.log("Something wrong happened");
}
);
}
I can work with the results inside SSHConnection.then() but this isn't functional for me.
I would like to have something similar below to work.
// Main function
(async function() {
let res = sendQuery(`SELECT 23+2 AS Sum;`);
console.log(res); // Outputs Sum: 25
})();
So to my question. Is there a way to access the results from a query inside of a promise.then(), from the outside?
I think the problem is you need to add another return statement to your code.
function sendQuery(sql) {
return SSHConnection.then(
function(connection) {
return connection.query(
sql,
function(err, results, fields) {
return results; // <---------- I want to return this from the function 'sendQuery()'
}
);
},
function(error) {
console.log("Something wrong happened");
}
);
}
This should return the results from the query properly IF connection.query returns a promise. I'm not sure if it does. If it does then you can just execute the function like so.
// Main function
(async function() {
let res = await sendQuery(`SELECT 23+2 AS Sum;`);
console.log(res); // Outputs Sum: 25
})();
If connection.query does not return a promise then I suppose you could wrap it in a promise like so.
function sendQuery (sql) {
return SSHConnection.then(
function (connection) {
return new Promise((resolve, reject) => {
connection.query(
sql,
function (err, results, fields) {
if (err)reject(err)
resolve(results) // <---------- I want to return this from the function 'sendQuery()'
}
)
})
},
function (error) {
console.log('Something wrong happened')
}
)
}
love the name by the way...really liked that movie. As to your question, I'd suggest a couple of things:
if you have a lot of queries coming, you might consider moving the connection independent of the query, so that the connection setup and teardown isn't part of the cost of time for the query itself. If you have a single connection, single DB, etc., then you could instantiate the connection once, at startup, and then leave the connection open, and reference it in your queries.
your question:
function sendQuery(sql) {
const resValues = await SSHConnection.then(
function(connection) {
connection.query(
sql,
function(err, results, fields) {
return results; // <---------- I want to return this from the function 'sendQuery()'
}
);
},
function(error) {
console.log("Something wrong happened");
}
);
return(resValues); // or handle error cases
}
Note I added a return value from the ".then()" call, which captures your "results" return value, and then returns that from the parent function (sendQuery)
I need assistance with the following: I have 4 MySQL queries which I wish to run in parallel but it is essential that all queries are completed before continuing in the main function; The four queries get different parts of the data (they are structured differently and access different tables) that is then used in the main function. Namely, I used async.parallel (even tried adding the await keyword) but still ended up continuing before the final callback is called in async.parallel. I even considered cluster and worker threads, but each query function is too different from one another.
The functions in async.parallel are MySQL connection queries in the form of:
connection.query(q, function(err, res) {
if(err) {
callback(err, [])
} else {
// do something
callback(null, res)
}
})
Each function is shaped differently and this is only a reduced form for the sake of brevity. Example of the function:
async main_function() {
await async.parallel({
"1": function(callback) { conn.query(q1, function(...) { console.log("1"); callback(null, 1) }) },
"2": function(callback) { conn.query(q2, function(...) { console.log("2"); callback(null, 2) }) },
"3": function(callback) { conn.query(q3, function(...) { console.log("3"); callback(null, 3) }) },
"4:" function(callback) { conn.query(q4, function(...) { console.log("4"); callback(null, 4) }) }
}, function(err, results) { /*final callback; do something*/ console.log("Finished"); })
console.log("Continuing");
//continue
}
Expected output:
3
2
4
1
Finished
Continuing
Actual output:
Continuing
3
2
4
1
Finished
If there's an entire other method (even without async.parallel), I'm open to it. I will likely have multiple distinct parallel MySQL queries later in the function and in other parts of the program, so having a parametrized form such as with async.parallel is extremely useful.
Javascript being single-threaded means that solutions like 'a while loop that checks whether the queries are finished' do not work. This main_function will be called very often so forking will cause issues.
EDIT: The Jakub's solution is rather simple as in my program works as the following:
async main_function() {
await Promise.all([
new Promise((resolve, reject) => {
connection.query(q1 , function (error, result) {
if (error != null) {
console.log("mysql query error", error);
return reject(error);
} else {
console.log("1");
resolve(result);
}
})
}),
new Promise((resolve, reject) => {
connection.query(q1 , function (error, result) {
if (error != null) {
console.log("mysql query error", error);
return reject(error);
} else {
console.log("2");
resolve(result);
}
})
}), ... // Two more times
]).then((values) => {
data[0] = values[0];
data[1] = values[1];
data[2] = values[2];
data[3] = values[3];
}).catch(function (err) { console.log(err); });
console.log("Continuing"); //continue
}
This is very crude right now, and if anybody wishes to suggest cleaner/better solutions for future programmers, go ahead.
These query functions are callback based (judging from your code), you can promisify them:
import { promisify } from 'utils';
async main_function() {
const res = await Promise.all([
promisify(conn.query)(q1),
promisify(conn.query)(q2),
// all your other queries
]);
console.log("Continuing");
//continue
}
Just started to learn express js framework ,here is my simple database query execution part its invoked with this url localhost:3000/api/test.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
return results;
});
Does it really asynchronous?? suppose another user request this url does he need to wait for the previous query execution??.
I've heard about async package ,but don't know how this is applicable in my case
UPDATE
I got proper result in console.log(); but when i return the result i got undefined error
Here is my model.js
module.exports = {
getUser:function () {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
});
}
}
From my controller.js
var model = require('../models/user.js');
module.exports = {
getData : function(req, res){
//invoke model
console.log(model.getUser());
}
}
Node is non-blocking and will serve this request as and when it's called.
If another user hits this endpoint then it will execute again regardless if the first query has completed or not (unless the SQL has locked the table, in which case all consecutive connections/queries will wait and may timeout because of it). This happens on a connection basis.
You should make sure to check your SQL server (MySQL?) configs here to make sure there are enough max_connections to be able to cope with whatever load you are expecting.
Remember that the biggest bottleneck to an application is usually the database.
Your query above will need a callback to return the data asynchronously.
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is:', results[0].id);
//cb=callback function passed in to context
if (cb) cb(results);
});
Updated answer from updated question
In your model.js:
module.exports = {
getUser:function (cb) {
db.query('SELECT * FROM user', function (error, results, fields) {
if (error) throw error;
console.log('The result is: ', results[0].id);
if (cb) cb(results);
});
}
}
In your controller.js:
module.exports = {
getData : function(req, res){
//invoke model
model.getUser(function(results) {
console.log(results);
});
}
}
When you deal with callback, the safe and clean way to handle them is Promises. It's now standard in JavaScript and don't require any module.
And yes it is asynchronous. Behind, there'll be network access and dialogs with the database server. Only when they're done chatting will the callback be called.
module.exports = {
getUser: function () {
// wrap asynchronously called callback in Promise
new Promise((resolve, reject) => {
db.query("SELECT * FROM user", (error, results, fields) => {
if (error) {
reject(error); // similar to "throw"
}
else {
resolve({ results, fields }); // similar to "return"
}
});
});
}
};
How do you use it:
Vanilla notation:
// similar to "try"
model.getUser()
.then((answer) => {
console.log("answer", answer);
})
// similar to "catch"
.catch((error) => {
console.log("error", error);
});
async-await notation (only available in last versions of nodejs & browsers):
// you must be in an async environement to use "await"
async function wrapper() {
try {
var answer = await model.getUser(); // wait for Promise resolution
console.log("answer", answer);
}
catch(error) {
console.log("error", error);
}
}
// async function return automatically a Promise so you can chain them easily
wrapper();
I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.
Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.
I keep running into this pattern when coding in Meteor where I find myself making multiple method calls nested within each other - first method fires, then in the callback, a second one fires which is dependent on the first one's result, etc. Is there a better pattern for using multiple methods without nested method calls inside callbacks? The code quickly gets messy.
Meteor.call('unsetProduct', product._id, omitObj, function(err, result) {
if(!err) {
Meteor.call('editProduct', product._id, object, function(err, result) {
if(!err) {
//if no error, then continue to update the product template
Meteor.call('editProductTemplate', self._id, obj, function(err, result) {
if(!err) {
//call some other method
}
else {
FormMessages.throw(err.reason, 'danger');
}
});
}
else {
FormMessages.throw(err.reason, 'danger');
}
});//end edit product
}
else {
AppMessages.throw(err.reason, 'danger');
}
});`
Take a look at reactive-method package. I think it does exactly what you need: it wraps asynchronous Meteor.calls into synchronous code. With it, your code would look cleaner, like
try {
const result = ReactiveMethod.call('unsetProduct', product._id, omitObj);
} catch (error) {
AppMessages.throw(err.reason, 'danger');
}
try {
const nestedResult = ReactiveMethod.call('editProduct', product._id, object);
} catch (error) {
FormMessages.throw(err.reason, 'danger');
}
try {
const evenMoreNestedResult = ReactiveMethod.call('editProductTemplate', self._id, obj);
} catch (error) {
FormMessages.throw(err.reason, 'danger');
}
Which will look nicer when you add some logic inside try statements.