We Keep Coding

Using push for inserting JS object into an array wont work at certain context

In the below code, users.push used within ‘db.each’ wont work. However, if I move ‘users.push’ outside then it seems to work.
How can I push the new objects from db.each into the users array?
let db = new sqlite3.Database('./db/main.db', (err) => {
if (err) console.error(err.message);
console.log('Connected to the main database.');
});
var users = [];
db.serialize(() => {
db.each(`SELECT email, name FROM users`, (err, row) => {
if (err) console.error(err.message);
let user = {
email: row.email,
name: row.name
}
users.push(user);
});
});
console.log(JSON.stringify(users));
db.close();
I am using express and sqlite3 node packages.

It's because db.serializeand db.each are asynchronous functions (and return immediately, thus executing console.log before the db callbacks are executed).
Here should be a working example :
db.serialize(() => {
db.each(`SELECT email,
name
FROM users`, (err, row) => {
if (err) {
console.error(err.message);
}
let user = {
email : row.email,
name : row.name
}
users.push(user);
console.log(JSON.stringify(users));
db.close();
});
});

First error: asynchronicity not handled properly
As Antoine Chalifour pointed out, you call console.log(JSON.stringify(users)); before users gets modified in the asynchronous callback. Refer to his answer for fix and explanations.
Second error: errors not handled
You wrote if (err) { console.error(err.message); } then go on with the rest of the function. That is bad, because an error might happen and you'd just continue with your program. You should instead write something like:
if (err) {
console.error(err);
return;
}
or:
if (err) throw err;

Creating error/result function in node

I'm having trouble understanding how to create functions that would return in the format of (err, result) for an Express app.
My current db query function is:
pool.query(
'SELECT id FROM users WHERE email = ? LIMIT 1',
[email],
(results) => { // I'd like this to be (err, results)
if(results instanceof Error){...}
}
})
In my db.js file, pool looks like this:
module.exports = {
query: (query, args, cb) => {
pool.getConnection( (err, connection) => {
if(err){
new Error('No database connections available in pool')
} else {
connection.query(query, args, (error, results, fields) => {
connection.release()
// I got a MySQL error here and I'd like to handle it in my callback function
if(error){
new Error('Bad query')
} else {
cb(results)
}
})
}
})
}
}
For this and other functions, I'd like to return a proper Error if there is one, and have my callback listen for err, result as parameters.
I tried using new Error('Bad query') but that came back as the first variable in my callback no matter what (which is how I ended up with instanceof Error.
How do you structure a callback and response so that your callback can be in the err, result format and check for/handle errors properly on functions you're creating? (I understand how to use it for modules already in this format - I'm talking about writing/formatting your own code.)
Thanks!

You can do it like this:
module.exports = {
query: (query, args, cb) => {
pool.getConnection( (err, connection) => {
if(err){
cb(new Error('No database connections available in pool'));
} else {
connection.query(query, args, (error, results, fields) => {
connection.release();
// I got a MySQL error here and I'd like to handle it in my callback function
if(error){
cb(new Error('Bad query'));
} else {
cb(null, results);
}
});
}
});
}
}
You always pass the error value as the first argument to the callback and, if there is a result, you pass it as the second. Then, within the callback, you check to see if err is non-null and, if so, there is an error. If it's null, then the second argument contains the result.
Note that by not returning or including the actual err value that the database gave you, you may be hiding useful information (like why the query failed).
Then, where you use this, you do something like this:
let query = 'SELECT id FROM users WHERE email = ? LIMIT 1';
pool.query(query, [email], (err, result) => {
if (err) {
// handle error here
} else {
// process result here
}
});

Retrieve MongoDB result outside callback function scope

I'm trying to get the document retrieved by MongoClient findOne method (in r parameter) outside the scope of callback function. How can i achieve that?
Maybe my approach to the usage of MongoDB driver for Node.js is not appropiate.
function loadUser(name) {
var result = {};
function connection(err, db) {
assert.equal(null, err);
function callback(err, r) {
assert.equal(null, err);
db.close();
result = r; // This does not work
}
db.collection('users').findOne({'user.name':name}, callback);
}
MongoClient.connect(url, connection);
return result;
}

The way you're doing it, result will not be the correct object because it's returned before MongoDB can find it and the value assigned.
You should do something like:
function loadUser(name, cb) {
function connection(err, db) {
assert.equal(null, err);
function callback(err, r) {
assert.equal(null, err);
db.close();
cb(err, r) // user
}
db.collection('users').findOne({'user.name':name}, callback);
}
MongoClient.connect(url, connection);
return;
}
And the usage of loadUser would be:
loadUser("example", function(err, user){
console.log(user);
//Now do what you need with user
});
Also notice that if you are always searching for users, it would be better to just open the connection once, and close it once application is terminated.

Your 'result' variable lives in the scope of the loadUser function. The interaction with MongoDB will be asynchronous, so by the time your callback fires, the loadUser function will have finished and the result variable will no longer exist.
You could simply move the result variable to the global scope. You will probably want to restructure your code a little too though, so that the callback notifies whatever is waiting for the 'result' to return.

You can use EventEmitter to process asynchronous tasks, which makes data flow more maintainable especially if you have a lot of conditional tasks to perform :
"use strict";
var MongoClient = require('mongodb').MongoClient;
const EventEmitter = require('events');
class MyEmitter extends EventEmitter {}
var url = 'mongodb://localhost:27017/myproject';
const myEmitter = new MyEmitter();
// connect to database
var db = MongoClient.connect(url, function(err, dbs) {
if (err) {
console.log(err);
return;
}
console.log("Connected correctly to server");
db = dbs;
//get user when connected
myEmitter.emit('getUser');
});
// getUser event
myEmitter.on('getUser', function() {
db.collection('users').findOne({ 'user.name': "test" }, function callback(err, r) {
if (err) {
myEmitter.emit('processError', err);
} else {
myEmitter.emit('processResult', r);
}
db.close();
});
});
// process result event
myEmitter.on('processResult', function(result) {
//result received, process it here
console.log("result : " + result);
});
// error event
myEmitter.on('processError', function(err) {
//an error occured, process the error here
console.log("error : " + err);
});

Nodejs express and promises not doing what I expect

I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.

Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.

Mongoose Find Results and Async

I am writing a NodeJS script that will run every hour through Heroku's scheduler. I am quering the Mongo instance I have (mongohq/compose) and then doing something with those results. I am working with Mongoose.js and the find() command. This returns an array of results. With those results I need to perform additional queries as well as some additional async processing (sending email, etc).
Long story short, due to node's async nature I need to wait until all the processing is complete before I call process.exit(). If I do not the script stops early and the entire result set is not processed.
The problem is that I have a christmas tree effect of calls at this point (5 nested asnyc calls).
Normally I'd solve this with the async.js library but I'm having a problem seeing this through with this many callbacks.
How can I make sure this entire process finishes before exiting the script?
Here's the code that I'm working with (note: also using lodash below as _):
Topic.find({ nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
finish();
} else {
_.forEach(topics, function (topic, callback) {
User.findById(topic.user, function (err, user) {
if (err) {
// TODO: impl logging
console.error(err);
} else {
// Create a new moment object (not moment.js, an actual moment mongoose obj)
var m = new Moment({ name: moment().format("MMM Do YY"), topic: topic});
m.save(function(err) {
if(err) {
// TODO: impl logging
console.error(err);
} else {
// Send an email via postmark
sendReminderTo(topic, user, m._id);
// Update the topic with next notification times.
// .. update some topic fields/etc
topic.save(function (err) {
if(err) {
console.error(err);
} else {
console.log("Topic updated.");
}
})
}
})
}
});
console.log("User: " + topic.user);
});
}
});

Part of what is making your code confusing is the usage of else statements. If you return your errors, you won't need the else statement and save 4 lines of indentation for every callback. That in and of itself will make things drastically more readable.
Using async:
Topic.find({nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
return finish(err);
}
async.each(topics, function(topic, topicCallback) {
async.auto({
user: function (callback) {
User.findById(topic.user, callback);
},
moment: function(callback) {
var m = new Moment({name: moment().format("MMM Do YY"), topic: topic});
m.save(callback);
},
topic: ["moment", "user", function (callback, results) {
var m = results.moment;
var user = results.user;
sendReminderTo(topic, user, m._id);
topic.save(callback);
}]
}, function(err) {
if (err) {
return topicCallback(err);
}
console.log("Topic updated.")
return topicCallback();
});
}, function(err) {
if (err) {
console.error(err);
return finish(err);
}
return finish();
});
});