We Keep Coding

NodeJS for loops with Mongo / HTTP call firing late

I'm fairly new to NodeJS and am still trying to wrap my head around the async nature of it, as well as when things are fired.
I've labeled each of the loops in the order that I expect them to fire with console.log() messages below. There are 4 loops in total that I expect to iterate in order.
When I hit the 'test call' end point, I begin by iterating through a list of data.
exports.testCall = function (req, res) {
var list = myData;
// Iterate through each list item and update MongoDB
for (var i=0; i<list.length; i++) {
var country = list[i].country;
var codeGroups = list[i].codes;
console.log('Loop #1');
for (var j=0; j<codeGroups.length; j++) {
console.log('Loop #2');
queryAndUpdateCodeGroup(codeGroups[j], country);
}
}
}
For each of the above 'codeGroups', I want to run a process that makes a call to my HTTP client, retrieves the data for that code group, and then update my MongoDB with that data.
function queryAndUpdateCodeGroup(group, country) {
myHTTPClient.itemLookup({
browseGroup: group,
country: country
}).then(function success (results) {
console.log('Loop #3');
for (var i=0; i<results.length; i++) {
var childrenArray = results[i].childArray;
var subCatArray = [];
var id = results[i].currentID;
var icon = '';
var name = getNameByCode(country, results[i].currentID);
for (var j=0; j<childrenArray.length; j++) {
subCatArray.push({name: childrenArray[j].name, itemID: id, icon: ''})
}
checkItem(name, id, icon, subCatArray);
}
}).catch(function error (err) {
console.log('ERR: ', JSON.stringify(err));
return err;
handleError(res, err);
});
}
Once the HTTP request is successful, I run this checkItem function which will either create a new entry in Mongo or will update an existing entry. Once this is complete, I want to start the process over for the remaining results in "loop #3". When Loop #3 has completed, I want to return to Loop #2 and repeat until this has completed for loop #2. When loop #2 has completed, I want to repeat this process in loop #1.
function checkItem(name, id, icon, subcats) {
console.log('Loop #4');
MyCategories.findOne({itemID: id}, function (err, cat) {
if (cat === null) {
console.log('ADDING NEW ITEM: ', name);
var newItem = new MyCategories({
name: name,
itemID: id,
icon: icon,
subCats: subcats
});
newItem.save(function (err) {
if (err) {
handleError(err);
return;
}
})
.then(function (res) {
console.log('New Item Success: ', newCatSuccess);
});
} else {
//Update item with latest info
var conditions = { itemID: currNodeId };
var update = { $set: { subCats: subcats }};
MyCategories.update(conditions, update, function (err, updated) {
if (err) {
handleError(err);
} else {
console.log('ITEM HAS BEEN UPDATED SUCCESSFULLY.');
}
});
}
});
}
Loop #1, and loop #2 seem to fire back and forth until completion, but loop #3 fires after #1 and #2 have completed, and #4 doesn't fire at all. How can I 'wait' until each step has completed before I continue on?

Callback until for loop with query is done

I posted a question before and realized my problem actually was async functions. I managed to work out most of it, but I got one little problem left. Using async I used waterfall to create an order for the some queries...
exports.getMenu = function(id_restaurant, callback){
async.waterfall([
async.apply(firstQuery, id_restaurant),
secondQuery,
thirdQuery,
fourthQuery,
formMenu
], function(err, result){
if(err){
console.log(err);
}
callback(result);
});
};
Everything works until fourthQuery, where I have to loop to get all dishes of a menu.
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if(err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes[i] = results2;
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
};
So what i can see that happens from printing the value of 'i' each loop is that it always has the value of 2. Because that's 'size' value. Also, even though it's saving results of index '2' I believe the callback is being done even before the for loop is done, because my fifth function is recieving an empty array.
How can i make my code wait to callback until my for loop is done?
NOTE: Sorry, part of my code is in spanish, tried to translate the important parts of it.

There are a few ways to handle this, one is to look into promise architecture. Promise.all will let you supply one callback to handle the values from each child promise.
To use what you've already got, however, I'd push the values into your dishes array, rather than assigning them specifically to i indexes, then check the size of that array at the end of each connection. When the array length matches the size, fire the callback. (as seen below)
If you need a way to tie each result to that specific i value, I'd recommend pushing them as an object
dishes.push({'index': i, 'dish': results2})
Afterward, if you need the array of just dishes, you can sort the array by that index value and run a map function.
dishes.sort(function(a,b){ return a.index - b.index; })
dishes = dishes.map(function(a){ return a.dish })
Here's the code adjusted:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback) {
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if (err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if (err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2)
if(dishes.length == size){
console.log("I'm sending " + dishes);
callback(null, dishes, array_nombresSecc)
}
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
;
};

Since you're already using the async, I would suggest replacing the for() loop in fourthQuery with async.each().
The updated fourthQuery would look like this:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
async.each(array_secciones,
function(item, itemCallback) {
// Function fun for each item in array_secciones
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [item],
function(err, results2) {
if(err) {
console.log(err);
return itemCallback(true);
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2);
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[dishes.length-1]));
// this prints the result but only if it has a value over 2
return itemCallback();
});
},
function(err) {
// Function run after all items in array are processed or an error occurs
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
});
}); // pool
};
Alternatively, you can use async.map(), which handles gathering the results in the final callback so doesn't rely on the dishes variable.

Inserting more than one record with Node.js and Tedious

I have an array of customer objects, that I wish to insert to the SQL database.
The customer objects are retrieved from the req data.
I am using Tedious for the request, and Tedious Connectionpool in order to have multiple connections at the same time.
When looping over the objects i am getting an error when trying to insert, the error being
{ [RequestError: Violation of PRIMARY KEY constraint `'PK__Customer__A4AE64D873A5400C'. Cannot insert duplicate key in object 'dbo.Customer'. The duplicate key value is (2).]`
Note that I only have 3 object being send in the req at this time. It looks to me that it is only the last object that are being handled and inserted. But since I am new to using tedious with Node.js i cant spot my mistake. Any suggestions ?
router.post('/',jsonParser, function(req, res) {
var customers = req.body.customers;
var companies = req.body.companies;
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var TYPES = require('tedious').TYPES;
var config = {
userName: '*************',
password: '*************',
server: '***********.database.windows.net',
// When you connect to Azure SQL Database, you need these next options.
options: {encrypt: true, database: '*******'}
};
var poolConfig = {
min: 1,
max: 3,
log: true
};
var pool = new ConnectionPool(poolConfig, config);
for (var i = 0; i < customers.length; i++) {
console.log('Inserting '+customers[i].firstname);
var firstname = customers[i].firstname;
var count = i;
pool.acquire(function (err, connection) {
if (err)
console.error(err);
//use the connection as normal
var request = new Request("INSERT INTO dbo.Customer" +
" (Firstname,CustomerId)" +
"VALUES" +
" (#Firstname,#CustomerId);", function (err, rowCount) {
if (err)
console.error(err);
console.log('rowCount: ' + rowCount);
//release the connection back to the pool when finished
connection.release();
});
request.addParameter('Firstname', TYPES.VarChar,firstname);
request.addParameter('CustomerId', TYPES.Int, count);
request.on('row', function (columns) {
console.log('value: ' + columns[0].value);
});
connection.execSql(request);
});
pool.on('error', function (err) {
console.error(err);
});
}
});

The scope of your variables count and firstName are global. By the time the pool.acquire( function get's executed the for loop has completed and it is inserting the last customer twice. One possible solution would be to put an anonymous function inside the for loop e.g. (it doesn't have to be anonymous though)
for (var i = 0; i < customers.length; i++) {
(function(count, firstName) {
...do insert here...
}(i, customers[i].firstname));
}

node.js return data from find collection in loop

I am trying to return data from this function. Console.log(documents) successfully shows the data in console. But this works only in body of the function. I can't return this data to the template. What should I do? Should I use some async package for node.js, or can be accomplished somehow like this?
Thank you.
var projects = req.user.projects;
var docs = [];
db.collection('documents', function(err, collection) {
for (i = 0; i < projects.length; i++) {
collection.find({'_projectDn': projects[i].dn},function(err, cursor) {
cursor.each(function(err, documents) {
if(documents != null){
console.log(documents);
//or docs += documents;
}
});
});
}
});
console.log(documents); // undefined
res.render('projects.handlebars', {
user : req.user,
documents: docs
});

Those db functions are async, which means that when you try to log it, the function hasn't finished yet. You can log it using a callback, for example:
function getDocuments(callback) {
db.collection('documents', function(err, collection) {
for (i = 0; i < projects.length; i++) {
collection.find({
'_projectDn': projects[i].dn
}, function(err, cursor) {
cursor.each(function(err, documents) {
if (documents !== null) {
console.log(documents);
callback(documents);// run the function given in the callback argument
}
});
});
}
});
}
//use the function passing another function as argument
getDocuments(function(documents) {
console.log('Documents: ' + documents);
});

Node/Express - How to wait until For Loop is over to respond with JSON

I have a function in my express app that makes multiple queries within a For Loop and I need to design a callback that responds with JSON when the loop is finished. But, I'm not sure how to do this in Node yet. Here is what I have so far, but it's not yet working...
exports.contacts_create = function(req, res) {
var contacts = req.body;
(function(res, contacts) {
for (var property in contacts) { // for each contact, save to db
if( !isNaN(property) ) {
contact = contacts[property];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err) { console.log(err) };
}); // .save
}; // if !isNAN
}; // for
self.response();
})(); // function
}; // contacts_create
exports.response = function(req, res, success) {
res.json('finished');
};

There are a few problems with your code besides just the callback structure.
var contacts = req.body;
(function(res, contacts) {
...
})(); // function
^ you are redefining contacts and res in the parameter list, but not passing in any arguments, so inside your function res and contacts will be undefined.
Also, not sure where your self variable is coming from, but maybe you defined that elsewhere.
As to the callback structure, you're looking for something like this (assuming contacts is an Array):
exports.contacts_create = function(req, res) {
var contacts = req.body;
var iterator = function (i) {
if (i >= contacts.length) {
res.json('finished'); // or call self.response() or whatever
return;
}
contact = contacts[i];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err)
console.log(err); //if this is really a failure, you should call response here and return
iterator(i + 1); //re-call this function with the next index
});
};
iterator(0); //start the async "for" loop
};
However, you may want to consider performing your database saves in parallel. Something like this:
var savesPending = contacts.length;
var saveCallback = function (i, err) {
if (err)
console.log('Saving contact ' + i + ' failed.');
if (--savesPending === 0)
res.json('finished');
};
for (var i in contacts) {
...
newContact.save(saveCallback.bind(null, i));
}
This way you don't have to wait for each save to complete before starting the next round-trip to the database.
If you're unfamiliar with why I used saveCallback.bind(null, i), it's basically so the callback can know which contact failed in the event of an error. See Function.prototype.bind if you need a reference.