NodeJS for loops with Mongo / HTTP call firing late - javascript

I'm fairly new to NodeJS and am still trying to wrap my head around the async nature of it, as well as when things are fired.
I've labeled each of the loops in the order that I expect them to fire with console.log() messages below. There are 4 loops in total that I expect to iterate in order.
When I hit the 'test call' end point, I begin by iterating through a list of data.
exports.testCall = function (req, res) {
var list = myData;
// Iterate through each list item and update MongoDB
for (var i=0; i<list.length; i++) {
var country = list[i].country;
var codeGroups = list[i].codes;
console.log('Loop #1');
for (var j=0; j<codeGroups.length; j++) {
console.log('Loop #2');
queryAndUpdateCodeGroup(codeGroups[j], country);
}
}
}
For each of the above 'codeGroups', I want to run a process that makes a call to my HTTP client, retrieves the data for that code group, and then update my MongoDB with that data.
function queryAndUpdateCodeGroup(group, country) {
myHTTPClient.itemLookup({
browseGroup: group,
country: country
}).then(function success (results) {
console.log('Loop #3');
for (var i=0; i<results.length; i++) {
var childrenArray = results[i].childArray;
var subCatArray = [];
var id = results[i].currentID;
var icon = '';
var name = getNameByCode(country, results[i].currentID);
for (var j=0; j<childrenArray.length; j++) {
subCatArray.push({name: childrenArray[j].name, itemID: id, icon: ''})
}
checkItem(name, id, icon, subCatArray);
}
}).catch(function error (err) {
console.log('ERR: ', JSON.stringify(err));
return err;
handleError(res, err);
});
}
Once the HTTP request is successful, I run this checkItem function which will either create a new entry in Mongo or will update an existing entry. Once this is complete, I want to start the process over for the remaining results in "loop #3". When Loop #3 has completed, I want to return to Loop #2 and repeat until this has completed for loop #2. When loop #2 has completed, I want to repeat this process in loop #1.
function checkItem(name, id, icon, subcats) {
console.log('Loop #4');
MyCategories.findOne({itemID: id}, function (err, cat) {
if (cat === null) {
console.log('ADDING NEW ITEM: ', name);
var newItem = new MyCategories({
name: name,
itemID: id,
icon: icon,
subCats: subcats
});
newItem.save(function (err) {
if (err) {
handleError(err);
return;
}
})
.then(function (res) {
console.log('New Item Success: ', newCatSuccess);
});
} else {
//Update item with latest info
var conditions = { itemID: currNodeId };
var update = { $set: { subCats: subcats }};
MyCategories.update(conditions, update, function (err, updated) {
if (err) {
handleError(err);
} else {
console.log('ITEM HAS BEEN UPDATED SUCCESSFULLY.');
}
});
}
});
}
Loop #1, and loop #2 seem to fire back and forth until completion, but loop #3 fires after #1 and #2 have completed, and #4 doesn't fire at all. How can I 'wait' until each step has completed before I continue on?

Related

Mongoose function shows variable as undefined?

Been staring at this for awhile and not sure why it is not working. I am trying to create an object within an embedded object, however, it seems like I am not passing the params down correctly as it says listings[j] is undefined. What do I need to pass into that piece for it to work?
function createBid(req, res) {
//get current listing object info
db.Listing.findById(req.params.listingId, function(err, foundListing) {
// console.log(foundListing._id );
if (err) {
console.log('Error', err);
}
res.json(foundListing);
// get array of all users
db.User.find({}, function(err, users) {
// for loop iterates through all users' listings
for (var i = 0; i < users.length; i++) {
var listings = users[i].listings
// for loop iterates through all listings ids
for (var j = 0; j < listings.length; j++) {
// finds match
// comparing _id with _id returning false. Not sure why, will return later
if (listings[j].topic === foundListing.topic && listings[j].created === foundListing.created) {
console.log("Found match: " + foundListing.topic);
// get current user id to add to bid object
db.User.findById(req.user, function(err, user) {
if (err) {
console.log(err);
}
var newBid = new db.Bid(req.body); // add data validation later
newBid.uid = user._id
// pushes new bid object into embedded listing
listings[j].bids.push(newBid);
listings[j].save(function(err, savedBid) {
console.log('newBid created: ', newBid);
res.json(newBid);
});
});
}
}
}
})
if (err) {
console.log(err);
}
});
};

Async write after select Pouchdb

I'm trying to check if an element exist before inserting it in my bdd.
I have to do this in order to (in the future) modify this existing element.
I'm using PouchDb and PouchDb-find with Node 6.9.1.
Actually I'm doing this:
for(var i = 0; i < 10;i++ ){
(function(_count, _pdb){
var count = _count;
var db = _pdb;
db.find({
selector: {numeroCandidat: parseInt(results[count].no_apb)}
}).then((result) => {
if(result.docs.length != 0){
console.log("l'étudiant existe");
}else{
console.log("l'étudiant n'existe pas");
var etudiant = {
"numeroCandidat": results[count].no_apb,
"nom": results[count].nom,
"numeroGroupe": "gr" + results[count].groupe,
"filiere": results[count].libelle,
};
db.post(etudiant).then((response) =>{
// handle response
console.log("STUDENT CREATED");
}).catch(function (err) {
console.log(err);
});
}
}).catch(function (err) {
});
})(i, this.pdb);
};
But the problem is : Due to the asynchronous version of my select query... if an element exists two times it appends that the second select occurred BEFORE the insertion of the first element, and I have this element two times in my database. I don't know how to deal with this one.
SO.. I'v found a workaround !
Simply create a function that I call recursivly after writting into my database.
Goodbye for loop.
var createStudents = function(_count, _pdb, _students){
if(_count >= 10) return;
console.log(_count);
var count = _count;
var db = _pdb;
var students = _students.slice(0);
db.find({
selector: {numeroCandidat: parseInt(students[count].no_apb)}
}).then((result) => {
if(result.docs.length != 0){
console.log("l'étudiant existe");
createStudents(++count,db,results);
}else{
var etudiant = {
"numeroCandidat": students[count].no_apb,
"nom": students[count].nom,
"numeroGroupe": "gr" + students[count].groupe,
"filiere": students[count].libelle,
"etudiantComms": [
{"commentaire": students[count].commentaire}
]
};
db.post(etudiant).then((response) =>{
// handle response
console.log("STUDENT CREATED");
createStudents(++count,db,results);
}).catch(function (err) {
console.log(err);
});
}
}).catch(function (err) {
});
}
createStudents(0,this.pdb,results);

Log only shows one of four rows of data

I am writing a small Node js application for automatic vehicle location system.
Here is the code for where I am getting trouble.
markerData contains 4 rows but only in the log I can see the last row.
for (var i = 0, len = markerData.length; i < len; i++) {
var thisMarker = markerData[i];
sql.connect(config, function (err) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
}
});
});
}
Please help me to solve the issue.
As var is not a block level variable in terms of scope, when `sql' module takes time to connect to the database asynchronously, the synchronous loop may change the value of the variable that's why you have the last row printed since the variable holds the reference to the last object at the time of successful connection.
Instead of _.each, I would recommend to use async module with async.each since you have few asynchronous operation to get rid of a synchronous loop.
You can check for samples here,
http://justinklemm.com/node-js-async-tutorial/
Here is your updated code with async.each
-> Install async module with npm install async --save
-> Then add the below reference in the required place,
// Reference
var async = require('async');
-> Modified code:
sql.connect(config, function (err) {
if(err) {
console.log('Connection error: ');
console.log(err);
} else {
async.each(markerData, function(thisMarker, callback) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if(err) {
console.log(err);
callback();
} else {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
} else {
console.log('Recordset empty for id: ' + thisMarker.id);
}
callback();
}
});
}, function(err){
if(err) {
console.log(err);
}
});
}
});
I'm not entirely sure how your library works, but presumably recordset2 is an array of records. recordset2[0] is therefore the first record. If you want the next one you should probably try recordset2[1] and so on and so forth.
Arrays: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array
You'll probably need to loop through all the elements in the array at some point. use a for loop for that:
for (var i = 0; i < recordset2.length; i++ {
console.log(recordset2[i])
}
That will print out everything your query returns.

Callback until for loop with query is done

I posted a question before and realized my problem actually was async functions. I managed to work out most of it, but I got one little problem left. Using async I used waterfall to create an order for the some queries...
exports.getMenu = function(id_restaurant, callback){
async.waterfall([
async.apply(firstQuery, id_restaurant),
secondQuery,
thirdQuery,
fourthQuery,
formMenu
], function(err, result){
if(err){
console.log(err);
}
callback(result);
});
};
Everything works until fourthQuery, where I have to loop to get all dishes of a menu.
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if(err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes[i] = results2;
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
};
So what i can see that happens from printing the value of 'i' each loop is that it always has the value of 2. Because that's 'size' value. Also, even though it's saving results of index '2' I believe the callback is being done even before the for loop is done, because my fifth function is recieving an empty array.
How can i make my code wait to callback until my for loop is done?
NOTE: Sorry, part of my code is in spanish, tried to translate the important parts of it.
There are a few ways to handle this, one is to look into promise architecture. Promise.all will let you supply one callback to handle the values from each child promise.
To use what you've already got, however, I'd push the values into your dishes array, rather than assigning them specifically to i indexes, then check the size of that array at the end of each connection. When the array length matches the size, fire the callback. (as seen below)
If you need a way to tie each result to that specific i value, I'd recommend pushing them as an object
dishes.push({'index': i, 'dish': results2})
Afterward, if you need the array of just dishes, you can sort the array by that index value and run a map function.
dishes.sort(function(a,b){ return a.index - b.index; })
dishes = dishes.map(function(a){ return a.dish })
Here's the code adjusted:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback) {
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if (err) {
console.log(err);
callback(true);
return;
}
for (var i = 0; i < size; i++) {
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [array_secciones[i]],
function(err, results2) {
if (err) {
console.log(err);
callback(true);
return;
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2)
if(dishes.length == size){
console.log("I'm sending " + dishes);
callback(null, dishes, array_nombresSecc)
}
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[i]));
// this prints the result but only if it has a value over 2
});
};
}); // pool
;
};
Since you're already using the async, I would suggest replacing the for() loop in fourthQuery with async.each().
The updated fourthQuery would look like this:
function fourthQuery(array_totalP, array_nombresSecc, array_secciones, callback){
var size = array_nombresSecc.length;
var array_secciones = array_secciones;
var array_nombresSecc = array_nombresSecc;
var dishes = [];
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
async.each(array_secciones,
function(item, itemCallback) {
// Function fun for each item in array_secciones
connection.query("SELECT name, price FROM menu_product WHERE id_seccion = ? AND active = 1", [item],
function(err, results2) {
if(err) {
console.log(err);
return itemCallback(true);
}
console.log("Result query 4 " + JSON.stringify(results2));
dishes.push(results2);
console.log("VALOR PLATILLOS EN i : " + JSON.stringify(dishes[dishes.length-1]));
// this prints the result but only if it has a value over 2
return itemCallback();
});
},
function(err) {
// Function run after all items in array are processed or an error occurs
console.log("I'm sending " + dishes); // this logs an empty array
callback(null, dishes, array_nombresSecc);
});
}); // pool
};
Alternatively, you can use async.map(), which handles gathering the results in the final callback so doesn't rely on the dishes variable.

node.js compare two arrays with objects

I need to remove all documents from my mongo db, which dont exists in new array with objects.
So I have array with objects like :
var items = [
{product_id:15, pr_name: 'a', description : 'desc'},
{product_id:44, pr_name: 'b', description : 'desc2'}
{product_id:32, pr_name: 'c', description : 'desc3'}];
and I have array with db values which I get by calling Model.find({}).
So now I do it in a 'straight' way:
async.each(products, function (dbProduct, callback) { //cycle for products removing
var equals = false;
async.each(items, function(product, callback){
if (dbProduct.product_id === product.product_id){
product.description = dbProduct.description;// I need to save desc from db product to new product
equals = true;
}
callback();
});
if (!equals) {
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
Product.remove({ _id: dbProduct._id }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
}
});
But its blocks the whole app and its very slow, because I have around 5000 values in my items array and in database too. So its very huge cycle numbers.
Maybe there can be a faster way?
UPDATE1
Using code below, from TbWill4321 answer:
var removeIds = [];
// cycle for products removing
async.each(products, function (dbProduct, callback) {
for ( var i = 0; i < items.length; i++ ) {
if (dbProduct.product_id === product.product_id) {
// I need to save desc from db product to new product
product.description = dbProduct.description;
// Return early for performance
return callback();
}
}
// Mark product to remove.
removeIds.push( dbProduct._id );
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
return callback();
}, function() {
Product.remove({ _id: { $in: removeIds } }, function (err) {
if (err) return updateDBCallback(err);
// Continue Here.
// TODO
});
});
Its takes around 11 sec(blocks whole web-app) and takes 12 362 878 cycles for me.
So maybe somebody can advise me something?
The Async library does not execute synchronous code in an asynchronous fashion.
5000 items is not a huge number for JavaScript, as I've worked on Big Data set's with 5 million+ points and it doesn't take long. You can get better performance by structuring like this:
var removeIds = [];
// cycle for products removing
async.each(products, function (dbProduct, callback) {
for ( var i = 0; i < items.length; i++ ) {
if (dbProduct.product_id === product.product_id) {
// I need to save desc from db product to new product
product.description = dbProduct.description;
// Return early for performance
return callback();
}
}
// Mark product to remove.
removeIds.push( dbProduct._id );
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
return callback();
}, function() {
Product.remove({ _id: { $in: removeIds } }, function (err) {
if (err) return updateDBCallback(err);
// Continue Here.
// TODO
});
});
Among the many problems you may have, off the top of my head you may want to start off by changing this bit:
Product.remove({ _id: dbProduct._id }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
Being within a .each() call, you'll make one call to the database for each element you want to delete. It's better to store all the ids in one array and then make a single query to delete all elements that have an _id that is in that array. Like this
Product.remove({ _id: {$in: myArrayWithIds} }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
On another note, since async will execute synchronously, node.js does offer setImmediate() (docs here), that will execute the function from within the event loop. So basically you can "pause" execution of new elements and serve any incoming requests to simulate "non-blocking" processing.

Categories