Node.js looping async calls - javascript

I'm busy working on an endpoint for a reporting system. Node being async is giving me issues, although I'd rather not force it to be synchronous.
We're using MongoDB and Mongoose. I've got to query regex over collection A, then for each document that gets returned, query multiple contained documents to populate a JSON object/array to be returned.
I can use populate for most of the data, except the final looped queries which is where the async kicks in and returns my report early. Is there an elegant way to do this? Or should I be splitting into a different function and calling that multiple times to stick to the functions should do only one thing rule?
Example Code:
A.find({ name: regex }).populate({ path: 'B', populate: { path: 'B.C', model: 'C' } }).exec(function(err, A) {
var report = [];
A.map(function(a)){
report[a.name] = [];
D.aggregate([
{
$match: {
id: B._id
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
], function(err, result) {
C.map(function(c){
report[a.name].push({
'field1': c.field1,
'field2': c.field2,
'field3': c.field3,
'count': result.count
});
});
});
}
return report;
});
The issue here is with the logic / async. Not with the syntax, hence the semi-pseudo code.
Any help or advice would be greatly appreciated.

You need to familiarize yourself with promises, and with async in general.
Because you are returning an array, that's the value you are going to get.
You have a few options when dealing with Async, but in your case, you want to look at two solutions:
// callbacks
getSetOfIDs((err, ids) => {
let remaining = ids.length;
let things = [];
let failed = false;
ids.forEach(id => {
getThingByID(id, (err, thing) => {
if (failed) { return; }
if (err) {
failed = true;
handleFailure(err);
} else {
remaining -= 1;
things.push(thing);
if (!remaining) {
handleSuccess(things);
}
}
});
});
});
Note, I'm not returning things, I'm passing it into a callback.
You can use higher-order functions to clean this sort of thing up.
// cleaned up callbacks
function handleNodeCallback (succeed, fail) {
return function (err, data) {
if (err) {
fail(err);
} else {
succeed(data);
}
};
}
function handleAggregateCallback (succeed, fail, count) {
let items = [];
let failed = false;
const ifNotFailed = cb => data => {
if (!failed) { cb(data); }
};
const handleSuccess = ifNotFailed((item) => {
items.push(item);
if (items.length === count) { succeed(items); }
});
const handleFailure = ifNotFailed((err) => {
failed = true;
fail(err);
});
return handleNodeCallback(handleSuccess, handleFailure);
}
A little helper code later, and we're ready to go:
// refactored callback app code (note that it's much less scary)
getSetOfIDs((err, ids) => {
const succeed = (things) => app.display(things);
const fail = err => app.apologize(err);
if (err) { return fail(err); }
let onThingResponse = handleAggregateCallback(succeed, fail, ids.length);
ids.forEach(id => getThingByID(id, onThingResponse));
});
Note that aside from higher-order functions, I'm never returning anything, I'm always passing continuations (things to do next, with a value).
The other method is Promises
// Promises
getSetOfIDs()
.then(ids => Promise.all(ids.map(getThingByID)))
.then(things => app.display(things))
.catch(err => app.apologize(err));
To really get what's going on here, learn Promises, the Promise.all static method, and array.map().
Both of these sets of code theoretically do the exact same thing, except that in this last case getSetOfIDs and getThingByID don't take callbacks, they return promises instead.

usually in async calls, after return statement any operations are cancelled.
maybe you can return report object only when all is done and well.
A.find({ name: regex }).populate({ path: 'B', populate: { path: 'B.C', model: 'C' } }).exec(function(err, A) {
var report = [];
A.map(function(a)){
report[a.name] = D.aggregate([
{
$match: {
id: B._id
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
], function(err, result) {
if(err){
return [];
}
var fields = []
C.map(function(c){
fields.push({
'field1': c.field1,
'field2': c.field2,
'field3': c.field3,
'count': result.count
});
});
return fields;
});
}
return report;
});

Just use promises:
A.find({ name: regex }).populate({ path: 'B', populate: { path: 'B.C', model: 'C' } }).exec(function(err, A) {
var report = [];
return Promise.all([
A.map(function(a)){
return new Promise(function(resolve, reject) {
report[a.name] = [];
D.aggregate([{ $match: { id: B._id }},{$group: {_id: null,count: { $sum: 1 }}}],
function(err, result) {
if(err) {
reject(err)
} else {
C.map(function(c){
report[a.name].push({
'field1': c.field1,
'field2': c.field2,
'field3': c.field3,
'count': result.count
});
});
resolve(report)
}
});
}
})])
})
.then(function(report){
console.log(report)
})
.catch(function(err){
console.log(err)
})

Related

array push results in empty array javascript

I am trying to store API results into an array.
The data is displayed in console, but on pushing the data into an array, the array is still empty.
Here's the code:
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
var favorites = [];
dbConn.then( function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function(err, result) {
if(err) throw err;
if(!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({msg:'record not found'});
}
else {
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', {id: tweet.id}, function(err, data, response) {
if(!err){
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
}
});
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e){console.log(e)})
});
It looks like you're defining the favorites array within the scope of the function callback. Try putting var favorites = []; above you app.post() call instead.
Also, keep in mind that it will only have a value after the callback is complete, so any synchronous code later down the line will only see the empty array value.
I've updated your code to get favorites by storing separately the promise and call it afterwards:
UPDATE
As you can see in the demo, i have 2x console.log at the bottom, the first one(C1) is contained in the promise favoritesPromise () and the second (C2) is after the promise.
Synchronous actions will never wait for asynchronus actions to take place, therefore in my example C2 will always be outputted before C1, even if console.log(1 ... ) is before console.log(2 ... ), they'll appear reversed in the console.
In the promise i added a setTimeout of 1ms to mock a request, it was all it took to achieve the current output. Another thing you can test is removing the setTimeout then output will change a bit, your promise becomes synchronus until it reaches resolve(favorites), that means favorites has all the data by now, but when resolve takes place, it becomes async, and in your console you will still see C2 first (but now with data) and C1 second.
In my earlier answer i tried to implement this reasoning to your code.
Keep it async folks!
var favorites = [];
var favoritesPromise = () => {
return new Promise((resolve, reject) => {
console.log('Retrieving data from the internet.');
// This timeout mocks your request to anything that is async or promie
setTimeout(() => {
console.log('Request done')
let resultFavorite_tweets = [{
id: 1,
name: 'a dog'
}, {
id: 2,
name: 'a cat'
}];
resultFavorite_tweets.forEach(item => {
favorites.push(item.name);
})
resolve(favorites);
// if you have an error use
// reject(err)
}, 1);
});
}
favoritesPromise().then(favList => {
console.log(1, 'this will always contain data from the internet, but will always be last', favList);
})
console.log(2, 'this will be empty (unless you remove setTimeout), but will always be first', favorites);
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
const favoritesPromise = () => {
return new Promise((resolve, reject) => {
var favorites = [];
dbConn.then(function(database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne({
_id: ObjectId(req.userId)
}, function(err, result) {
if (err) reject(err);
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({
msg: 'record not found'
});
} else {
result.favorite_tweets.forEach(function(tweet) {
T.get('statuses/show', {
id: tweet.id
}, function(err, data, response) {
if (!err) {
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
reject(err);
}
});
resolve(data);
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e) {
reject(e)
})
});
}
// Here you call the promise to retrieve "favorites"
favoritesPromise().then(favoritesList => {
console.log('your favorites array', favoritesList)
})
})
Try next code
app.post('/fetchFavoriteTweets/', verifyToken, function (req, res) {
var favorites = [];
dbConn.then(function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function (err, result) {
if (err) throw err;
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({ msg: 'record not found' });
}
else {
// Counter
let count = result.favorite_tweets.length;
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', { id: tweet.id }, function (err, data, response) {
// Decrease count
count -= 1;
if (!err) {
favorites.push(data);
// Check if count is zero
if (count === 0) {
console.log(favorites);
res.status(200).json({msg:'success', data:favorites});
}
} else {
console.log(err);
}
});
});
}
});
}).catch(function (e) { console.log(e) })
});

How to wait for inside for each to complete before iterating to next loop?

Hi I want to implement nested for each. But for loop should not iterate to next until nested for loop completes its execution. I have tried with asyn.forEachOf but first for loop executed without waiting for second for loop. I have implemented like this
async.forEachOf(playerIds, function (playerId, key, playerCallback) {
console.log(playerId);
//Find coach rating for every player's goals
DateInfo.find({
coachId: userId,
playerId: playerId,
coachRemoved: false
}).select('objectiveId frequencyId').populate({
path: 'objectiveId', match: {'deleted': {$ne: 2}}, select: {'name': 1}
}).exec(function (error, coachDateInfos) {
if (error) {
response.status(200).json({message: error});
} else {
console.log(coachDateInfos);
playerCallback()
}
})
});
I haven't posted inside async function. I have two player Ids it prints two player ids first then it prints coachDateInfos.
That's because a for-loop is a synchronous action while Model.find().exec() is an asynchronous action. You can't mix the two together. You can do the following (no async lib):
app.get(async (req, res) => {
const promises = []
playerIs.forEach(playerId => {
const promise = DateInfo
.find({coachId: userId, playerId, coachRemoved: false})
.select('objectiveId frequencyId')
.populate({path: 'objectiveId', match: {'deleted': {$ne: 2}}, select: {'name': 1})
.exec()
promises.push(promise)
})
try {
res.json(await Promise.all(promises))
} catch (err) {
res.status(200).json(err);
}
})
var arrayOfFuncs = [];
for (var key in playerIds) {
let player = playerIds[key];
let func_1 = function(playerId,cb)
{
console.log(playerId);
//Find coach rating for every player's goals
DateInfo.find({
coachId: userId,
playerId: playerId,
coachRemoved: false
}).select('objectiveId frequencyId').populate({
path: 'objectiveId', match: {'deleted': {$ne: 2}}, select: {'name': 1}
}).exec(function (error, coachDateInfos) {
if (error) {
//response.status(200).json({message: error});
cb(error, null);
} else {
console.log(coachDateInfos);
cb(null, coachDateInfos);
}
})
}
arrayOfFuncs.push(func_1.bind(null,player));
}
async.parallel(arrayOfFuncs, function(errString, allResults) {
if(errString) {
console.log(errString);
} else {
console.log(allResults);
}
})

Callback NodeJS & insert data with mongoose

I know this topic as already asked many times before but I didn't find the right answer to do what I want.
Actually, I try to save two different list of JSON object in MongoDB via Mongoose. To perform both at the same time I use 'async'.
However, when I save it with the command insertMany() I get an error because he calls the callback of async before finishing the insertMany(). Therefore answer[0] is not defined.
What will be the proper way of doing it ?
Here is my code with the async:
const mongoose = require("mongoose");
const async = require("async");
const utils = require("../utils");
const experimentCreate = function(req, res) {
let resData = {};
let experimentList = req.body.experiment;
let datasetList = req.body.datasetList;
async.parallel(
{
dataset: function(callback) {
setTimeout(function() {
answer = utils.createDataset(datasetList);
callback(answer[0], answer[1]);
}, 100);
},
experiment: function(callback) {
setTimeout(function() {
answer = utils.createExp(experimentList);
callback(answer[0], answer[1]);
}, 100);
}
},
function(err, result) {
if (err) {
console.log("Error dataset or metadata creation: " + err);
sendJSONresponse(res, 404, err);
} else {
console.log("Experiment created.");
resData.push(result.dataset);
resData.push(result.experiment);
console.log(resData);
sendJSONresponse(res, 200, resData);
}
}
);
};
Then the two functions called createExp and createDataset are the same in another file. Like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
return [err, null];
} else {
console.log("All dataset created.");
return [null, ds];
}
});
};
There's a few problems with your code. For one, you're not returning anything in your createDataset function. You're returning a value in the callback of insertMany but it doesn't return that value to the caller of createDataset as it's within another scope. To solve this issue, you can wrap your Dataset.insertMany in a promise, and resolve or reject depending on the result of Data.insertMany like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
return new Promise((resolve, reject) => {
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
reject(err);
} else {
console.log("All dataset created.");
resolve(ds);
}
});
});
};
Now your return object is no longer going to be an array so you won't be able to access both the error and the result via answer[0] and answer[1]. You're going to need to chain a then call after you call createDataset and use callback(null, answer) in the then call (as that means createDataset executed successfully) or use callback(err) if createDataset throws an error like below:
dataset: function(callback) {
setTimeout(function() {
utils.createDataset(datasetList).then(answer => {
callback(null, answer);
}).catch(err => callback(err)); // handle error here);
}, 100);
}
Note: You'll most likely need to alter your createExp code to be structurally similar to what I've produced above if it's also utilizing asynchronous functions.

Promise resolve not waiting for promise inside it

I am trying to make a rest call and update a list and then resolve the promise with the updated list.
function addTestCaseToTestRail(){
return new Promise(function(resolve){
compareTestRailAndProtractor().then(function(tests){
var testsLength = tests.tests.length;
var url = testRailURL+testRailData.addTestEndPoint;
for(var i=0; i<testsLength; i++){
if(tests.tests[i].id==undefined){
var newId=""
var options = {
url:url,
headers:headers,
body:{
"title":tests.tests[i].name,
"custom_jira_component" : 465
},
json: true
}
request.post(options, function(err, httpResponse, body){
if (err) {
console.error(err);
return;
}
newId = body.id;
});
tests.tests[i].id = newId;
}
}
resolve(tests);
});
});
}
function test(){
addTestCaseToTestRail().then(function(tests){
console.log(tests);
});
}
test()
The request is getting posted and I am able to create tests in test rail but the resolve(tests) does not have the newId assignment.
This is the output I am getting. Not sure why resolve does not wait for the rest call to complete.
{ tests:
[ { id: '', name: 'test1'},
{ id: '', name: 'test2'},
{ id: '', name: 'test3'},
{ id: '', name: 'test4'},
{ id: '', name: 'test6'},
{ id: '', name: 'test5'} ] }
compareTestRailAndProtractor returns a Promise. You can use async/await within .then() and Promise constructor within for loop to await request callback, which is issue at code at Question, as the for loop does not await the callback function
function addTestCaseToTestRail() {
return compareTestRailAndProtractor()
.then(async function(tests) {
var testsLength = tests.tests.length;
var url = testRailURL + testRailData.addTestEndPoint;
for (var i = 0; i < testsLength; i++) {
await new Promise((resolve, reject) => {
if (tests.tests[i].id == undefined) {
var newId = ""
var options = {
url: url,
headers: headers,
body: {
"title": tests.tests[i].name,
"custom_jira_component": 465
},
json: true
}
request.post(options, function(err, httpResponse, body) {
if (err) {
reject(err);
}
newId = body.id;
tests.tests[i].id = newId;
resolve();
});
} else {
resolve()
}
});
}
return tests
})
}
function test() {
addTestCaseToTestRail()
.then(function(tests) {
console.log(tests);
})
.catch(function(err) {
console.error(err)
})
}
test()
I think it's because of the non-blocking nature of Javascript. The thing is that "resolve(test)" gets executed before your "post" event has a response (which is the one where you take the Id from).
I would recommend you to take a look at https://caolan.github.io/async/ Async is an amazing library for handling async processes.
Good luck!
EDIT: You can also take a look at async/await JS operators

The right way to do multiple MongoDB operations

If I need to perform two or three different operations on a few collections, is there a better way than chaining together find/update operations? For example:
db.collection('contactinfos').findOneAndUpdate(
{ _id: ObjectID(contactID) },
{ $set: { sharedWith } }
).then(response => {
db.collection('users').update(
{ _id: { $in: sharedWith.map(id => ObjectID(id)) } },
{ $addToSet: { hasAccessTo: contactID } },
{ multi: true }
).then(response => {
db.collection('users').update(
{ _id: { $in: notSharedWith.map(id => ObjectID(id)) } },
{ $pull: { hasAccessTo: contactID } },
{ multi: true }
).then(response => {
return res.send({ success: true });
}).catch(err => {
logger.error(`in updating sharing permissions for ${contactID} by user ${_id}`, err);
return res.status(400).send({ reason: 'unknown' });
});
}).catch(err => {
logger.error(`in updating sharing permissions for ${contactID} by user ${_id}`, err);
return res.status(400).send({ reason: 'unknown' });
});
}).catch(err => {
logger.error(`in updating sharing permissions for ${contactID} by user ${_id}`, err);
return res.status(400).send({ reason: 'unknown' });
});
It just seems messy and there has to be some better way of doing it. Furthermore, if there is an error after the first findOneAndUpdate that prevents the other updates from running, then there will be inconsistent data across documents. The documents contain ID references to other documents for faster lookup.
Also, is there a way to catch all errors within a chain of promises?
From your callback hell I can see you do not use response argument of .then() method anywhere. If you do not need results of one query to perform another, consider using Promise.all() method:
const updateContactInfo = db.collection('contactinfos')
.findOneAndUpdate(
{ _id: ObjectID(contactID) },
{ $set: { sharedWith } }
);
const updateUsers = db.collection('users')
.update(
{ _id: { $in: sharedWith.map(id => ObjectID(id)) } }, //hint: use .map(ObjectId) instead.
{ $addToSet: { hasAccessTo: contactID } },
{ multi: true }
);
const updateUsers2 = db.collection('users')
.update(
{ _id: { $in: notSharedWith.map(id => ObjectID(id)) } }, //hint: use .map(ObjectId) instead.
{ $pull: { hasAccessTo: contactID } },
{ multi: true }
);
Promise
.all([updateContactInfo, updateUsers, updateUsers2])
.then((values) => {
const updateContactInfoResult = values[0];
const updateUsersResult = values[1];
const updateUsers2Result = values[2];
return res.send({ success: true });
})
.catch((reason) => {
logger.error(`msg`, reason);
return res.status(400).send({ reason: 'unknown' });
});
Promise.all() will continue executing following .then() only if all the promises do resolve, otherwise it'll fall into the .catch() method. As of error handling, you can easily chain multiple .catch() methods, which is nicely explained here.
If you cannot have any data inconsistency, either:
Get some SQL database with transactions (easier solution)
Look into MongoDB Two-Phase Commit
And if it is acceptable to happen, let's say once per 1kk times, do include checking it's consistency within your app's logic.

Categories