Build an array of items in an async process [duplicate] - javascript

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I am a newbie in JavaScript and I am a bit stuck with the async way JS executes code...
Here is my code :
var marketPlaces = []
body.rows.forEach(function(doc) {
marketPlacesDB.get(doc.id, function(err, body){
if (err) {
callback(err, null)
} else {
console.log("Ajout d'une marketplace")
marketPlaces.push({id: body._id, name: body.name})
}
})
})
console.log("Retour des résultats")
callback(null, { marketPlaces: marketPlaces })
body.rows is an array containing ids of objects I would like to return in the marketPlaces array. For each element, I need to make a new request to the database to get the details of the objects (here only the "name").
The result is an empty array because the foreach loop ends before the callbacks of the get function return.
I can't figure out how to make this "synchronous".
Thanks for your answers.
Philippe.

If they didn't give you the synchronous API, you can't.
But you can still make it works 'synchronous' by adding a big callback. (I'm a non-native English speaker, dunno what word should I use here)
let counter = 0;
const max = marketPlacesDB.getLength(); // base on the real situation
function regularCallback() {
/* your callback */
if(++counter == max)
bigCallback();
};
function bigCallback() {
// continue on your original work
}

You can't make it synchronous if marketPlaceDb is not providing api. But you can make it work with asynchronous version too:
var async = require('async')
function getMarketPlaces(callback) {
var marketPlaces = []
async.eachSeries(body.rows, doc, function (next) {
marketPlacesDB.get(doc.id, function(err, body){
if (err) {
next(err, null) // finish async operation
} else {
marketPlaces.push({id: body._id, name: body.name})
next() // go next iteration
}
})
}, function (err) {
// eachSeries done
// here is marketPlaces
console.log("Retour des résultats")
callback(err, { marketPlaces: marketPlaces })
})
}
getMarketPlaces(console.log)
I used 'async' library from npm and eachSeries method to iterate array asynchronously.

Thanks to Ozgur using async library seems to be the most elegant way to answer my question.
The correct code is :
var marketPlaces = []
async.eachOfSeries(body.rows, function (item, key, next) {
marketPlacesDB.get(item.id, function(err, body){
if (err) {
next(err, null)
} else {
marketPlaces.push({id: body._id, name: body.name})
next()
}
})
}, function (err) {
console.log("Retour des résultats")
callback(err, { marketPlaces: marketPlaces })
})

Related

Async, waterfall issue

Here i am trying to retrieve all the objects and push them into the json file. For some reason there is only one record being pushed into file when it should contain more objects. The response is being sent even before the execution. Can you help me out with this or let me know where I am going wrong? Here is my code:
exports.createjoson = (req, res) => {
const Responsearray = [];
async.waterfall(
[
function(waterfallCb) {
// ... first function
},
function(results, waterfallCb1) {
//second function
async.eachLimit(
results,
100,
function(singleResult, eachCallback) {
async.waterfall(
[
async function(innerWaterfallCb) {
try {
NewsModel.find(
{ _id: singleResult.newsId }, // #individual article
async (err, newsResult) => {
if (err) {
return innerWaterfallCb(
// #displaying error
"error in fetching news data"
);
}
const map = new Map();
for (const item of newsResult) {
if (!map.has(item.source)) {
map.set(item.source, true);
Response = {
newsId: item._id,
title: item.title,
comment: singleResult.comment
};
}
}
resPond = await Response;
Responsearray.push(resPond);
let data = JSON.stringify(Responsearray);
await fs.writeFileSync("abc.json", data);
}
);
} catch (error) {
innerWaterfallCb(error);
}
}
],
function(err) {
if (err) {
return eachCallback(err);
}
eachCallback(null);
}
);
},
function(err) {
if (err) {
return waterfallCb1(err);
}
waterfallCb1(null);
}
);
}
],
function(err) {
if (err) {
return res.status(200).json({ status: "400", message: err });
}
res.status(200).json({ status: "200", message: "success" });
}
);
};
There are a number of problems with the code:
fs.writeFileSync will overwrite the file, not append to it, so only the last data you write will be in abc.json. Also it does not return a Promise so there is no need to use await on it. It runs synchronously so will not return until it's complete (that's what the Sync in its function name means). To append instead of overwrite the file, you can set the flag option to "a" to append (the default is "w").
There doesn't seem to be a call to return innerWaterfallCb(null) anywhere - only in error conditions. The inner waterfall function shouldn't be marked async since it doesn't need to do any await calls really. But you should call return innerWaterfallCb(null) after the file is written.
It may be better to just collect the data in responseArray and write the file once at the end of the outer waterfall instead of writing it repeatedly deep inside the inner waterfall.
Variables should start with lowercase letters (like responseArray not ResponseArray since uppercase first letters indicate classes or modules usually.
Don't mix async/await with the async module (waterfall and eachLimit). If you're using proper Promises and async/await then there should be no need to use the async module. It would be cleaner to remove the use of waterfall entirely and rewrite to use Promise objects properly.

I can't return an object value, but it is showed up by console.log [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 3 years ago.
I'm trying to return a string value of an object generated by aws translate, the structure's object is
{
TranslatedText: "Hola",
SourceLanguageCode: "en",
TargetLanguageCode: "es"
}
the function
this.translate.translator.translateText(params, (err, data) => {
if (err) {
console.log(err, err.stack)
return err.stack;
}
if(data) {
console.log("translation :");
console.log(data.TranslatedText);
return data.TranslatedText;
}
});
I can see the string in console, but it is not returning it.
I think that I'm misunderstanding some async job here, and maybe the returned value is actually getting an undefined, but I'm not clear.
As it is in a callback function, so you won't be able to return the data like the way you are doing. Try using promises instead.
If doesn't make sense to return values from callbacks. Do what you want to do with the return value inside your callback.
Sounds like translateText is an async function. Therefore, await it like so:
this.translate.translator.translateText(params, (err, data) => {
if (err) {
console.log(err, err.stack)
return err.stack;
}
if(data) {
console.log("translation :");
console.log(data.TranslatedText);
return data.TranslatedText;
}
});

nodeJS design pattern to break out of a async.waterfall or promise.then.then [duplicate]

This question already has answers here:
How to properly break out of a promise chain?
(3 answers)
Closed 7 years ago.
I have 4 async functions to execute in my node app in a waterfall style.
async.waterfall will tidy it up, but based on the result of the first function, I either want to error, carry on down the waterfall or break out to the finished function.
There is no way to do this at the moment (other than 'Error' with a success value, which just smells bad).
I was looking at using promises, (Bluebird), but again, I don't see a way to break out, if I DON'T return another promise in the .then, then all the following .thens are executed but with null parameters.
Is there another pattern / utility that can help?
or am I over thinking this and just call the first method normally , and then based on the result run the async.waterfall, or return the result!
Assuming I have 4 functions to call, a,b,c,d.....
async.waterfall([
function(cb) {
a("some data", cb)
},
function(data, cb) {
if(data) {
// HERE we want to skip the remaining waterfall a go to the final func
} else {
b("some data", cb);
}
},
c,
d
],
function(err, data) {
//do something
});
Or if they were promises...
a("some data")
.then(function(data) {
if(data) {
// here I want to stop executing the rest of the promises...
} else {
return b("some data")
}
})
.then(c)
.then(d)
.catch(function(err) {
//log error
})
.finally(function() {
// do something
})
I'm thinking I should just do this...
function doit(data, done) {
a("some data", function(err, data) {
if(err) { return done(err) };
if(data) { return done(null, data) };
//Here we run the waterfall if the above doesn't exist.
async.waterfall([
function(cb) {
b("some data", cb)
},
c,
d
],
function(err, data) {
done(err, data);
});
}
With promises, you can just attach 'followup promises' to the execution branch that you need:
a("some data")
.then(function(data) {
if(data) {
return "foo";//other promises won't be executed here
} else {
return b("some data").then(c).then(d)
}
})
.catch(function(err) {
//log error
})
.finally(function() {
// do something
})

Mongoose Find Results and Async

I am writing a NodeJS script that will run every hour through Heroku's scheduler. I am quering the Mongo instance I have (mongohq/compose) and then doing something with those results. I am working with Mongoose.js and the find() command. This returns an array of results. With those results I need to perform additional queries as well as some additional async processing (sending email, etc).
Long story short, due to node's async nature I need to wait until all the processing is complete before I call process.exit(). If I do not the script stops early and the entire result set is not processed.
The problem is that I have a christmas tree effect of calls at this point (5 nested asnyc calls).
Normally I'd solve this with the async.js library but I'm having a problem seeing this through with this many callbacks.
How can I make sure this entire process finishes before exiting the script?
Here's the code that I'm working with (note: also using lodash below as _):
Topic.find({ nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
finish();
} else {
_.forEach(topics, function (topic, callback) {
User.findById(topic.user, function (err, user) {
if (err) {
// TODO: impl logging
console.error(err);
} else {
// Create a new moment object (not moment.js, an actual moment mongoose obj)
var m = new Moment({ name: moment().format("MMM Do YY"), topic: topic});
m.save(function(err) {
if(err) {
// TODO: impl logging
console.error(err);
} else {
// Send an email via postmark
sendReminderTo(topic, user, m._id);
// Update the topic with next notification times.
// .. update some topic fields/etc
topic.save(function (err) {
if(err) {
console.error(err);
} else {
console.log("Topic updated.");
}
})
}
})
}
});
console.log("User: " + topic.user);
});
}
});
Part of what is making your code confusing is the usage of else statements. If you return your errors, you won't need the else statement and save 4 lines of indentation for every callback. That in and of itself will make things drastically more readable.
Using async:
Topic.find({nextNotificationDate: {$lte: moment().utc()}}, function (err, topics) {
if (err) {
console.error(err);
return finish(err);
}
async.each(topics, function(topic, topicCallback) {
async.auto({
user: function (callback) {
User.findById(topic.user, callback);
},
moment: function(callback) {
var m = new Moment({name: moment().format("MMM Do YY"), topic: topic});
m.save(callback);
},
topic: ["moment", "user", function (callback, results) {
var m = results.moment;
var user = results.user;
sendReminderTo(topic, user, m._id);
topic.save(callback);
}]
}, function(err) {
if (err) {
return topicCallback(err);
}
console.log("Topic updated.")
return topicCallback();
});
}, function(err) {
if (err) {
console.error(err);
return finish(err);
}
return finish();
});
});

Set Variable to result of Mongoose Find

I'm trying to do something like this
function retrieveUser(uname) {
var user = User.find({uname: uname}, function(err, users) {
if(err)
console.log(err);
return null;
else
return users[0];
return user;
But this returns a document instead of a user object. The parameter users is an array of user objects matching the query, so how would I store one of the objects into a variable that my function could return?
The function User.find() is an asynchronous function, so you can't use a return value to get a resultant value. Instead, use a callback:
function retrieveUser(uname, callback) {
User.find({uname: uname}, function(err, users) {
if (err) {
callback(err, null);
} else {
callback(null, users[0]);
}
});
};
The function would then be used like this:
retrieveUser(uname, function(err, user) {
if (err) {
console.log(err);
}
// do something with user
});
Updated on 25th Sept. 2019
Promise chaining can also be used for better readability:
Model
.findOne({})
.exec()
.then((result) => {
// ... rest of the code
return Model2.findOne({}).exec();
})
.then((resultOfModel2FindOne) => {
// ... rest of the code
})
.catch((error) => {
// ... error handling
});
I was looking for an answer to the same question.
Hopefully, MongooseJS has released v5.1.4 as of now.
Model.find({property: value}).exec() returns a promise.
it will resolve to an object if you use it in the following manner:
const findObject = (value) => {
return Model.find({property: value}).exec();
}
mainFunction = async => {
const object = await findObject(value);
console.log(object); // or anything else as per your wish
}
Basically, MongoDB and NodeJS have asynchronous functions so we have to make it to synchronous functions then after it will work properly as expected.
router.get('/', async function(req, res, next) {
var users = new mdl_users();
var userData = []; // Created Empty Array
await mdl_users.find({}, function(err, data) {
data.forEach(function(value) {
userData.push(value);
});
});
res.send(userData);
});
In Example, mdl_users is mongoose model and I have a user collection(table) for user's data in MongoDB database and that data storing on "userData" variable to display it.In this find function i have split all documents(rows of table) by function if you want just all record then use direct find() function as following code.
router.get('/', async function(req, res, next) {
var users = new mdl_users();
var userData = await mdl_users.find();
res.send(userData);
});

Categories