I'm working with NodeJS and the Async api to create an api function that returns a list of stories. The get can be shallow (only contains Object ID's referencing other objects) or deep (all ID references are dereferenced by replacing the ID with the object referenced by it). The shallow get works fine, however when I run the deep copy, it hangs. You can see in my callbacks I placed console.log(#) to log which callback is fired, but none are fired.
I feel like the issue lies within if I'm mistaking how async handles the callback function parameter for the .each, .serial and .parallel functions. I need a function that will be fired once async completes all of its tasks, but the callback function is instead called after every operation each, serial or parallel completed.
router.get('/stories', function(req, res, next) {
var db = req.db,
options = {
deep : req.query.deep != null ? parseInt(req.query.deep) : false,
offset : req.query.offset || 0,
limit : req.query.limit || 0
};
Story.listStories(db, options, function(err, stories){
if (!options.deep){
res.json(new Response(err, stories));
res.end();
}else{
if (err || stories == null){
res.json(new Response(err, null));
res.end();
return;
}
async.each(stories,
function(story, cb1){
var articles = [],
galleries = [];
async.series([
function(cb2){
async.parallel([
//Extract the story's articles and their outlets
function(cb3){
async.each(story.articles,
function(article_id, cb4){
Article.getArticle(db, article_id, function(err, article){
if (err){
cb4(err);
return;
}
Outlet.getOutlet(db, article.outlet, function(err, outlet){
if (err){
cb4(err);
return;
}
article.outlet = outlet;
articles.push(article);
});
});
},
function(err){console.log(4);
if (err)
cb3(err);
});
}
],
function(err){console.log(3); //Parallel callback
if (err)
cb1(err);
});
},
function(cb2){
story.articles = articles;
}
],
function(err){console.log(2);
if (err)
cb1(err);
});
},
function(err){console.log(1);
res.json(new Response(err, stories));
res.end();
}
);
}
});
});
You're calling those async callbacks (cb1, cb2, cb3, cb4, and etc) only for error cases. you need to call for non-error cases also. Example:
if (err) {
return cb1(err);
}
cb1(null); // or cb1()
Related
I am kind of confused with the logic of results which go from one task to the other task in async.auto. For example in the following code logic I added some data to models in task1, which is initially an output from initialtask and in finalTask added data to models from task1 is reflected in results.initialTask1 as well. Similarly added data in task2 is reflected in results.initialTask1 in finalTask.
To sum up all of results.initialTask1, results.task1[0], results.task2[0], results.task3[0] are identical in finalTask. Is this the logic of async.auto? Or is it something like reference by pointer in C++ which causes whatever changes for models in task1, it reflects in models in initialTask as well?
async.auto({
initialTask: function(callback) {
//Do some operations
callback(null, name, initialModels);
},
task1: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
task2: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
task3: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
finalTask: ['task1', 'task2', 'task3', function(callback, results) {
//Here the followings are the same: results.initialTask[1], results.task1[0], results.task2[0], results.task3[0]
}]
});
I'm looking for any answer which helps me make sure that is the logic or not? I'm not necessarily looking for any official documents or ...
This is expected behavior. Basically async.auto will execute all the functions in the order it deems necessary. So in your case initialTask will be called first. Then task1, task2, and task3 will be called in parallel. Finally finalTask will be called with the results. The reason all the values are the same is related to JavaScript's call-by-sharing, meaning if you change a function parameter itself, then it won't affect the item that was fed into the parameter. If you change the internals of the parameter, it will carry up to the item.
More info here.
Example:
async.auto({
// this function will just be passed a callback
readData: async.apply(fs.readFile, 'data.txt', 'utf-8'),
showData: ['readData', function(results, cb) {
// results.readData is the file's contents
// ...
}]
}, callback);
async.auto({
get_data: function(callback) {
console.log('in get_data');
// async code to get some data
callback(null, 'data', 'converted to array');
},
make_folder: function(callback) {
console.log('in make_folder');
// async code to create a directory to store a file in
// this is run at the same time as getting the data
callback(null, 'folder');
},
write_file: ['get_data', 'make_folder', function(results, callback) {
console.log('in write_file', JSON.stringify(results));
// once there is some data and the directory exists,
// write the data to a file in the directory
callback(null, 'filename');
}],
email_link: ['write_file', function(results, callback) {
console.log('in email_link', JSON.stringify(results));
// once the file is written let's email a link to it...
// results.write_file contains the filename returned by write_file.
callback(null, {'file':results.write_file,
'email':'user#example.com'});
}]
}, function(err, results) {
console.log('err = ', err);
console.log('results = ', results);
});
async.auto is very useful and powerful function which is provided by Async Lib .it have 3 fields
1-task
2- concurrency
3-callback
In Async.auto, Each function depends on its parent function except the first function, if any function will get any error during execution .then their child function or say .. their below-defined function will not get executed further, an error will occur with callback and the main callback will immediately return with an error
1- Task :- an Object
2- concurrency :- An optional integer for determining the maximum number of tasks that can be run in parallel. By default, as many as possible.
3- callback:- return the response
exapmle-
AnyService.prototype.forgetPassword = function (res, email, isMobile, callback) {
Logger.info("In AnyService service forgetPassword email...", email);
db.User.findOne({
email: email.toLowerCase(),
deleted: false
}, function (err, user) {
if (!user) {
configurationHolder.responseHandler(res, null, configurationHolder.LoginMessage.registerFirst, true, 403)
} else {
async.auto({
token: function (next, results) {
return gereratePasswordToken(next, email, user, isMobile);
},
sendMail: ['token', function (next, result) {
return SendMailService.prototype.forgetPasswordMail(next, result.token, email, user.fullName);
}]
}, function (err, result) {
if (err == null && result != null) {
configurationHolder.ResponseUtil.responseHandler(res, null, configurationHolder.LoginMessage.forgotPassword, false, 200)
} else {
callback(new Error(configurationHolder.errorMessage.oops))
}
})
}
});
}
Ok I have a JS method that uses Lodash and mongoose to find docs in a Mongoose collection. This looks fine, but it that appears to not finish the callback function before moving on to the next doc to look for. Below is my function:
importParts: function(participants, event_id, done){
_.forEach(participants, function (participant) {
var race = {event_id: event_id, chip_number_64: participant.chip_number_64};
Runner.findOne({contact_id: participant.contact_id}, function (err, doc) {
if (err) {
logger.error('CANNOT FIND AND UPDATE RUNNER BECAUSE OF: ', err);
done(err);
}
logger.info("IN FINDONE");
if (doc === null) {
logger.info("IN FINDONE1");
participant.races = [race];
Runner.create(participant, function (err, row, rowsAffected) {
if (err) {
logger.error('FAILED TO IMPORT PARTICIPANT: ', doc);
logger.error(err);
done(err);
}
logger.info("IN FINDONE2");
});
}
});
};
done(null);
}
For some reason the above code does not honor the callback function and appears to asynchronously return back to the main method that is calling this one. It's as if the callback is not being honored till after a set amount of time or something is happening asynchronously that shouldn't because I have everything wrapped in callbacks. So I am just trying to find out why the callback isn't completing when the query is executed? Also, this still happens even without the forEach iteration included in above code.
I am using NodeJS to create an express endpoint that will retrieve the metadata from my images stored on my server. I have the following code for the logic of the endpoint:
/*
* Gallery Controller
*/
var fs = require('fs'),
_ = require('underscore'),
im = require('imagemagick');
/**
* List paths to images stored in selected gallery
*/
exports.list = function(req, res) {
var dir = 'public/images/' + req.params.id;
fs.readdir(dir, function(err, files) {
if (err) return res.send({error: 'No gallery found with provided id'}, 404);
if (files.length > 0) {
var collection = [],
myData = {};
files.forEach(function(file) {
if(file === '.DS_Store') return;
im.readMetadata( dir + '/' + file, function(err, metadata) {
if (err) throw err;
myData = metadata;
console.log(myData); // logs as object with expected data
});
console.log(myData); // logs as empty {}
collection.push(myData);
});
console.log(collection); // logs as [ {}, {} ]
res.json(collection, 200);
} else {
res.json({error: 'Selected gallery is empty'}, 404);
}
});
};
I've listed what the logs appear as in the terminal, why am I getting this scoping issue? I can't seem to wrap my head around it. If I try to return the metadata obj and assign it to the var, I get the following error: TypeError: Converting circular structure to JSON
Use the async module, it'll improve your life in many ways.
The problem you are having is a common one I see, and it is that your loop is asynchronous, but you are treating it as something serial.
Instead of doing files.forEach, you want to loop them asynchronously and then do some more stuff when the looping is done. You can use async.each for that.
async.each(files, function (file, next) {
if (file === '.DS_Store') return next();
im.readMetadata(path.join(dir, file), function (e, data) {
collection.push(data);
next(err);
});
}, function (err) {
if (err) throw err;
console.log(collection);
});
As an alternative, an even more appropriate solution might be to use async.map.
async.map(files, function (file, next) {
if (file === '.DS_Store') return next();
im.readMetadata(path.join(dir, file), next);
}, function (err, collection) {
if (err) throw err;
console.log(collection);
});
You need to restructure your code:
files.forEach(function(file, i) {
if (file === '.DS_Store') return; // see text
im.readMetadata( dir + '/' + file, function(err, metadata) {
if (err) throw err;
collection.push(metadata);
if (i === files.length - 1) {
res.json(collection); // see text
}
});
});
The reason is that the metadata is only available when the callback function to readMetadata is called; that's how asynchronous I/O works in Node.
In that callback, you add the metadata to the collection. If the iteration of the forEach has reached the final element (i is the index of the current element, when its value is one less than the size of the array, it's the last element), the response is sent.
Two issues:
if .DS_Store is the last/only file in the directory, this code will fail because it will never send back a response; I'll leave it to you to deal with that case ;)
res.json will, by default, return a 200 status so you don't have to specify it; if you do want to specify a status, it needs to be res.json(200, collection) (arguments swapped)
I'm attempting to use the async module with NodeJS for the first time, and I'm running into a problem when I try and dynamically construct an array of functions for the async.parallel function to run:
methods = [];
for (key in entries) {
methods.push(function(callback) {
return callback(null, key);
});
}
return async.parallel(methods, function(err, results) {
console.log(results);
return render_views(req, res, 'view_blog_all', {
entries: entries
});
});
The output I keep seeing is:
[ 'powerful_sms_communication_for_teams',
'powerful_sms_communication_for_teams',
'powerful_sms_communication_for_teams' ]
And my 'entries' object I'm looping over definitely has three different keys. Something I'm missing?
ASYNC MODULE:
https://github.com/caolan/async
This is a common problem people run into with asynchronous logic. The key is to remember that your callback will not run until you call async.parallel, and at that point, key will the last key value in the loop.
One way to solve this is to capture the key value within a new scope using an IIFE.
methods = [];
for (key in entries) {
(function(key){
methods.push(function(callback) {
return callback(null, key);
});
})(key);
}
You can also use async.map with a normal array and single iterator instead.
return async.map(
entries,
function(key, callback){
callback(null, key);
},
function(err, results) {
console.log(results);
return render_views(req, res, 'view_blog_all', {
entries: entries
});
}
);
Here's a rewrite using https://github.com/caolan/async#map
async.map(entries, function(entry, callback) {
//do your stuff, fully parallel and async ;)
callback(null, entry);
}, function(err, results) {
console.log(results);
return render_views(req, res, 'view_blog_all', {
entries: entries
});
});
Each time I throw a query, the nest depth increase by one, just like the code below. If I knew how to define a query as a function not in the action, the readability of my code would increase.
exports.getAll = function (req, res) {
client.query('SELECT * FROM tag', function (err, result, fields) {
client.destroy();
if (err) {
throw err;
}
var tag = result[0].tag;
client.query('SELECT COUNT(follow_id) AS following_tag_num FROM follow WHERE user_id = ?', [req.session.user.user_id], function (err, result, fields) {
client.destroy();
if (err) {
throw err;
}
res.render('hoge', {
title: 'Welcome to Hoge',
userInfo: req.session.user,
tag: tag,
following_tag_num: result[0].following_tag_num
});
});
});
}
Just make the handler a named function:
client.query(
'SELECT COUNT(follow_id) AS following_tag_num FROM follow WHERE user_id = ?',
[req.session.user.user_id],
handleResult
);
function handleResult(err, result, fields) {
client.destroy();
if (err) {
throw err;
}
res.render('hoge', {
title : 'Welcome to Hoge',
userInfo : req.session.user,
tag : tag,
following_tag_num: result[0].following_tag_num
});
}
You might look into several node flow control modules that are available to help curb the nesting. I like one called async. It provides a variety of ways to de-nest your nested code.
var async = require('async');
async.waterfall([
function(callback) {
client.query(sql, callback);
},
function(results, callback) {
// do something with results, then call callback
}],
function(err, data) {
// if any errors occur above, err is not null
// otherwise 'data' is whatever got passed to the last callback
});
async.waterfall takes a list of functions, and passes the results of each one on to the next, finally calling the second parameter, another function, with the final result. Results are passed not by returning them, but by a callback function. async also supports running several functions in parallel, in series, and a variety of different common patterns used in node.