Related
I've got my array of queries
var array = [ 'UPDATE EVALUATION SET mark = "16" WHERE id_eval = "21" AND id_usr = "125"',
'UPDATE EVALUATION SET mark = "9" WHERE id_eval = "22" AND id_usr = "125"',
'UPDATE EVALUATION SET mark = "8" WHERE id_eval = "34" AND id_usr = "125"'
]
However, when I try to execute them all at once with async, my web page says Waiting for localhost... and it keeps on loading forever. What am I doing wrong?
async.forEach(array, function(query, callback) {
connection.query(query, function(err, rows, fields) {
if(err) {
return console.error(err);
}
callback();
});
}, function(err){
if(err) {
return console.log(err);
}
});
Just make sure that you return the response after the forEach callback is called:
async.forEach(array, function(query, callback) {
connection.query(query, function(err, rows, fields) {
if(err) {
console.error(err);
}
callback();
});
}, function(err){
if(err) {
console.log(err);
}
res.redirect('/next-page');
});
That way, the redirection will occur only at the end of all the queries.
Some things you should verify:
Verify you didn't call res.end() or res.redirect() or anything similar before the above code.
Verify that your DB query method actually expect only 2 arguments: query and callback, and not anything in between (e.g. the query parameters).
Verify this piece of code is actually called when you expect it. Try debugging the request all the way.
Currently there's no real error handling here. You should consider returning an HTTP error if something went wrong. This should also help you debugging this code in the future.
Usually you have a request handler like this:
const server = http.createServer((req, res) => {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
res.end('Hello World\n');
});
Your browser holds the connection until res.endwas called!
In your environment it should look somethinge like
const server = http.createServer((req, res) => {
// ...
async.forEach(array, function(query, callback){
connection.query(query, function(err, rows, fields) {
// you may do some work here but leave it *alyways* via callback!
callback(err);
});
}, function(err){
if(err){ // this may be errors from above
return console.log(err);
}
// Exit here !
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
res.end('All done.\n');
});
});
...or a bit cleaner
function doSomeDatabaseUpdates(onFinished) {
var array = [...];
async.forEach(array, function(query, callback){
connection.query(query, function(err, rows, fields) {
// you may do some work here but leave it *alyways* via callback!
callback(err);
});
},
function IGetCalledAfterAboveCallbackWereExecuted(err){
if(err){ // this may be errors from above
return console.log(err);
}
// Exit here !
onFinished();
});
}
const server = http.createServer((req, res) => {
// ... do some work
if (doSomeUpdated === true) {
doSomeDatabaseUpdates(function calledAfterUpdates() {
res.end("updates something");
});
} else {
res.end("nothing to do");
}
});
I am trying to do a waterfall async but i don't get the expected output that i want.
Basically my waterfall works as expected if i use an array instead of the query
so i guess i am doing something wrong on the callback of the query but i don't know what.
Code when it works with what i expect using array:
function range(start, end) {
var foo = [];
for (var i = start; i <= end; i++) {
foo.push(i);
}
return foo;
}
users = range(1,2)
obj = [1,2];
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
results = {sku_config:'A',img:'http//blabla',sku_config:'B',img:'http//bloblo'}
callback(null, results);
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
request
loop
request
Finish
loop
request
loop
request
Finish
User For Loop Completed
But when i try to query the data with mysql here comes the problem
code:
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
loop
Finish
loop
loop
Finish
User For Loop Completed
request
request
request
request
All the request gets executed at the end :(
If you have idea on what i could fix.
Thanks
The first problem you have is that your callbacks have the exact same name, this could cause major problems. The callbacks you are meaning to call can not be differentiated, which could cause your program to execute pieces of code that shouldn't be executed until later.
The second problem is that the callback is placed outside of the request.get function. The nature of node js means that it does not wait until the request.get function returns and instead just calls the callback straight away. By placing the callback inside of the request.get function it is forced to wait until the request function returns and then the callback is called. A revised version of your code is below.
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, seriesCallback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
seriesCallback(null);
});
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
});
}, function(err){
console.log("User For Loop Completed");
});
Your callback(null); inside async.eachSeries are after request.
To fix just put inside request like this.
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
callback(null);
});
Plus to be clear what you actually calling rename callback functions. For example callback inside eachSeries call next
function(obj,callback) {
async.eachSeries(obj, function (sku, next) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
next(null);
});
}, function(responsetoendofloop){
callback(null);
});
}
Hope this helps.
High level
I'm new to JS and Node. I'm working on an API endpoint that should return a status value. To compute the status value I need to make two sequential mongo queries where the second set of queries depend on the first query. The second set of queries will give me a status for each value found in the first query, of which I will pick one based on some logic. What is the best way to do it in NodeJS?
Specifics
Here are parts of my first attempt.
function getItemStatus(key, value, callback) {
MongoClient.connect(mongo_url, function(err, db) {
if (err) { return console.dir(err); }
db.collection('status', function(err, coll) {
if (err) { return console.dir(err); }
coll.distinct("_id.metric", function(err, metrics) {
if (err) { return console.dir(err); }
console.log('metrics : ', metrics);
_.foreach(metrics, function(metric) {
var query = {"_id": {
"$gte" : {key: key, value: value, created: new Date("1800-01-01T00:00:00"), metric : metric},
"$lte" : {key: key, value: value, created: new Date("2100-01-01T00:00:00"), metric : metric}}};
coll.find(query, {sort: {"_id.created": -1}, limit: 1})
I make a connection, query for a set of metric values using a distinct query. For each metric I then want to ask for the latest status. Ideally I'd like to have the entire set of statuses so that I could write a function taking this set and deciding on which status will be returned. My problem is passing the statuses back "up the chain" so that I can process the set of statuses.
In a synchronous situation I would simply write something like this
val metrics = getDistinctMetrics(key, value)
val statuses = metrics.map(getStatusForMetric)
val status = filterStatuses(statuses)
How can I accomplish this in JavaScript/NodeJS?
UPDATED to highlight the fact that the first queries will trigger several queries in the second step, i.e. one for each result found by the first query.
As I understand your question you want to execute queries parallel or in a waterfall mode and do some logic on the final result. You should look into a library allowing parallel/waterfall execution. Like this
Waterfall: Waterfall
async.waterfall([
function(callback) {
callback(null, 'one', 'two');
},
function(arg1, arg2, callback) {
// arg1 now equals 'one' and arg2 now equals 'two'
callback(null, 'three');
},
function(arg1, callback) {
// arg1 now equals 'three'
callback(null, 'done');
}
], function (err, result) {
// result now equals 'done'
});
Parallel: Parallel
async.parallel({
collectionOne: function (callback) {
collectionOne.find(query, function (err, result) {
if (err) {
return handleError(res, err);
}
callback(null, result);
})
},
collectionTwo: function (callback) {
collectionTwo.find(query, function (err, result) {
if (err) {
return handleError(res, err);
}
callback(null, result);
})
},
collectionThree: function (callback) {
collectionThree.find(query, function (err, result) {
if (err) {
return handleError(res, err);
}
callback(null, result);
})
},
collectionFour: function (callback) {
collectionFour.find(query, function (err, result) {
if (err) {
return handleError(res, err);
}
callback(null, result);
})
},
}, function (err, results) {
return res.status(200).json(results);
});
And in the final callback you can doo some logic or return response.
In your sample code, you are making network calls inside another network calls, which can lead to callback hell which can lead to misbehave of queries, in order to overcome that, you can use promises.
This will help you in avoiding callback hell as well as your query will also be resolved.
Sample code:-
new Promise (function(resolve, reject){
return db.collection.find(function).exec()
.then(function(result_of_first_query){
return db.collection.findOne(function).exec() //in this yopu can user the result of first query
}).then(function(result_of_second_query){
resolve(result_of_second_query);
})
})
You can add more queries with each .then
I am kind of confused with the logic of results which go from one task to the other task in async.auto. For example in the following code logic I added some data to models in task1, which is initially an output from initialtask and in finalTask added data to models from task1 is reflected in results.initialTask1 as well. Similarly added data in task2 is reflected in results.initialTask1 in finalTask.
To sum up all of results.initialTask1, results.task1[0], results.task2[0], results.task3[0] are identical in finalTask. Is this the logic of async.auto? Or is it something like reference by pointer in C++ which causes whatever changes for models in task1, it reflects in models in initialTask as well?
async.auto({
initialTask: function(callback) {
//Do some operations
callback(null, name, initialModels);
},
task1: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
task2: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
task3: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
finalTask: ['task1', 'task2', 'task3', function(callback, results) {
//Here the followings are the same: results.initialTask[1], results.task1[0], results.task2[0], results.task3[0]
}]
});
I'm looking for any answer which helps me make sure that is the logic or not? I'm not necessarily looking for any official documents or ...
This is expected behavior. Basically async.auto will execute all the functions in the order it deems necessary. So in your case initialTask will be called first. Then task1, task2, and task3 will be called in parallel. Finally finalTask will be called with the results. The reason all the values are the same is related to JavaScript's call-by-sharing, meaning if you change a function parameter itself, then it won't affect the item that was fed into the parameter. If you change the internals of the parameter, it will carry up to the item.
More info here.
Example:
async.auto({
// this function will just be passed a callback
readData: async.apply(fs.readFile, 'data.txt', 'utf-8'),
showData: ['readData', function(results, cb) {
// results.readData is the file's contents
// ...
}]
}, callback);
async.auto({
get_data: function(callback) {
console.log('in get_data');
// async code to get some data
callback(null, 'data', 'converted to array');
},
make_folder: function(callback) {
console.log('in make_folder');
// async code to create a directory to store a file in
// this is run at the same time as getting the data
callback(null, 'folder');
},
write_file: ['get_data', 'make_folder', function(results, callback) {
console.log('in write_file', JSON.stringify(results));
// once there is some data and the directory exists,
// write the data to a file in the directory
callback(null, 'filename');
}],
email_link: ['write_file', function(results, callback) {
console.log('in email_link', JSON.stringify(results));
// once the file is written let's email a link to it...
// results.write_file contains the filename returned by write_file.
callback(null, {'file':results.write_file,
'email':'user#example.com'});
}]
}, function(err, results) {
console.log('err = ', err);
console.log('results = ', results);
});
async.auto is very useful and powerful function which is provided by Async Lib .it have 3 fields
1-task
2- concurrency
3-callback
In Async.auto, Each function depends on its parent function except the first function, if any function will get any error during execution .then their child function or say .. their below-defined function will not get executed further, an error will occur with callback and the main callback will immediately return with an error
1- Task :- an Object
2- concurrency :- An optional integer for determining the maximum number of tasks that can be run in parallel. By default, as many as possible.
3- callback:- return the response
exapmle-
AnyService.prototype.forgetPassword = function (res, email, isMobile, callback) {
Logger.info("In AnyService service forgetPassword email...", email);
db.User.findOne({
email: email.toLowerCase(),
deleted: false
}, function (err, user) {
if (!user) {
configurationHolder.responseHandler(res, null, configurationHolder.LoginMessage.registerFirst, true, 403)
} else {
async.auto({
token: function (next, results) {
return gereratePasswordToken(next, email, user, isMobile);
},
sendMail: ['token', function (next, result) {
return SendMailService.prototype.forgetPasswordMail(next, result.token, email, user.fullName);
}]
}, function (err, result) {
if (err == null && result != null) {
configurationHolder.ResponseUtil.responseHandler(res, null, configurationHolder.LoginMessage.forgotPassword, false, 200)
} else {
callback(new Error(configurationHolder.errorMessage.oops))
}
})
}
});
}
I want to know how to handle dependencies in using the async library in Node.js, look at the following example:
db.open(function(error, client) {
client.collection(('my-collection', function(error, collection) {
collection.insert({ "email": "test#gmail.com" }, function(error, docs) {
// Do stuff.
});
});
});
Using the async library:
async.parallel([
function(callback) {
db.open(function(error, client) {
callback(error, client);
});
},
function(callback) {
// How do I access the "client" variable at this point?
}
],
function(results){
// Do stuff.
});
You are using async parallell, which runs all functions together and they can finish in any order.
you can use the async waterfall function which passes varibles from one callback to the next function eg.
async.waterfall([
function(callback){
callback(null, 'one', 'two');
},
function(arg1, arg2, callback){
callback(null, 'three');
},
function(arg1, callback){
// arg1 now equals 'three'
callback(null, 'done');
}
], function (err, result) {
// result now equals 'done'
});
or you can use the auto function, which allowes you to specify which other functions must finish first eg.
async.auto({
get_data: function(callback){
// async code to get some data
},
make_folder: function(callback){
// async code to create a directory to store a file in
// this is run at the same time as getting the data
},
write_file: ['get_data', 'make_folder', function(callback){
// once there is some data and the directory exists,
// write the data to a file in the directory
callback(null, filename);
}],
email_link: ['write_file', function(callback, results){
// once the file is written let's email a link to it...
// results.write_file contains the filename returned by write_file.
}]
});
so what you could do in your case is this:
async.auto({
dbReady: function(callback) {
db.open(function(error, client) {
callback(error, client);
});
},
connected: ['dbReady', function(callback, results){
// How do I access the "client" variable at this point?
console.log(results.dbReady);
}
},
function(results){
// Do stuff.
});
Have a look at this to see all the available functions and their uses