I am trying to do a waterfall async but i don't get the expected output that i want.
Basically my waterfall works as expected if i use an array instead of the query
so i guess i am doing something wrong on the callback of the query but i don't know what.
Code when it works with what i expect using array:
function range(start, end) {
var foo = [];
for (var i = start; i <= end; i++) {
foo.push(i);
}
return foo;
}
users = range(1,2)
obj = [1,2];
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
results = {sku_config:'A',img:'http//blabla',sku_config:'B',img:'http//bloblo'}
callback(null, results);
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
request
loop
request
Finish
loop
request
loop
request
Finish
User For Loop Completed
But when i try to query the data with mysql here comes the problem
code:
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
loop
Finish
loop
loop
Finish
User For Loop Completed
request
request
request
request
All the request gets executed at the end :(
If you have idea on what i could fix.
Thanks
The first problem you have is that your callbacks have the exact same name, this could cause major problems. The callbacks you are meaning to call can not be differentiated, which could cause your program to execute pieces of code that shouldn't be executed until later.
The second problem is that the callback is placed outside of the request.get function. The nature of node js means that it does not wait until the request.get function returns and instead just calls the callback straight away. By placing the callback inside of the request.get function it is forced to wait until the request function returns and then the callback is called. A revised version of your code is below.
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, seriesCallback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
seriesCallback(null);
});
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
});
}, function(err){
console.log("User For Loop Completed");
});
Your callback(null); inside async.eachSeries are after request.
To fix just put inside request like this.
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
callback(null);
});
Plus to be clear what you actually calling rename callback functions. For example callback inside eachSeries call next
function(obj,callback) {
async.eachSeries(obj, function (sku, next) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
next(null);
});
}, function(responsetoendofloop){
callback(null);
});
}
Hope this helps.
Related
I have multiple functions created inside module.exports
I'm trying to invoke async retry on a function which is trying to do a POST request to an API and wait for response. It will retry for 3 times with timeout of 200s.
But everytime i run it the first iteration is executed, its able to call the function sendEnergy and initaite a POST request. But on fail when its retrying its throwing error
Error: Invalid arguments for async.retry
Here is my code-
module.exports = {
sendMessageToNewModel: (body, id) => {
url = '';
switch (id) {
case 'energy':
url = 'http://myapi.test.api/appliance'; //for appliance energy
break;
case 'device-energy':
url = 'http://myapi.test.api/device'; //for device energy
break;
}
async.retry({ times: 3, interval: 200 }, module.exports.sendEnergy(url, body), function (err, result) {
console.log(result)
});
},
sendEnergy: (url, body) => {
const options = {
url: url,
json: true,
body: body
};
request.post(options, (err, res, body) => {
if (!err) {
return res.statusCode;
}
});
}
}
Check this line
async.retry({ times: 3, interval: 200 }, module.exports.sendEnergy(url, body), function (err, result) {
console.log(result)
});"
module.exports.sendEnergy(url, body) = invoke sendEnergy(url, body)
which return nothing
According to the doc, it must be a function.
Without changing the structure , i could think of one way right now.
let sendEnergyRetry = sendEnergy.bind(null,url,body);
async.retry({ times: 3, interval: 200 }, sendEnergyRetry , function (err, result) {
console.log(result)
});"
I'm writing a function for my Node.js server (using express as a framework, MongoDB as the database). I have to call two mongoose queries:
(1) Find data filtered by parameters of one collection
(2) Aggregate data of another collection (needs result from the first query because one attribute gets appended to these results)
I now use the async.waterfall a function that makes it possible to ensure that the second operation only gets triggered after the first one has finished.
The variable resultResponse (at the end of my code snippet) has the right value when I run the function.
The problem I am struggling with:
The server throws the error TypeError: callback is not a function for the first callback.
This type error is also the answer I receive via postman calling /getRoutes.
So res.send() doesn't send the variable resultResponse back (which is correct), but the error.
Does anyone know, how I can solve this problem? Thanks a lot!
let async = require('async');
app.get('/getRoutes', function (req, res) {
var param = req.query;
let resultResponse;
async.waterfall([
(1)
function (param, callback) {
let dataCallback;
// first mongoose operation
Route.find(param).lean().exec(function (err, data) {
if (err)
throw err;
dataCallback = data;
}
.then(callback(null, dataCallback)));
},
TypeError: callback is not a function
(2)
function (aRoutes, callback) {
let oRoutes = [];
for (let i in aRoutes) {
// second mongoose operation
Rating.aggregate([{$match: {route: aRoutes[i]._id}}
, {
$group:
{_id: null, rating: {$avg: '$rating'}}
}
]).then(function (response) {
let oneRoute = aRoutes[i];
let avgRating = response[0].rating;
oneRoute.avg_rating = avgRating;
oRoutes.push(oneRoute);
if (i == (aRoutes.length - 1)) {
callback(null, oRoutes);
}
});
}
}],
function (err, result) {
if (err) throw err;
resultResponse = result; //console.log -> right result here
});
res.send(resultResponse); });
Your first function will be seeking callback function as the first parameter. Check here for more details. param is defined globally so you don't need to pass.
function (callback) {
let dataCallback;
// first mongoose operation
Route.find(param).lean().exec(function (err, data) {
if (err)
throw err;
dataCallback = data;
}).then(function(){
callback(null, dataCallback);
});
}
OR
Simply return data insteadof creating undefined object.
function (callback) {
// first mongoose operation
Route.find(param).lean().exec(function (err, data) {
if (err)
throw err;
return data;
}).then(function(dataCallback){
callback(null, dataCallback);
});
}
I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.
Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.
I am calling function dorequest many times per request to node server.
I have problem with request to webpage running on apache2.2.21. Almost of these request are done without any problems, but several request ending with error ECONNRESET and I don't know why. If I use apapche2.4 then everything going well.
var request = require('request');
function dorequest(set, callback){
request.get(url, function optionalCallback(err, httpResponse, body){
if (err){
console.log(url);
throw err;
} else {
//do some stuffs
}
});
}
Probably your apache server simply drops your request because there are too many connections at the same time initiated by dorequest function.
You can execute those request consequently by calling one in the callback of another by calling the next request in the callback for the previous one, but since there are quite a lot of them and for estetic reasons I would recommend to use async library - it's awesome and really handy when dealing with things like that.
function dorequest(set, callback){
request.get(url, function optionalCallback(err, httpResponse, body){
if (err){
callback(err);
} else {
//do some stuffs
}
callback(err, res);
});
}
var maxRequestAtATime = 30;
async.mapLimit(arrayOfOptions, maxRequestAtATime, dorequest, function(err, results){
// results is now an array of stats for each request
});
If the options of a request depend on the options of the previous one, you should use async.waterfall.
I updated script and use async.queue function for that and still have some err on apache.
function dorequest(set, callback)
{
console.log('add request');
q.push({set: set, callback: callback}, function (err) { });
}
var q = async.queue(function (task, callback) {
setTimeout(function () {
console.log('hello ' + task.set.url, ' lenght: ',q.length());
if (task.set.method=='get')
{
myrequest.get(task.set.url, function optionalCallback(err, httpResponse, body)
{
if (err)
{
console.log(task.set.url);
throw err;
}
else
{
//console.log(set.url,body);
if (typeof task.callback !='undefined') task.callback(body);
callback();
}
});
}
else
{
if (!task.set.data) task.set.data={};
myrequest.post(task.set.url, function optionalCallback(err, httpResponse, body)
{
if (err)
{
console.log(task.set.url);
throw err;
}
else
{
//console.log(set.url,body);
if (typeof task.callback !='undefined') task.callback(body);
callback();
}
}).form(task.set.data);
}
},500);
},1);
I need to request data from two web servers. The tasks are independent; therefore, I am using aync.parallel. Now I am only writing 'abc', 'xyz', and 'Done' to the body of my web page.
Since tasks are performed at the same time, can I run into a strange output? E.g.,
xab
cyz
The code.
var async = require('async');
function onRequest(req, res) {
res.writeHead(200, {
"Content-Type" : "text/plain"
});
async.parallel([ function(callback) {
res.write('a');
res.write('b');
res.write('c\n');
callback();
}, function(callback) {
res.write('x');
res.write('y');
res.write('z\n');
callback();
} ], function done(err, results) {
if (err) {
throw err;
}
res.end("\nDone!");
});
}
var server = require('http').createServer(onRequest);
server.listen(9000);
If you want to be absolutely certain in the order in which the results are printed, you should pass your data (abc\n and xyz\n) through the callbacks (first parameter is the error) and handle/write them in the final async.parallel callback's results argument.
async.parallel({
one: function(callback) {
callback(null, 'abc\n');
},
two: function(callback) {
callback(null, 'xyz\n');
}
}, function(err, results) {
// results now equals to: results.one: 'abc\n', results.two: 'xyz\n'
});