We Keep Coding

Async, waterfall issue

Here i am trying to retrieve all the objects and push them into the json file. For some reason there is only one record being pushed into file when it should contain more objects. The response is being sent even before the execution. Can you help me out with this or let me know where I am going wrong? Here is my code:
exports.createjoson = (req, res) => {
const Responsearray = [];
async.waterfall(
[
function(waterfallCb) {
// ... first function
},
function(results, waterfallCb1) {
//second function
async.eachLimit(
results,
100,
function(singleResult, eachCallback) {
async.waterfall(
[
async function(innerWaterfallCb) {
try {
NewsModel.find(
{ _id: singleResult.newsId }, // #individual article
async (err, newsResult) => {
if (err) {
return innerWaterfallCb(
// #displaying error
"error in fetching news data"
);
}
const map = new Map();
for (const item of newsResult) {
if (!map.has(item.source)) {
map.set(item.source, true);
Response = {
newsId: item._id,
title: item.title,
comment: singleResult.comment
};
}
}
resPond = await Response;
Responsearray.push(resPond);
let data = JSON.stringify(Responsearray);
await fs.writeFileSync("abc.json", data);
}
);
} catch (error) {
innerWaterfallCb(error);
}
}
],
function(err) {
if (err) {
return eachCallback(err);
}
eachCallback(null);
}
);
},
function(err) {
if (err) {
return waterfallCb1(err);
}
waterfallCb1(null);
}
);
}
],
function(err) {
if (err) {
return res.status(200).json({ status: "400", message: err });
}
res.status(200).json({ status: "200", message: "success" });
}
);
};

There are a number of problems with the code:
fs.writeFileSync will overwrite the file, not append to it, so only the last data you write will be in abc.json. Also it does not return a Promise so there is no need to use await on it. It runs synchronously so will not return until it's complete (that's what the Sync in its function name means). To append instead of overwrite the file, you can set the flag option to "a" to append (the default is "w").
There doesn't seem to be a call to return innerWaterfallCb(null) anywhere - only in error conditions. The inner waterfall function shouldn't be marked async since it doesn't need to do any await calls really. But you should call return innerWaterfallCb(null) after the file is written.
It may be better to just collect the data in responseArray and write the file once at the end of the outer waterfall instead of writing it repeatedly deep inside the inner waterfall.
Variables should start with lowercase letters (like responseArray not ResponseArray since uppercase first letters indicate classes or modules usually.
Don't mix async/await with the async module (waterfall and eachLimit). If you're using proper Promises and async/await then there should be no need to use the async module. It would be cleaner to remove the use of waterfall entirely and rewrite to use Promise objects properly.

{Status: 202} when running a lambda from code

This is my code in Javascript:
var params = {
FunctionName: "theTable",
InvokeArgs: JSON.stringify({ "name": "KirklandWA" })
};
lambda.invokeAsync(params, function (err, data) {
if (err) console.log(err, err.stack); // an error occurred
else {
console.log(data);
}
});
This is in Lambda:
exports.handler = async (event, context) => {
return "theReturnedValue";
};
What is happening is that it is not returning the theReturnedValue, instead returns
{Status: 202}
Status: 202
The code in Lambda is getting invoked, I made sure of it at Cloudwatch.

You're invoking with invokeAsync which will only return status code as stated in the documentation. Use invoke with InvocationType: "RequestResponse" instead
Reference:
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#invoke-property
var lambda = new AWS.Lambda({});
var params = {
FunctionName: "function_name",
InvocationType: "RequestResponse"
};
response = lambda.invoke(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});

Verify that your lambda this selected as NodeJs 8.10 runtime, once having verified you must return a promise directly:
exports.handler = async (event, context) => {
return await new Promise((resolve, reject) => {
resolve({ message: "hello world" });
});
};
I was looking invokeAsync method definition in the aws-sdk documentation and the status code is fine:
Callback parameters:
err (Error) — the error object returned from the request. Set to null if the request is successful.
data (Object) — the de-serialized data returned from the request. Set to null if a request error occurs. The data object has the
following properties:
Status — (Integer) It will be 202 upon success.
I prepare you a basic example, please review handler.js file:
https://github.com/ns4lin4s/stackoverflow
Dont forget, that you must specify response body:
So when you finish, let me know how that works!

The problem is that your lambda function is not returning anything back to the caller.
Your handle funtion has a third parameter which is a callback function that is used to return results to the caller.
The callback function takes in two values, an Error and results
callback(Error error, Object result);
If you provide the Error values the lambda will throw the an error you provide to the user, if you dont provide an Error value but the results, its the results that will be returned
Its all documented very well here
Here are basic examples
callback(); // Indicates success but no information returned to the caller.
callback(null); // Indicates success but no information returned to the caller.
callback(null, "success"); // Indicates success with information returned to the caller.
callback(error); // Indicates error with error information returned to the caller.
Your handler function should be.
exports.handler = async (event, context,callback) => {
callback(null, "theReturnedValue");
};

async waterfall not following order when mysql query

I am trying to do a waterfall async but i don't get the expected output that i want.
Basically my waterfall works as expected if i use an array instead of the query
so i guess i am doing something wrong on the callback of the query but i don't know what.
Code when it works with what i expect using array:
function range(start, end) {
var foo = [];
for (var i = start; i <= end; i++) {
foo.push(i);
}
return foo;
}
users = range(1,2)
obj = [1,2];
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
results = {sku_config:'A',img:'http//blabla',sku_config:'B',img:'http//bloblo'}
callback(null, results);
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
request
loop
request
Finish
loop
request
loop
request
Finish
User For Loop Completed
But when i try to query the data with mysql here comes the problem
code:
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
loop
Finish
loop
loop
Finish
User For Loop Completed
request
request
request
request
All the request gets executed at the end :(
If you have idea on what i could fix.
Thanks

The first problem you have is that your callbacks have the exact same name, this could cause major problems. The callbacks you are meaning to call can not be differentiated, which could cause your program to execute pieces of code that shouldn't be executed until later.
The second problem is that the callback is placed outside of the request.get function. The nature of node js means that it does not wait until the request.get function returns and instead just calls the callback straight away. By placing the callback inside of the request.get function it is forced to wait until the request function returns and then the callback is called. A revised version of your code is below.
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, seriesCallback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
seriesCallback(null);
});
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
});
}, function(err){
console.log("User For Loop Completed");
});

Your callback(null); inside async.eachSeries are after request.
To fix just put inside request like this.
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
callback(null);
});
Plus to be clear what you actually calling rename callback functions. For example callback inside eachSeries call next
function(obj,callback) {
async.eachSeries(obj, function (sku, next) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
next(null);
});
}, function(responsetoendofloop){
callback(null);
});
}
Hope this helps.

Results of tasks in async.auto

I am kind of confused with the logic of results which go from one task to the other task in async.auto. For example in the following code logic I added some data to models in task1, which is initially an output from initialtask and in finalTask added data to models from task1 is reflected in results.initialTask1 as well. Similarly added data in task2 is reflected in results.initialTask1 in finalTask.
To sum up all of results.initialTask1, results.task1[0], results.task2[0], results.task3[0] are identical in finalTask. Is this the logic of async.auto? Or is it something like reference by pointer in C++ which causes whatever changes for models in task1, it reflects in models in initialTask as well?
async.auto({
initialTask: function(callback) {
//Do some operations
callback(null, name, initialModels);
},
task1: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
task2: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
task3: ['initialTask', function(callback, results) {
var models = results.initialTask[1];
//Add some more data to models
callback(null, models);
}],
finalTask: ['task1', 'task2', 'task3', function(callback, results) {
//Here the followings are the same: results.initialTask[1], results.task1[0], results.task2[0], results.task3[0]
}]
});
I'm looking for any answer which helps me make sure that is the logic or not? I'm not necessarily looking for any official documents or ...

This is expected behavior. Basically async.auto will execute all the functions in the order it deems necessary. So in your case initialTask will be called first. Then task1, task2, and task3 will be called in parallel. Finally finalTask will be called with the results. The reason all the values are the same is related to JavaScript's call-by-sharing, meaning if you change a function parameter itself, then it won't affect the item that was fed into the parameter. If you change the internals of the parameter, it will carry up to the item.
More info here.
Example:
async.auto({
// this function will just be passed a callback
readData: async.apply(fs.readFile, 'data.txt', 'utf-8'),
showData: ['readData', function(results, cb) {
// results.readData is the file's contents
// ...
}]
}, callback);
async.auto({
get_data: function(callback) {
console.log('in get_data');
// async code to get some data
callback(null, 'data', 'converted to array');
},
make_folder: function(callback) {
console.log('in make_folder');
// async code to create a directory to store a file in
// this is run at the same time as getting the data
callback(null, 'folder');
},
write_file: ['get_data', 'make_folder', function(results, callback) {
console.log('in write_file', JSON.stringify(results));
// once there is some data and the directory exists,
// write the data to a file in the directory
callback(null, 'filename');
}],
email_link: ['write_file', function(results, callback) {
console.log('in email_link', JSON.stringify(results));
// once the file is written let's email a link to it...
// results.write_file contains the filename returned by write_file.
callback(null, {'file':results.write_file,
'email':'user#example.com'});
}]
}, function(err, results) {
console.log('err = ', err);
console.log('results = ', results);
});

async.auto is very useful and powerful function which is provided by Async Lib .it have 3 fields
1-task
2- concurrency
3-callback
In Async.auto, Each function depends on its parent function except the first function, if any function will get any error during execution .then their child function or say .. their below-defined function will not get executed further, an error will occur with callback and the main callback will immediately return with an error
1- Task :- an Object
2- concurrency :- An optional integer for determining the maximum number of tasks that can be run in parallel. By default, as many as possible.
3- callback:- return the response
exapmle-
AnyService.prototype.forgetPassword = function (res, email, isMobile, callback) {
Logger.info("In AnyService service forgetPassword email...", email);
db.User.findOne({
email: email.toLowerCase(),
deleted: false
}, function (err, user) {
if (!user) {
configurationHolder.responseHandler(res, null, configurationHolder.LoginMessage.registerFirst, true, 403)
} else {
async.auto({
token: function (next, results) {
return gereratePasswordToken(next, email, user, isMobile);
},
sendMail: ['token', function (next, result) {
return SendMailService.prototype.forgetPasswordMail(next, result.token, email, user.fullName);
}]
}, function (err, result) {
if (err == null && result != null) {
configurationHolder.ResponseUtil.responseHandler(res, null, configurationHolder.LoginMessage.forgotPassword, false, 200)
} else {
callback(new Error(configurationHolder.errorMessage.oops))
}
})
}
});
}

Understanding Node.JS async.parallel

I need to request data from two web servers. The tasks are independent; therefore, I am using aync.parallel. Now I am only writing 'abc', 'xyz', and 'Done' to the body of my web page.
Since tasks are performed at the same time, can I run into a strange output? E.g.,
xab
cyz
The code.
var async = require('async');
function onRequest(req, res) {
res.writeHead(200, {
"Content-Type" : "text/plain"
});
async.parallel([ function(callback) {
res.write('a');
res.write('b');
res.write('c\n');
callback();
}, function(callback) {
res.write('x');
res.write('y');
res.write('z\n');
callback();
} ], function done(err, results) {
if (err) {
throw err;
}
res.end("\nDone!");
});
}
var server = require('http').createServer(onRequest);
server.listen(9000);

If you want to be absolutely certain in the order in which the results are printed, you should pass your data (abc\n and xyz\n) through the callbacks (first parameter is the error) and handle/write them in the final async.parallel callback's results argument.
async.parallel({
one: function(callback) {
callback(null, 'abc\n');
},
two: function(callback) {
callback(null, 'xyz\n');
}
}, function(err, results) {
// results now equals to: results.one: 'abc\n', results.two: 'xyz\n'
});