We Keep Coding

How to use async for callbacks using nodejs?

I have search function once i have search String from clinet i want to loop through files and match the string from files in fs, I have problem in loop i want to get all match result and send result to client. Below trying to achieve but getting an error pasted in question. New to async library any help will be appreciated.
app.js
app.get('/serverSearch', function (req, res) {
var searchTxt = req.query.searchTxt;
dirDirectory.readDirectory(function(logFiles){
// res.json(logFiles);
if(logFiles){
searchFileService.readFile(searchTxt,logFiles,function(lines,err){
console.log('Logs',lines);
if (err)
return res.send();
res.json(lines);
})
}
});
console.log('Search text', searchTxt);
});
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var searchStr;
var result = [];
//Async Method
function readFile(str, logFiles, callback) {
async.series([
//Load user to get `userId` first
function(callback) {
searchStr = str;
for (var i = 0; i < logFiles.length; i++) {
if (logFiles[i].filename !== '.gitignore') {
fs.readFile('logs/dit/' + logFiles[i].filename, 'utf8', function(err, data) {
if (err) {
return console.log(err);
}
inspectFile(data);
});
}
callback(result);
}
},
//Load posts (won't be called before task 1's "task callback" has been called)
function() {
function inspectFile(data, callback) {
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
result.push(line);
// then log it
return line;
}
});
}
}
], function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return err;
});
};
Error
if (fn === null) throw new Error("Callback was already called.");

You should be using async.map instead of series. You are miss understanding what series does, series process request top down. You are attempting to break this chain by accessing a function within the series itself. Which is a no, no.
for example:
async.series([
function() {
let i = 0;
do {
console.log("I'm first in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
},
function() {
let i = 0;
do {
console.log("I'm next in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
}
]);
The output of this would be:
I'm next in the series: 0
I'm next in the series: 1
I'm next in the series: 2
until the callback is fired, which then tells async to move to the next function in the series array.
The output then would be:
I'm last in the series: 0
I'm last in the series: 1
I'm last in the series: 2
At no point in this series should you be accessing the function within the series after the current. So you should never be trying to cross access that.
With async.map you can actually perform on operation on each entity within your array, which is essentially what you are trying to do.
var results = [];
async.map(logFiles, function(logfile, callback) {
if (logfile.filename !== '.gitignore') {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(err, null);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(null, results);
}
});
}
}), function(error, result) {
results.map(result => {
console.log(result);
});
});
Also you should use util.inspect instead of console.log, it's much cleaner and has more options.
The documentation on this is a bit rough, but here it is. https://caolan.github.io/async/docs.html#map hope this helps!

You should use async.eachSeries method:
function readFile(str, logFiles, callback) {
async.eachSeries(array, function(item, cb){
//Process item
cb(error,item);
}, function(err){
if (err) {
console.log("Some error in one of the items");
callback(err);
} else {
console.log("All arrays items have been treated successfully");
callback(null);
}
});
}
And I would recommend to load the user and posts before using the async.eachSeries function.

Q promises chaining to do things in correct order

I have a asynchronous function that needs to be called multiple times in the correct order. It's about uploading images to a server but like I said the images should be uploaded in the correct order.
My function looks like this:
function uploadImage(sku,fileName) {
console.log("Uploading image for sku="+sku+", imageName="+fileName);
var deferred = Q.defer();
var readStream = fs.createReadStream(rootPath+'/data/'+sku+"/"+fileName);
var req = request.post('http://localhost:3000/'+sku+'/upload/'+fileName);
readStream.pipe(req);
req.on('end', function() {
console.log("Image imageName="+fileName+" uploaded successfully.");
db.updateLastUploadedImage(sku,fileName).then(function (res) {
if(res) {
console.log("Table watches for sku="+sku+" updated.");
deferred.resolve(sku);
}
});
});
req.on('error',function(err) {
deferred.reject(err);
});
return deferred.promise;
}
I tried to bring it on with chaining the promises like documented in https://github.com/kriskowal/q but it's not working well. Somehow I do not come to the "then" block.
So I tried to make a recursive function but it also does not go inside the "then" block of the function call.
Only this method works but its not running through the correct order.
function uploadImages(sku) {
var promises = [];
for(var x=0; x<10; x++) {
promises.push(uploadImage(sku,(x+1)+".jpg")));
}
return Q.all(promises).then(function (res) {
return sku;
});
}
My recursive solution looks like this:
function uploadImages(sku,current,max) {
var deferred = Q.defer();
if(current<=max) {
uploadImage(sku,current+'.jpg').then(function (res) {
if(res) {
uploadImages(sku,current+1,max);
}
}, function (err) {
deferred.reject();
});
} else {
deferred.resolve(sku);
}
return deferred.promise;
}
What I'm looking for is something like this (but thats not the way to implement):
return uploadImage(sku,"1.jpg").then(function(res) {
return uploadImage(sku,"2.jpg").then(function(res) {
return uploadImage(sku,"3.jpg").then(function(res) {
return uploadImage(sku,"4.jpg").then(function(res) {
return uploadImage(sku,"5.jpg").then(function(res) {
return uploadImage(sku,"6.jpg").then(function(res) {
return uploadImage(sku,"7.jpg").then(function(res) {
return uploadImage(sku,"8.jpg").then(function(res) {
return uploadImage(sku,"9.jpg").then(function(res) {
return uploadImage(sku,"10.jpg").then(function(res) {
return sku;
});
});
});
});
});
});
});
});
});
});
So what is the best practice for my purpose?

There is no concept of "correct order" for async calls because they are just that -- asynchronous and they can end at any point.
In your the callback in Q.all(promises).then(...) you should have the responses in the order that you made them, but the order of your console logs may not be in the same order due their asynchronous nature.
In your case, you can probably do it recursively:
function uploadFiles(sku, current, max) {
return uploadImage(sku, current + '.jpg').then(function (sku) {
if (current > max) { return sku; }
return uploadFiles(sku, current + 1, max);
});
}
// use it
uploadFiles('SOME_SKU', 1, 10).then(function (sku) {
// ALL DONE!
});

Try to catch exceptions from the promise.
return Q.all(promises).then(function (res) {
return sku;
})
.catch(function (error) {
// Handle any error from all above steps
});

Javascript promises chaining

I'm using Parse Cloud functions for mobile apps, but all of them follow asynchronous nature. Hence to overcome that nature, I started using javascript Promises.
But promises also not giving me the desired output.
The problem is : second-then is executed after the third-then. And in third-then the parameter of getRecommendedGroup groups is getting []
getRecommendedGroup(request.params.email, groups, function(data){
this is a callback
});
Basically groups is an output of 2nd then.
So how should I write my code inorder to make 2nd-then execute before 3rd-then.
Below is the code snippet
Parse.Cloud.define("getGroup", function(request, response) {
var grpMember = new Parse.Query("GroupMember"),
groupIds = [],
groupObj = {};
grpMember.equalTo("user", request.params.email);
//from the groupMember we're taking groupId
grpMember.find()
.then(function(grpMemberResponse) {
grpMemberResponse.forEach(function(value, index) {
var memberObj = value;
groupIds.push(memberObj.get("groupId").id);
});
return groupIds;
})
//with this groupId we're retriving group and pushing it to an groupArr
.then(function(groupIds) {
var groupArr = [];
var promises = [];
groupIds.forEach(function(value, index) {
alert("GroupId :: " + value);
promises.push(findGroup(value, function(group) {
groupArr.push(group);
});
});
return Parse.Promise.when(promises);
})
.then(function(groups) {
alert("GroupArr :: " + JSON.stringify(groups));
getRecommendedGroup(request.params.email, groups, function(data) {
groupObj["own"] = groups;
groupObj["recommended"] = data;
response.success(groupObj);
});
});
});
var findGroup = function(objectId, callback) {
var groupQuery = new Parse.Query("Group");
groupQuery.get(objectId, {
success: function(group) {
alert("Group Obj Id ::" + group.id);
callback(group);
},
error: function(error) {
alert("Error while finding Group " + error);
}
});
};

The second then is running second, but it is calling an asynch function. In order to sequence properly, those promises being launched by the loop must be fulfilled. Fortunately, Promise.when() does that...
.then(function(groupIds) {
var findPromises = []
groupIds.forEach(function(value, index) {
findPromises.push(findGroup(value, function(group));
)};
return Parse.Promise.when(findPromises);
}).then() {
// arguments is now a (var arg) array of results from running
// all of the findPromises promises
});
Edit - In order for this to work, the code needs to push a promise to do the find. That means findGroup must return a promise...
function findGroup(objectId) {
var groupQuery = new Parse.Query("Group");
return groupQuery.get(objectId);
}

Bluebird Promising the result of a heavy function

I've been using Bluebird a lot recently on a HAPI API development. I've just run into my first real problem, that perhaps my understanding or naivety has me stumped.
The following code is an example of what I am facing:-
var Promise = require('bluebird'),
stuff = require('../stuff');
module.exports = {
getSomething: function(request, reply) {
var p = Promise.resolve();
p = p.then(function() {
return db.find() //etc. etc.
});
p = p.then(function(resultFromPromise) {
//problems begin here
var data = stuff.doSomeReallyLongAndBoringFunction(resultFromPromise);
return data;
});
p.then(function(data) {
//no data here.
});
};
};
I've commented where the problems usually begin. the stuff.doSomeReallyLongAndBoringFunction() returns an object (using more promises concidently) and it's this object I want to access, but //no data here always fires before data returns. stuff.doSomeReallyLongAndBoringFunction() continues to run regardless and completes successfully, but after the code goes async, I don't know how to promise that function's return value back.
Can anyone offer any guidance? Please accept my apologies for any naivety in the question!
Help as always, is appreciated
NB just for clarity, stuff.doSomeReallyLongAndBoringFunction() does not return a Promise itself. Although, I did try return new Promise(reject, resolve) { }); manual wrap. It is simply a function that uses promises itself (successfully) to get data.
Update 1
stuff.doSomeReallyLongAndBoringFunction() is too big to post directly, but it does something like this:-
var Promise = require('bluebird'),
rp = require('request-promise');
module.exports = {
doSomeReallyLongAndBoringFunction: function() {
var p = Promise.resolve();
p = p.then(function() {
return db.find() //etc. etc.
});
p.then(function() {
rp(options).then(function(response){
//get some data from remote location
}).then(function(dataFromService) {
//do some jiggery pokery with said data
var marshalledData = dataFromService;
db.something.create({
Field: 'something'
}).exec(function(err, saved) {
return marshalledData;
});
});
}).catch(function(err) {
});
};
};
Update 2
Thank you Justin for your help. Here is the actual code, perhaps this may help?
Promise.resolve()
.then(function() {
if(typeof utils.intTryParse(place) !== 'number') {
return foursquare.createPlaceFromFoursquare(sso, place, request, reply);
} else {
return { Place: { PlaceId: place }};
}
}).then(function(placeObj) {
console.log('Place set as', placeObj); //always returns undefined, despite function actually completing after async op...
});

If your doSomeReallyLongAndBoringFunction is really running asynchronously, then it doesn't make sense to run it the way you have setup.
Edit - Here's a simple explanation of the way your code looks to be running vs a refactored version. It's been simplified , so you'll need to fill in the relevant sections with your actual implementation.
var Promise = require('bluebird');
function myAsync() {
setTimeout(function(){
return 'done sleeping';
}, 2000);
};
//The way your code is running
Promise.resolve()
.then(function(){
return 'hello';
})
.then(function(done){
console.log(done);
return myAsync(); //your error is here
})
.then(function(done){
console.log(done);
});
//refactored
Promise.resolve()
.then(function(){
return 'hello';
})
.then(function(done){
console.log(done);
return new Promise(function(resolve) {
setTimeout(function(){
resolve('done sleeping');
}, 2000);
});
})
.then(function(done){
console.log(done);
});

just for clarity, stuff.doSomeReallyLongAndBoringFunction() does not return a Promise itself.
And that's your problem. As it does something asynchronous and you want to get its result, it should return a promise. In fact, that's the case for every asynchronous function, especially then callbacks! It should be something like
module.exports = {
doSomeReallyLongAndBoringFunction: function() {
return db.find()
// ^^^^^^
.then(function() {
return rp(options).then(function(response){
// ^^^^^^
//get some data from remote location
}).then(function(dataFromService) {
//do some jiggery pokery with said data
var marshalledData = dataFromService;
return db.something.create({
// ^^^^^^
Field: 'something'
}).execAsyc();
});
}).catch(function(err) {
});
}
};
Your getSomething method has the same issues, and should look like this:
var createPlace = Promise.promisify(foursquare.createPlaceFromFoursquare);
module.exports = {
getSomething: function(request) {
var p;
if (typeof utils.intTryParse(place) !== 'number')
p = createPlace(sso, place, request); // this returns a promise!
else
p = Promise.resolve({Place: {PlaceId: place}});
return p.then(function(placeObj) {
// ^^^^^^
console.log('Place set as', placeObj);
});
}
};
See also these generic rules for promise development.

doSomeReallyLongAndBoringFunction needs to look like this:
doSomeReallyLongAndBoringFunction: function(param) {
var resolver = Promise.defer();
/*
* do some asynchronous task and when you are finished
* in the callback, do this:
*/
resolver.resolve(resultFromAsyncTask);
/*
*
*
*/
return resolver.promise;
}

Asynchronus call inside recursive function

I am building a native desktop application in javascript using CEF, And I have API to access filesystem from CEF. I have a senario,
in which I need to get names of all files(there could be trees of directories) within a particular directory. I need to get result array, I am using jquery promises. What I don't understand is: when do I resolve the promise to get final result array?
/*read all directories under this and get path*/
var result = [];
function _processEntries(dirPath) {
var dirEntry = new NativeFileSystem.DirectoryEntry(dirPath), deferred = new $.Deferred();
/*async call*/
dirEntry.createReader().readEntries(
function (entries) {
for (var i = 0; i < entries.length; i++) {
if (entries[i].isDirectory) {
_processEntries(entries[i].fullPath).done(function () {
deferred.resolve(result);
});
} else {
result.push(entries[i].fullPath);
}
}
},
function (error) {
console.log("Failed while reading dir:", error);
}
);
return deferred.promise();
}
// Calling function
_processEntries("C:/Users/abc").done(function(result){
console.log("FILES ARRAY:",result);
});
Please suggest any other technique if I am doing it wrong :)

Mmmm i see few errors you resolve the entire promise when the first child returns, you must wait for all your child directory finish there respective promise, and your are resolve a global variable not internal, look this example with the changes:
function _processEntries(dirPath) {
var result = [];
var dirEntry = new NativeFileSystem.DirectoryEntry(dirPath), deferred = new $.Deferred();
/*async call*/
dirEntry.createReader().readEntries(
function (entries) {
var promises = [];
for (var i = 0; i < entries.length; i++) {
if (entries[i].isDirectory) {
promises.push(_processEntries(entries[i].fullPath));
} else {
result.push(entries[i].fullPath);
}
}
if(promises.length === 0) {
deferred.resolve(result);
} else {
$.when.apply($,promises).done(function() {
result.concat(arguments);
deferred.resolve(result);
})
}
},
function (error) {
console.log("Failed while reading dir:", error);
}
);
return deferred.promise();
}

when do I resolve the promise
Immediately when the asynchronous call has ended. You would have the deferred antipattern otherwise. Avoiding that, you can work with promises everywhere, and use proper composition with them.
Also, you should not initialise your result variable outside of the _processEntries function - you'd get huge problems with calling the function multiple times.
function getDirectoryEntries(dirPath) {
// a promise helper function
var dirEntry = new NativeFileSystem.DirectoryEntry(dirPath),
deferred = new $.Deferred();
dirEntry.createReader().readEntries(deferred.resolve, deferred.reject);
return deferred.promise();
}
function _processEntries(dirPath) {
return getDirectoryEntries(dirPath).then(function (entries) {
var promises = [];
for (var i = 0; i < entries.length; i++) {
if (entries[i].isDirectory) {
promises.push(_processEntries(entries[i].fullPath));
} else {
promises.push($.when(entries[i].fullPath));
}
}
return $.when.apply($, promises).then(function() {
return [].concat.apply([], arguments);
});
});
}
Call it like this:
_processEntries("C:/Users/abc").then(function(result){
console.log("FILES ARRAY:",result);
}, function (error) {
console.log("Failed while reading dir:", error);
});