We Keep Coding

javascript api call, send and handle data in specific order

I have this API call where i make sure the data return in the same order i send it. However, i realized thats not really what i want, i want to make sure the data is send and taken care of one at a time.
data[n] has returned before data[n+1] is send.
the reason for this is:
If i do it as seen below, the server still gets it in a random order, and therefor saves the data in my DB in a random order. (or well not random, heavier data gets processed slower)
var promiseArray = [];
for (var i = 0; i < data.length; i++) {
var dataPromise = $http.post('/api/bla/blabla', $httpParamSerializer(data[i]))
.then (function (response) {
//return data for chaining
return response.data;
});
promiseArray.push(dataPromise);
}
$q.all(promiseArray).then(function (dataArray) {
//succes
}).catch (function (errorResponse) {
//error
});
how can i make sure the data is send and processed and returned, one at a time in a smooth way ?

You could do something like this:
var i = -1;
processNextdata();
function processNextdata() {
i++;
if(angular.isUndefined(data[i]))
return;
$http.post('/api/bla/blabla', $httpParamSerializer(data[i]))
.then(processNextdata)
}
Update:
Callback after every result:
var i = -1;
processNextdata();
function processNextdata() {
i++;
if(angular.isUndefined(data[i]))
return;
$http.post('/api/bla/blabla', $httpParamSerializer(data[i]))
.then(function(result) {
// do something with a single result
return processNextdata();
}, errorCallback);
}
Callback after everything is done:
var i = -1, resultData = [];
processNextdata()
.then(function(result) {
console.log(result);
}, errorCallback);
function processNextdata() {
i++;
if(angular.isUndefined(data[i]))
return resultData;
$http.post('/api/bla/blabla', $httpParamSerializer(data[i]))
.then(function(result) {
resultData.push(result.data);
return processNextdata();
}, $q.reject);
}

When using the Promise.all([...]) method, the documentation shows the following:
The Promise.all(iterable) method returns a promise that resolves when all of the promises in the iterable argument have resolved, or rejects with the reason of the first passed promise that rejects.
What this tells us is that there is no expected order of synchronized operations, but in fact the promises run parallel to one another and can complete in any order.
In your case, there is an expected order that you want your promises to run in, so using Promise.all([...]) won't satisfy your requirements.
What you can do instead is execute individual promises, then if you have some that can run in parallel use the Promise.all([...]) method.
I would create a method that takes a request as an argument, then returns the generated promise:
function request (req) {
return new Promise(function (resolve, reject) {
request({
url: url
, port: <port>
, body: req
, json: <true/false>
, method: '<POST/GET>'
, headers: {
}
}, function (error, response, body) {
if (error) {
reject(error);
} else {
resolve(body);
}
});
});
You can then call this function and store the result:
var response = request(myRequest);
Alternatively, you could create an array of your requests and then call the function:
var requests = [request1, request2, ..., requestN];
var responses = [];
for (var i = 0; i < requests.length; i++) {
responses.push(request(requests[i]));
}

Promises for loop angular confused

I understand using promises in simple scenarios but currently really confused on how to implement something when using a for loop and some updates to local sqlite database.
Code is as follows
surveyDataLayer.getSurveysToUpload().then(function(surveys) {
var q = $q.defer();
for (var item in surveys) {
var survey = surveys[item];
// created as a closure so i can pass in the current item due to async process
(function(survey) {
ajaxserviceAPI.postSurvey(survey).then(function(response) {
//from response update local database
surveyDataLayer.setLocalSurveyServerId(survey, response.result).then(function() {
q.resolve; // resolve promise - tried only doing this when last record also
})
});
})(survey) //pass in current survey used to pass in item into closure
}
return q.promise;
}).then(function() {
alert('Done'); // This never gets run
});
Any help or assistance would be appreciated. I'm probably struggling on how best to do async calls within loop which does another async call to update and then continue once completed.
at least promises have got me out of callback hell.
Cheers

This answer will get you laid at JS conferences (no guarantees though)
surveyDataLayer.getSurveysToUpload().then(function(surveys) {
return Promise.all(Object.keys(surveys).map(function(key) {
var survey = surveys[key];
return ajaxserviceAPI.postSurvey(survey).then(function(response){
return surveyDataLayer.setLocalSurveyServerId(survey, response.result);
});
}));
}).then(function() {
alert('Done');
});

This should work (explanations in comments):
surveyDataLayer.getSurveysToUpload().then(function(surveys) {
// array to store promises
var promises = [];
for (var item in surveys) {
var survey = surveys[item];
// created as a closure so i can pass in the current item due to async process
(function(survey) {
var promise = ajaxserviceAPI.postSurvey(survey).then(function(response){
//returning this promise (I hope it's a promise) will replace the promise created by *then*
return surveyDataLayer.setLocalSurveyServerId(survey, response.result);
});
promises.push(promise);
})(survey); //pass in current survey used to pass in item into closure
}
// wait for all promises to resolve. If one fails nothing resolves.
return $q.all(promises);
}).then(function() {
alert('Done');
});
Awesome tutorial: http://ponyfoo.com/articles/es6-promises-in-depth

You basically want to wait for all of them to finish before resolving getSurveysToUpload, yes? In that case, you can return $q.all() in your getSurveysToUpload().then()
For example (not guaranteed working code, but you should get an idea):
surveyDataLayer.getSurveysToUpload().then(function(surveys) {
var promises = [];
// This type of loop will not work in older IEs, if that's of any consideration to you
for (var item in surveys) {
var survey = surveys[item];
promises.push(ajaxserviceAPI.postSurvey(survey));
}
var allPromise = $q.all(promises)
.then(function(responses) {
// Again, we want to wait for the completion of all setLocalSurveyServerId calls
var promises = [];
for (var index = 0; index < responses.length; index++) {
var response = responses[index];
promises.push(surveyDataLayer.setLocalSurveyServerId(survey, response.result));
}
return $q.all(promises);
});
return allPromise;
}).then(function() {
alert('Done'); // This never gets run
});

Populate an array within an asynchronous function in Node.js

I recently started with Mongoose and Node.js and I have some difficulties in handling the result of an asynchronous Mongoose query. I have a collection called 'Groups' and a group can hold members located in a seperate collection called 'Members'. To accomplish this each group has a field named 'members' which is an array of ObjectId's. In addition each member has a profile which is refered to by an ObjectId. So each member has a field named 'profile' and the profiles are stored in a collection called 'Profiles'.
Now I want to get a list of profiles in a given group. Therefore I wrote this code:
// Function that returns a profile by a given objectId
function getprofile(profileId, profile) {
Profile
.findById(profileId)
.exec(function(err, res) {
if(err) { return null; }
if(!res) { return null; }
return res;
});
}
// Main function that returns a list of all (unique) profiles from members within a group
Group
.findById(req.params.id)
.populate('members')
.exec(function(err, group) {
if(err) { return handleError(res, err); }
var tmpProfiles = [];
var allProfiles = [];
for(var i=0; i<group.members.length; i++) {
var memberProfile = group.members[i].profile;
// Check if the member's profile is already in the array
if(objectIndexOf(tmpProfiles, memberProfile) == -1) {
tmpProfiles.push(memberProfile);
allProfiles.push(getprofile(memberProfile)); // add the profile to the array of profiles
}
}
return res.json(allProfiles); // this returns an empty array
});
The problem is that the 'Profile.findById' function and the 'Group.findById' function are both asynchronous and for this reason the function returns an array with only 'null' values. I know that I can solve this by using the 'Async' library but that is not an option for me so I have to go through the 'callback hell'. Can someone put me in the right direction by solving this problem? I already found some solutions which uses recursive calling of functions but I have no idea how to implement it in this case.

You can take advantage of the fact that mongoose methods return Promises.
Note: you will need to either be using Node >= v0.12 or have a Promise library required in the module.
// Function that returns a profile by a given objectId
function getprofile(profileId, profile) {
return Profile.findById(profileId); // return the Promise of this `findById` call
}
// Main function that returns a list of all (unique) profiles from members within a group
Group
.findById(req.params.id)
.populate('members')
.then(function(group) {
var tmpProfiles = [];
var promises = []; // store all promises from `getprofile` calls
for(var i = 0; i < group.members.length; i++) {
var memberProfile = group.members[i].profile;
// Check if the member's profile is already in the array
if(objectIndexOf(tmpProfiles, memberProfile) == -1) {
tmpProfiles.push(memberProfile);
promises.push(getprofile(memberProfile));
}
}
return Promise.all(promises);
})
.then(function(allProfiles) {
res.json(allProfiles);
})
.catch(function(err) {
//todo: handle error
console.log(err);
});

I support your learning by doing callback hell without caolan/async!
Here's an async answer anyway, to compare to your OP and the promises answer
Group...exec(function(err, group){
async.waterfall([
function(done1){
var tmpUnique = [];
async.filter(group.members, function(member, done2){
var isUnique = (tmpUnique.indexOf(member.profile) === -1);
if(isUnique){ tmpUnique.push(member.profile) };
done2(isUnique);
}, function(err, uniqueMembers){
done1(err, uniqueMembers);
})
},
function(uniqueMembers, done1){
async.map(uniqueMembers, function(member, done2){
getProfile(member.profile, done2);
// rewrite getProfile(id, callback){} to callback(err, profile)
// and maybe rename the key member.profile if its an id to get the real profile?
}, function(err, uniqueProfiles){
done1(err, uniqueProfiles)
});
},
], function(err, uniqueProfiles){
//callback(err, uniqueProfiles) or do something further
})
})

Nodejs Bluebird Promises - How to know if all promises are executed

According to this answer, the promise has been created, but the method 'then'( also 'done') will be executed without waiting for the output from subprocess, I need to have a method which is to be called after completely executing all subprocess, how this can be accomplished using bluebird api?
Sample Code
var Promise = require('bluebird')
var exec = require('child_process').exec
// Array with input/output pairs
var data = [
['input1', 'output1'],
['input2', 'output2'],
.....
]
var PROGRAM = 'cat'
Promise.some(data.map(function(v) {
var input = v[0]
var output = v[1]
new Promise(function(yell, cry) {
exec('echo "' + input + '" | ' + PROGRAM, function(err, stdout) {
if(err) return cry(err)
yell(stdout)
})
}).then(function(out) {
if(out !== output) throw new Error('Output did not match!')
})
}),data.length)
.then(function() {
// Send succes to user if all input-output pair matched
}).catch(function() {
// Send failure to the user if any one pair failed to match
})
Here the 'then' function is executed immediately even before the subprocess is completed.

Promise.some() expects an array of promises as its first argument. You are passing the results of data.map() to it, but your callback to data.map() never returns anything so therefore .map() doesn't construct an array of promises and therefore Promise.some() has nothing to wait on so it calls it's .then() handler immediately.
Also, if you're going to wait for all the promises, then you might as well use Promise.all() instead.
This is what I think you want.
Changes:
Switch to Promise.all() since you want to wait for all the promises.
Return the new Promise so .map() will create the array of promises.
Moved the output check into the original promise so it can reject rather than throw and it just seems like that moves all the result checking into one place.
Added all missing semi-colons.
Changed cry and yell to resolve and reject so code would be more familiar to outsiders expecting normal promise names.
Here's the code:
var Promise = require('bluebird');
var exec = require('child_process').exec;
// Array with input/output pairs
var data = [
['input1', 'output1'],
['input2', 'output2']
];
var PROGRAM = 'cat';
Promise.all(data.map(function(v) {
var input = v[0];
var output = v[1];
return new Promise(function(resolve, reject) {
exec('echo "' + input + '" | ' + PROGRAM, function(err, stdout) {
if(err) {
reject(err);
} else if (stdout !== output) {
reject(new Error('Output did not match!'));
} else {
resolve(stdout);
}
});
});
})).then(function() {
// Send succes to user if all input-output pair matched
}).catch(function() {
// Send failure to the user if any one pair failed to match
});

Call async function while pushing the same function on array

I have this code:
var queue = [];
var allParserd = [];
_.each(webs, function (web) {
queue.push(function () {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
allParserd.push(parsed);
});
});
});
Promise.all(queue).then(function (data) {
console.log(allParserd);
});
Basically I need to fetch all my webs and be sure to give the result after that every parsing is done. the function parseWebsite return the correct data, but in this way is not called and allParsed return just as an empty array. I'm sure that I miss some things, I've started to use the promises just from some days.
If you need some more information just tell me.
P.s.
I want that all the functions to start at the same time; I don't want to wait for each one response for going forward.

Tagged with Bluebird so let's use it:
First, let's convert your callback API to promises:
Promise.promisifyAll(WebsitesUtils);
Now, let's use .map to map every item in webs to it being parsed parseWebsite:
Promise.map(webs, function(item){
return WebsitesUtils.parseWebsiteAsync(item); // note the suffix
}).then(function(results){
// all the results are here.
}).catch(function(err){
// handle any errors
});
As you can see - this is trivial to do with Bluebird.

Promise.all doesn't take a queue of functions to execute. It expects an array of promises which represent the results of the many concurrently running (still pending) requests.
The first step is to have a function that actually returns a promise, instead of only executing a callback. We can use
function parseWebsite(web) {
return new Promise(function(fulfill, reject) {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
if (err)
reject(err);
else
fulfill(parsed);
});
});
}
or simply use promisification that does this generically:
var parseWebsite = Promise.promisify(WebsitesUtils.parseWebsite, WebsitesUtils);
Now we can go to construct our array of promises by calling that function for each site:
var promises = [];
_.each(webs, function (web) {
promises.push(parseWebsite(web));
});
or just
var promises = _.map(webs, parseWebsite);
so that in the end we can use Promise.all, and get back our allParsed array (which even is in the same order as webs was!):
Promise.all(promises).then(function(allParsed) {
console.log(allParsed);
});
Bluebird even provides a shortcut function so you don't need promises:
Promise.map(webs, parseWebsite).then(function(allParsed) {
console.log(allParsed);
});

Here's how might do it with async:
var async = require('async');
var webs = ...
async.map(webs, function(web, callback) {
WebsitesUtils.parseWebsite(web, callback);
}, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});
and if parseWebsite() isn't a prototype method dependent on WebsitesUtils then you could simplify it further:
async.map(webs, WebsitesUtils.parseWebsite, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});