We Keep Coding

Recursive call to a async function in nodejs

I perform an async search call. The search results are not returned in one go. I get max 200 results at once. For next set of results I have to extract bookmark from the first set of results. So the query should happen sequentially. I have written following code but its not working. The next loop (for loop) is executing without finishing the first query. I dont know how to resolve this.
var getQueryResult = function(resourcetypeStr, startTime, endTime, bookmark){
var promise = new Promise(function(resolve, reject) {
var options = {
OPTIONS
};
search(DESIGN_DOC, INDEX, options) //Async function
.then(function (data) {
resolve(data);
})
.catch(function(error){
logger.error("Error querying database " + resourcetypeStr + ": " + error);
return reject(error);
});
});
return promise;
};
var getRemainingData = function(resourcetypeStr, startTime, endTime, bookmark, number){
var promise = new Promise(function(resolve, reject){
var result;
for(i = 0; i<number; i++ ){
var data = getQueryResult(resourcetypeStr, startTime, endTime, bookmark).then (function(data){
if(result){
result = result.concat(data);
}else{
result = data;
}
if(data.row.lenth ===0){
resolve(result);
}
bookmark = data.bookmark;
return data;
});
}
});
return promise;
};
var getResources = function (resourcetypeStr, list, startTime, endTime) {
var promise = new Promise(function(resolve, reject) {
var options = {
OPTIONS
};
return search(DESIGN_DOC, INDEX, options)//Async function
.then(function (data){
if(data.total_rows > 200){
debugger;
getRemainingData(resourcetypeStr, startTime, endTime, data.bookmark, function(result){
resolve(data.concat(result));
});
}else{
resolve(data);
}
})
.catch(function(error){
console.log("reject error :"+error);
return reject(error);
});
});
return promise;
};
I also tried
function getRemainingData(resourcetypeStr, uuidQueryString, startTime, endTime, bookmark, number, callback) {
var result; // clone collection
(function getOne(resourcetypeStr, uuidQueryString, startTime, endTime, bookmark) {
console.log("getOne is called");
getRemainingData(resourcetypeStr, uuidQueryString, startTime, endTime, bookmark).then(function(data){
if(result){
result = result.concat(dataToJson(data));
}else{
result = dataToJson(data);
}
if(data.row.lenth ===0){
callback(result);
}else{
setTimeout(getOne(resourcetypeStr, uuidQueryString, startTime, endTime, data.bookmark), 0);
}
} );
})();
}
with callback but this is overflowing stack. I am not sure why as in my test case there are only 300 results so it should ideal run only once here.

Have you tried something like this?
var promise = new Promise(function(resolve, reject){
var result,
callback = function callback(data) {
if (result) {
result = result.concat(data);
} else {
result = data;
}
if (data.row.length ===0) {
resolve(result);
}
else {
getQueryResult(resourcetypeStr, startTime, endTime, bookmark).then(callback);
}
};
getQueryResult(resourcetypeStr, startTime, endTime, bookmark).then(callback);
});
return promise;
When the getQueryResult returns, if the length is 0, we're done and can resolve the outer promise, else call getQueryResult again with the same callback until the length is 0. SO instead of a for loop, have each resolving getQueryResult call the next one if needed.
If this is similar to your second try, check the values of data.row.length, since this can only go infinite if data.row.length is never 0.

Why isn't my promise getting resolved?

I have two functions - A helper function for downloading files which is as follows
var downloadHelper = function(url, saveDir) {
var deferred = Q.defer();
setTimeout(function() {
deferred.resolve("success");
}, 2000);
return deferred.promise;
}
Now I have a list of files to be downloaded in parallel. I have the logic for that function as follows:
var downloadAll = function() {
var fileDownloadList = []
for(var key in config.files) {
var deferred = Q.defer();
var saveLocation = __base + config.localDir
downloadHelper(
config.files[key],
saveLocation
).then(function() {
deferred.resolve("downloaded: " + fileUrl);
}).catch(function(err) {
deferred.reject(err);
});
fileDownloadList.push(deferred.promise);
}
Q.all(fileDownloadList).done(function() {
console.log("All downloaded");
},function(err) {
console.log(err);
});
setTimeout(function() {
console.log(fileDownloadList);
}, 10000);
}
The done is never getting called!
For debugging purposes, I added a setTimeout that will be called after 10 seconds and what I see is that out of 2 files, the second promise is resolved and the first one is still in pending state.
Any ideas?
Thanks in advance

One way to make your code work
for(var key in config.files) {
(function() {
var deferred = Q.defer();
var saveLocation = __base + config.localDir
downloadHelper(
config.files[key],
saveLocation
).then(function() {
deferred.resolve("downloaded: " + fileUrl);
}).catch(function(err) {
deferred.reject(err);
});
fileDownloadList.push(deferred.promise);
}());
}
But since downloadhelper returns a promise, no need to create yet another one
for (var key in config.files) {
var saveLocation = __base + config.localDir
fileDownloadList.push(downloadHelper(
config.files[key],
saveLocation
).then(function () {
return("downloaded: " + fileUrl);
}));
}
You'll see I removed
.catch(function(err) {
deferred.reject(err);
})
That's redundant, it's the same as not having the catch at all

Resolve promise when all images are loaded

I was preloading images with the following code:
function preLoad() {
var deferred = $q.defer();
var imageArray = [];
for (var i = 0; i < $scope.abbreviations.length; i++) {
imageArray[i] = new Image();
imageArray[i].src = $scope.abbreviations[i].imgPath;
}
imageArray.forEach.onload = function () {
deferred.resolve();
console.log('Resolved');
}
imageArray.forEach.onerror = function () {
deferred.reject();
console.log('Rejected')
}
return deferred.promise;
}
preLoad();
I thought images were all loading correctly because I could see the 'Resolved' log.
Later somebody pointed out that the code above doesn't guarantee that all images are loaded before resolving the promise. In fact, only the first promise is resolved.
I was advised to use $q.all applied to an array of promises instead.
This is the resulting code:
function preLoad() {
var imageArray = [];
var promises;
for (var i = 0; i < $scope.abbreviations.length; i++) {
imageArray[i] = new Image();
imageArray[i].src = $scope.abbreviations[i].imgPath;
};
function resolvePromises(n) {
return $q.when(n);
}
promises = imageArray.map(resolvePromises);
$q.all(promises).then(function (results) {
console.log('array promises resolved with', results);
});
}
preLoad();
This works, but I want to understand:
what's happening in each function;
why I need $q.all to make sure all images are loaded before resolving the promises.
The relevant docs are somewhat cryptic.

Check out this plunkr.
Your function:
function preLoad() {
var promises = [];
function loadImage(src) {
return $q(function(resolve,reject) {
var image = new Image();
image.src = src;
image.onload = function() {
console.log("loaded image: "+src);
resolve(image);
};
image.onerror = function(e) {
reject(e);
};
})
}
$scope.images.forEach(function(src) {
promises.push(loadImage(src));
})
return $q.all(promises).then(function(results) {
console.log('promises array all resolved');
$scope.results = results;
return results;
});
}
The idea is very similar to Henrique's answer, but the onload handler is used to resolve each promise, and onerror is used to reject each promise.
To answer your questions:
1) Promise factory
$q(function(resolve,reject) { ... })
constructs a Promise. Whatever is passed to the resolve function will be used in the then function. For example:
$q(function(resolve,reject) {
if (Math.floor(Math.random() * 10) > 4) {
resolve("success")
}
else {
reject("failure")
}
}.then(function wasResolved(result) {
console.log(result) // "success"
}, function wasRejected(error) {
console.log(error) // "failure"
})
2) $q.all is passed an array of promises, then takes a function which is passed an array with the resolutions of all the original promises.

I'm not used to angular promise library, but the idea is as follows:
function getImagePromise(imgData) {
var imgEl = new Image();
imgEl.src = imgData.imgPath;
return $q(function(resolve, reject){
imgEl.addEventListener('load', function(){
if ((
'naturalHeight' in this
&& this.naturalHeight + this.naturalWidth === 0
)
|| (this.width + this.height == 0)) {
reject(new Error('Image not loaded:' + this.src));
} else {
resolve(this);
}
});
imgEl.addEventListener('error', function(){
reject(new Error('Image not loaded:' + this.src));
});
})
}
function preLoad() {
return $q.all($scope.abbreviations.map(getImagePromise));
}
// using
preLoad().then(function(data){
console.log("Loaded successfully");
data.map(console.log, console);
}, function(reason){
console.error("Error loading: " + reason);
});

stoping async tasks running in parallel

I'm trying to run through (using foreach) an array of objects and then for each I'd like to call a function that uses request to get a file and then unzips it with zlib, but one at a time, given the nature of node this is currently done asynchronously.
I'd like it to be done something like this...
- foreach - first object
- call function for first object
- when function has completed
- go to the next object in the array
I have tried using the SYNC module to try and solve this but with no luck.
Any ideas on how I can achieve this?
// the function i am trying to run for each in sync
var downloadUnzipFile = function(mID) {
try {
// Read File
console.log("Started download/unzip of merchant: " + mID + " # " + new Date().format('H:i:s').toString());
request(linkConst(mID))
// Un-Gzip
.pipe(zlib.createGunzip())
// Write File
.pipe(fs.createWriteStream(fileName(mID)))
.on('error', function(err) {
console.error(err);
})
.on('finish', function() {
console.log("CSV created: " + fileName(mID));
console.log("Completed merchant: " + mID + " # " + new Date().format('H:i:s').toString());
//console.log("Parsing CSV...");
//csvReader(fileName);
});
} catch (e) {
console.error(e);
}
}
module.exports = function(sMerchants) {
var oMerchants = JSON.parse(JSON.stringify(sMerchants));
sync(function() {
oMerchants.forEach(function eachMerchant(merchant) {
downloadUnzipFile(merchant.merchant_aw_id);
})
})
};

var promiseQueue = (function() {
'use strict';
var promiseQueue = function() {
var queue = [Promise.resolve(true)];
var add = function(cb) {
var args = Array.prototype.slice.call(arguments);
args.shift();
queue.unshift(new Promise(function(resolve) {
queue[0].then(function() {
resolve(cb.apply(null, args));
queue.pop();
});
}));
};
return {
add: add
}
}
return promiseQueue;
}());
usage EXAMPLE:
This is the asynch function that will be called
var theFun = function (time, n) { // use whatever arguments you like that will be called with your function
return new Promise(function(resolve) {
//asynch function goes here INSTEAD of the setTimeout and it's contents, I repeat, INSTEAD of the setTimeout
setTimeout(function() { // this is for demonstrating ONLY
console.log('resolving', n, time); // this is for demonstrating ONLY
resolve(time); // this is for demonstrating ONLY
}, time); // this is for demonstrating ONLY
// remember to resolve("someValueNotImportantAsItIsntUsedAnywhere") on completion of your asynch function
});
}
This is how the items get added to the queue - I did it this way because of MY use case
var pq = promiseQueue();
for(var i = 0; i < 5; i++ ) {
var r = 1000 - i * 150;
console.log('adding ', i, r);
pq.add(theFun, r, i);
}
Hope you find this of some use

First, your function needs to take a callback so it can communicate when it has finished:
var downloadUnzipFile = function(mID, next) {
try {
// Read File
console.log("Started download/unzip of merchant: " + mID + " # " + new Date().format('H:i:s').toString());
request(linkConst(mID))
// Un-Gzip
.pipe(zlib.createGunzip())
// Write File
.pipe(fs.createWriteStream(fileName(mID)))
.on('error', function(err) {
console.error(err);
})
.on('finish', function() {
console.log("CSV created: " + fileName(mID));
console.log("Completed merchant: " + mID + " # " + new Date().format('H:i:s').toString());
//console.log("Parsing CSV...");
//csvReader(fileName);
next();
});
} catch (e) {
console.error(e);
next();
}
}
Then, we need to recursively call each one when the previous has finished:
module.exports = function(sMerchants, next) {
var oMerchants = JSON.parse(JSON.stringify(sMerchants));
var i = 0;
var run = function() {
if(i < oMerchants.length)
downloadUnzipFile(i++, run);
else
next();
};
};
Note that I also added a callback to the exported function, so it can communicate when it is finished. If this is unnecessary, you can drop it.

This may work for you, uses Promise. Need to add resolve and reject callbacks to your downloadUnzipFile-
var exports = (function () {
'use strict';
var pre = document.getElementById('out');
function log(str) {
pre.appendChild(document.createTextNode(str + '\n'));
}
function downloadUnzipFile(id, resolve, reject) {
log('Start: ' + id);
try {
setTimeout(function () {
resolve(id);
}, 3000);
} catch (e) {
reject(e);
}
}
function done(id) {
log('Done: ' + id);
}
function error(e) {
log(e.message);
}
function getPromise(mID) {
return new Promise(function (resolve, reject) {
downloadUnzipFile(mID, resolve, reject);
});
}
return function (sMerchants) {
JSON.parse(sMerchants).reduce(function (next, mID) {
if (!next) {
next = getPromise(mID);
} else {
next = next.then(function (id) {
done(id);
return getPromise(mID);
}, error);
}
return next;
}, null).then(done, error);
};
}());
exports(JSON.stringify([1, 2, 3, 4, 5]));
<script src="https://cdnjs.cloudflare.com/ajax/libs/json2/20150503/json2.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/es5-shim/4.1.7/es5-shim.min.js"></script>
<script src="https://rawgit.com/jakearchibald/es6-promise/master/dist/es6-promise.min.js"></script>
<pre id="out"></pre>
I added the browser shims to support older browsers that may be viewing this, you shouldn't need them on node.js but you may need to require a Promise shim if you are using an old node.js.

asynchronous execution in protractor end to end tests

I have written a function , which is returning a value. In my main i am calling the function like this:
var fn_return_val = lesson.select_lesson(lesson1_text);
console.log("returned value is : " + fn_return_val);
And my function implementation is like(other file.js) :
module.exports = {
select_lesson:
function select_lesson(lesson_name) {
console.log('\n ************************* Lessson name: ' + lesson_name);
var desiredOption, status;
var repeter = element.all(by.repeater('item in items'));
repeter.then(function (items) {
items.forEach(function (icon) {
console.log('\n ************************* item');
icon.getText().then(function (txt) {
if (txt == lesson_name) {
desiredOption = icon;
}
})
}).then(function clickOption() {
if (desiredOption) {
var el = desiredOption.all(by.css('[ng-click="launchActivity()"]'));
var el_progress = desiredOption.all(by.css('.pna-progress'));
var abc = el.getAttribute('value').then(function (txt) {
status = txt;
return status
});
el_progress.getAttribute('style').then(function (progress) {
console.log('\n ************************* Lessson progress : ' + progress);
});
el.click();
}
});
});
}
};
The problem is function is returning "undefined" value, and the print statement console.log("returned value is : " + fn_return_val);
is executing before the function implementation
Can anyone help me on resolving this?

This is all about promises and protractor's Control Flow.
You need to resolve the promise and log the results inside then:
lesson.select_lesson(lesson1_text).then(function(fn_return_val) {
console.log("returned value is : " + fn_return_val);
});
And you also need to return from a function:
function select_lesson(lesson_name) {
...
// return here
return repeter.then(function (items) {
...
}).then(function clickOption() {
...
});
});
}