We Keep Coding

Async promises recursion with unknow supply of async values

I'm banging my head around async promises recursion. I have bunch of promises that resolve when async data is download (combined by Promise.all). But sometimes in the data that I just download there is link to another data, that must be download (recursion). The best explanation is showing code I guess. Comments are in code.
(I have tried various combinations to no avail.)
var urls = ['http://czyprzy.vdl.pl/file1.txt', 'http://czyprzy.vdl.pl/file2.txt', 'http://czyprzy.vdl.pl/file3.txt'];
var urlsPromise = [];
var secondPart = [];
var thirdPart = [];
function urlContent(url, number) {
return new Promise(function (resolve) {
var dl = request(url, function (err, resp, content) {
if (err || resp.statusCode >= 400) {
return resolve({number : number, url : url, error : 'err'});
}
if (!err && resp.statusCode == 200) {
if (content.indexOf('file') !== -1) // if there is 'file' inside content we need (would like to :) download this new file by recursion
{
content = content.slice(content.indexOf('file') + 4);
content =+ content; // (number to pass later on, so we know what file we are working on)
url = 'http://czyprzy.vdl.pl/file' + content + '.txt'; // (we build new address)
//urlsPromise.push(urlContent(url, content)); // this will perform AFTER Promise.all(urlsPromise) so we simply can't do recurention (like that) here
secondPart.push(urlContent(url, content)); // if we use another promise array that put resolved items to that array everything will work just fine - but only till first time, then we would need to add another (thirdPart) array and use another Promise.all(thirdPart)... and so on and so on... --- the problem is I don't know how many files there will be, so it means I have no idea how many 'parts' for Promise.all I need to create, some kind of asynchronous loop/recursion would save me here, but I don't know how to do that properly so the code can run in proper order
}
return resolve({number : number, url : url}); // this goes to 'urlsPromise' array
}
});
});
}
if (urls.length !== 0) {
for (var i = 0; i < urls.length; i++)
{urlsPromise.push(urlContent(urls[i], i + 1));}
}
Promise.all(urlsPromise).then(function(urlsPromise) {
console.log('=======================================');
console.log('urlsPromise:\n');
console.log(urlsPromise); // some code/calculations here
}).then(function() {
return Promise.all(secondPart).then(function(secondPart) {
console.log('=======================================');
console.log('secondPart:\n');
console.log(secondPart); // some code/calculations here
secondPart.forEach(function(item)
{
thirdPart.push(urlContent(item.url, item.number + 3));
});
});
}).then(function() {
return Promise.all(thirdPart).then(function(thirdPart) {
console.log('=======================================');
console.log('thirdPart:\n');
console.log(thirdPart); // some code/calculations here
});
}).then(function()
{
console.log();
console.log('and so on and so on...');
});
//// files LINKING (those files do exist on live server - just for testing purposes):
// file1->file4->file7->file10 /-/ file1 content: file4 /-/ file4 content: file7 /-/ file7 content: file10
// file2->file5->file8->file11 /-/ file2 content: file5 /-/ file5 content: file8 /-/ file8 content: file11
// file3->file6->file9->file12 /-/ file3 content: file6 /-/ file6 content: file9 /-/ file9 content: file12
//// the console.log output looks like this:
// =======================================
// urlsPromise:
// [ { number: 1, url: 'http://czyprzy.vdl.pl/file4.txt' },
// { number: 2, url: 'http://czyprzy.vdl.pl/file5.txt' },
// { number: 3, url: 'http://czyprzy.vdl.pl/file6.txt' } ]
// =======================================
// secondPart:
// [ { number: 4, url: 'http://czyprzy.vdl.pl/file7.txt' },
// { number: 5, url: 'http://czyprzy.vdl.pl/file8.txt' },
// { number: 6, url: 'http://czyprzy.vdl.pl/file9.txt' } ]
// =======================================
// thirdPart:
// [ { number: 7, url: 'http://czyprzy.vdl.pl/file10.txt' },
// { number: 8, url: 'http://czyprzy.vdl.pl/file11.txt' },
// { number: 9, url: 'http://czyprzy.vdl.pl/file12.txt' } ]
// and so on and so on...

The await keyword can massively simplify this. You won't need to use a self recursive function. This demo fakes the server call with a randomly sized array.
https://jsfiddle.net/mvwahq19/1/
// setup: create a list witha random number of options.
var sourceList = [];
var numItems = 10 + Math.floor(Math.random() * 20);
for (var i = 0; i < numItems; i++)
{
sourceList.push(i);
}
sourceList.push(100);
var currentIndex = 0;
// a function which returns a promise. Imagine it is asking a server.
function getNextItem() {
var item = sourceList[currentIndex];
currentIndex++;
return new Promise(function(resolve) {
setTimeout(function() {
resolve(item);
}, 100);
});
}
async function poll() {
var collection = [];
var done = false;
while(!done) {
var item = await getNextItem();
collection.push(item);
console.log("Got another item", item);
if (item >= 100) {
done = true;
}
}
console.log("Got all items", collection);
}
poll();
You can write a normal for loop except the contents use await.

This answer was provided thanks to trincot - https://stackoverflow.com/users/5459839/trincot
When I asked him this question directly, he support me with time and knowledge and give this excellent answer.
CODE:
//// files LINKING (those files do exist on live server - just for testing purposes):
// file1->file4(AND file101)->file7->file10 /-/ file1 content: file4 /-/ file4 content: file7 /-/ file7 content: file10 /-/ file10 content: EMPTY /-/ file101 content: EMPTY
// file2->file5(AND file102)->file8->file11 /-/ file2 content: file5 /-/ file5 content: file8 /-/ file8 content: file11 /-/ file11 content: EMPTY /-/ file102 content: EMPTY
// file3->file6(AND file103)->file9->file12 /-/ file3 content: file6 /-/ file6 content: file9 /-/ file9 content: file12 /-/ file12 content: EMPTY /-/ file103 content: EMPTY
var urls = ['http://czyprzy.vdl.pl/file1.txt', 'http://czyprzy.vdl.pl/file2.txt', 'http://czyprzy.vdl.pl/file3.txt'];
var urlsPromise = [];
function requestPromise(url) {
return new Promise(function(resolve, reject) {
request(url, function (err, resp, content) {
if (err || resp.statusCode != 200) reject(err || resp.statusCode);
else resolve(content);
});
});
}
async function urlContent(url, number) {
var arr = [];
let content = await requestPromise(url);
while (content.indexOf(';') !== -1)
{
var semiColon = content.indexOf(';');
var fileLink = content.slice(content.indexOf('file'), semiColon + 1);
content = content.replace(fileLink, ''); // we need to remove the file link so we won't iterate over it again, we will add to the array only new links
var fileLinkNumber = fileLink.replace('file', '');
fileLinkNumber = fileLinkNumber.replace(';', '');
fileLinkNumber =+ fileLinkNumber;
url = 'http://czyprzy.vdl.pl/file' + fileLinkNumber + '.txt'; // we build new address
arr.push({url, fileLinkNumber});
}
if (content.indexOf('file') !== -1)
{
var fileLinkNumber = content.slice(content.indexOf('file') + 4);
fileLinkNumber =+ fileLinkNumber;
url = 'http://czyprzy.vdl.pl/file' + fileLinkNumber + '.txt';
arr.push({url, fileLinkNumber});
}
var newArr = arr.map(function(item)
{
return urlContent(item.url, item.fileLinkNumber); // return IS important here
});
return [].concat(arr, ...await Promise.all(newArr));
}
async function doing() {
let urlsPromise = [];
for (let i = 0; i < urls.length; i++) {
urlsPromise.push(urlContent(urls[i], i + 1));
}
let results = [].concat(...await Promise.all(urlsPromise)); // flatten the array of arrays
console.log(results);
}
//// this is only to show Promise.all chaining - so you can do async loop, and then wait for some another async data - in proper chain.
var test_a = ['http://czyprzy.vdl.pl/css/1.css', 'http://czyprzy.vdl.pl/css/2.css', 'http://czyprzy.vdl.pl/css/cssa/1a.css', 'http://czyprzy.vdl.pl/css/cssa/2a.css'];
var promisesTest_a = [];
function requestStyle(url)
{
return new Promise(function(resolve, reject)
{
request(url, function(error, response, content)
{
if (response.statusCode === 200 && !error)
{resolve(content);}
else
{reject(error);}
});
});
}
for (var i = 0; i < test_a.length; i++)
{promisesTest_a.push(requestStyle(test_a[i]));}
Promise.all(promisesTest_a).then(function(promisesTest_a)
{
console.log(promisesTest_a);
}).then(function()
{
console.log('\nNow we start with #imports...\n');
}).then(function()
{
return doing();
}).then(function()
{
console.log('ALL DONE!');
});
COMMENT:
At first the explanation what is [...] - destructured rest parameters (just in case if you don't know it).
var arr = [];
var array1 = ['one', 'two', 'three']
var array2 = [['four', 'five', ['six', 'seven']], 'eight', 'nine', 'ten'];
arr = array1.concat(array2);
console.log(arr); // it does not flattern the array - it just concatenate them (join them together)
console.log('---');
// however
arr = array1.concat(...array2);
console.log(arr); // notice the [...] - as you can see it flatern the array - 'four' and 'five' are pull out of an array - think of it as level up :) remember that it pull up WHOLE array that is deeper - so 'six' and 'seven' are now 1 level deep (up from 2 levels deep, but still in another array).
console.log('---');
// so
arr = [].concat(...arr);
console.log(arr); // hurrrray our array is flat (single array without nested elements)
console.log();
All files (links) that are ready to be download (those 3 starting ones in a urls array) are downloaded almost immediately (synchronous loop over array that contain them - one after the other, but very fast, right away cause we simply iterate over them in synchronous way).
Then, when we have their contents (cause we Await till content is downloaded - so we got a resolved promise data here) we start to look for info about other possible urls (files) related to the one we already got, to download them (via async recursion).
When we found all the info about possible additional urls/files (presented in an array of regexs - matches), we push it to data array (named arr in our code) and download them (thanks to the mutation of url).
We download them by return the async urlContent function that need to Await for requestPromise promise (so we have the resolve/rejected data in urlContent so if needed we can mutate it - build proper url to get the next file/content).
And so on, so on, till we "iterate" (download) over all files. Every time the urlContent is called, it return an array of promises (promises variable) that initially are pending. When we await Promise.all(promises) the execution only resumes at that spot when ALL those promises have been resolved. And so, at that moment, we have the values for each of these promises. Each of these is an array. We use one big concat to nit all those arrays together into one big array, also including the elements of arr (we need to remmeber that it can be more then 1 file to download from file we have already download - that is why we store values in data array - named arr in code - which store promiseReques function resolved/rejected values). This "big" array is the value, with which a promise is resolved. Recall that this promise is the one that was returned already by this current function context, at the time the first await was executed.
This is important part - so it (urlContent) returns (await) a single promise and that (returned) promise is resolved with an array as value. Note that an async function returns the promise to the caller immediately, when the first await is encountered. The return statement in an async function determines what the value is with which that returned promise is resolved. In our case that is an array.
So urlContent at every call return an promise - resolved value in a array - [...] (destructured rest parameters - returns a promise that eventually resolves to an array), that is collected by our async doing function (cause 3 urls was fired at start - every one has it own urlContent function... path), that collect (Await!) all those arrays from Promise.all(urlsPromise), and when they are resolved (we await for them to be resolved and passed by Promise.all) it 'return' your data (results variable). To be precise, doing returns a promise (because it is async). But the way that we call doing, we show we are not interested in what this promise resolves to, and in fact, since doing does not have a return statement, that promise resolves to UNDEFINED (!). Anyway, we don't use it - we merely output the results to the console.
One thing that can be confusing with async functions is that the return statement is not executed when the function returns (what is in a name, right!? ;). The function has already returned when it executed the first await. When eventually it executes the return statement, it does not really return a value, but it resolves "its own" promise; the one it had returned earlier. If we would really want to separate output from logic, we should not do console.log(results) there, but do return results, and then, where we call doing, we could do doing.then(console.log); Now we do use the promise returned by doing!
I would reserve the verb "to return" for what the caller of a function gets back from it synchronously.
I would use "to resolve" for the action that sets a promise to a resolved state with a value, a value that can be accessed with await or .then().

Using response from one API call to do another API Call

I'm using AngularJS to build a site where one of the functions is to present Billboard(it's a music chart) listings for a specified date.
I want to present the songs in order, together with an image of the song.
First I'm calling this API:
http://billboard.modulo.site/
where I give a date and get a response of the top 10 songs for that date and data about each song.
The response from the Billboard API also includes a spotify id and I want to use that ID and call the Spotify Web API to get an image of that song, to complement the information I present about each song.
This is how it looks like in my controller:
var spotifyID = [];
$scope.spotifyImg = [];
musicService.getBillboard($scope.date).then(function(data){ //Response is top 10 songs for given date
$scope.status = "";
$scope.songlist = data;
for (var i = 0; i < data.length; i++) {
spotifyID[i] = data[i].spotify_id; //data[i].spotify_id returns the ID of the track, as given by the billboard API
}
$scope.getImages();
});
$scope.getImages = function() {
for (var i = 0; i < spotifyID.length; i++) {
if(spotifyID[i] !== null) {
musicService.getSpotify(spotifyID[i]).then(function(data){
$scope.spotifyImg[i] = data.album.images[0].url; //returns the appropriate image from the Spotify Web API
});
}
}
console.log($scope.spotifyImg);
}
And in my view it would look something like this:
<div ng-repeat = "song in songlist">
<div>{{ song.rank }}</div>
<div>
<img ng-src=" {{ spotifyImg[$index] }}"/>
</div>
</div>
However, it does not work.
When I'm checking the $scope.spotifyImg array in the console, it is of length 11 and only has one element in index 10 and that is the image of the last song(that is the 10th song).
I'm a bit confused as to why the $scope.spotifyImg array only contains one element in index 10. Also why is the array of length 11 when the spotifyID is of length 10?
Any ideas of how I could solve this?

The problem is that getSpotify is run asynchronous, when the responses to these calls come in, i is probably set to spotifyID.length - 1 which means that all callback functions set the $scope.spotifyImg[spotifyID.length - 1] element.
Try this:
$scope.spotifyImg = [];
musicService.getBillboard($scope.date).then(function(data){
$scope.status = "";
$scope.songlist = data;
for (var i = 0; i < data.length; i++) {
$scope.getImage(data[i].spotify_id, i);
}
});
$scope.getImage = function(id, index) {
musicService.getSpotify(id).then(function(data){
$scope.spotifyImg[index] = data.album.images[0].url;
});
}

create separate function and put the content of for loop inside that function and call that function inside the loop
$scope.getImages = function() {
for (var i = 0; i < spotifyID.length; i++) {
if (spotifyID[i] !== null) {
sampleFunc(i);
}
}
}
function sampleFunc(i) {
musicService.getSpotify(spotifyID[i]).then(function(data) {
$scope.spotifyImg[i] = data.album.images[0].url; //returns the appropriate image from the Spotify Web API
});
}
I think reason you get only last index of an array is when you are calling promise inside loop, loop does't wait until the promise returns. it just keep executing.At the time promise returns loop is executed and it;s getting last index of the array. That's why you need to separately call the promise from for loop

You can use IIFE
(function(i){
musicService.getSpotify(spotifyID[i]).then(function (data) {
$scope.spotifyImg[i] = data.album.images[0].url;
});
})(i)
So,your getImages function should be like this.
$scope.getImages = function () {
for (var i = 0; i < spotifyID.length; i++) {
if (spotifyID[i] !== null) {
(function(i){
musicService.getSpotify(spotifyID[i]).then(function (data) {
$scope.spotifyImg[i] = data.album.images[0].url;
});
})(i)
}
}
}

Try using this code
Js code
var spotifyID = [];
$scope.spotifyImg = [];
musicService.getBillboard($scope.date).then(function(data) { //Response is top 10 songs for given date
$scope.status = "";
$scope.songlist = data;
for (var i = 0; i < data.length; i++) {
spotifyID[i] = data[i].spotify_id; //data[i].spotify_id returns the ID of the track, as given by the billboard API
}
$scope.getImages(0);
});
$scope.getImages = function(index) {
if (index == spotifyID.length) {
return;
}
musicService.getSpotify(spotifyID[index]).then(function(data) {
$scope.spotifyImg[index] = data.album.images[0].url; //returns the appropriate image from the Spotify Web API
// call recursive here
$scope.getImages(index++);
});
}
}
call your getimages function recursively so that will add you images in array.

Return array of array of objects with deferred

I am trying to get an array of "records" that each contain an array of "field objects" (each with properties .name, .value, and .fid).
Each array of records is obtained via an AJAX call to a database via an API call.
I want to be able to iterate through these later with some other code, but, for the moment, I'm trying to get the resultant array of arrays.
I'll be honest, I'm struggling with how to do this with promises...
If I was doing this synchronously I'd use a FOR TO loop and push each iteration into an array, but async is causing me issues...
My current code is as follows. This returns a single array of fields, rather tham an array of arrays. I think I need to push each 'thisArray' to another array. Anyone willing to help?:
NOTE:
ridList is a simple array of DB record IDs, e.g.: ["391", "392", "393", "394", "395", "398", "399", "400", "396", "397", "437"]
AVMI_db is the URL the AJAX call gets its data from.
///////////////////////////////////////////////////////////////
// Get record info for each array member
///////////////////////////////////////////////////////////////
function AVMI_getMultipleRecordInfoFromArray(ridList, AVMI_db) {
var promises = [];
$.each(ridList, function (index,value) {
var def = new $.Deferred();
var thisArray = [];
$.get(AVMI_db, { //******* ITERATIVE AJAX CALL *******
act: 'API_GetRecordInfo',
rid: value
}).then(function(xml2) {
$(xml2).find('field').each(function() {
var $field = {};
$field.fid = $(this).find('fid').text();
$field.name = $(this).find('name').text();
$field.value = $(this).find('value').text();
thisArray.push($field);
});
def.resolve(thisArray);
});
promises.push(def);
});
return $.when.apply(undefined, promises).promise();
};

You can create an array which is populated from as you iterate over the calls and return the array out after the promises are executed.
function AVMI_getMultipleRecordInfoFromArray(ridList, AVMI_db) {
var promises = [];
var returnValue = []; // to be returned by the promise
$.each(ridList, function (index,value) {
var def = new $.Deferred();
var thisArray = [];
$.get(AVMI_db, { //******* ITERATIVE AJAX CALL *******
act: 'API_GetRecordInfo',
rid: value
}).then(function(xml2) {
$(xml2).find('field').each(function() {
var $field = {};
$field.fid = $(this).find('fid').text();
$field.name = $(this).find('name').text();
$field.value = $(this).find('value').text();
thisArray.push($field);
});
returnValue.push(thisArray);
def.resolve(thisArray);
});
promises.push(def);
});
return $.when.apply(undefined, promises).then(function() {
return returnValue;
}).promise();
};

Maybe I'm missing something here, please feel free to comment if I didn't understood what you want. I undertood that you want to get an array of arrays instead of an array of fields. For this purpose, you gave the answer in your own question:
I think I need to push each 'thisArray' to another array.
But what puzzles me and makes me think I could have misunderstood something, is that if you can do code a bit complex as this, I think you could have made a simple change as that (or do you got this code somewhere ?), but anyway...
I can't test this because I would have to create the whole API call, but it should be as simple as:
///////////////////////////////////////////////////////////////
// Get record info for each array member
///////////////////////////////////////////////////////////////
function AVMI_getMultipleRecordInfoFromArray(ridList, AVMI_db) {
var promises = [];
$.each(ridList, function (index,value) {
var def = new $.Deferred();
var thisArray = [];
$.get(AVMI_db, { //******* ITERATIVE AJAX CALL *******
act: 'API_GetRecordInfo',
rid: value
}).then(function(xml2) {
$(xml2).find('field').each(function() {
var $field = [];
$field.push($(this).find('fid').text());
$field.push($(this).find('name').text());
$field.push($(this).find('value').text());
thisArray.push($fields);
});
def.resolve(thisArray);
});
promises.push(def);
});
return $.when.apply(undefined, promises).promise();

Only continue for loop when async get request finishes

I am writing a tool that will loop through a list of id's (represented by id in id_list). We check a cache object to see if we already have a value for the id. If we don't already have a value for the given id, we'll need to make a get request to get the associated value and then add it to the cache.
In the time it takes to do one async get request, the entire loop runs. This means the cache is never actually used. Is there anyway I can require the get request to finish before continuing the loop? Normally I would chain the request through the onSuccess function of the previous, but since there's a change, no request will be made.
cache = {};
var rating;
for (id in id_list){
if (id in cache){
rating = cache[id];
}else{
rating = $.get(~~~async get request happens here~~~);
cache[id] = rating;
}
$(".result").append(rating);//display result in ui
}

You can't use a for loop if you want it to wait between each iteration. A common design pattern is to create a local function for a given iteration and then call it each time the async operation finishes.
Assuming id_list is an object with properties, you could do it like this:
var cache = {};
var ids = Object.keys(id_list);
var cntr = 0;
function next() {
var id;
if (cntr < ids.length) {
id = ids[cntr++];
// see if we can just get the value from the cache
if (id in cache) {
$(".result").append(cache[id]);
// schedule next iteration of the loop
setTimeout(next, 1);
} else {
// otherwise get rating via Ajax call
$.get(...).then(function(rating) {
$(".result").append(rating);
// put rating in the cache
cache[id] = rating;
next();
});
}
}
}
next();
Or, if id_list is an array of ids, you can change it to this:
var cache = {};
var cntr = 0;
var id_list = [...];
function next() {
var id;
if (cntr < id_list.length) {
id = id_list[cntr++];
// see if we can just get the value from the cache
if (id in cache) {
$(".result").append(cache[id]);
// schedule next iteration of the loop
setTimeout(next, 1);
} else {
// otherwise get rating via Ajax call
$.get(...).then(function(rating) {
$(".result").append(rating);
// put rating in the cache
cache[id] = rating;
next();
});
}
}
}
next();

How To handle result set of websql in html 5?

I m creating mobile web application using html5 and javascript.I m having two javascript files. AttributesDatabase.js and AttributeView.js.From AttributeView.js i m calling one function from AttributeDatabase.js in that i m executing one select query.Now the query result should go to AtttributeView.js.But the Websql transaction is asynchronous call that is what it is not returning proper result.Is there any way to handle the websql result.
Please help if any way there?
Edited
AttributeView.js
var AttributeDAOObj = new AttributeDAO();
AttributeDAOObj.GetAttributeList();
alert(AttributeDAOObj.GetAttributeList()); //This alert is coming as undefined.
AttributeDAO.js
this.GetAttributeList = function () {
var baseDAOObj = new BaseDAO();
var query = "SELECT AttributeName FROM LOGS";
// this.Successcalbackfromsrc = this.myInstance.Successcalback;
var parm = { 'query': query, 'Successcalback': this.myInstance.Successcalback };
baseDAOObj.executeSql(parm);
}
//To Create database and execute sql queries.
function BaseDAO() {
this.myInstance = this;
//Creating database
this.GetMobileWebDB = function () {
if (dbName == null) {
var dbName = 'ABC';
}
var objMobileWebDB = window.openDatabase(dbName, "1.0", dbName, 5 * 1024 * 1024);
return objMobileWebDB;
}
//Executing queries and getting result
this.executeSql = function (query) {
var objMobileWebDB = this.myInstance.GetMobileWebDB();
objMobileWebDB.transaction(function (transaction) {
//In this transaction i m returning the result.The result value is coming.
transaction.executeSql(query, [], function (transaction, result) { return result; }, this.Errorclback);
});
}
}

The problem is in you succes call back (like in the comment to your question, stated by DCoder)
function (transaction, result) { return result; }
this is returning where to?
So this is how to do it (or at least one way)
you can do for example:
function (transaction,result){
console.log("yes, I have some result, but this doesn't say anything, empty result gives also a result");
// so check if there is a result:
if (result != null && result.rows != null) {
if (result.rows.length == 0) {
// do something if there is no result
}else{
for ( var i = 0; i < result.rows.length; i++) {
var row = result.rows.item(i);
var id = result.rows.item(i).id; //supposing there is an id in your result
console.log('Yeah! row id = '+id);
}
}
}else{
// do something if there is no result
}
};
note the code above can be compacter, but this is how to understand it better.
another way is to put this function is a seperate piece of code, so you keep the sql statement more compact and readable. Like you call you error callback this can be in your function (with this. in front of it) or a completely seperate function.