We Keep Coding

Push inside forEach with query not working properly

I'm working with mongodb stitch/realm and I'm trying to modify objects inside an array with a foreach and also pushing ids into a new array.
For each object that i'm modifying, I'm also doing a query first, after the document is found I start modifying the object and then pushing the id into another array so I can use both arrays later.
The code is something like this:
exports = function(orgLoc_id, data){
var HttpStatus = require('http-status-codes');
// Access DB
const db_name = context.values.get("database").name;
const db = context.services.get("mongodb-atlas").db(db_name);
const orgLocPickupPointCollection = db.collection("organizations.pickup_points");
const orgLocStreamsCollection = db.collection("organizations.streams");
const streamsCollection = db.collection("streams");
let stream_ids = [];
data.forEach(function(stream) {
return streamsCollection.findOne({_id: stream.stream_id}, {type: 1, sizes: 1}).then(res => { //if I comment this query it will push without any problem
if(res) {
let newId = new BSON.ObjectId();
stream._id = newId;
stream.location_id = orgLoc_id;
stream.stream_type = res.type;
stream.unit_price = res.sizes[0].unit_price_dropoff;
stream._created = new Date();
stream._modified = new Date();
stream._active = true;
stream_ids.push(newId);
}
})
})
console.log('stream ids: ' + stream_ids);
//TODO
};
But when I try to log 'stream_ids' it's empty and nothing is shown. Properties stream_type and unit_price are not assigned.
I've tried promises but I haven't had success

It's an asynchronous issue. You're populating the value of the array inside a callback. But because of the nature of the event loop, it's impossible that any of the callbacks will have been called by the time the console.log is executed.
You mentioned a solution involving promises, and that's probably the right tack. For example something like the following:
exports = function(orgLoc_id, data) {
// ...
let stream_ids = [];
const promises = data.map(function(stream) {
return streamsCollection.findOne({ _id: stream.stream_id }, { type: 1, sizes: 1 })
.then(res => { //if I comment this query it will push without any problem
if (res) {
let newId = new BSON.ObjectId();
// ...
stream_ids.push(newId);
}
})
})
Promise.all(promises).then(function() {
console.log('stream ids: ' + stream_ids);
//TODO
// any code that needs access to stream_ids should be in here...
});
};
Note the change of forEach to map...that way you're getting an array of all the Promises (I'm assuming your findOne is returning a promise because of the .then).
Then you use a Promise.all to wait for all the promises to resolve, and then you should have your array.
Side note: A more elegant solution would involve returning newId inside your .then. In that case Promise.all will actually resolve with an array of the results of all the promises, which would be the values of newId.

Creating a recursive function with async / await with setTimeout

I have a process in my code where I need to get a list of technician drive times. I use the Google Maps API to get the driving time between the origin and destination. As most of you know, the API requires to have a timeout of roughly 1 second or more to work without generating errors. I have created a recursive function to retrieve the list of times that I need using a setTimeout within the method, like so:
function GetTechDriveTimes(info, destAddress) {
let techs = this.state.Techs
.filter(tech => tech.Address != "" && !tech.Notes.includes('Not'))
.map(tech => {
let techObj = {
TechName: tech.FirstName + " " + tech.LastName,
TechAddress: tech.Address + " " + tech.City + " " + tech.State + " " + tech.Zip,
KioskID: info.ID.toUpperCase(),
DriveTime: "",
};
return techObj
});
let temp = [...techs]; // create copy of techs array
const directionsService = new google.maps.DirectionsService();
recursion();
let count = 0;
function recursion() {
const techAddress = temp.shift(); // saves first element and removes it from array
directionsService.route({
origin: techAddress.TechAddress,
destination: destAddress,
travelMode: 'DRIVING'
}, function (res, status) {
if (status == 'OK') {
let time = res.routes[0].legs[0].duration.text;
techs[count].DriveTime = time;
} else {
console.log(status);
}
if (temp.length) { // if length of array still exists
count++;
setTimeout(recursion, 1000);
} else {
console.log('DONE');
}
});
}
return techs;
}
After this method is complete, it will return an array with the techs and their respective drive times to that destination. The problem here is that using setTimeout obviously doesn't stop execution of the rest of my code, so returning the array of technicians will just return the array with empty drive times.
After timeout is complete I want it to return the array within the method it was called like this:
function OtherMethod() {
// there is code above this to generate info and destAddress
let arr = GetTechDriveTimes(info, destAddress);
// other code to be executed after GetTechDriveTimes()
}
I've looked online for something like this, and it looks like I would need to use a Promise to accomplish this, but the difference from what I found online is that they weren't using it inside of a recursive method. If anyone has any ideas, that would help me a lot. Thanks!

You could use promises, but you can also create a callback with the "other code to be executed after GetTechDriveTimes" and send it to the function:
function OtherMethod() {
// there is code above this to generate info and destAddress
// instead of arr = GetTechDriveTimes, let arr be the parameter of the callback
GetTechDriveTimes(info, destAddress, function(arr) {
// other code to be executed after GetTechDriveTimes()
});
}
function GetTechDriveTimes(info, destAddress, callback) {
...
if (temp.length) { // if length of array still exists
...
} else {
console.log('DONE');
callback(techs); // send the result as the parameter
}
...

Async promises recursion with unknow supply of async values

I'm banging my head around async promises recursion. I have bunch of promises that resolve when async data is download (combined by Promise.all). But sometimes in the data that I just download there is link to another data, that must be download (recursion). The best explanation is showing code I guess. Comments are in code.
(I have tried various combinations to no avail.)
var urls = ['http://czyprzy.vdl.pl/file1.txt', 'http://czyprzy.vdl.pl/file2.txt', 'http://czyprzy.vdl.pl/file3.txt'];
var urlsPromise = [];
var secondPart = [];
var thirdPart = [];
function urlContent(url, number) {
return new Promise(function (resolve) {
var dl = request(url, function (err, resp, content) {
if (err || resp.statusCode >= 400) {
return resolve({number : number, url : url, error : 'err'});
}
if (!err && resp.statusCode == 200) {
if (content.indexOf('file') !== -1) // if there is 'file' inside content we need (would like to :) download this new file by recursion
{
content = content.slice(content.indexOf('file') + 4);
content =+ content; // (number to pass later on, so we know what file we are working on)
url = 'http://czyprzy.vdl.pl/file' + content + '.txt'; // (we build new address)
//urlsPromise.push(urlContent(url, content)); // this will perform AFTER Promise.all(urlsPromise) so we simply can't do recurention (like that) here
secondPart.push(urlContent(url, content)); // if we use another promise array that put resolved items to that array everything will work just fine - but only till first time, then we would need to add another (thirdPart) array and use another Promise.all(thirdPart)... and so on and so on... --- the problem is I don't know how many files there will be, so it means I have no idea how many 'parts' for Promise.all I need to create, some kind of asynchronous loop/recursion would save me here, but I don't know how to do that properly so the code can run in proper order
}
return resolve({number : number, url : url}); // this goes to 'urlsPromise' array
}
});
});
}
if (urls.length !== 0) {
for (var i = 0; i < urls.length; i++)
{urlsPromise.push(urlContent(urls[i], i + 1));}
}
Promise.all(urlsPromise).then(function(urlsPromise) {
console.log('=======================================');
console.log('urlsPromise:\n');
console.log(urlsPromise); // some code/calculations here
}).then(function() {
return Promise.all(secondPart).then(function(secondPart) {
console.log('=======================================');
console.log('secondPart:\n');
console.log(secondPart); // some code/calculations here
secondPart.forEach(function(item)
{
thirdPart.push(urlContent(item.url, item.number + 3));
});
});
}).then(function() {
return Promise.all(thirdPart).then(function(thirdPart) {
console.log('=======================================');
console.log('thirdPart:\n');
console.log(thirdPart); // some code/calculations here
});
}).then(function()
{
console.log();
console.log('and so on and so on...');
});
//// files LINKING (those files do exist on live server - just for testing purposes):
// file1->file4->file7->file10 /-/ file1 content: file4 /-/ file4 content: file7 /-/ file7 content: file10
// file2->file5->file8->file11 /-/ file2 content: file5 /-/ file5 content: file8 /-/ file8 content: file11
// file3->file6->file9->file12 /-/ file3 content: file6 /-/ file6 content: file9 /-/ file9 content: file12
//// the console.log output looks like this:
// =======================================
// urlsPromise:
// [ { number: 1, url: 'http://czyprzy.vdl.pl/file4.txt' },
// { number: 2, url: 'http://czyprzy.vdl.pl/file5.txt' },
// { number: 3, url: 'http://czyprzy.vdl.pl/file6.txt' } ]
// =======================================
// secondPart:
// [ { number: 4, url: 'http://czyprzy.vdl.pl/file7.txt' },
// { number: 5, url: 'http://czyprzy.vdl.pl/file8.txt' },
// { number: 6, url: 'http://czyprzy.vdl.pl/file9.txt' } ]
// =======================================
// thirdPart:
// [ { number: 7, url: 'http://czyprzy.vdl.pl/file10.txt' },
// { number: 8, url: 'http://czyprzy.vdl.pl/file11.txt' },
// { number: 9, url: 'http://czyprzy.vdl.pl/file12.txt' } ]
// and so on and so on...

The await keyword can massively simplify this. You won't need to use a self recursive function. This demo fakes the server call with a randomly sized array.
https://jsfiddle.net/mvwahq19/1/
// setup: create a list witha random number of options.
var sourceList = [];
var numItems = 10 + Math.floor(Math.random() * 20);
for (var i = 0; i < numItems; i++)
{
sourceList.push(i);
}
sourceList.push(100);
var currentIndex = 0;
// a function which returns a promise. Imagine it is asking a server.
function getNextItem() {
var item = sourceList[currentIndex];
currentIndex++;
return new Promise(function(resolve) {
setTimeout(function() {
resolve(item);
}, 100);
});
}
async function poll() {
var collection = [];
var done = false;
while(!done) {
var item = await getNextItem();
collection.push(item);
console.log("Got another item", item);
if (item >= 100) {
done = true;
}
}
console.log("Got all items", collection);
}
poll();
You can write a normal for loop except the contents use await.

This answer was provided thanks to trincot - https://stackoverflow.com/users/5459839/trincot
When I asked him this question directly, he support me with time and knowledge and give this excellent answer.
CODE:
//// files LINKING (those files do exist on live server - just for testing purposes):
// file1->file4(AND file101)->file7->file10 /-/ file1 content: file4 /-/ file4 content: file7 /-/ file7 content: file10 /-/ file10 content: EMPTY /-/ file101 content: EMPTY
// file2->file5(AND file102)->file8->file11 /-/ file2 content: file5 /-/ file5 content: file8 /-/ file8 content: file11 /-/ file11 content: EMPTY /-/ file102 content: EMPTY
// file3->file6(AND file103)->file9->file12 /-/ file3 content: file6 /-/ file6 content: file9 /-/ file9 content: file12 /-/ file12 content: EMPTY /-/ file103 content: EMPTY
var urls = ['http://czyprzy.vdl.pl/file1.txt', 'http://czyprzy.vdl.pl/file2.txt', 'http://czyprzy.vdl.pl/file3.txt'];
var urlsPromise = [];
function requestPromise(url) {
return new Promise(function(resolve, reject) {
request(url, function (err, resp, content) {
if (err || resp.statusCode != 200) reject(err || resp.statusCode);
else resolve(content);
});
});
}
async function urlContent(url, number) {
var arr = [];
let content = await requestPromise(url);
while (content.indexOf(';') !== -1)
{
var semiColon = content.indexOf(';');
var fileLink = content.slice(content.indexOf('file'), semiColon + 1);
content = content.replace(fileLink, ''); // we need to remove the file link so we won't iterate over it again, we will add to the array only new links
var fileLinkNumber = fileLink.replace('file', '');
fileLinkNumber = fileLinkNumber.replace(';', '');
fileLinkNumber =+ fileLinkNumber;
url = 'http://czyprzy.vdl.pl/file' + fileLinkNumber + '.txt'; // we build new address
arr.push({url, fileLinkNumber});
}
if (content.indexOf('file') !== -1)
{
var fileLinkNumber = content.slice(content.indexOf('file') + 4);
fileLinkNumber =+ fileLinkNumber;
url = 'http://czyprzy.vdl.pl/file' + fileLinkNumber + '.txt';
arr.push({url, fileLinkNumber});
}
var newArr = arr.map(function(item)
{
return urlContent(item.url, item.fileLinkNumber); // return IS important here
});
return [].concat(arr, ...await Promise.all(newArr));
}
async function doing() {
let urlsPromise = [];
for (let i = 0; i < urls.length; i++) {
urlsPromise.push(urlContent(urls[i], i + 1));
}
let results = [].concat(...await Promise.all(urlsPromise)); // flatten the array of arrays
console.log(results);
}
//// this is only to show Promise.all chaining - so you can do async loop, and then wait for some another async data - in proper chain.
var test_a = ['http://czyprzy.vdl.pl/css/1.css', 'http://czyprzy.vdl.pl/css/2.css', 'http://czyprzy.vdl.pl/css/cssa/1a.css', 'http://czyprzy.vdl.pl/css/cssa/2a.css'];
var promisesTest_a = [];
function requestStyle(url)
{
return new Promise(function(resolve, reject)
{
request(url, function(error, response, content)
{
if (response.statusCode === 200 && !error)
{resolve(content);}
else
{reject(error);}
});
});
}
for (var i = 0; i < test_a.length; i++)
{promisesTest_a.push(requestStyle(test_a[i]));}
Promise.all(promisesTest_a).then(function(promisesTest_a)
{
console.log(promisesTest_a);
}).then(function()
{
console.log('\nNow we start with #imports...\n');
}).then(function()
{
return doing();
}).then(function()
{
console.log('ALL DONE!');
});
COMMENT:
At first the explanation what is [...] - destructured rest parameters (just in case if you don't know it).
var arr = [];
var array1 = ['one', 'two', 'three']
var array2 = [['four', 'five', ['six', 'seven']], 'eight', 'nine', 'ten'];
arr = array1.concat(array2);
console.log(arr); // it does not flattern the array - it just concatenate them (join them together)
console.log('---');
// however
arr = array1.concat(...array2);
console.log(arr); // notice the [...] - as you can see it flatern the array - 'four' and 'five' are pull out of an array - think of it as level up :) remember that it pull up WHOLE array that is deeper - so 'six' and 'seven' are now 1 level deep (up from 2 levels deep, but still in another array).
console.log('---');
// so
arr = [].concat(...arr);
console.log(arr); // hurrrray our array is flat (single array without nested elements)
console.log();
All files (links) that are ready to be download (those 3 starting ones in a urls array) are downloaded almost immediately (synchronous loop over array that contain them - one after the other, but very fast, right away cause we simply iterate over them in synchronous way).
Then, when we have their contents (cause we Await till content is downloaded - so we got a resolved promise data here) we start to look for info about other possible urls (files) related to the one we already got, to download them (via async recursion).
When we found all the info about possible additional urls/files (presented in an array of regexs - matches), we push it to data array (named arr in our code) and download them (thanks to the mutation of url).
We download them by return the async urlContent function that need to Await for requestPromise promise (so we have the resolve/rejected data in urlContent so if needed we can mutate it - build proper url to get the next file/content).
And so on, so on, till we "iterate" (download) over all files. Every time the urlContent is called, it return an array of promises (promises variable) that initially are pending. When we await Promise.all(promises) the execution only resumes at that spot when ALL those promises have been resolved. And so, at that moment, we have the values for each of these promises. Each of these is an array. We use one big concat to nit all those arrays together into one big array, also including the elements of arr (we need to remmeber that it can be more then 1 file to download from file we have already download - that is why we store values in data array - named arr in code - which store promiseReques function resolved/rejected values). This "big" array is the value, with which a promise is resolved. Recall that this promise is the one that was returned already by this current function context, at the time the first await was executed.
This is important part - so it (urlContent) returns (await) a single promise and that (returned) promise is resolved with an array as value. Note that an async function returns the promise to the caller immediately, when the first await is encountered. The return statement in an async function determines what the value is with which that returned promise is resolved. In our case that is an array.
So urlContent at every call return an promise - resolved value in a array - [...] (destructured rest parameters - returns a promise that eventually resolves to an array), that is collected by our async doing function (cause 3 urls was fired at start - every one has it own urlContent function... path), that collect (Await!) all those arrays from Promise.all(urlsPromise), and when they are resolved (we await for them to be resolved and passed by Promise.all) it 'return' your data (results variable). To be precise, doing returns a promise (because it is async). But the way that we call doing, we show we are not interested in what this promise resolves to, and in fact, since doing does not have a return statement, that promise resolves to UNDEFINED (!). Anyway, we don't use it - we merely output the results to the console.
One thing that can be confusing with async functions is that the return statement is not executed when the function returns (what is in a name, right!? ;). The function has already returned when it executed the first await. When eventually it executes the return statement, it does not really return a value, but it resolves "its own" promise; the one it had returned earlier. If we would really want to separate output from logic, we should not do console.log(results) there, but do return results, and then, where we call doing, we could do doing.then(console.log); Now we do use the promise returned by doing!
I would reserve the verb "to return" for what the caller of a function gets back from it synchronously.
I would use "to resolve" for the action that sets a promise to a resolved state with a value, a value that can be accessed with await or .then().

How to run then() when all json data gone through the loop

In my code I deal with multiple JSON requests that need to be parsed in an order.
let allRadio_data, allHistory_data, iTunes_data;
$.when(parseRadioList())
.then(function(list) {
radioList_data = list;
return $.when(
parseEachRadioData(radioList_data),
ParseEachHistoryData(radioList_data)
);
})
.then(function() {
console.log(allRadio_data);
console.log(allHistory_data);
return $.when(_parseiTunes());
})
.then(function(iTunesInfo) {
iTunes_data = iTunesInfo;
return _cacheOptions();
})
function _cacheOptions() {
// FINAL function
}
/////////
function parseRadioList() {
return $.getJSON("https://api.myjson.com/bins/xiyvr");
}
function _parseiTunes() {
return $.getJSON("https://itunes.apple.com/search?term=jackson&limit=10&callback=?")
}
function parseEachRadioData(radioList) {
allRadio_data = [];
$.each(radioList, function(index, radio) {
$.when($.getJSON(radio.url + "/stats?sid=" + radio.stream_id + "&json=1&callback=?"))
.then(function(data) {
allRadio_data.push(data);
});
})
}
function ParseEachHistoryData(radioList) {
allHistory_data = [];
$.each(radioList, function(index, radio) {
$.when($.getJSON(radio.url + "/played?sid=" + radio.stream_id + "&type=json&callback=?"))
.then(function(data) {
allHistory_data.push(data);
});
})
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
Right now the code is running, but where I do console.log(allRadio_data); it is empty. However, if I do settimeout() to delay it for a second the data is completed. This means then() is not running on time.
This is the structure I am looking for:
JSON file 1 is parsed. parseRadioList()
JSON1 is an array of multiple entries of JSON URLS.
Run through the URLs within JSON1 array and do getJSON for each. parseEachRadioData(radioList_data) & ParseEachHistoryData(radioList_data)
Push data of each JSON in one general Array.
Once completed, parse JSON2 _parseiTunes()
Any Idea how to make this code running in the right structure.
Thanks in advance.

For a start, parseEachRadioData and ParseEachHistoryData don't return anything at all, let alone a Promise - so it's impossible to wait on them
Also, you're overusing $.when ... in fact you never need to use it, just use regular promises since jQuery $.getJSON etc return a usable Promise-like object
i.e. your code could be
let allRadio_data, allHistory_data, iTunes_data;
parseRadioList()
.then(function(list) {
radioList_data = list;
return Promise.all([
parseEachRadioData(radioList_data),
ParseEachHistoryData(radioList_data)
]);
})
.then(function(result) {
allRadio_data = result[0];
allHistory_data = result[1];
console.log(allRadio_data);
console.log(allHistory_data);
return _parseiTunes();
})
.then(function(iTunesInfo) {
iTunes_data = iTunesInfo;
return _cacheOptions();
})
function _cacheOptions() {
// FINAL function
}
/////////
function parseRadioList() {
return $.getJSON("https://api.myjson.com/bins/xiyvr");
}
function _parseiTunes() {
return $.getJSON("https://itunes.apple.com/search?term=jackson&limit=10&callback=?")
}
function parseEachRadioData(radioList) {
return Promise.all(radioList.map(radio => $.getJSON(radio.url + "/stats?sid=" + radio.stream_id + "&json=1&callback=?")));
}
function ParseEachHistoryData(radioList) {
return Promise.all(radioList.map(radio => $.getJSON(radio.url + "/played?sid=" + radio.stream_id + "&type=json&callback=?")));
}

from a first view
$.when(parseRadioList()) // your code will wait her
.then(function (list) {
radioList_data = list;
return $.when(
parseEachRadioData(radioList_data), // this two method are void they will complete
ParseEachHistoryData(radioList_data) // immediately without waithing for getJson...
);
})

Uncaught TypeError: Cannot read property 'then' of undefined when chaining 3 getJson calls

I'm new to chaining JavaScript promises. I read all of the answers regarding above error. Added lots of return, but still, do not understand why it is returning undefined.
I have 3 getJson calls (user, logo and stream). Data from all three are colected in thisNameInfo array and used to build html.
In one of the prevous versions all then statments were chained in one signle line. That did not produce error, but the html was build before the getJson call was executed. After reading this thread how to chain then functions I added 3 call routines (callUser, callLogo and callStream).
It passes first callUser and gives me Cannot read property 'then' of undefined for 'then' after the callLogo. Point of error is underline in the code with ``````````````.
Thanks for help.
If you have suggestiong how to better pass data from getJson calls to function that build html I would love to hear it.
Here is my code:
var allStreamers = ["freecodecamp", "animeexpo", "brunofin"];
// build html for one stereamer
var buildStreamerHtml = function(thisNameInfo){
//build html using thisNameInfo
... some code goes here
$("#display").append(html);
};
// get user
var user = function(name, thisNameInfo){
// return promise or "then" will return undefined!!!
return $.getJSON(
"https://wind-bow.glitch.me/twitch-api/users/" + name,
function(data) {
// if user does not exist data.error if 404 and data.message exist
if (data.message) {
thisNameInfo.userExist = "no";
thisNameInfo.title = data.message;
thisNameInfo.url = "#";
thisNameInfo.logo = "";
} else{
thisNameInfo.userExist = "yes";
}
});
};
// get logo, title and url
var logo = function(name, thisNameInfo){
if (thisNameInfo.userExist === "yes"){
// get logo and title with link to url
// return promise or "then" will return undefined!!!
return $.getJSON("https://wind-bow.glitch.me/twitch-api/channels/" + name,
function(dataChannel) {
thisNameInfo.url = dataChannel.url;
thisNameInfo.title = dataChannel.display_name;
thisNameLogo.logo = dataChannel.logo;
});
}
};
// get stream title and number of watchers
var stream = function(name, thisNameInfo){
if (thisNameInfo.userExist === "yes"){
// return promise or "then" will return undefined!!!
return $.getJSON("https://wind-bow.glitch.me/twitch-api/streams/" + name,
function(dataStreams) {
if (dataStreams.stream) {
thisNameLogo.status = "Online";
thisNameLogo.streamTitle = dataStreams.stream.channel.status;
thisNameLogo.viewers = dataStreams.stream.viewers;
} else {
thisNameLogo.status = "Offline";
}
});
}
};
var callUser = function(name, thisNameInfo){
return user(name, thisNameInfo).then(callLogo(name, thisNameInfo));
};
var callLogo = function(name, thisNameInfo){
return logo(name, thisNameInfo).then(callStream(name, thisNameInfo));
}; ``````````````````````````````````````
var callStream = function(name, thisNameInfo){
return stream(name, thisNameInfo);
};
// link together all asinhronious calls for one streamer
var getStreamerInfo = function(name){
"use strict";
// this variable builds up by assinhronious calls
// then its value is usedd by buildStreamerHtml
console.log("getStreamerInfo name: " + name);
var thisNameInfo = {};
callUser(name, thisNameInfo).then(buildStreamerHtml(thisNameInfo));
};
// loop through all streamers and display them
allStreamers.forEach(getStreamerInfo);
The undefine points after the second promise callLogo

It looks like your issue could be that you are not passing callback functions to each then().
When you pass callLogo(name, thisNameInfo) to then(), you are actually calling the function immediately and passing it's return value:
return user(name, thisNameInfo).then(callLogo(name, thisNameInfo));
Instead, you need to pass a callback function that will be called once the Promise resolves:
return user(name, thisNameInfo).then(function() {
callLogo(name, thisNameInfo)
});
You need to do this anytime you are using then().