I am lost in the promised land and could really use some guidance. I have exhausted searching numerous SO questions (2-3 hours of reading solutions + docs) related to this seemingly common issue and feel I just am not getting it.
Overview
Below I have code that takes in an Object type (resources), grabs a few values from this Object and then calculates distance and duration from the GoogleMaps Distance Matrix. The results of the function googleRequest() are a promise containing two values (distance and duration).
I would like to get these two values back within the for loop, execute pushToRows(), and then return an array called final_rows.
Problem
final_rows shows UNDEFINED for the duration and distance keys within each row. I speculate this is occurring because I am attempting to access the values in dist_dur inappropriately. I would appreciate any help on resolving this issue. Thanks.
Code
final_rows = []
function getDistTime(resources){
for (var i = 0; i < resources.data.length; i++) {
var origin1 = $("#citystate").val();
var destinationA = resources.data[i]['DEMOBILIZATION CITY'] + ',' + resources.data[i]['DEMOBILIZATION STATE'];
var dist_time_data = googleRequest(origin1, destinationA).then((values) => {
return values
})
pushToRows(resources.data[i], dist_time_data)
}
// console.log(final_rows)
}
function pushToRows(resources, dist_dur){
resources["DISTANCE_MI"] = dist_dur[0];
resources["ACTUAL_DUR_HR"] = dist_dur[1];
resources["FINANCE_DUR_HR"] = (dist_dur[0] / 45.0).toFixed(2)
final_rows.push(resources)
}
So, what you would need to do is just store promises in an array in the for loop and then wait for these promises to resolve using Promise.all but this would parallelize your requests to google distance api.
function getDistTime(resources){
const promiseArr = [];
for (var i = 0; i < resources.data.length; i++) {
var origin1 = $("#citystate").val();
var destinationA = resources.data[i]['DEMOBILIZATION CITY'] + ',' + resources.data[i]['DEMOBILIZATION STATE'];
promiseArr.push(googleRequest(origin1, destinationA));
}
// Not sure how would you use the data pushed in rows but since you are not waiting for promises to be resolved, data would be updated later on
return Promise.all(promiseArr)
.then((resultsArr) => {
resultsArr.forEach((result, i) => pushToRows(resources.data[i], result));
})
}
function pushToRows(resources, dist_dur){
resources["DISTANCE_MI"] = dist_dur[0];
resources["ACTUAL_DUR_HR"] = dist_dur[1];
resources["FINANCE_DUR_HR"] = (dist_dur[0] / 45.0).toFixed(2)
final_rows.push(resources)
}
I would recommend to use async-await which are syntactic sugar to promises but make your code easy to understand and remove the complications that come with promise chaining.
If you move your pushToRows() inside where you return values, you will have access to that data.
googleRequest(origin1, destinationA).then((values) => {
pushToRows(resources.data[i], values);
});
Until that promise resolves, dist_time_data would be undefined
You could also convert to Promise.all() which takes an array of promises and resolves when all of the promises are complete:
function getDistTime(resources){
const promises = [];
for (var i = 0; i < resources.data.length; i++) {
var origin1 = $("#citystate").val();
var destinationA = resources.data[i]['DEMOBILIZATION CITY'] + ',' + resources.data[i]['DEMOBILIZATION STATE'];
promises.push(googleRequest(origin1, destinationA));
}
return Promise.all(promises).then((results) => {
return results.map((result, i) => {
return {
...resources.data[i],
DISTANCE_MI: result[0],
ACTUAL_DUR_HR: result[1],
FINANCE_DUR_HR: (result[0] / 45.0).toFixed(2)
};
});
});
}
getDistTime(resources).then(result => {
//result is now "final_rows"
});
I need to read a grid and take that data and call a $getJSON url. The grid could have over 100 lines of data. The getJSON returns a list of comma separated values that I add to an array. Once the loop is finished I take the array and process it for the duplicates. I need to use the duplicates in another process. I know that I can't determine the order of the data that is coming back but I need to know that all of the calls have been make.
for (let i = 0; i < rowscount; i++){
$.getJSON(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" +
terms,
function (data) {
var pmids = data.esearchresult.idlist;
var pmidlist = pmids.join();
pmid_List.push(pmidlist);
if (i == rowscount - 1) {
// call the related function
}
});
}
I can't figure out how to be sure that the process has finished. The call to the related function has been done early at times.
Well if we keep track of how many have completed we can fire off the code when the last one is done.
let complete = 0;
for (let i = 0; i < rowscount; i++){
$.getJSON(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" +
terms,
function (data) {
var pmids = data.esearchresult.idlist;
var pmidlist = pmids.join();
pmid_List.push(pmidlist);
complete += 1;
if (complete == rowscount) {
// call the related function
}
});
}
I'd use fetch and Promise.all
const link = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=";
Promise.all(Array.from({
length: 3
}, () => fetch(link + 'foo').then(e => e.json()))).then(e => {
//called when all requests are done
console.log(e);
})
Try this
function getJson(url, i) {
return $.getJSON(url, function (data) {
//var pmids = data.esearchresult.idlist;
//var pmidlist = pmids.join();
//pmid_List.push(pmidlist);
console.log('completed', i)
return data;
});
}
function run() {
let promises = []
for (let i = 0; i < rowscount; i++) {
const terms = 'foot';
const url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" + terms;
promises.push(getJson(url, i));
}
return promises;
}
Promise.all(run()).then(() => console.log('All are completed'));
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
I'm using the following code to update the state but the Promise.all() array isn't always resolved before the final function is called. Sometimes, the entire array is populated but it usually isn't.
I'm logging the Promise.all() array to the console just so I can see what's happening. What is really peculiar is the state is always updated properly but calling the individual elements in the array often returns "undefined". I need to use the individual elements to create a new array for displaying later which is why I'm accessing them.
Please help me either figure out how to access the array when it's finished updating or a better way to process the entire thing.
class X {
componentDidMount() {
const numTiles = 3;
for (let i = 0; i < numTiles; i++) {
Promise.all([
this.fetchSong(), // returns JSON from SQLite DB
this.fetchArtists(), // returns JSON from SQLite DB
])
.then(values => {
this.testing(values);
});
}
}
testing(arr) {
console.log("arr: ", arr);
console.log("arr[0]: ", arr[0]);
console.log("arr[0].id: ", arr[0].id);
console.log("arr[0].name: ", arr[0].name);
console.log("arr[0].artist: ", arr[0].artist);
console.log("arr[1]: ", arr[1]);
console.log("arr[1][0]: ", arr[1][0]);
console.log("arr[1][1]: ", arr[1][1]);
console.log("arr[1][0].artist: ", arr[1][0].artist);
console.log("arr[1][1].artist: ", arr[1][1].artist);
}
}
Edit: code for fetchSong() and fetchArtists() added.
fetchSong() {
let id = Math.floor(Math.random() * 2000) + 1; // get random number for id
return new Promise((resolve, reject) => {
Bingo.getSong(id).then(song => {
resolve(song);
});
});
}
fetchArtists() {
return new Promise((resolve, reject) => {
let arr = [];
for (let j = 0; j < 2; j++) {
let id = Math.floor(Math.random() * 10) + 1;
Bingo.getArtist(id).then(artist => {
arr.push(artist);
});
resolve(arr);
};
});
}
The console screenshot below shows the Promise.all() array has been populated but the array elements are still missing.
The problem is in the implementation of fetchArtists, which resolved before any of the single-artist-fetching promises were resolved.
I've also simplified fetchSong, but the gist of fetchArtists now is you store the promises that will resolve to single artists, then wait for all of them to resolve. There's no reason to add a .then() there, since as you know, Promise.all() will resolve with an array of resolved values anyhow.
fetchSong() {
const id = Math.floor(Math.random() * 2000) + 1;
return Bingo.getSong(id);
}
fetchArtists() {
const fetchPromises = [];
for (let j = 0; j < 2; j++) {
const id = Math.floor(Math.random() * 10) + 1;
fetchPromises.push(Bingo.getArtist(id));
}
return Promise.all(fetchPromises);
}
I'm working with Node.js since 5 years and from 2 years on big projects with this framework. For two years, I'm confronted to a problem: how to work asynchronously and faster with non-async third party applications who's stacks requests, like MySQL, MongoDB or Apache SolR ?
I'm used to work with promises and to prepared several promises requests, like this:
const promises = []
for (let i = 0; i < 1000; i += 1) {
const promise = mySqlRequest()
promises.push(promise)
}
Promise.all(promises)
.then()
.catch()
This example will work but will send 1000 requests at the same time to MySQL server, who's will stacks these requests and become very slow, will consume very large quantity of RAM.
The best solution is to do only one big request, but in some case it's impossible and I'm forced to make recursive function,
which comes down to be synchronous and slower.
So, what the best way to work fast and asynchronous with Node.js and a stacking third party ?
If sending all requests at once doesn't work and sending them one by one doesn't work either, you'd need something similar to a thread-pool where some arbitrary number of tasks execute simultaneously. This is easily implementable using promises, for example like this:
Promise.pooled = function(arr, num = 5) {
return new Promise(function(resolve, reject) {
var i = -1;
var error = false;
var end = function() {
num--;
if(num === 0) resolve();
}
var next = function() {
if(error) return;
i++;
if(i >= arr.length)
end();
else
arr[i]().then(next).catch(onerr);
}
var onerr = function() {
if(error) return
error = true
reject.call(arguments)
}
for(var j = 0; j < num; j++)
next()
});
}
What this allows you is pass an array of functions as first argument, those functions should take no parameter and return a promise. It will then execute exactly num simultaneously. If one of the promises fail, it will fail its own promise aswell and stop executing (this is changeable easily).
Example:
Promise.after = function(ms) {
return new Promise(function(resolve) {
setTimeout(resolve, ms)
});
}
Promise.pooled = function(arr, num = 5) {
return new Promise(function(resolve, reject) {
var i = -1;
var error = false;
var end = function() {
num--;
if(num === 0) resolve();
}
var next = function() {
if(error) return;
i++;
if(i >= arr.length)
end();
else
arr[i]().then(next).catch(onerr);
}
var onerr = function() {
if(error) return
error = true
reject.call(arguments)
}
for(var j = 0; j < num; j++)
next()
});
}
var test = [
afterH(1000),
afterH(500),
afterH(800),
afterH(600),
afterH(3000),
afterH(300),
afterH(900),
afterH(2000),
afterH(1500),
afterH(900),
afterH(700),
afterH(600),
afterH(700)
];
// helper function, returns a function which when invoked returns a promise
function afterH(ms) {
return function() {
console.log("Starting one job")
return Promise.after(ms);
}
}
Promise.pooled(test, 3).then(function() {console.log("All jobs finished") }).catch(function() {console.log("Job failed")})
So I have this code:
Rating.find({user: b}, function(err,rating) {
var covariance=0;
var standardU=0;
var standardV=0;
while (rating.length>0){
console.log("the avarage rating u is:" + avarageRatingU)
console.log("the avarage rating v is:" + avarageRatingV)
currentMovie = rating.pop();
var u=currentMovie.value-avarageRatingU;
standardU = standardU + Math.pow(u,2);
var v=0;
Rating.find({movieid:currentMovie.movieid, user:a}, function(err,ratings) {
if (err) throw err;
if(ratings.length>0){
v=ratings.pop().value-avarageRatingV;
standardV=standardV+Math.pow(v,2);
covariance =covariance+u*v;
console.log(covariance);
}
})
}
console.log(covariance)
callback(null,covariance);
//sim = covariance/(Math.sqrt(standardU)*Math.sqrt(standardV));
})
The problem is when I print covariance, it will print 0, because the print happends before the calculation. I thought about using async.series, however, I can not have the callback function inside the while loop.
Any tips would be appreciated.
Ty :)
Can't you call your callback inside the loop somehow like this? Not sure if this helps you.
while (rating.length>0){
console.log("the avarage rating u is:" + avarageRatingU)
console.log("the avarage rating v is:" + avarageRatingV)
currentMovie = rating.pop();
var u=currentMovie.value-avarageRatingU;
standardU = standardU + Math.pow(u,2);
var v=0;
Rating.find({movieid:currentMovie.movieid, user:a}, function(err,ratings) {
if (err) throw err;
if(ratings.length>0){
v=ratings.pop().value-avarageRatingV;
standardV=standardV+Math.pow(v,2);
covariance =covariance+u*v;
console.log(covariance);
}
if(rating.length == 0){
console.log(covariance);
callback(null, covariance);
}
})
}
Using either promises or the async library would be the way to go in this case instead of the traditional while loop. Since you are already familiar with async, fetch all the data you need ahead of time with async.map (https://github.com/caolan/async#map).
Rating.find({user: b}, function(err,rating) {
var covariance=0;
var standardU=0;
var standardV=0;
aync.map(rating, function(currentMovie, cb) {
Rating.find({ movieid:currentMovie.movieid, user:a }, function(err,ratings) {
cb(err, ratings);
});
}, function(err, results) {
if (!err) {
// compute covariance with all movie data
}
});
})
This is my second promises solution tonight :). So, let me try:
//supposing you are using node js
var Q = require('q'); //https://github.com/dscape/nano
Rating.find({user: b}, function(err,rating) {
var covariance=0;
var standardU=0;
var standardV=0;
var promises = [];
while (rating.length>0){
console.log("the avarage rating u is:" + avarageRatingU)
console.log("the avarage rating v is:" + avarageRatingV)
currentMovie = rating.pop();
var u=currentMovie.value-avarageRatingU;
standardU = standardU + Math.pow(u,2);
var v=0;
var def = Q.defer();
promises.push(def);
Rating.find({movieid:currentMovie.movieid, user:a}, function(err,ratings) {
if (err) {
def.reject();
throw err;
}
if(ratings.length>0){
v=ratings.pop().value-avarageRatingV;
standardV=standardV+Math.pow(v,2);
covariance =covariance+u*v;
def.resolve();
//console.log(covariance);
}
});
}
Q.allSettled(promises, function() {
console.log(covariance);
});
callback(null,covariance);
//sim = covariance/(Math.sqrt(standardU)*Math.sqrt(standardV));
});
Try using a Promise! Now that Node.js 4.x is out, all of JavaScript ES6 (and Promises) comes with it. You can look at the Mozilla Foundation documentation for promises here.
If you've never used a promise before, all it really does is allow you to return a value before the value is available. Later, the promise is able to become resolved or rejected with a real value.
In this case, you're going to want to encapsulate your inner Rating.find() call inside a new Promise( function(resolve, reject) { ... } ) and attach a then() clause onto your promise which prints out the value of the covariance. When you retrieve the real covariance from your inner Rating.find() function, simply use the resolve parameter (which is a function which takes a single value) and pass the covariance so it gets passed into the function in your Promise's then() clause, which will print the covariance.
Shoot me a comment if there's any confusion!
Assuming that rating (or ratings) is an array you need to iterate over and call an async function on each item (e.g. Rating.find), async.eachSeries would probably make the most sense.
async.eachSeries(ratings, function iterator(rating, callback) {
// invoke async function here for each rating item
// and perform calculations
}, function done() {
// all async functions have completed, print your result here
});
More documentation here: https://github.com/caolan/async
This is a standard problem of calling an asynchronous function within a loop.
Let's look at a simpler example first. Say I want a function that prints the value of i in a for loop, and I want it to run asynchronously:
for (i = 0; i < 5; i++) {
setTimeout(function() { console.log(i); }, 1);
}
Now this will actually only print out 5, 5 times. This is because the asynchronous function setTimeout returns after the loop has ended, and the value of i at the time that it executes is 5. If we want it to print out 0 - 4, then we need to do the following:
for (i = 0; i < 5; i++) {
(function(i) {
setTimeout(function() { console.log(i); }, 1);
})(i);
}
Notice how the anonymous function takes in i as a parameter. This creates a closure that "saves" the value of i at the time it was called.
Now back to your problem:
covariance = covariance + u * v;
This occurs before your console.log(covariance) statement, meaning that your print statement actually is occurring after the calculation. That means that covariance really is equal to 0 at this point.
Your covariance variable is initialized to 0. So are u and v. 0 + 0 * 0 = 0. Okay, great: so if u and v aren't being set properly, at least the math checks out.
Let's go back to your loop:
while (rating.length>0){
currentMovie = rating.pop();
var u=currentMovie.value-avarageRatingU;
var v=0;
Rating.find({movieid:currentMovie.movieid, user:a}, function(err,ratings) {
if (err) throw err;
if(ratings.length>0){
v=ratings.pop().value-avarageRatingV;
standardV=standardV+Math.pow(v,2);
covariance =covariance+u*v;
...
Here, we can see that your asynchronous Rating.find callback is pulling the last known value of u, i.e. its value at the end of the while loop. There isn't really any reason for v to be defined outside of the Rating.find callback.
If you want the value of u for each loop, try wrapping it in an anonymous self-executing function to "save" the value, like so:
Rating.find({user: b}, function(err,rating) {
var covariance = 0;
var standardU = 0;
var standardV = 0;
var u = 0;
while (rating.length > 0){
console.log("the avarage rating u is:" + avarageRatingU)
console.log("the avarage rating v is:" + avarageRatingV)
currentMovie = rating.pop();
u = currentMovie.value - avarageRatingU;
standardU = standardU + Math.pow(u, 2);
(function(u) {
Rating.find({ movieid: currentMovie.movieid, user: a }, function(err, ratings) {
if (err) throw err;
if (ratings.length > 0) {
var v = ratings.pop().value - avarageRatingV;
standardV = standardV + Math.pow(v,2);
covariance = covariance + u * v;
console.log(covariance);
}
});
})(u);
}
console.log(covariance)
callback(null,covariance);
//sim = covariance/(Math.sqrt(standardU)*Math.sqrt(standardV));
});
I've also moved the declaration of u outside of your loop (declaring them inside is bad practice, since you're reinstantiating the variable each time). I moved the declaration of v inside the Rating.find callback, as it's not even being used outside of there.
How about this:
Rating.find({user: b}, function(err,rating) {
var covariance=0;
var standardU=0;
var standardV=0;
var c = rating.length; // loop counter
var r = 0; // received counter
function aux_callback (covariance) {
r++;
if (r==c)
callback(null,covariance);
}
while (rating.length>0){
console.log("the avarage rating u is:" + avarageRatingU)
console.log("the avarage rating v is:" + avarageRatingV)
currentMovie = rating.pop();
var u=currentMovie.value-avarageRatingU;
standardU = standardU + Math.pow(u,2);
var v=0;
Rating.find({movieid:currentMovie.movieid, user:a}, function(err,ratings) {
if (err) throw err;
if(ratings.length>0){
v=ratings.pop().value-avarageRatingV;
standardV=standardV+Math.pow(v,2);
covariance =covariance+u*v;
console.log(covariance);
aux_callback(covariance);
}
})
}
//sim = covariance/(Math.sqrt(standardU)*Math.sqrt(standardV));
})
You need to trigger the callback from the lambda function inside the loop to make sure it does not happen before you run that code. Given that you seem to need all the loop to be completed, I created two counters, c to keep track of how many times the lambda has to run and r to count how many times you have called aux_callback. This code should wait until you have calculated all the covariances before calling callback
Which orm are you using?, I think a better approach is to group your results by "movieid", and after that compute the covariance for each group without make a query for each "rating" result.
Rating.find({user: b}, function(err, rating){
var covariance=0;
var standardU=0;
var standardV=0;
var ratingsByMovieId = groupByMovieId(rating);
while(rating.length > 0){
console.log("the avarage rating u is:" + avarageRatingU)
console.log("the avarage rating v is:" + avarageRatingV)
currentMovie = rating.pop();
var u=currentMovie.value-avarageRatingU;
standardU = standardU + Math.pow(u,2);
var v=0;
ratingsByMovieId[currentMovie.movieid].forEach(function(ratings){
// this code isn't asynchronous :)
// because the code isn't asynchronous you don't need a callback.
});
}
});
function groupByMovieId(ratings) {
var groups = {};
ratings.forEach(function(rating){
var movieId = rating.movieid;
groups[movieId] = groups[movieId] || [];
groups[movieId].push(rating);
});
return groups;
}
I'm not tested this code jeje :)