We Keep Coding

Asynchronous confusion. Processing an API response and losing my grasp on what is happening in this code

I've gotten lost inside my own head about how to best structure this function.
This code below is mostly correct, I just stubbed out a few of the functions at the bottom, not important.
What is supposed to happen:
My node app runs at a certain time and calls refreshDevices()
The function fetches a JSON payload from an API
The function loops over elements in that payload and checks if there is a record already
If there is no record, one is created, if there is then it is updated
After this is run a log entry is created logging the number of changes.
My issue is the log entry happens prior to any of the record checks taking place... my log shows 0 added & 0 updated.
I've tried re-writing the code to use an inline async/await function but I think that is effectively the same thing as below.
How am I supposed to structure this function to do what I've said.
I am finding the mix of async and sync very confusing to wrap my head around.
I even tried adding the log entry bit to happen in a .finally()
block at the end, but that was executing before the checks happened.
I have tried this using both forEach and map.
refreshDevices(_timespan) {
const status = {
added: 0,
updated: 0,
};
// Standard API fetch & json() conversion
fetch(`https://api.example.com/test&timespan=${timespan}`)
.then((response) => response.json())
.then((JSON) => {
JSON.forEach((device) => {
findDevice(device._id)
.then(record => {
if (!record) {
addDeviceRecord(device);
status.added++;
} else {
deviceSeen(device._id);
status.updated++;
}
})
.catch(error => {
console.log('Error!');
});
});
systemlog.log({
message: 'Refresh',
data: {
success: true,
note: `Added: ${status.added} & updated: ${status.updated}`,
},
});
})
.catch((error) => {
systemlog.log({
message: 'Refresh',
type: systemlog.logType.ERROR,
data: {
success: false,
note: error,
},
});
});
}
function findDevice(id) {
return DB('device').findOne({ id }); // this returns is a promise
}
function addDeviceRecord(record) {
return DB('device').insertOne(record); // this returns is a promise
}
function deviceSeen(id) {
return DB('device').updateOne({ id }, { $set: { lastSeen: new Date() } }); // this returns is a promise
}
This question

As you have identified, your system.log call happens immediately after launching a litany of promises which haven't resolved.
Inside your refreshDevices(_timespan) function add an empty array to add each new promise in the forEach.
Then modify your code like
JSON.forEach((device) => {
promises.push( // Add this promise to the array
findDevice(device._id)
.then(record => {
if (!record) {
addDeviceRecord(device);
status.added++;
} else {
deviceSeen(device._id);
status.updated++;
}
})
.catch(error => {
console.log('Error!');
});
) // end of .push() arg
});
Then use Promise.all() to wait for all promises to be resolved before calling systemLog.log, sort of like (pseudo-pseudo code :)
Promise.all(promises).then(
systemlog.log({
message: 'Refresh',
data: {
success: true,
note: `Added: ${status.added} & updated: ${status.updated}`,
},
});
)

You could use Async/await to make the code more readable and use for of to iterate over data.
for of loop will wait for the first iteration to complete and then go for the second one and so on.
On the other hand forEach is based on callback and so it will execute and register a callback for each iteration in the callback queue and exit and don't wait for the callbacks to complete.
Try this
async refreshDevices(_timespan) {
const status = {
added: 0,
updated: 0,
};
// Standard API fetch & json() conversion
try {
let response = await fetch(`https://api.example.com/test&timespan=${timespan}`)
let data= await response.json();
for (const device of data) {
try {
let record = await findDevice(device._id);
if (!record) {
addDeviceRecord(device);
status.added++;
} else {
deviceSeen(device._id);
status.updated++;
}
} catch (error) {
console.log('Error!');
}
}
systemlog.log({
message: 'Refresh',
data: {
success: true,
note: `Added: ${status.added} & updated: ${status.updated}`,
},
});
} catch (error) {
systemlog.log({
message: 'Refresh',
type: systemlog.logType.ERROR,
data: {
success: false,
note: error,
},
});
}
}
function findDevice(id) {
return DB('device').findOne({ id }); // this returns is a promise
}
function addDeviceRecord(record) {
return DB('device').insertOne(record); // this returns is a promise
}
function deviceSeen(id) {
return DB('device').updateOne({ id }, { $set: { lastSeen: new Date() } }); // this returns is a promise
}
Alternatively you could also use promise.all to wait for all promises to be resolved.
async refreshDevices(_timespan) {
const status = {
added: 0,
updated: 0,
};
// Standard API fetch & json() conversion
try {
let response = await fetch(`https://api.example.com/test&timespan=${timespan}`)
let data = await response.json();
let promises = data.map(device => {
return new Promise( async (resolve, reject) => {
try {
let record = await findDevice(device._id);
if (!record) {
addDeviceRecord(device);
status.added++;
} else {
deviceSeen(device._id);
status.updated++;
}
resolve();
} catch (error) {
console.log('Error!');
reject(error);
}
});
});
let result = await Promise.all(promises);
systemlog.log({
message: 'Refresh',
data: {
success: true,
note: `Added: ${status.added} & updated: ${status.updated}`,
},
});
} catch (error) {
systemlog.log({
message: 'Refresh',
type: systemlog.logType.ERROR,
data: {
success: false,
note: error,
},
});
}
}
function findDevice(id) {
return DB('device').findOne({ id }); // this returns is a promise
}
function addDeviceRecord(record) {
return DB('device').insertOne(record); // this returns is a promise
}
function deviceSeen(id) {
return DB('device').updateOne({ id }, { $set: { lastSeen: new Date() } }); // this returns is a promise
}

Figuring the complexity of Promise.all

I have been struggling for quite some time to get this multiple async nodejs request apis to work but unfortunately i am not able to get them work.
Index.js Code:
service.get(
"/restraunts",
versionRoutes({
"1.0.0": getRestrauntsList
})
);
function getRestrauntsList(req, res, next) {
console.log("Started getRestrauntsList");
file1
.appEnvironment(req, res, next)
.then(function(result) {
return file2.getRestrauntsList(req, res, next);
})
.then(function(result) {
res.status(200).send(result);
return;
})
.catch(function(errorResult) {
res.status(500).send(errorResult);
return;
});
}
File2.js
module.exports = {
getRestrauntsList: function(req, res, next) {
console.log("getRestrauntsList started..");
var cities = [1, 2, 3, 4, 5];
let restrauntsList = [];
let urlArray = [];
var restrauntsListPromise = cities.map(function(id) {
return new Promise(function(resolve, reject) {
var options = {
method: "GET",
url: "someurl/" + id + "/restaurants",
headers: {
"AUTH-TOKEN": "TOKEN"
}
};
request(options, function(error, response, body) {
if (error) {
if ("message" in error) {
errorMsg = error.message;
var result = {
status: "error",
message: errorMsg
};
} else {
var result = {
status: "error",
message: "Resource Timeout."
};
}
reject(result);
return promise;
}
console.log(
"Response: " + JSON.stringify(response)
);
if (response.statusCode === 200 || response.statusCode === 201) {
body = JSON.parse(body);
if (body.success) {
let result = {
status: "success",
data: body.result
};
resolve(result);
} else {
let result = {
status: "error",
message: body.error
};
reject(result);
}
} else {
let result = {
status: "error",
message: body.error
};
reject(result);
}
});
});
});
console.log('restrauntsListPromise:' + JSON.stringify(restrauntsListPromise));
Promise.all(restrauntsListPromise).then(function(result) {
var content = result.map(function(restraunts) {
return restrauntsList.push(restraunts.body);
});
// res.send(content);
resolve({
restrauntsList: restrauntsList
});
return promise;
});
},
};
Ideally i expect to get the response of all the apis in the
restrauntsListPromise
and then using Promise.all i should iterate all the promises and formulate my required object.
The response of my code however is
restrauntsListPromise:[{},{},{},{},{}]
and then
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Ideally what should happen is i should be able to pass the combined result of all the five apis calls as a single object back to the calling promise here
.then(function(result) {
res.status(200).send(result);
return;
})
The problem being the method getRestrauntsList finishes execution and then after some time, i get the responses of the apis.

The problem being the method getRestrauntsList finishes execution and then after some time, i get the responses of the apis.
This is because you're not returning a promise from the getRestrauntsList().
There are few items that needs to addressed to make it work
1. Remove the unused variables
return promise; // both inside promise.all[] and request()
There is no declared variable named promise. So, you can remove it.
2. Accessing .body instead of .data
You're resolving as resolve({status: "success", data: body.result}); But When you are iterating, you are accessing using .body instead of .data. You need to be using .data. Also, you can eliminate restrauntsList array since you're using a .map()
3. Calling resolve() to return values.
You can't use resolve() to return value within Promise.all[] since you didn't create a promise using new Promise((resolve, reject) => { ... });. By default, a return within a promise will be a promise. so, a simple return will suffice. But if you want to be explicit, you can also return using Promise.resolve()
Making those changes,
return Promise.all(restrauntsListPromise).then(function (result) {
return {
restrauntsList: result.map(function (restraunts) {
return restraunts.data;
})
};
//or using Promise.resolve();
// return Promise.resolve({
// restrauntsList: result.map(function (restraunts) {
// return restraunts.data;
// })
// });
});

You are looking for
return Promise.all(restrauntsListPromise).then(function(result) { /*
^^^^^^ */
var contents = result.map(function(restaurants) {
return restaurants.body;
// ^^^^^^^^^^^^^^^^^^^^^^^^
});
return {restaurantsList: contents};
// ^^^^^^
});
You need to return the promise chain from the getRestrauntsList method, you should return the value from the map callback instead of using push on an array, and you will need to return from the then callback - there is no resolve function as you're not inside a new Promise constructor that you only need for callback APIs.

Return data from node promises

I am currently trying to fetch data from the Spotify API with promises, and yesterday I got tremendous help for another question, regarding the same topic: "Loop the object returned from node promise and feed to the next .then".
What I do is first getting the tracks from my playlist, and then call another api which fetches the artists. Lastly I call another api which gets the artist images.
Now my question is: how do I return the data that I get from my promises?
This is my function that gets the playlist urls:
function getPlaylists(access_token) {
var options = {
url: 'https://api.spotify.com/v1/me/playlists',
headers: { 'Authorization': 'Bearer ' + access_token },
json: true
};
return new Promise(function(resolve, reject) {
request.get(options, function(error, response, body) {
var playlists = body.items;
var playlistArray = [];
playlists.forEach(function(playlist) {
var name = playlist.name;
var url = playlist.tracks.href;
playlistArray.push(url);
});
if(!error) {
resolve(playlistArray);
} else {
reject(error);
}
});
});
}
This one gets the artists:
function getArtists(url,access_token) {
var params = {
url: url,
headers: { 'Authorization': 'Bearer ' + access_token },
json: true
};
return new Promise(function(resolve, reject) {
request.get(params, function(error, response, body) {
var tracks = body.items;
var artistArray = [];
tracks.forEach(function(artists) {
let allArtists = artists.track.artists;
allArtists.forEach(function(artist) {
artistArray.push(artist);
});
})
if(!error) {
resolve(artistArray);
} else {
reject(error);
}
});
})
}
And this one gets the artist image:
function getArtistImages(artistId) {
var options = {
url: 'https://api.spotify.com/v1/artists/' + artistId,
json: true
};
return new Promise(function(resolve, reject) {
request.get(options, function(error, response, body) {
if(error != null) {
reject(error);
} else {
resolve(body);
}
});
})
}
EDIT EDIT
The way I call these functions is like this:
getPlaylists(access_token)
.then(playlists => Promise.all(playlists.map(playlist =>
getArtists(playlist, access_token)))
.then(artists => {
artists.map(artist => {
artist.map(a => {
console.log(a);
let component = renderToString(
<App>
<Table artists={a} />
</App>
);
res.send(
component
)
})
})
}));
It only returns the first result - obviously because it only gets to loop through the forEach loop once, before "res.send()", so how do I make sure that it loops through all artists, before I render the view? I believe I have to do another Promise.all(), but I am not sure where - does anyone have an idea?
I appreciate it :)

Its old post but I think it can be helpfull for someone.
I wrote a plugin to perform foreach based on promises supporting concurrency. I see you dont need concurrency what turn things more simple to apply other solutions.
I wrote a code by your code using my plugin. It works!
'use strict';
var request = require('request')
var promiseForeach = require('promise-foreach')
function getPlaylists(access_token) {
var options = {
url: 'https://api.spotify.com/v1/me/playlists',
headers: { 'Authorization': 'Bearer ' + access_token },
json: true
};
return new Promise(function (resolve, reject) {
request.get(options, function (error, response, body) {
var playlists = body.items;
var playlistArray = [];
playlists.forEach(function (playlist) {
var name = playlist.name;
var url = playlist.tracks.href;
playlistArray.push(url);
});
if (!error) {
resolve(playlistArray);
} else {
reject(error);
}
});
});
}
function getArtists(url, access_token) {
var params = {
url: url,
headers: { 'Authorization': 'Bearer ' + access_token },
json: true
};
return new Promise(function (resolve, reject) {
request.get(params, function (error, response, body) {
var tracks = body.items;
var artistArray = [];
tracks.forEach(function (artists) {
let allArtists = artists.track.artists;
allArtists.forEach(function (artist) {
artistArray.push(artist);
});
})
if (!error) {
promiseForeach.each(artistArray,
[function (artist) {
return getArtistImages(artist.id)
}],
function (arrayOfResultOfTask, currentList) {
return {
artistId: currentList.id,
artistName: currentList.name,
artistImages: arrayOfResultOfTask[0].images
}
},
function (err, newList) {
if (err) {
console.error(err)
return;
}
resolve(newList)
})
} else {
reject(error);
}
});
})
}
function getArtistImages(artistId) {
var options = {
url: 'https://api.spotify.com/v1/artists/' + artistId,
headers: { 'Authorization': 'Bearer ' + access_token },
json: true
};
return new Promise(function (resolve, reject) {
request.get(options, function (error, response, body) {
if (error != null) {
reject(error);
} else {
resolve(body);
}
});
})
}
var access_token = 'YOUR-TOKEN';
getPlaylists(access_token)
.then(playlists => {
promiseForeach.each(playlists,
[function (playlist) {
return getArtists(playlist, access_token)
}],
function (arrayOfResultOfTask, currentList) {
return {
playlistURL: currentList,
artist: arrayOfResultOfTask[0]
}
//return renderToString(
// <App>
// <Table artists={render} />
// </App>
//);
},
function (err, newList) {
if (err) {
console.error(err)
return;
}
res.send(newList)
})
});
The plugin: https://www.npmjs.com/package/promise-foreach
I hope it helps someone!

If I understand your problem correctly, it sounds like you are tripping up on how to actually use the result of Promises.
Asynchronous applications:
Promises encapsulate an asynchronous result, which is made available through the callback passed to then().
This asynchronous-ism will cascade throughout your application.
There are a number of ways applications manage the reality of asynchronous results: events, callbacks, observables, promises...
Example (using events):
This is a crude and untested example of how the data from an asynchronous request can get injected into your view. When my asynchronous request calls my then() callback, I update my model which triggers an event to re-render my view.
There are totally issues with this example (i.e. what if I don't want to rerender my whole view? what if my getArtists() returns sooner than my view can render it's loading state?). But for simplicity, we won't go there.
+function(){
var _view = $('#viewport');
var _model = {...}
var _spotifyClient = new SpotifyClient(); // this contains method similar to those you included in your question
_view.on('load', onLoad);
_view.on('model:update', onModelUpdate);
function onLoad() {
_spotifyClient
.getArtists()
.then(function(result) {
// when the getArtists() request has responded, I can update my model.
updateModel({ isLoading: false, artists: result });
})
// this will happen immediately after starting the "getArtists()" request.
udpateModel({ isLoading: true });
}
function updateModel(mod) {
for(var p in mod) {
_model[p] = mod[p];
}
_view.trigger('model:update', _model);
}
function onModelUpdate(model) {
refreshView();
}
function refreshView() {
var viewModel = buildTemplateModel(_model);
renderTemplate(_view, viewModel);
}
}();
I encourage you to research some view frameworks such as angular, knockout, or react. I also encourage you to research Reactive Extensions which provides an observable interface and many utilities regarding asynchronous streams.
Note on side effects:
Having the result of a promise trigger an event can be classified as a "side-effect". You should keep side-effects to a minimum in our application, and they really only belong in the controller / main part of your application.
If you are using promises in your application, reusable library classes and functions, that operate on promises, should return promises.

You need to call res.send on the array of all results, not on each one separately:
getPlaylists(access_token).then(playlists =>
Promise.all(playlists.map(playlist =>
getArtists(playlist, access_token)
))
).then(artists => {
res.send(artists.map(artist =>
artist.map(a => {
console.log(a);
return renderToString(
<App>
<Table artists={a} />
</App>
);
})
))
});
Also your parenthesis were slightly misnested (didn't match the indentation of the then calls) but that didn't lead to the problem.

Recursively calling asynchronous function that returns a promise

I'm trying to recursively call AWS's SNS listEndpointsByPlatformApplication. This returns the first 100 endpoints then a token in NextToken if there are more to return (details: AWS SNS listEndpointsByPlatformApplication).
Here's what I've tried:
var getEndpoints = function(platformARN, token) {
return new models.sequelize.Promise(function(resolve, reject) {
var params = {
PlatformApplicationArn: platformARNDev
};
if (token != null) {
params['NextToken'] = token;
}
sns.listEndpointsByPlatformApplication(params, function(err, data) {
if (err) {
return reject(err);
}
else {
endpoints = endpoints.concat(data.Endpoints); //save to global var
if ('NextToken' in data) {
//call recursively
return getEndpoints(platformARN, data.NextToken);
}
else {
console.log('trying to break out!');
return resolve(true);
}
}
});
});
}
I'm calling it with:
getEndpoints(platformARNDev, null)
.then(function(ret) {
console.log('HERE!');
}, function(err) {
console.log(err);
});
Problem is: the first call happens, then the recursive call happens, and I get the message trying to break out! but the HERE! never gets called. I've got something wrong with how my promises are returning I think.
Grateful for pointers.

The problem is that you try and resolve/reject partially completed query. Here is a complete working example with dummy service. I incapsulated the data grabbing into it's own recursive function and only do resolve/reject when i've completely fetched all the data or stumbled upon an error:
// This is the mock of the service. It yields data and token if
// it has more data to show. Otherwise data and null as a token.
var dummyData = [0, 1, 2, 3, 4];
function dummyAsyncCall(token, callback) {
token = token || 0;
setTimeout(function() {
callback({
dummyDataPart: dummyData[token],
token: (typeof (dummyData[token]) == 'undefined') ? null : (token + 1)
});
});
}
// Here is how you would recursively call it with promises:
function getAllData() {
//data accumulator is sitting within the function so it doesn't pollute the global namespace.
var dataSoFar = [];
function recursiveCall(token, resolve, reject) {
dummyAsyncCall(token, function(data) {
if (data.error) {
reject(data.error);
}
if (!data.token) {
//You don't need to return the resolve/reject result.
resolve(dataSoFar);
} else {
dataSoFar = dataSoFar.concat(data.dummyDataPart);
recursiveCall(data.token, resolve, reject);
}
});
}
return new Promise(function(resolve, reject) {
// Note me passing resolve and reject into the recursive call.
// I like it this way but you can just store them within the closure for
// later use
recursiveCall(null, resolve, reject);
});
}
//Here is the call to the recursive service.
getAllData().then(function(data) {
console.log(data);
});
Fiddle with me

That's because you dont need to return resolve/reject, just call resolve/reject when the recursive call completes. A rough code would look like this
var getEndpoints = function(platformARN, token) {
return new models.sequelize.Promise(function(resolve, reject) {
var params = {
PlatformApplicationArn: platformARNDev
};
if (token != null) {
params['NextToken'] = token;
}
sns.listEndpointsByPlatformApplication(params, function(err, data) {
if (err) {
reject(err);
}
else {
endpoints = endpoints.concat(data.Endpoints); //save to global var
if ('NextToken' in data) {
//call recursively
getEndpoints(platformARN, data.NextToken).then(function () {
resolve(true);
}).catch(function (err) {
reject(err);
});
}
else {
console.log('trying to break out!');
resolve(true);
}
}
});
});
}
(caution: this is just a rough code, may work or may not, but is to give a general idea)
I've added a code snippet below, to support this concept, and it works great, check it out.
i = 0;
$('#output').empty();
function pro() {
return new Promise(function(resolve, reject) {
if (i > 3) {
resolve();
return;
}
window.setTimeout(function() {
console.log(i);
$('#output').append(i).append('<br/>');
i += 1;
pro().then(function() {
resolve()
}).catch(function() {
reject()
});
}, 2000);
});
}
pro().then(function () { $('#output').append("now here"); })
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
<div id="output"></div>

All functions in Q.all block are not being promised

I have the following code below in a then block
The issue I'm facing is at the end when i do the res.json(optionData1) its not returning the fully completed js data object i.e. the output after the processData function is missing
Am i using Q.all in the correct way?
var processUserInfo = function(categoryToProcess, inputToProcess, optionComingIn) {
var d = Q.defer();
if (optionData1['option'] == optionComingIn) {
if (optionData1[categoryToProcess].hasOwnProperty(inputToProcess)) {
optionData1[categoryToProcess][inputToProcess]++;
} else {
optionData1[categoryToProcess][inputToProcess] = 1;
}
d.resolve(optionData1);
}
}
var processData = function(item, optionComingIn) {
var d = Q.defer();
return User.find(
{_id: item},
{gender: 1, country:1},
function(req, foundUser) {
processUserInfo('gender', foundUser[0]['gender'], optionComingIn)
.then(function(resolve,reject) {
d.resolve();
});
});
return d.promise;
}
Q.all(foundQ[0]['people'].map(function(item) { // Or Q.allSettled
processCounts(item['optionSelected']);
processData(item['userID'], item['optionSelected']);
}))
.then(function(){
res.json(optionData1); //Doesnt give me the full result
});
Thanks
UPDATE: Using the return method as in the answer below got everything working.

Here is code which may work - too much "unknown" in your code snippet to be sure
modified processData to return a promise that resolves when user.Find is done
added a return in the .map, so the promise returned by processData is waited on in Q.all
So ... here's the fixed code (processuserInfo unchanged so omitted form the answer)
var processData = function (item, optionComingIn) {
// return a promise to wait for
return Q.promise(function(resolve, reject) {
User.find({
_id: item
}, {
gender: 1,
country: 1
},
function (req, foundUser) {
processUserInfo('gender', foundUser[0]['gender'], optionComingIn);
resolve();
}
);
});
}
Q.all(foundQ[0]['people'].map(function (item) { // Or Q.allSettled
processCounts(item['optionSelected']);
return processData(item['userID'], item['optionSelected']);
// return added
}))
.then(function () {
res.json(optionData1); //Doesnt give me the full result
});