I'm struggling a bit with JS promises.
I am using a library to pull data from Spotify that returns promises.
In my main function I can use an await to build an object from the response data and push it to an array (called nodes):
var nodes = [];
main();
async function main() {
var id = '0gusqTJKxtU1UTmNRMHZcv';
var artist = await getArtistFromSpotify(id).then(data => buildArtistObject(data));
nodes.push(artist);
When I debug here then all is good, nodes has my object.
However, when I introduce a 2nd await underneath to make another call:
nodes.forEach((node, i) => {
if (node.done == false) {
console.log(node.toString());
var related_artists = await getRelatedArtists(node.spotify_id);
I get the following error: SyntaxError: await is only valid in async function
I thought the first await statement would be resolved and the execution would continue until the next..?
Any help would be greatly appreciated.
EDIT
The other functions, if that helps, are just as follows:
function getArtistFromSpotify(id) {
let response = spotify
.request('https://api.spotify.com/v1/artists/' + id).then(function (data) {
return data;
})
.catch(function (err) {
console.error('Error occurred: ' + err);
return null;
});
return response;
}
function getRelatedArtists(id) {
let response = spotify
.request('https://api.spotify.com/v1/artists/' + id + '/related-artists').then(function (data) {
return data;
})
.catch(function (err) {
console.error('Error occurred: ' + err);
return null;
});
return response;
}
function buildArtistObject(data) {
var artist = {
node_id: nodes.length,
name: null,
genres: null,
popularity: null,
spotify_id: null,
done: false
}
artist.name = data.name;
artist.genres = data.genres;
artist.popularity = data.popularity > 0 ? data.popularity : 0;
artist.spotify_id = data.id;
return artist;
}
The code below has multiple problems.
var nodes = [];
main();
async function main() {
var id = '0gusqTJKxtU1UTmNRMHZcv';
var artist = await getArtistFromSpotify(id).then(data => buildArtistObject(data));
nodes.push(artist);
// ...
First of all, main mutates global scope nodes. Not only is this an antipattern even in synchronous code (functions should not rely on, or modify, global variable names; use parameters and return values instead), in asynchronous code, nodes will never be available for use anywhere but within main. See How do I return the response from an asynchronous call?.
Secondly, try to avoid combining then and await. It's confusing.
It's also a little odd that an array of nodes is used, yet only one artist is pushed onto it...
As for this code:
nodes.forEach((node, i) => {
if (node.done == false) {
console.log(node.toString());
var related_artists = await getRelatedArtists(node.spotify_id);
// ...
The error is self-explanatory. You must add async to the enclosing function if you want it to be asynchronous: nodes.forEach(async (node, i) => { // .... But that spawns a new promise chain per node, meaning future code that's dependent on the result won't be able to await all of the promises in the loop resolving. See Using async/await with a forEach loop. The likely solution is for..of or Promise.all.
While I'm not 100% sure what your final goal is, this is the general pattern I'd use:
async function main() {
const id = '0gusqTJKxtU1UTmNRMHZcv';
const data = await getArtistFromSpotify(id);
const artist = await buildArtistObject(data);
const nodes = [artist]; // odd but I assume you have more artists somewhere...
for (const node of nodes) {
if (!node.done) {
const relatedArtists = await getRelatedArtists(node.spotify_id);
}
}
/* or run all promises in parallel:
const allRelatedArtists = await Promise.all(
nodes.filter(e => !e.done).map(e => getRelatedArtists(e.spotify_id))
);
*/
// ...
}
main();
Since your code isn't runnable and some of the intent is unclear from the context, you'll likely need to adapt this a bit, so consider it pseudocode.
You have some misunderstandings of how to use promises -
let response = spotify
.request(url)
.then(function(data) { return data }) // this does nothing
.catch(function (err) { // don't swallow errors
console.error('Error occurred: ' + err);
return null;
})
return response
You'll be happy there's a more concise way to write your basic functions -
const getArtist = id =>
spotify
.request('https://api.spotify.com/v1/artists/' + id)
const getRelatedArtists = id =>
spotify
.request('https://api.spotify.com/v1/artists/' + id + '/related-artists')
Now in your main function, we can await as many things as needed. Let's first see how we would work with a single artist ID -
async function main(artistId) {
const artistData = await getArtist(artistId)
const relatedData = await getRelatedArtists(artistId)
return buildArtist(artistData, relatedData)
}
If you have many artist IDs -
async function main(artistIds) {
const result = []
for (const id of artistIds) {
const artistData = await getArtist(artistId)
const relatedData = await getRelatedArtists(artistId)
result.push(buildArtist(artistData, relatedData))
}
return result
}
Either way, the caller can handle errors as
main([693, 2525, 4598])
.then(console.log) // display result
.catch(console.error) // handle errors
Which is the same as -
main([693, 2525, 4598]).then(console.log, console.error)
The pattern above is typical but sub-optimal as the caller has to wait for all data to fetch before the complete result is returned. Perhaps you would like to display the information, one-by-one as they are fetched. This is possible with async generators -
async function* buildArtists(artistIds) {
for (const id of artistIds) {
const artistData = await getArtist(artistId)
const relatedData = await getRelatedArtists(artistId)
yield buildArtist(artistData, relatedData) // <- yield
}
}
async function main(artistIds) {
for await (const a of buildArtists(artistIds)) // <- for await
displayArtist(a)
}
main([693, 2525, 4598]).catch(console.error)
Related
I'd like to reuse the same code in a loop. This code contains promises. However, when iterating, this code results in an error.
I've tried using for and while loops. There seems to be no issue when I use the for loop for a single iteration.
Here is a minimal version of my code:
var search_url = /* Some initial URL */
var glued = "";
for(var i = 0; i < 2; i++)
{
const prom = request(search_url)
.then(function success(response /* An array from a XMLHTTPRequest*/) {
if (/* Some condition */)
{
search_url = /* Gets next URL */
glued += processQuery(response[0]);
} else {
console.log("Done.")
}
})
.catch(function failure(err) {
console.error(err.message); // TODO: do something w error
})
}
document.getElementById('api-content').textContent = glued;
I expect the results to append to the variable glued but instead, I get an error: failure Promise.catch (async) (anonymous) after the first iteration of the loop.
Answer:
You can use the Symbol.iterator in accordance with for await to perform asynchronous execution of your promises. This can be packaged up into a constructor, in the example case it's called Serial (because we're going through promises one by one, in order)
function Serial(promises = []) {
return {
promises,
resolved: [],
addPromise: function(fn) {
promises.push(fn);
},
resolve: async function(cb = i => i, err = (e) => console.log("trace: Serial.resolve " + e)) {
try {
for await (let p of this[Symbol.iterator]()) {}
return this.resolved.map(cb);
} catch (e) {
err(e);
}
},
[Symbol.iterator]: async function*() {
this.resolved = [];
for (let promise of this.promises) {
let p = await promise().catch(e => console.log("trace: Serial[Symbol.iterator] ::" + e));
this.resolved.push(p);
yield p;
}
}
}
}
What is the above?
It's a constructor called Serial.
It takes as an argument an array of Functions that return Promises.
The functions are stored in Serial.promises
It has an empty array stored in Serial.resolved - this will store the resolved promise requests.
It has two methods:
addPromise: Takes a Function that returns a Promise and adds it to Serial.promises
resolve: Asynchronously calls a custom Symbol.iterator. This iterator goes through every single promise, waits for it to be completed, and adds it to Serial.resolved. Once this is completed, it returns a map function that acts on the populated Serial.resolved array. This allows you to simply call resolve and then provide a callback of what to do with the array of responses. A.e. .resolve()((resolved_requests) => //do something with resolved_requests)
Why does it work?
Although many people don't realize this Symbol.iterator is much more powerful than standard for loops. This is for two big reasons.
The first reason, and the one that is applicable in this situation, is because it allows for asynchronous calls that can affect the state of the applied object.
The second reason is that it can be used to provide two different types of data from the same object. A.e. You may have an array that you would like to read the contents of:
let arr = [1,2,3,4];
You can use a for loop or forEach to get the data:
arr.forEach(v => console.log(v));
// 1, 2, 3, 4
But if you adjust the iterator:
arr[Symbol.iterator] = function* () {
yield* this.map(v => v+1);
};
You get this:
arr.forEach(v => console.log(v));
// 1, 2, 3, 4
for(let v of arr) console.log(v);
// 2, 3, 4, 5
This is useful for many different reasons, including timestamping requests/mapping references, etc. If you'd like to know more please take a look at the ECMAScript Documentation: For in and For Of Statements
Use:
It can be used by calling the constructor with an Array of functions that return Promises. You can also add Function Promises to the Object by using
new Serial([])
.addPromise(() => fetch(url))
It doesn't run the Function Promises until you use the .resolve method.
This means that you can add promises ad hoc if you'd like before you do anything with the asynchronous calls. A.e. These two are the same:
With addPromise:
let promises = new Serial([() => fetch(url), () => fetch(url2), () => fetch(url3)]);
promises.addPromise(() => fetch(url4));
promises.resolve().then((responses) => responses)
Without addPromise:
let promises = new Serial([() => fetch(url), () => fetch(url2), () => fetch(url3), () => fetch(url4)])
.resolve().then((responses) => responses)
Data:
Since I can't really replicate your data calls, I opted for JSONPlaceholder (a fake online rest api) to show the promise requests in action.
The data looks like this:
let searchURLs = ["https://jsonplaceholder.typicode.com/todos/1",
"https://jsonplaceholder.typicode.com/todos/2",
"https://jsonplaceholder.typicode.com/todos/3"]
//since our constructor takes functions that return promises, I map over the URLS:
.map(url => () => fetch(url));
To get the responses we can call the above data using our constructor:
let promises = new Serial(searchURLS)
.resolve()
.then((resolved_array) => console.log(resolved_array));
Our resolved_array gives us an array of XHR Response Objects. You can see that here:
function Serial(promises = []) {
return {
promises,
resolved: [],
addPromise: function(fn) {
promises.push(fn);
},
resolve: async function(cb = i => i, err = (e) => console.log("trace: Serial.resolve " + e)) {
try {
for await (let p of this[Symbol.iterator]()) {}
return this.resolved.map(cb);
} catch (e) {
err(e);
}
},
[Symbol.iterator]: async function*() {
this.resolved = [];
for (let promise of this.promises) {
let p = await promise().catch(e => console.log("trace: Serial[Symbol.iterator] ::" + e));
this.resolved.push(p);
yield p;
}
}
}
}
let searchURLs = ["https://jsonplaceholder.typicode.com/todos/1", "https://jsonplaceholder.typicode.com/todos/2", "https://jsonplaceholder.typicode.com/todos/3"].map(url => () => fetch(url));
let promises = new Serial(searchURLs).resolve().then((resolved_array) => console.log(resolved_array));
Getting Results to Screen:
I opted to use a closure function to simply add text to an output HTMLElement.
This is added like this:
HTML:
<output></output>
JS:
let output = ((selector) => (text) => document.querySelector(selector).textContent += text)("output");
Putting it together:
If we use the output snippet along with our Serial object the final functional code looks like this:
let promises = new Serial(searchURLs).resolve()
.then((resolved) => resolved.map(response =>
response.json()
.then(obj => output(obj.title))));
What's happening above is this:
we input all our functions that return promises. new Serial(searchURLS)
we tell it to resolve all the requests .resolve()
after it resolves all the requests, we tell it to take the requests and map the array .then(resolved => resolved.map
the responses we turn to objects by using .json method. This is necessary for JSON, but may not be necessary for you
after this is done, we use .then(obj => to tell it to do something with each computed response
we output the title to the screen using output(obj.title)
Result:
let output = ((selector) => (text) => document.querySelector(selector).textContent += text)("output");
function Serial(promises = []) {
return {
promises,
resolved: [],
addPromise: function(fn) {
promises.push(fn);
},
resolve: async function(cb = i => i, err = (e) => console.log("trace: Serial.resolve " + e)) {
try {
for await (let p of this[Symbol.iterator]()) {}
return this.resolved.map(cb);
} catch (e) {
err(e);
}
},
[Symbol.iterator]: async function*() {
this.resolved = [];
for (let promise of this.promises) {
let p = await promise().catch(e => console.log("trace: Serial[Symbol.iterator] ::" + e));
this.resolved.push(p);
yield p;
}
}
}
}
let searchURLs = ["https://jsonplaceholder.typicode.com/todos/1", "https://jsonplaceholder.typicode.com/todos/2", "https://jsonplaceholder.typicode.com/todos/3"].map(url => () => fetch(url));
let promises = new Serial(searchURLs).resolve()
.then((resolved) => resolved.map(response =>
response.json()
.then(obj => output(obj.title))));
<output></output>
Why go this route?
It's reusable, functional, and if you import the Serial Constructor you can keep your code slim and comprehensible. If this is a cornerstone of your code, it'll be easy to maintain and use.
Using it with your code:
I will add how to specifically use this with your code to fully answer your question and so that you may understand further.
NOTE glued will be populated with the requested data, but it's unnecessary. I left it in because you may have wanted it stored for a reason outside the scope of your question and I don't want to make assumptions.
//setup urls:
var search_urls = ["https://jsonplaceholder.typicode.com/todos/1", "https://jsonplaceholder.typicode.com/todos/2"];
var request = (url) => () => fetch(url);
let my_requests = new Serial(search_urls.map(request));
//setup glued (you don't really need to, but if for some reason you want the info stored...
var glued = "";
//setup helper function to grab title(this is necessary for my specific data)
var addTitle = (req) => req.json().then(obj => (glued += obj.title, document.getElementById('api-content').textContent = glued));
// put it all together:
my_requests.resolve().then(requests => requests.map(addTitle));
Using it with your code - Working Example:
function Serial(promises = []) {
return {
promises,
resolved: [],
addPromise: function(fn) {
promises.push(fn);
},
resolve: async function(cb = i => i, err = (e) => console.log("trace: Serial.resolve " + e)) {
try {
for await (let p of this[Symbol.iterator]()) {}
return this.resolved.map(cb);
} catch (e) {
err(e);
}
},
[Symbol.iterator]: async function*() {
this.resolved = [];
for (let promise of this.promises) {
let p = await promise().catch(e => console.log("trace: Serial[Symbol.iterator] ::" + e));
this.resolved.push(p);
yield p;
}
}
}
}
//setup urls:
var search_urls = ["https://jsonplaceholder.typicode.com/todos/1", "https://jsonplaceholder.typicode.com/todos/2"];
var request = (url) => () => fetch(url);
let my_requests = new Serial(search_urls.map(request));
//setup glued (you don't really need to, but if for some reason you want the info stored...
var glued = "";
//setup helper function to grab title(this is necessary for my specific data)
var addTitle = (req) => req.json().then(obj => (glued += obj.title, document.getElementById('api-content').textContent = glued));
// put it all together:
my_requests.resolve().then(requests => requests.map(addTitle));
<div id="api-content"></div>
Final Note
It's likely that we will be seeing a prototypal change to the Promise object in the future that allows for easy serialization of Promises. Currently (7/15/19) there is a TC39 Proposal that does add a lot of functionality to the Promise object but it hasn't been fully vetted yet, and as with many ideas trapped within the Proposal stage, it's almost impossible to tell when they will be implemented into Browsers, or even if the idea will stagnate and fall off the radar.
Until then workarounds like this are necessary and useful( the reason why I even went through the motions of constructing this Serializer object was for a transpiler I wrote in Node, but it's been very helpful beyond that! ) but do keep an eye out for any changes because you never know!
Hope this helps! Happy Coding!
Your best bet is probably going to be building up that glued variable with recursion.
Here's an example using recursion with a callback function:
var glued = "";
requestRecursively(/* Some initial URL string */, function() {
document.getElementById('api-content').textContent = glued;
});
function requestRecursively(url, cb) {
request(url).then(function (response) {
if (/* Some condition */) {
glued += processQuery(response[0]);
var next = /* Gets next URL string */;
if (next) {
// There's another URL. Make another request.
requestRecursively(next, cb);
} else {
// We're done. Invoke the callback;
cb();
}
} else {
console.log("Done.");
}
}).catch(function (err) {
console.error(err.message);
});
}
I'm trying to do a relatively simple, in theory, function using Firebase Functions.
Specifically:
Add + 1 to a realtime database variable for all users
Send a notification to all users
I'm still trying to understand async/await which is probably why I'm struggling so much with this.
Here is what I'm doing:
exports.gcIncrement = functions.database
.ref('gthreads/{threadId}/messages/{messageId}')
.onCreate(async (snapshot, context) => {
const data = snapshot.val();
const threadId = context.params.threadId;
const uid = context.auth.uid;
adb.ref('gchats/' + threadId).once('value').then(async (gchatData) => {
const parent = gchatData.val();
incrementUser(parent.users, uid, threadId); //parent.users is an object with 1-30 users.
sendGCNotification(parent.users, data);
return true;
}).catch(error => console.log(error))
});
And then I have the function incrementUser:
function IncrementUser(array, uid, threadId) {
for (const key in array) {
if (key != uid) {
const gcMessageRef =
adb.ref('users/' + key + '/gthreads/' + threadId + '/' + threadId+'/unread/');
gcMessageRef.transaction((int) => {
return (int || 0) + 1;
}
}
}
and the function sendGCNotification:
function sendGCNotification(array, numbOfMsg, data) {
let payload = {
notification: {
title: 'My App - ' + data.title,
body: "This is a new notification!",
}
}
const db = admin.firestore()
for (const key in array) {
if (!data.adminMessage) {
if (array[key] === 0) {
const devicesRef = db.collection('devices').where('userId', '==', key)
const devices = await devicesRef.get();
devices.forEach(result => {
const tokens = [];
const token = result.data().token;
tokens.push(token)
return admin.messaging().sendToDevice(tokens, payload)
})
}
}
}
}
I currently get the error:
'await' expression is only allowed within an async function.
const devices = await devicesRef.get();
But even when I get it error-free, it doesn't seem work. The Firebase Functions log says:
4:45:26.207 PM
gcIncrement
Function execution took 444 ms, finished with status: 'ok'
4:45:25.763 PM
gcIncrement
Function execution started
So it seems to run as expected but not fulfill the code as expected. Any ideas? Thank you!
All uses of await have to occur within the main body of a function that's marked async. Your function sendGCNotification is not async. You'll have to mark it async, and also make sure that any promises within it have been awaited, or return a promise that resolves when all the async work is done.
Also, in IncrementUser you are not handling the promise returned by gcMessageRef.transaction(). You need to handle every promise that you generate from all the async work, and make sure they are all a part of the final promise that you return or await from your top-level function.
If you want to learn more about promises and async/await in Cloud Functions code, I suggest you use my video series. Specifically, the one titled "How does async/await work with TypeScript and ECMAScript 2017?". Even if you aren't using TypeScript, async/await work the same way.
My requirement is pretty straightforward -
I want to insert a document in MongoDB database. But before I have to check if the slug already exists in database. Then perform an operation to rename the slug if the slug is already exists.
What I have been trying is to perform an async await callback to check the slug is already exists then insert the document.
mongoClient.connect(function (err, mongoClient) {
let db = mongoClient.db("articles");
let category_information = async (db, category_info) => {
let slug_information = await db.collection('categories').find({slug: category_info.slug});
slug_information.count((err, count) => {
if (count > 0) {
let new_slug = `${category_info.slug}_${new Date().getTime()}`;
console.log(new_slug);
return new_slug;
}
else
return category_info.slug;
})
};
let category_promise = category_information(db, category_info);
category_promise.then(value => {
console.log(value);
category_info.slug = value;
});
db.collection('categories')
.insertOne(category_info, (err, data) => {
assert.equal(null, err);
res.status(200);
res.json('success');
});
mongoClient.close();
});
In console I get undefined value from Promise. Can you please figure out my code?
I am new in MongoDB. So also, do you have the solution of the problem in MongoDB way? I mean, can I perform these two queries within a single query?
Thanks!
You don't need to await find() since it's actually the command coming after, in this case count() that is executing the query.
Next I wonder where and how category_info is defined. It's missing in the code above. But I'll assume you have set it properly in your code.
You must return something from your async function (a promise preferably). Right now you only return from the count-callback.
With async/await you should be able to:
const count = await slug_information.count();
if (count > 0) {
let new_slug = `${category_info.slug}_${new Date().getTime()}`;
console.log(new_slug);
return new_slug;
} else {
return category_info.slug;
}
Basically, if you use a callback like (err, count)=>{..} then you say "I won't be using promises here!", no promise will come and you have nothing to wait for.
Next: category_promise.then(... this bit is async, so you cannot know that it'll resolve before you start your insertOne( query. Actually you can be almost sure it hasn't.
So you either chain another then:
category_promise.then(value => {
console.log(value);
return category_info.slug = value;
}).then( ()=>{
db.collection('categories')
.insertOne( ...
});
or just async the whole thing:
const MongoClient = require("mongodb").MongoClient;
const category_info = { slug: "abc" };
async function run(req, res, next) {
const mongoClient = await MongoClient.connect("mongodb://localhost:27017");
let db = mongoClient.db("categories");
// With async/await this is almost superfluous, but lets roll with it.
let category_information = async (db, category_info) => {
const count = await db.collection("articles")
.find({ slug: category_info.slug })
.count();
if (count > 0) {
let new_slug = `${category_info.slug}_${new Date().getTime()}`;
console.log(new_slug);
return new_slug;
} else {
return category_info.slug;
}
};
category_info.slug = await category_information(db, category_info);
// note that insertOne() does not return the inserted document.
let data = await db.collection("categories").insertOne(category_info);
res.status(200).json(data);
mongoClient.close();
}
run(); // or app.get("/some-route", run);
This code runs, but I haven' tested every case (count and so on), so grain of salt and all that.
Consider this code
const response = await fetch('<my url>');
const responseJson = await response.json();
responseJson = _.sortBy(responseJson, "number");
responseJson[0] = await addEnabledProperty(responseJson[0]);
What addEnabledProperty does is to extend the object adding an enabled property, but this is not important. The function itself works well
async function addEnabledProperty (channel){
const channelId = channel.id;
const stored_status = await AsyncStorage.getItem(`ChannelIsEnabled:${channelId}`);
let boolean_status = false;
if (stored_status == null) {
boolean_status = true;
} else {
boolean_status = (stored_status == 'true');
}
return _.extend({}, channel, { enabled: boolean_status });
}
Is there a way to use _.map (or another system), to loop trough entire responseJson array to use addEnabledProperty against each element?
I tried:
responseJson = _.map(responseJson, function(channel) {
return addEnabledProperty(channell);
});
But it's not using async so it freeze the app.
I tried:
responseJson = _.map(responseJson, function(channel) {
return await addEnabledProperty(chanell);
});
But i got a js error (about the row return await addEnabledProperty(chanell);)
await is a reserved word
Then tried
responseJson = _.map(responseJson, async function(channel) {
return await addEnabledProperty(channell);
});
But I got an array of Promises... and I don't understand why...
What else!??
EDIT: I understand your complains about I didn't specify that addEnabledProperty() returns a Promise, but, really, I didn't know it. In fact, I wrote "I got an array of Promises... and I don't understand why "
To process your response jsons in parallel you may use Promise.all:
const responseJson = await response.json();
responseJson = _.sortBy(responseJson, "number");
let result = await Promise.all(_.map(responseJson, async (json) =>
await addEnabledProperty(json))
);
Since addEnabledProperty method is async, the following also should work (per #CRice):
let result = await Promise.all(_.map(responseJson, addEnabledProperty));
I found that I didn't have to put the async / await inside of the Promise.all wrapper.
Using that knowledge, in conjunction with lodash chain (_.chain) could result in the following simplified version of the accepted answer:
const responseJson = await Promise.all( _
.chain( response.json() )
.sortBy( 'number' )
.map( json => addEnabledProperty( json ) )
.value()
)
How about using partial.js(https://github.com/marpple/partial.js)
It cover both promise and normal pattern by same code.
_p.map([1, 2, 3], async (v) => await promiseFunction());
You can use Promise.all() to run all the promises in your array.
responseJson = await Promise.all(_.map(responseJson, (channel) => {
return addEnabledProperty(channel);
}));
If you want to iterate over some object, you can use keys first to get an array of keys and then just loop over your keys while awaiting for necessary actions.
This works when you want to wait until every previous iteration step is finished before getting into the next one.
Here is a consolidated asyncMap function that you can use for your object:
async function asyncMap(obj, cb) {
const result = {};
const keysArr = keys(obj);
let keysArrLength = keysArr.length;
while (keysArrLength-- > 0) {
const key = keysArr[keysArrLength];
const item = obj[key];
// eslint-disable-next-line no-await-in-loop
result[key] = await cb(item, key);
}
return result;
}
And then, for your example case:
responseJson = await asyncMap(responseJson, addEnabledProperty);
Otherwise use Promise.all like was proposed above to run all the iteration steps in parallel
I am creating an API that when GET, a series of calls to the News API are made, news article titles are extracted into a giant string, and that string is processed into an object to be delivered to a wordcloud on the front-end. So far, I've been able to use underscore's _.after and request-promise to make my app wait till all API calls have completed before calling processWordBank() which takes the giant string and cleans it up into an object. However, once processWordBank() is called, I don't understand where the flow of the program is. Ideally, processWordBank() returns obj to cloudObj in the router, so that the obj can be passed to res.json() and spit out as the response. I believe my use of _.after has put me in a weird situation, but it's the only way I've been able to get async calls to finish before proceeding to next desired action. Any suggestions?
(I've tried to leave out all unnecessary code but let me know if this is insufficient)
// includes...
var sourceString = ""
// router
export default ({ config }) => {
let news = Router()
news.get('/', function(req, res){
var cloudObj = getSources()
res.json({ cloudObj })
})
return news
}
// create list of words (sourceString) by pulling news data from various sources
function getSources() {
return getNewsApi()
}
// NEWS API
// GET top 10 news article titles from News API (news sources are determined by the values of newsApiSource array)
function getNewsApi() {
var finished = _.after(newsApiSource.length, processWordBank)
for(var i = 0; i < newsApiSource.length; i++) {
let options = {
uri: 'https://newsapi.org/v1/articles?source=' + newsApiSource[i] + '&sortBy=' + rank + '&apiKey=' + apiKey,
json: true
}
rp(options)
.then(function (res) {
let articles = res.articles // grab article objects from the response
let articleTitles = " " + _.pluck(articles, 'title') // extract title of each news article
sourceString += " " + articleTitles // add all titles to the word bank
finished() // this async task has finished
})
.catch(function (err) {
console.log(err)
})
}
}
// analyse word bank for patterns/trends
function processWordBank(){
var sourceArray = refineSource(sourceString)
sourceArray = combineCommon(sourceArray)
sourceArray = getWordFreq(sourceArray)
var obj = sortToObject(sourceArray[0], sourceArray[1])
console.log(obj)
return obj
}
A big issue in your asynchronous flow is that you use a shared variable sourceString to handle the results. When you have multiple calls to getNewsApi() your result is not predictable and will not always be the same, because there is no predefined order in which the asynchronous calls are executed. Not only that, but you never reset it, so all subsequent calls will also include the results of the previous calls. Avoid modifying shared variables in asynchronous calls and instead use the results directly.
I've been able to use underscore's _.after and request-promise to make my app wait till all API calls have completed before calling processWordBank()
Although it would possible to use _.after, this can be done very nicely with promises, and since you're already using promises for your requests, it's just a matter of collecting the results from them. So because you want to wait until all API calls are completed you can use Promise.all which returns a promise that resolves with an array of the values of all the promises, once all of them are fulfilled. Let's have a look at a very simple example to see how Promise.all works:
// Promise.resolve() creates a promise that is fulfilled with the given value
const p1 = Promise.resolve('a promise')
// A promise that completes after 1 second
const p2 = new Promise(resolve => setTimeout(() => resolve('after 1 second'), 1000))
const p3 = Promise.resolve('hello').then(s => s + ' world')
const promises = [p1, p2, p3]
console.log('Waiting for all promises')
Promise.all(promises).then(results => console.log('All promises finished', results))
console.log('Promise.all does not block execution')
Now we can modify getNewsApi() to use Promise.all. The array of promises that is given to Promise.all are all the API request you're doing in your loop. This will be created with Array.protoype.map. And also instead of creating a string out of the array returned from _.pluck, we can just use the array directly, so you don't need to parse the string back to an array at the end.
function getNewsApi() {
// Each element is a request promise
const apiCalls = newsApiSource.map(function (source) {
let options = {
uri: 'https://newsapi.org/v1/articles?source=' + source + '&sortBy=' + rank + '&apiKey=' + apiKey,
json: true
}
return rp(options)
.then(function (res) {
let articles = res.articles
let articleTitles = _.pluck(articles, 'title')
// The promise is fulfilled with the articleTitles
return articleTitles
})
.catch(function (err) {
console.log(err)
})
})
// Return the promise that is fulfilled with all request values
return Promise.all(apiCalls)
}
Then we need to use the values in the router. We know that the promise returned from getNewsApi() fulfils with an array of all the requests, which by themselves return an array of articles. That is a 2d array, but presumably you would want a 1d array with all the articles for your processWordBank() function, so we can flatten it first.
export default ({ config }) => {
let news = Router()
new.get('/', (req, res) => {
const cloudObj = getSources()
cloudObj.then(function (apiResponses) {
// Flatten the array
// From: [['source1article1', 'source1article2'], ['source2article1'], ...]
// To: ['source1article1', 'source1article2', 'source2article1', ...]
const articles = [].concat.apply([], apiResponses)
// Pass the articles as parameter
const processedArticles = processWordBank(articles)
// Respond with the processed object
res.json({ processedArticles })
})
})
}
And finally processWordBank() needs to be changed to use an input parameter instead of using the shared variable. refineSource is no longer needed, because you're already passing an array (unless you do some other modifications in it).
function processWordBank(articles) {
let sourceArray = combineCommon(articles)
sourceArray = getWordFreq(sourceArray)
var obj = sortToObject(sourceArray[0], sourceArray[1])
console.log(obj)
return obj
}
As a bonus the router and getNewsApi() can be cleaned up with some ES6 features (without the comments from the snippets above):
export default ({ config }) => {
const news = Router()
new.get('/', (req, res) => {
getSources().then(apiResponses => {
const articles = [].concat(...apiResponses)
const processedArticles = processWordBank(articles)
res.json({ processedArticles })
})
})
}
function getNewsApi() {
const apiCalls = newsApiSource.map(source => {
const options = {
uri: `https://newsapi.org/v1/articles?source=${source}&sortBy=${rank}&apiKey=${apiKey}`,
json: true
}
return rp(options)
.then(res => _.pluck(res.articles, 'title'))
.catch(err => console.log(err))
})
return Promise.all(apiCalls)
}