I'm trying to make several asynchronous backend calls to generate a JSON response in my express API. Because of the nature of the API, I have 3 requests that are being made that are dependent on each other in some way.
Request 1: Returns an Array of values that are used to make request 2. Each value will be used as a mapping for the remaining requests. That is to say, it will be a unique identifier used to map the response from the requests in Request 3.
Request 2 (Parallel Batch): A request is made using each value from the Array returned in request 1. Each of these returns a value to be used in each of the Request 3s. That is to say, it's a 1-to-1
Request 3 (Parallel Batch): This request takes the response from Request 2, and makes a 1-to-1 follow up request to get more data on that specific mapping (the id from request 1)
I would like the final data I send to the consumer to look like this:
{
id1: details1,
id2: details2,
id3: details3,
...
}
Here is the code I have so far...
app.get("/artists/:artist/albums", (req, res) => {
console.log("#############")
const artistName = req.params.artist
let response = {};
let s3Promise = s3.listAlbums(artistName)
let albumDetailsPromises = []
s3Promise
.then((data) => {
data.map((album) => {
// Each album name here will actually be used as the unique identifier for
// the final response
// Build an Array of promises that will first fetch the albumId, then use
// that album id to fetch the details on the album
albumDetailsPromises.push(
discogs.getAlbumId(artistName, album).then( // Returns a promise
({ data }) => {
let masterId = data.results[0].id
let recordName = data.results[0].title
// Storing the album name to carry as a unique id alongside the promise
return [album, discogs.getAlbumDetails(masterId) // Returns a promise ]
}
)
)
})
})
.then(() => {
// When all the albumIds have been fetched, there will still exist a promise in the
// second index of each element in the albumDetailsPromises array
Promise.all(albumDetailsPromises)
.then((namedPromises) => {
namedPromises.map(
(album) => {
let albumName = album[0] // Unique Id
let albumDetailPromise = album[1]
// Resolving the albumDetailsPromise here, and storing the value on
// a response object that we intend to send as the express response
albumDetailPromise
.then(
({ data }) => {
response[albumName] = data
})
.catch(err => response[albumName] = err)
})
})
})
.catch((err) => console.log(err))
})
As of now, everything seems to be working as expected, I just can't seem to figure out how to "await" the response object being updated at the end of all these Promises. I've omitted res.send(response) from this example because it's not working, but that's of course my desired outcome.
Any advice is appreciated! New to javascript...
I would recommend rewriting this using async/await as it helps to reduce nesting. You can also extract the logic the get the album-details into a separate function, as this also increases the readability of the code. Something like this (this still needs error-handling, but it should give you a start):
app.get("/artists/:artist/albums", async (req, res) => {
const artistName = req.params.artist;
const albumNames = await s3.listAlbums(artistName);
const result = {};
const albumDetailPromises = albumNames.map(albumName => requestAlbumDetails(discogs, artistName, albumName));
const resolvedAlbumDetails = await Promise.all(albumDetailPromises);
// map desired response structure
for(const albumDetail of resolvedAlbumDetails) {
result[albumDetail.albumName] = albumDetail.albumDetails;
}
res.json(result);
});
async function requestAlbumDetails(service, artistName, albumName) {
const albumInfo = await service.getAlbumId(artistName, albumName);
const masterId = albumInfo.results[0].id;
const albumDetails = await service.getAlbumDetails(masterId);
return { albumName, albumDetails };
}
To answer your question how you could do it with your code:
You'd need to wait for all details to be fulfilled using another Promise.all call and then just send the response in the then-handler:
Promise.all(albumDetailsPromises)
.then((namedPromises) => {
const detailsPromises = namedPromises.map(
(album) => {
let albumName = album[0];
let albumDetailPromise = album[1];
return albumDetailPromise
.then(({ data }) => {
response[albumName] = data;
})
.catch(err => response[albumName] = err);
});
return Promise.all(detailsPromises)
.then(() => res.json(response));
})
Refactored using async/await...
app.get("/artists/:artist/albums", async (req, res) => {
const artistName = req.params.artist
let response = {};
let albums = await s3.listAlbums(artistName)
const promises = albums.map(async (album) => {
let result = await discogs.getAlbumId(artistName, album)
try {
let masterId = result.data.results[0].id
let tempRes = await discogs.getAlbumDetails(masterId)
return [album, tempRes.data]
} catch (error) {
return [album, { "msg": error.message }]
}
})
responses = await Promise.all(promises)
responses.map(data => { response[data[0]] = data[1] })
res.send(response)
})
Related
I am attempting to build a bot that will periodically poll an API using axios for the price of multiple cryptocurrencies across multiple exchanges. I need to be able to then look at this stored data to analyse price differences of a given token between multiple exchanges to calculate if there is profit to be made if I were to purchase it on one and then sell on another.
So far I am able to get access to the price data and console.log it as the request is returned, however I need to add this data to an array so that it can be analysed at a later point. I understand that I will need to use promises to handle this but I find it slightly confusing.
The following code immediately outputs an empty array and then outputs all the individual prices. How can I gain access to those values at a later point in the code?
const axios = require('axios');
const { exchanges } = require('./resources/exchanges.json');
const { currencies } = require('./resources/currencies.json');
const buyCurrency = currencies.buy[0];
const poll = async () => {
const data = new Array();
await currencies.sell.forEach(async sellCurrency => {
await exchanges.forEach(async exchange => {
try {
const allOtherExchanges = exchanges.filter(x => x !== exchange).map(x => `,${x}`).join();
const response = await axios.get(`https://api.0x.org/swap/v1/quote?buyToken=${buyCurrency}&sellToken=${sellCurrency}&sellAmount=1000000000000000000&excludedSources=0x${allOtherExchanges}`)
if (response && response.data.price) {
console.log(exchange, sellCurrency, response.data.price)
data.push({
exchange,
price: response.data.price
});
}
} catch {}
});
});
console.log(data);
};
poll()
One of the solutions would be as follows:
const axios = require('axios');
const { exchanges } = require('./resources/exchanges.json');
const { currencies } = require('./resources/currencies.json');
const buyCurrency = currencies.buy[0];
const poll = async () => {
const data = new Array();
const promises = [];
currencies.sell.forEach(sellCurrency => {
exchanges.forEach(exchange => {
const allOtherExchanges = exchanges.filter(x => x !== exchange).map(x => `,${x}`).join();
promises.push(
axios.get(`https://api.0x.org/swap/v1/quote?buyToken=${buyCurrency}&sellToken=${sellCurrency}&sellAmount=1000000000000000000&excludedSources=0x${allOtherExchanges}`)
.then(response => {
if (response && response.data &&response.data.price) {
console.log(exchange, sellCurrency, response.data.price)
data.push({
exchange,
price: response.data.price
});
}
}).catch(err => console.error(err))
);
});
});
await Promise.all(promises);
console.log(data);
};
poll();
I'm creating a JS function that will make a call to an API, loop through the returned data and perform another call to retrieve more information about the initial data (for example where the first call return an ID, the second call would return the name/address/number the ID corresponds to). Positioning the async and await keywords though, have proven to be way more challenging than I imagined:
useEffect(() => {
const getAppointments = async () => {
try {
const { data } = await fetchContext.authAxios.get('/appointments/' + auth.authState.id);
const updatedData = await data.map(value => {
const { data } = fetchContext.authAxios.get('/customerID/' + value.customerID);
return {
...value, // de-structuring
customerID: data
}
}
);
setAppointments(updatedData);
} catch (err) {
console.log(err);
}
};
getAppointments();
}, [fetchContext]);
Everything get displayed besides the customerID, that results undefined. I tried to position and add the async/await keywords in different places, nothing works. What am I missing?
map returns an array, not a promise. You need to get an array of promises and then solve it (also, if your way worked, it would be inefficient waitting for a request to then start the next one.)
const promises = data.map(async (value) => {
const { data } = await fetchContext.authAxios.get('/customerID/' + value.customerID);
return {
...value,
customerID: data
};
});
const updatedData = await Promise.all(promises);
I'm trying to only make one http call at time but when I log the response from getUrl they are piling up and I start to get 409s (Too many requests)
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
fetch(fetchUrl).then(async res => {
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
cb(url);
});
}
let requests = urls.map((url, i) => {
return new Promise(resolve => {
getUrl(url, i, resolve);
});
});
const all = await requests.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult => [...chainResults, currentResult]),
);
}, Promise.resolve([]));
Basically I don't want the next http to start until the previous one has finished. Otherwise I hammer their server.
BONUS POINTS: Make this work with 5 at a time in parallel.
Since you're using await, it would be a lot easier to use that everywhere instead of using confusing .thens with reduce. It'd also be good to avoid the explicit Promise construction antipattern. This should do what you want:
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
Then your results variable will contain an array of response texts (or an error will have been thrown, and the code won't reach the bottom).
The syntax for an async function is an async keyword before the argument list, just like you're doing in your original code:
const fn = async () => {
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
// do something with results
};
To have a limited number of requests at a time, make a queue system - when a request completes, recursively call a function that makes another request, something like:
const results = [];
const queueNext = async () => {
if (!urls.length) return;
const url = urls.shift();
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
await queueNext();
}
await Promise.all(Array.from({ length: 5 }, queueNext));
// do something with results
You cannot use Array methods to sequentually run async operations because array methods are all synchronous.
The easiest way to achieve sequential async tasks is through a loop. Otherwise, you will need to write a custom function to imitate a loop and run .then after a async task ends, which is quite troublesome and unnecessary.
Also, fetch is already returning a Promise, so you don't have to create a Promise yourself to contain that promise returned by fetch.
The code below is a working example, with small changes to your original code (see comments).
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
// To imitate actual fetching
const fetch = (url) => new Promise(resolve => {
setTimeout(() => {
resolve({
ok: true,
text: () => new Promise(res => setTimeout(() => res(url), 500))
});
}, 1000);
});
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
return fetch(fetchUrl).then(async res => { // <-- changes here
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
return url; // <--- changes here
});
}
async function getAllUrls(urls){
const result = [];
for (const url of urls){
const response = await getUrl(url);
result.push(response);
}
return result;
}
getAllUrls(urls)
.then(console.log);
async/await is perfect for this.
Assuming you have an array of URLs as strings:
let urls = ["https://example.org/", "https://google.com/", "https://stackoverflow.com/"];
You simply need to do:
for (let u of urls) {
await fetch(u).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
The loop will not iterate until the current fetch() has resolved, which will serialise things.
The reason array.map doesn't work is as follows:
async function doFetch(url) {
return await fetch(url).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
let mapped = urls.map(doFetch);
is equivalent to:
let mapped;
for (u of urls) {
mapped.push(doFetch(u));
}
This will populate mapped with a bunch of Promises immediately, which is not what you want. The following is what you want:
let mapped;
for (u of urls) {
mapped.push(await doFetch(u));
}
But this is not what array.map() does. Therefore using an explicit for loop is necessary.
Many people provided answers using for loop. But in some situation await in for loop is not welcome, for example, if you are using Airbnb style guide.
Here is a solution using recursion.
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
async function serialFetch(urls) {
return await doSerialRecursion(
async (url) => {
return result = await fetch(url)
.then((response) => {
// handle response
})
.catch((err) => {
// handle error
});
},
urls,
0
);
}
async function doSerialRecursion(fn, array, startIndex) {
if (!array[startIndex]) return [];
const currResult = await fn(array[startIndex]);
return [currResult, ...(await doSerialRecursion(array, fn, startIndex + 1))];
}
const yourResult = await serialFetch(urls);
The doSerialRecursion function will serially execute the function you passed in, which is fetch(url) in this example.
I have a method that receives a profiles array and I have to map for each profile and inside this map I have to map again in the photos propriety, which contains the pictures ids for requesting to an API for getting this picture.
The question is, where can I safely access this profiles array with their loaded photos for each respective profile?
profiles.map((profile, i) => {
let photos = []
Promise.all(profile.photos.map(async idPhoto => {
const res = await fetch(...)
const img = await res.blob()
photos.push(img)
}))
.then(() => profiles[i].photos = [...photos])
})
With the outer map function the way it currently is, the Promise.all() calls are discarded, so there is no way for your code to detect when they are complete.
However, since you also do not appear to be using the return value of the outer map, we can make it return an array of Promises that resolve when the inner their array of Promises is all resolved. And then we can use the same Promise.all(array.map()) pattern as we use for the inner map.
const photoRequests = profiles.map(async (profile, i) => {
let photos = []
await Promise.all(profile.photos.map(async idPhoto => {
const res = await fetch(...)
const img = await res.blob()
photos.push(img)
}));
profiles[i].photos = [...photos];
})
// And now...
await Promise.all(photoRequests);
// After this it is safe to access.
// Or, if the outer map is not in an async method:
Promise.all(photoRequests).then(() => {
// It is safe to access profiles here
});
I've refactored the outer map to be an async function (aids readability IMO), but you can put it back if you prefer. Just have the outer map function return the result of the Promise.all call.
As to what else could be improved here, the having variables photos and profile.photos is a little confusing, so consider renaming photos. Also make it const while you're at it, as it's never reassigned.
Unless there's some other code that manipulates the photos array, the array spread syntax is not needed. Same for the index variable. Final code might look something like:
const photoRequests = profiles.map(async profile => {
const loadedPhotos = []
await Promise.all(profile.photos.map(async idPhoto => {
const res = await fetch(...)
const img = await res.blob()
loadedPhotos.push(img)
}));
profile.photos = loadedPhotos;
})
await Promise.all(photoRequests);
Or you could use the fact that Promise.all resolves to an array containing the resolve values of the individual promises it received:
const photoRequests = profiles.map(async profile => {
profile.photos = await Promise.all(
profile.photos.map(async idPhoto => {
const res = await fetch(...)
return res.blob()
})
);
})
await Promise.all(photoRequests);
I think it would be better to separate each mapping into its own function, it makes it easier to read. I refactored your code into this:
let fetchPhoto = async (photoId) => {
// const res = await fetch(...);
// return res.blob();
return { imageData: photoId } // mock blob result
};
let mapPhotoIdToImage = async (profile) => {
let photos = profile.photos.map(fetchPhoto)
photos = await Promise.all(photos);
profile.photos = photos;
return profile;
};
let profileList = [{photos: ['id1', 'id2']}];
let result = await profileList.map(mapPhotoIdToImage);
result:
[{ photos: [ { imageData: 'id1' }, { imageData: 'id2' } ] }]
I need to merge data from API. I do a first call to an endpoint that gives me a list of ids, then I do a request for each id. My goal is to return a list with the responses of all requests but I lost myself in promises ...
My code runs on NodeJS. Here is the code :
const fetch = require('node-fetch')
const main = (req, res) => {
fetch('ENDPOINT_THAT_GIVES_LIST_OF_IDS')
.then(response => response.json())
.then(response => {
parseIds(response)
.then(data => {
console.log(data)
res.json(data)
// I want data contains the list of responses
})
})
.catch(error => console.error(error))
}
const getAdditionalInformations = async function(id) {
let response = await fetch('CUSTOM_URL&q='+id, {
method: 'GET',
});
response = await response.json();
return response
}
const parseIds = (async raw_ids=> {
let ids= []
raw_ids.forEach(function(raw_id) {
let informations = {
// Object with data from the first request
}
let additionalInformations = await
getAdditionalInformations(raw_id['id'])
let merged = {...informations, ...additionalInformations}
ids.push(merged)
})
return ids
})
main()
I get this error : "await is only valid in async function" for this line :
let additionalInformations = await getAdditionalInformations(raw_id['id'])
Help me with promise and async/await please.
You're almost there, just a slight bit of error here with your parentheses:
// notice the parentheses'
const parseIds = async (raw_ids) => {
let ids= []
raw_ids.forEach(function(raw_id) {
let informations = {
// Object with data from the first request
}
let additionalInformations = await getAdditionalInformations(raw_id['id'])
let merged = {...informations, ...additionalInformations}
ids.push(merged)
})
return ids
}
You are missing an async after forEach
const parseIds = (async raw_ids=> {
let ids= []
raw_ids.forEach(async function(raw_id) {
let informations = {
// Object with data from the first request
}
let additionalInformations = await
getAdditionalInformations(raw_id['id'])
let merged = {...informations, ...additionalInformations}
ids.push(merged)
})
return ids
})
One suggestion: you are mixing promises (.then()) with async/await. Prefer async/await is more readable.
Note that getAdditionalInformations inside forEach doesn't wait for it to be done before going to the next entry of the array.
You can use plain old for(var i=0; .... instead