my fetch works perfectly with .then, but i want to step it up a notch by using async and await. It should wait for all 5 API calls, and then place the answer, instead, it shows answer on every API call
async function getPhotosFromAPI() {
for (let i = 1; i <= 5; i++) {
let albums = await fetch(
`https://jsonplaceholder.typicode.com/photos/?albumId=${i}`
);
let result = await albums.json();
let res = `<div class="album${i}"></div>`;
document.querySelector(".display-images").innerHTML += res;
for (let j = 1; j <= 5; j++) {
document.querySelector(
`.album${i}`
).innerHTML += `<img src="${result[j].url}"/>`;
}
}
console.log(result);
}
async function showPhotos() {
await getPhotosFromAPI();
document.getElementById("#loader").style.display = "none";
}
showPhotos();
document.getElementById("img").style.display = "block";
for (let i = 1; i <= 5; i++) {
fetch(`https://jsonplaceholder.typicode.com/photos/?albumId=${i}`)
.then((response) => response.json())
.then((json) => {
document.getElementById("img").style.display = "none";
const display = document.querySelector(".display-images");
const albumNo = document.querySelector(".album-no");
// document.getElementById('img').style.display = "block";
// document.getElementById('img').style.display = "none";]
display.innerHTML += `<div class="album-${i}>`;
for (let z = 1; z <= 5; z++) {
display.innerHTML += `<img id="img" alt="pic-from-album${json[i].albumId}" src="${json[z].url}"/>`;
}
display.innerHTML += `<div>`;
});
}
You should use a concurrent way of fetching like Promise.all to avoid round-trips
async function getPhotosFromAPI() {
let albums = await Promise.all(
Array(5).fill().map((elem, index) =>
fetch(`https://jsonplaceholder.typicode.com/photos/?albumId=${index+1}`)
)
)
let results = await Promise.all(
albums.map(album => album.json())
)
return results
}
//Display
You're asking for the code to wait until each fetch finishes by using await on fetch's return value (then again on the return value of json) in your loop. So it will do just that: wait until that request is complete before moving on to the next loop iteration.
If you don't want to do that, you need to start each fetch one after another and then wait for them all to complete. I'd probably break out the work for just one of them into a function, then call it five times, building an array of the promises it returns, then await Promise.all(/*...*/) those promises, something along these lines:
document.getElementById("img").style.display = "block";
// Fetch one item
const fetchOne = async (i) => {
const response = await fetch(`https://jsonplaceholder.typicode.com/photos/?albumId=${i}`);
if (!response.ok) {
throw new Error(`HTTP error ${response.status}`);
}
const data = await response.json();
document.getElementById("img").style.display = "none";
const display = document.querySelector(".display-images");
const albumNo = document.querySelector(".album-no");
// document.getElementById('img').style.display = "block";
// document.getElementById('img').style.display = "none";]
display.innerHTML += `<div class="album-${i}>`;
for (let z = 1; z <= 5; z++) {
display.innerHTML += `<img id="img" alt="pic-from-album${data[i].albumId}" src="${data[z].url}"/>`;
}
display.innerHTML += `<div>`;
};
// ...
await Promise.all(Array.from({length: 5}, (_, i) => fetchOne(i + 1)));
// All done
(I took the version with .then as my starting point for the above, since the two versions in your question were so different and you said the one with .then worked... Also note that I renamed the variable json to data, since it doesn't contain JSON [it's not a string], it contains the result of parsing JSON.)
Related
I have the following web data collector:
function start(urls) {
Promise.map(urls, requestPromise)
.map((htmlPage, index) => {
const $ = cheerio.load(htmlPage);
$(".fixedttitle2").each(function () {
mytxt = $(this).text();
myarray.push(mytxt);
});
mainarray[urls[index]] = myarray;
});
fs.writeFileSync("1.json", JSON.stringify(mainarray));
}
var urls = [];
for (i = 1; i <= 100; i++) {
urls = "https://thisurl.com/" + i.toString();
start(urls);
}
Now I want to check response of each request at first, How I can check the response code at first inorder to get rid of some URLs that return 500 Error? How I can handle it?
You might be looking for something like this.
scrape (née start) processes a single URL and returns a promise of [url, content], or if there's an error, [url, null].
main generates the list of URLs to scrape, then starts scrape for all of them.
Note that all 100 requests start at once; this may or may not be a problem for you.
Finally, when all of the scrape promises complete, their return values are gathered into response, and that's written into the JSON file.
This differs from the original in that the original kept re-writing the file as new content was scraped.
async function scrape(url) {
try {
const htmlPage = await requestPromise(url);
const $ = cheerio.load(htmlPage);
const texts = [];
$('.fixedttitle2').each(function () {
texts.push($(this).text());
});
return [url, texts];
} catch (err) {
console.error(`Error processing url: ${url}: ${err}`);
return [url, null];
}
}
async function main() {
const urls = [];
for (var i = 1; i <= 100; i++) {
urls.push(`https://thisurl.com/${i}`);
}
const response = await Promise.all(urls.map(scrape));
fs.writeFileSync('1.json', JSON.stringify(response));
}
If you'd like the requests to be done sequentially, you can await scrape() in the loop:
async function main() {
const response = [];
for (var i = 1; i <= 100; i++) {
const url = `https://thisurl.com/${i}`;
response.push(await scrape(url));
}
fs.writeFileSync('1.json', JSON.stringify(response));
}
You could also move the write file call into the loop if you wanted the same incremental behavior your original code had.
EDIT
You can also add a delay to the sequential loop:
// Basic promisified delay function
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
async function main() {
const response = [];
for (var i = 1; i <= 100; i++) {
const url = `https://thisurl.com/${i}`;
response.push(await scrape(url));
await delay(1000); // Wait for 1000 ms between scrapes
}
fs.writeFileSync('1.json', JSON.stringify(response));
}
I am working with OpenWeatherMapAPI to calculate the sum of precipitation for the previous 5 days. To do this I have 5 async functions with api calls using the fetch api. The data received, that concerns me, is the hourly historic weather data spanning a 24 hour period. Full code bellow. The json response is stored to a constant (Ex.const histData1) where it is then iterated through to sum all of one values over that given 24 hour period. Note: humidity is used as a proof of concept because it hasnt rained here in awhile
for (var i in histData1.hourly){
total1 += histData1.hourly[i].humidity;
};
When I send the resulting value, total1, to the console I receive the expected results.
My trouble is coming when trying to add all of these results together i.e. total1 + total2 + total3 + total4 + total5.
One solution I thought might work was returning the total[1,2,3,4,5] variable at each function and then summing them. Ex.
var rainTotals = getData1() + getData2() + getData3() + getData4() + getData5()
console.log(rainTotals)
This resulted in the following output: [object Promise][object Promise][object Promise][object Promise][object Promise] so it appears to be a datatype issue.
Can anyone shed light as to adding all of the humidity values up from the five separate functions. Feel free to roast my code, I'm pretty new at this.
Thanks!
//WORKS Provies a json of hourly weather data for (1)24 hr period starting 5 days ago.
const fiveDaysAgo = Math.floor((Date.now() / 1000)-432000);
const fivedayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + fiveDaysAgo +"&appid="
async function getData5(){
const response5 = await fetch(fivedayURL)
const histData5 = await response5.json();
var total5 = 0
for (var i in histData5.hourly){
total5 += histData5.hourly[i].humidity;
};
console.log(total5);
return total5;
}
getData5();
const fourDaysAgo = Math.floor((Date.now() / 1000)-345600);
const fourdayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + fourDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData4(){
const response4 = await fetch(fourdayURL)
const histData4 = await response4.json();
var total4 = 0;
for (var i in histData4.hourly){
total4 += histData4.hourly[i].humidity
};
console.log(total4);
return total4;
}
getData4();
const threeDaysAgo = Math.floor((Date.now() / 1000)-259200);
const threedayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + threeDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData3(){
const response3 = await fetch(threedayURL);
const histData3 = await response3.json();
var total3 = 0;
for (var i in histData3.hourly){
total3 += histData3.hourly[i].humidity;
};
console.log(total3);
return total3;
}
getData3();
const twoDaysAgo = Math.floor((Date.now() / 1000)-172800);
const twodayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + twoDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData2(){
const response2 = await fetch(twodayURL);
const histData2 = await response2.json();
var total2 = 0;
for (var i in histData2.hourly){
total2 += histData2.hourly[i].humidity;
};
console.log(total2);
return total2;
}
getData2();
const oneDaysAgo = Math.floor((Date.now() / 1000)-86400);
const onedayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + oneDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData1(){
const response1 = await fetch(onedayURL);
const histData1 = await response1.json();
var total1 = 0;
for (var i in histData1.hourly){
total1 += histData1.hourly[i].humidity;
};
console.log(total1);
return total1;
}
getData1();
var rainTotals = getData1() + getData2() + getData3() + getData4() + getData5()
console.log(rainTotals)//returns [object Promise][object Promise][object Promise][object Promise][object Promise]
There are a couple things happening here. Firstly, to answer your question, because your getDataX functions are declared asynchronous, they return Promises that will eventually either resolve or reject with the actual values that you're looking for.
When working with Promises, you need sum the total values after all of these promises have resolved.
Second, you have a bit of duplication in your code. You'll notice that there is very little difference between the innards of your getDataX function. To simplify this, you can extract the differences as function parameters, and encapsulate all the same code within one function.
This would result in an implementation like below:
function getDaysAgo(days) {
return Math.floor((Date.now() / 1000) - (86400 * days))
}
async function getDataForDaysAgo(days) {
let daysAgo = getDaysAgo(days)
const apiURL = `http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=${daysAgo}&appid=5ffab1cda2c6b2750c78515f41421805`
const apiResponse = await fetch(apiURL)
const responseJson = await apiResponse.json()
var total = 0
responseJson.hourly.forEach(hour => {
total += hour.humidity
});
console.log(`getDataForDaysAgo(${days}) returns ${total}`)
return total
}
async function getDataSums() {
var data1 = await getDataForDaysAgo(5)
var data2 = await getDataForDaysAgo(4)
var data3 = await getDataForDaysAgo(3)
var data4 = await getDataForDaysAgo(2)
var data5 = await getDataForDaysAgo(1)
return data1 + data2 + data3 + data4 + data5;
}
getDataSums().then(result => {
console.log(result)
})
Async functions always returns a promise.
What you can do is to use Promise.all to aggregate them into one array.
Promise.all([getData1(), getData2(), getData3(), getData4(), getData5()]).then((values) => {
var sum = 0;
for(var i=0; i<values.length; i++){
sum += values[i];
}
console.log(sum);
});
Source : Async Functions
You can use await to get the result of async function.
var rainTotals = await getData1() + await getData2() + await getData3() + await getData4() + await getData5();
Async function delays because it wait for await. 'return' give data immediatly so that's why data give empty object(when console.log(getData1()) only empty objects returned). So getding total data should be awaited. 'total' data in each getData function was pushed into an array. Each getData function get await in asyncAll function to log array at once.
// Data array
let arr = [];
//WORKS Provies a json of hourly weather data for (1)24 hr period starting 5 days ago.
const fiveDaysAgo = Math.floor((Date.now() / 1000)-432000);
const fivedayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + fiveDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData5(){
const response5 = await fetch(fivedayURL)
const histData5 = await response5.json();
var total5 = 0
for (var i in histData5.hourly){
total5 += histData5.hourly[i].humidity;
};
console.log(total5);
await arr.push(total5);
return total5;
}
getData5();
const fourDaysAgo = Math.floor((Date.now() / 1000)-345600);
const fourdayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + fourDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData4(){
const response4 = await fetch(fourdayURL)
const histData4 = await response4.json();
var total4 = 0;
for (var i in histData4.hourly){
total4 += histData4.hourly[i].humidity
};
console.log(total4);
await arr.push(total4);
return total4;
}
getData4();
const threeDaysAgo = Math.floor((Date.now() / 1000)-259200);
const threedayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + threeDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData3(){
const response3 = await fetch(threedayURL);
const histData3 = await response3.json();
var total3 = 0;
for (var i in histData3.hourly){
total3 += histData3.hourly[i].humidity;
};
console.log(total3);
await arr.push(total3);
return total3;
}
getData3();
const twoDaysAgo = Math.floor((Date.now() / 1000)-172800);
const twodayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + twoDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData2(){
const response2 = await fetch(twodayURL);
const histData2 = await response2.json();
var total2 = 0;
for (var i in histData2.hourly){
total2 += histData2.hourly[i].humidity;
};
console.log(total2);
await arr.push(total2);
return total2;
}
const oneDaysAgo = Math.floor((Date.now() / 1000)-86400);
const onedayURL = "http://api.openweathermap.org/data/2.5/onecall/timemachine?lat=29.8833&lon=-97.9414&dt=" + oneDaysAgo +"&appid=5ffab1cda2c6b2750c78515f41421805"
async function getData1(){
const response1 = await fetch(onedayURL);
const histData1 = await response1.json();
var total1 = 0;
for (var i in histData1.hourly){
total1 += histData1.hourly[i].humidity;
};
console.log(total1);
//Push data after data asynced
await arr.push(total1);
return total1;
}
//Moved getData function to asyncAll function.
var rainTotals = [];
// Awaited for getData functions, logged total data.
async function asyncAll() {
await getData1();
await getData2();
await getData3();
await getData4();
await getData5();
await console.log(arr.join(', '), 'async');
}
asyncAll();
I want to access shopify api using Node.js with request method. I get first 50 items but i need to send the last id of the products i get as a response so it can loop through all the products until we don't have another id (i check that if the last array is not 50 in length.)
So when i get the response of lastID i want to feed that again to the same function until the Parraylength is not 50 or not 0.
Thing is request works asynchronously and i don't know how to feed the same function with the result lastID in node.js.
Here is my code
let importedData = JSON.parse(body);
//for ( const product in importedData.products ){
// console.log(`${importedData.products[product].id}`);
//}
lastID = importedData.products[importedData.products.length-1].id;
let lastIDD = lastID;
console.log(`This is ${lastID}`);
importedData ? console.log('true') : console.log('false');
let Prarraylength = importedData.products.length;
console.log(Prarraylength);
//console.log(JSON.stringify(req.headers));
return lastIDD;
});```
You can use a for loop and await to control the flow of your script in this case.
I'd suggest using the request-native-promise module to get items, since it has a promise based interface, but you could use node-fetch or axios (or any other http client) too.
In this case, to show you the logic, I've created a mock rp which normally you'd create as follows:
const rp = require("request-promise-native");
You can see we're looping through the items, 50 at a time. We're passing the last id as a url parameter to the next rp call. Now this is obviously going to be different in reality, but I believe you can easily change the logic as you require.
const totalItems = 155;
const itemsPerCall = 50;
// Mock items array...
const items = Array.from({ length: totalItems}, (v,n) => { return { id: n+1, name: `item #${n+1}` } });
// Mock of request-promise (to show logic..)
// Replace with const rp = require("request-promise-native");
const rp = function(url) {
let itemPointer = parseInt(url.split("/").slice(-1)[0]);
return new Promise((resolve, reject) => {
setTimeout(() => {
let slice = items.slice(itemPointer, itemPointer + itemsPerCall);
itemPointer += itemsPerCall;
resolve( { products: slice });
}, 500);
})
}
async function getMultipleRequests() {
let callIndex = 0;
let lastID = 0;
const MAX_CALLS = 20;
const EXPECTED_ARRAY_LENGTH = 50;
for(let callCount = 1; callCount < MAX_CALLS; callCount++) {
// Replace with the actual url..
let url = "/products/" + lastID;
let importedData = await rp(url);
lastID = importedData.products[importedData.products.length - 1].id;
console.log("Call #: " + ++callIndex + ", Item count: " + importedData.products.length + ", lastID: " + lastID);
if (importedData.products.length < EXPECTED_ARRAY_LENGTH) {
console.log("Reached the end of products...exiting loop...");
break;
}
}
}
getMultipleRequests();
I'm new to Javascript and still wrapping my mind around writing promises.
function addToList(data) {
conversationList = data.Body.Conversations
console.log(conversationList)
for (i=0; i<conversationList.length; i++) {
fullInbox.push(conversationList[i])
}
console.log(fullInbox.length)
}
var fullInbox = []
var maxLenReturn = 200
var offset = 0
function fetchData(offset){
fetch(asynchCall)
.then(response=>{return response.json()})
.then(data=>{
var fullLength = data.Body.TotalConversationsInView
console.log(fullLength)
addToList(data)
if (offset < fullLength-maxLenReturn) {
offset+= maxLenReturn
fetchData(offset)
}
})
}
fetchData(offset)
// trying to make something like this work
.then( .... console.log(fullInbox.length))
I have a loop inside the fetchData function and want to wrap it in a promise so that when its done, I can print out the fullInbox
var promise1 = new Promise(function(resolve, reject) {
var fullInbox = []
var maxLenReturn = 200
var offset = 0
fetchData(offset);
resolve(fullInbox)
});
promise1.then(function(value) {
console.log('promise resolved')
console.log(value);
});
I think I need the resolve inside fetchData but unsure how to write it so that it will loop through everything before resolving.
From what I can see of your code, your addToList function is synchronous, so it doesn't need to do anything with promises. But the next time you call fetchData, you need to handle the promise. So, something like this would work:
function fetchData(offset){
return fetch(asynchCall)
.then(response=>{return response.json()})
.then(data=>{
var fullLength = data.Body.TotalConversationsInView
console.log(fullLength)
addToList(data)
let promise;
if (offset < fullLength-maxLenReturn) {
offset+= maxLenReturn
promise = fetchData(offset)
} else {
promise = Promise.resolve();
}
return promise;
})
}
And if you want to use the async/await style syntax, things get a lot nicer to read:
async function fetchData(offset){
let response = await fetch(asynchCall)
let data = await response.json();
let fullLength = data.Body.TotalConversationsInView
console.log(fullLength)
addToList(data)
if (offset < fullLength-maxLenReturn) {
offset+= maxLenReturn
await fetchData(offset)
}
}
Async/await is supported in all major current browsers, and you can polyfill for older browsers. Due to its cleaner syntax, I highly recommend using it. It will save you from writing bugs.
So, here is a shorter version, doing what you need it to do. It uses async/await. And each call to fetchData returns the inbox items from the offset it was passed all the way to the end. The use of default parameters allows you to avoid using global variables.
async function fetchData(offset = 0, maxLenReturn = 200) {
let response = await fetch(asyncCall)
let data = await response.json();
let inbox = data.Body.Conversations;
let fullLength = data.Body.TotalConversationsInView
if (offset < fullLength-maxLenReturn) {
offset+= maxLenReturn
inbox.push(...await fetchData(offset))
}
return inbox;
}
let fullInbox = fetchData()
I don't know if this helps.
var promise1 = new Promise(function(resolve, reject) {
var fullInbox = [];
var maxLenReturn = 200;
var offset = 150;
for(let i = offset; i < maxLenReturn; i++) {
fullInbox.push(i);
}
resolve(fullInbox);
}).then(value => console.log(value));
I am struggling with getting this resolved, as I am new to Promises.
I need to first read both the Summative and Formative from Firebase before I can determine the StudentPlacement
The way the code below, provides null as the StudentPlacement snapshot.val(), as it is not waiting for the x and y values.
exports.boxScoresUpdate = functions.database.ref('/Tests/{id}/TestScores').onWrite(event => {
let testScr = 0;
for (let i = 1; i <= section; i++) {
//
testScr += parseInt(nValue[i]);
var xIndex = 0;
var yIndex = 0;
admin.database().ref('TestScores').child(data.key).child('Summative').child(i).once("value").then(x => {
xIndex = x.val();
});
admin.database().ref('TestScores').child(data.key).child('Formative').child(i).once("value").then(y => {
yIndex = y.val();
});
admin.database().ref('StudentPlacement').child(data.key).child(xIndex + ":" + yIndex).once("value", snapshot => {
// SnapShot
console.log("Student Placement is: ", snapshot.val());
});
}
}
Can anyone please help me structure the trigger!?
You're waiting for both functions to finish before executing the next bit of code. Look into Promise.all.
for (let i = 1; i <= section; i++) {
const xIndexRef = admin.database().ref('TestScores').child(data.key).child('Summative').child(i).once("value");
const yIndexRef = admin.database().ref('TestScores').child(data.key).child('Formative').child(i).once("value");
Promise.all([xIndexRef, yIndexRef])
.then(results => {
const xSnapshot = results[0];
const ySnapshot = results[1];
return admin.database().ref('StudentPlacement').child(data.key).child(xSnapshot.val() + ":" + ySnapshot.val()).once("value");
})
.then(snapshot => {
console.log("Student Placement is: ", snapshot.val());
});
}
Promise.all waits for both xIndexRef and yIndexRef to complete their execution.
Once executed the results are returned into a thenable object.
You can access the results and complete your execution.