I am refactoring some code that crawls some web pages (removing "callback hell"), and want a three second delay between each request. Here is the request function:
const getHTML = function(page, i) {
return new Promise(function(resolve, reject) {
setTimeout(function () {
api.makeAPIGetRequest(page).then((html) => {
resolve(html);
}).catch((err) => {
reject(err);
})
}, i * 3000);
});
}
I am traversing an array of objects then an array:
let p = [
{
location: 'England',
pages: [1, 3, 5]
},
{
location: 'Scotland',
pages: [2, 4, 6]
}
];
The problem is is that the output is random (because of the delay):
Page 1 - Loaded
Page 2 - Loaded
Page 5 - Loaded
Page 4 - Loaded
Page 3 - Loaded
Page 6 - Loaded
It should be:
Page 1 - Loaded
Page 3 - Loaded
Page 5 - Loaded
Page 2 - Loaded
Page 4 - Loaded
Page 6 - Loaded
Here is my code:
p.map(async (data) => {
await crawlLocationPages(data);
})
function crawlLocationPages(data) {
return Promise.all(
data.pages.map(async (page, i) => {
await getHTML(page, i).then((html) => { // <-- waits 3 seconds
console.log('Page ' + page + ' - Loaded' );
});
})
).then(() => {
})
};
I would rather keep the object and array model as it is.
Any help is appreciated.
await doesn't work inside .map and .forEach, but it does work inside for loops. And of course, it has to be inside an async function.
const run = async () => {
for(let data of p){
await crawlLocationPages(data);
}
}
const crawlLocationPages = async data => {
for(let page of data.pages){
const html = await getHTML(page);
console.log('Page ' + page + ' - Loaded - HTML = ', html );
await pause();
}
}
const pause = () => new Promise( (resolve, reject) => setTimeout(resolve, 3000) );
run()
Solved it using ES6 generators and yield.
function* crawlGenerator() {
for (let i = 0; i <= (p.length - 1); i++) {
yield crawlLocationPages(p[i]);
}
}
let crawl = crawlGenerator();
crawl.next();
function crawlLocationPages(data) {
return Promise.all(
data.pages.map(async (page, i) => {
await getHTML(page, i).then((html) => { // <-- waits 3 seconds
console.log('Page ' + page + ' - Loaded' );
});
})
).then(() => {
crawl.next();
})
};
More information here: https://davidwalsh.name/async-generators
This approach might be less confusing but only works if each request take no longer than 3 seconds.
pages = p.flatMap(location => location.pages);
page = 0;
var interval = setInterval(() => {
if(page === pages.length){
clearInterval(interval);
}
api.makeAPIGetRequest(pages[page++]).then((html) => {
console.log('Page ' + page + ' - Loaded' );
}).catch((err) => {
console.error(err);
});
}, 3000);
Or call next inside then
function getPage(pages, i) {
const ts = Date.now();
api.makeAPIGetRequest(pages[i++]).then((res)=>{
console.log(res);
if(i < pages.length) {
const delay = Math.max(3000 - (Date.now() - ts), 0);
setTimeout(getPage(pages, i), delay);
}
})
}
pages = p.flatMap(location => location.pages);
getPages(pages, 0);
Related
Promise.all([
seperatingDMCM(),
compileDirectMessage(),
renderingResult(),
addResultButtonListener(),
]);
I have a progress bar in my UI and I have 4 functions mentioned above each of which returns a Promise. I have a loop inside the first function seperatingDMCM(), that will handle all the my data. I want to increment the progress bar with each increment of the loop. Meaning each loop iteration should be async, so the UI will update and only afterwards the loop will iterate. When the loop has ended I want to return a promise so that the other functions will begin to execute. I am facing an issue that progress bar is not working as it suppose to and is immediately being invoked when seperatingDMCM() returns the promise () and not asynchronously. This progress bar is immediately being updated to 100% on the next page instead of live updates with small increment on the current page.
This is my updateUI function:
function startProgressBar(i, arr) {
return new Promise((resolve) => {
setTimeout(() => {
i = (i * 100) / arr.length;
console.log(i);
let elem = document.getElementById("progressBar");
if (i < 100) {
elem.innerText = i + "%";
elem.style.width = i + "%";
elem.innerHTML = i + "%";
resolve();
}
}, 0);
});
}
This is my first function where I want to update the UI per loop iteration
function seperatingDMCM() {
const contentType = "Content type";
return new Promise((resolve) => {
rowObject.forEach(async (row, index) => {
const creatingInts = () => {
console.log("CreatedInt at", index);
row["Date created (UTC)"] = ExcelDateToJSDate(
row["Date created (UTC)"]
);
if (
row[contentType] !== "DM" &&
row.hasOwnProperty("Falcon user name")
) {
publicCommentsCount++;
interaction = { row : row};
compiledInteractions.push(interaction);
interaction = {};
} else {
dmData.push(row);
}
};
startProgressBar(index, rowObject).then(creatingInts());
});
quickSort(dmData, "Falcon URL");
console.log("SORTED", dmData);
console.log(workbook);
console.log("Rows", rowObject);
resolve();
});
}
So basically im working on a cron job in my app that fires every 3 hours and updating users 'score' by calling the RiotApi
basically the function so far
exports.updatePlayersPoints = async () => {
console.log('STARTED UPDATING');
try {
const players = await UserLadder.findAll();
await Promise.all(
players.map(async (player) => {
const p = await RiotAccount.findOne({
where: {
userId: player.userId,
},
include: RiotRegions,
});
const beginTime = new Date(player.dataValues.createdAt);
let data;
try {
const res = await axios.get(
`https://${
p.dataValues.riot_region.dataValues.name
}.api.riotgames.com/lol/match/v4/matchlists/by-account/${
p.dataValues.accountId
}?queue=420&beginTime=${beginTime.getTime()}&api_key=${
process.env.RIOT_KEY
}`
);
data = res.data;
} catch (error) {
if (!error.response.status === 404) {
console.error(error);
}
}
if (!data) {
return;
}
let totalScore = player.dataValues.userPoints;
await Promise.all(
data.matches.map(async (match, i) => {
if (i < 15) {
const { data } = await axios.get(
`https://${p.dataValues.riot_region.dataValues.name}.api.riotgames.com/lol/match/v4/matches/${match.gameId}?api_key=${process.env.RIOT_KEY}`
);
const calculateScore = () => {
return new Promise((resolve) => {
const { stats } = _.find(
data.participants,
(o) => o.championId === match.champion
);
const killsPts = stats.kills * 2;
const deathPts = stats.deaths * -1.5;
const assistsPts = stats.assists;
const wardsPts = stats.wardsPlaced / 4;
const firstBloodPts = stats.firstBloodKill ? 3 : 0;
const firstBloodAssistPts = stats.firstBloodAssist ? 3 : 0;
const firstTowerPts = stats.firstTowerKill ? 2 : 0;
const firstTowerAssistPts = stats.firstTowerAssist ? 2 : 0;
const score =
killsPts +
deathPts +
assistsPts +
wardsPts +
firstBloodPts +
firstBloodAssistPts +
firstTowerPts +
firstTowerAssistPts;
totalScore += score;
resolve();
});
};
await calculateScore();
}
})
);
const user = await UserLadder.findOne({
where: {
userId: player.userId,
},
});
user.userPoints = parseFloat(totalScore);
user.lastGameId = data.matches[0].gameId;
await user.save();
})
);
console.log('FINISHED UPDATING');
} catch (error) {
console.error(error);
}
};
Basically it just looks up the table userladder to find the players that are signed to the ladder and for each one of these players it fires a map function that makes a request to the riotapi to get the match history of this player and then later make an inside map function to map each one of these matches.
but basically I updated it to now keep track of the game id of the last call before 3 hours so it doesn't have to make request that was already done.
user.lastGameId = data.matches[0].gameId;
but now in my second map function that maps the matches I wasn't it so that if the last game from my database matches the game id that currently being mapped I want to stop the map function and not continue this record or the ones after because it also means they all have been already counted.
but I can not seem to find a way to do it.
i tried using break; but it didn't work
any ideas?
using for loop
I tried a small test with for loop so I tried
for (let i = 0; i < 15; i++) {
await new Promise(async (resolve, reject) => {
const match = data.matches[i];
console.log(match);
resolve();
if (i === 1) {
break;
}
});
}
but I still go the same error
SyntaxError: Illegal break statement
Instead of trying to "break" a map, you should filter the matches that you want to process before you execute the map.
Something like this:
await Promise.all(
const filteredMatches = data.matches.filter(match => match.gameId > previousId);
filteredMatches.map(async (match, i) => { ...
More on filter() in javascript.
Edit: If generated id's are random and are not ordered, you can store all previous id's in a Set, and then just ask if it has been previously added
await Promise.all(
const filteredMatches = data.matches.filter(match => mySet.has(match.gameId));
filteredMatches.map(async (match, i) => { ...
More on Set in javascript.
Here's my example code:
function recursiveFetch(num) {
// EXAMPLE that recursivley fetches all todos from example API
return new Promise(resolve => {
fetch("https://jsonplaceholder.typicode.com/todos/" + num)
.then((response) => {
return response.json();
})
.then((data) => {
if (num == 0) {
console.log("Done getting TODOs");
resolve(num);
} else {
recursiveFetch(num - 1);
}
});
});
}
new Promise(resolve => {
// Just using this for this example
resolve(10);
})
.then((num) => {
// This runs fine and returns a promise which is eventually resolved
return recursiveFetch(num);
})
.then((num) => {
// This never happens?
console.log("num is now " + num);
})
I can't tell why but the second .then is never run.
If I execute this code in the Firefox console I get the output Done getting TODOs but the "num is now " log is never called?
To fix your code as it is, you need
recursiveFetch(num - 1).then(resolve)
However, there are quite a few mistakes, how about this cleaned-up one:
async function fetchAll(num) {
let data = [];
for (let i = 1; i <= num; i++) {
let t = await fetch("https://jsonplaceholder.typicode.com/todos/" + i);
data.push(await t.json())
}
return data;
}
Promise
.resolve(3)
.then(fetchAll)
.then(console.log)
You need to add a resolve inside the "else" when you call recursively the function "recursiveFecth". Since the function is returning another promise you need to resolve then, otherwise it will exit inmediately.
I've tried it and it works:
function recursiveFetch(num) {
// EXAMPLE that recursivley fetches all todos from example API
return new Promise(resolve => {
fetch("https://jsonplaceholder.typicode.com/todos/" + num)
.then((response) => {
return response.json();
})
.then((data) => {
if (num == 0) {
console.log("Done getting TODOs");
resolve(num);
} else {
resolve(recursiveFetch(num - 1));
}
});
});
}
new Promise(resolve => {
// Just using this for this example
resolve(10);
})
.then((num) => {
// This runs fine and returns a promise which is eventually resolved
return recursiveFetch(num);
})
.then((num) => {
// This never happens?
console.log("num is now " + num);
})
You need to return the recursiveFetch in the inner promise
...
.then((data) => {
if (num == 0) {
console.log("Done getting TODOs");
resolve(num);
} else {
return recursiveFetch(num - 1);
}
});
In the following piece of code I try to demonstrate how concatMap preserves the order of events, even if the action performed on events do not complete in order.
Now I get the error that
delayerObservable.complete() is not a function
This basically is just taken from a tutorial. I call next(), and then I call complete(). It should work, at least that's what I thought.
I can achive the desired functionality by returning randomDelayer.first()
return randomDelayer.first()
but I would like to complete the observable from within, since I might want to send out more events than just one.
const myTimer = Rx.Observable.create((observer) => {
let counter = 0;
setInterval( () => {
observer.next(counter++);
console.log('called next with counter: '+ counter);
},2000);
});
const myRandomDelayer = myTimer.concatMap( (value) => {
const randomDelayer = Rx.Observable.create( (delayerObservable) => {
const delay = Math.floor(Math.random()*1000);
setTimeout(() => {
console.log(delayerObservable);
delayerObservable.next('Hello, I am Number ' + value + ' and this was my delay: ' + delay);
delayerObservable.complete(); // <<-- this does not work (not a function)
}, delay);
});
return randomDelayer;
});
myRandomDelayer.subscribe( (message) => {
console.log(message);
});
It seems that there are quite a few changes between version 4 and 6 of the rxjs-framework. The working version of the defective source is this:
const { Observable } = rxjs;
const { map, filter, concatMap, pipe } = rxjs.operators;
console.log('Starting....');
const myTimer = Observable.create((observer) => {
let counter = 0;
setInterval( () => {
counter++;
if (counter < 10){
console.log('nexting now with counter ' + counter);
observer.next(counter);
} else {
observer.complete();
}
},1000);
});
const myRandomDelayer = myTimer.pipe(
concatMap( (value) => {
const randomDelayer = Observable.create( (delayerObservable) => {
const delay = Math.floor(Math.random()*5000);
setTimeout(() => {
delayerObservable.next('Hello, I am Number ' + value + ' and this was my delay: ' + delay);
delayerObservable.complete();
}, delay);
});
return randomDelayer;
})
);
myRandomDelayer.subscribe( (message) => {
console.log(message);
});
For the following function, I have to add a timeout after every GET request in array ajaxUrls. All the XHR GET request are in array ajaxUrls.
function getAllSearchResultProfiles(searchAjaxUrl) {
var ajaxUrls = [];
for (var i = 0; i < numResults; i += resultsPerPage) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
return Promise.all(ajaxUrls.map(getSearchResultsForOnePage))
.then(function(responses) {
return responses.map(function(response) {
if (response.meta.total === 0) {
return [];
}
return response.result.searchResults.map(function(searchResult) {
return (searchResult);
});
});
})
.then(function(searchProfiles) {
return [].concat.apply([], searchProfiles);
})
.catch(function(responses) {
console.error('error ', responses);
});
}
function getSearchResultsForOnePage(url) {
return fetch(url, {
credentials: 'include'
})
.then(function(response) {
return response.json();
});
}
I want a certain timeout or delay after every GET request. I am facing difficulty in where exactly to add the timeout.
If you want to make requests in serial, you shouldn't use Promise.all, which initializes everything in parallel - better to use a reduce that awaits the previous iteration's resolution and awaits a promise-timeout. For example:
async function getAllSearchResultProfiles(searchAjaxUrl) {
const ajaxUrls = [];
for (let i = 0; i < numResults; i += resultsPerPage) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
const responses = await ajaxUrls.reduce(async (lastPromise, url) => {
const accum = await lastPromise;
await new Promise(resolve => setTimeout(resolve, 1000));
const response = await getSearchResultsForOnePage(url);
return [...accum, response];
}, Promise.resolve([]));
// do stuff with responses
const searchProfiles = responses.map(response => (
response.meta.total === 0
? []
: response.result.searchResults
));
return [].concat(...searchProfiles);
}
Note that only asynchronous operations should be passed from one .then to another; synchronous code should not be chained with .then, just use variables and write the code out as normal.
I find a simple for loop in an async function to be the most readable, even if not necessarily the most succinct for things like this. As long as the function is an async function you can also create a nice pause() function that makes the code very easy to understand when you come back later.
I've simplified a bit, but this should give you a good idea:
function pause(time) {
// handy pause function to await
return new Promise(resolve => setTimeout(resolve, time))
}
async function getAllSearchResultProfiles(searchAjaxUrl) {
var ajaxUrls = [];
for (var i = 0; i < 5; i++) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
let responses = []
for (url of ajaxUrls) {
// just loop though and await
console.log("sending request")
let response = await getSearchResultsForOnePage(url)
console.log("recieved: ", response)
responses.push(response)
await pause(1000) // wait one second
}
//responses.map() and other manilpulations etc...
return responses
}
function getSearchResultsForOnePage(url) {
//fake fetch
return Promise.resolve(url)
}
getAllSearchResultProfiles("Test")
.then(console.log)
If you want to add a delay in every request then add a setTimout() in your function which fetches data from api
function getSearchResultsForOnePage(url) {
return new Promise((resolve, reject) => {
fetch(url, {
credentials: 'include'
})
.then(response => reresponse.json())
.then(data => {
let timeout = 1000;
setTimeout(() => resolve(data), timeout);
});
}