I have an exercise to make a function executeFunctions which takes as arguments a list of async functions and an argument, e.g. number.
The functions have to happen one after another, so if fun1 ends, fun2 needs to start with the value which was returned from fun1.
The problem is that I can't use async and await. I wanted to do it using reduce, but I guess that it wants to execute const res1 and go further before it returns a value (because of setTimeout).
Is there any way to do it without async and await?
const fun1 = function(value) {
return setTimeout(() => value*2, 3000)
}
const fun2 = function(value) {
return setTimeout(() => value*4, 3000)
}
const cb2 = (value) => {
return value*10
}
const executeFunctions = (funTab, cb) => (n) => {
const res1= funTab[0](n)
console.log(res1)
const resReduce = funTab.reduce((prev,curr) => {
const res2 = curr(prev)
return prev+res2
}, res1)
return cb(resReduce)
};
executeFunctions([fun1,fun2], cb2)(2)
We can use Promise-chaining:
const fun1 = function(value) {
return Promise.resolve(value * 2);
}
const fun2 = function(value) {
return Promise.resolve(value * 2);
}
const fun3 = function(value) {
return Promise.resolve(value * 2);
}
const executeFunctions = (funcList) => (n) => {
let chain = Promise.resolve(n); // initial value
for (let i = 0; i < funcList.length; i++) {
chain = chain.then(funcList[i]); // keep chaining
}
return chain; // last promise
};
const main = () => {
// we need to wait for the last promise in order to print the result
executeFunctions([fun1, fun2, fun3])(2).then(x => console.log('solution is:', x));
}
main() // prints: "solution is: 16"
or, we can also use a modified version of the suggested reduce solution, by changing the implementation of executeFunctions as follows (the rest of the code should remain as in the previous snippet):
const executeFunctions = (funcList) => (n) => {
const init = Promise.resolve(n);
const res = funcList.reduce((p, c) => {
return p.then(c)
}, init);
return res;
};
I am trying to scrape the list of results from Google Maps.
Example: Visit https://www.google.com/maps/search/gym+in+nyc
Get all results in an array, loop Starts and click element#1, extract data, back to results page and continue loop.
const finalData = async function () {
const arr = [];
const resultList = [...[...document.querySelectorAll("[aria-
label^='Results for']")][0].children].filter((even, i) => !(i % 2));
for (const eachElement of resultList) {
let response = await scrapePage(eachElement);
arr.push(response);
}
return arr;
};
async function scrapePage(elem) {
// Clicks each element
let click = await elem.click();
// Grabs Just the Title
const titleText = await setTimeout(function () {
let title = document.querySelector(".section-hero-header-title
span").innerText;
return title;
}, 3000);
// setTimeout to cause delay before click the back button
setTimeout(function () {
document.querySelector(".section-back-to-list-button").click();
}, 5000);
return titleText;
}
const final = finalData().then((value) => {
return value;
});
I have no idea why when I try the above code in devtools, only the last result is clicked and why my const variable "final" is filled with array of random numbers.
The problem is that you had assumed that the await operator worked on setTimeout; setTimout, setInterval and related functions do not use promises at all.
When working with time orientated code, generally I would set up a helper function that uses a promise:
const delay = seconds => new Promise(
resolve => {
setTimeout(resolve, seconds);
}
);
Updating your code to wait for the timeouts from here is simple:
const finalData = async () => {
const arr = [];
const resultList = [...document.querySelector("[aria-label^='Results for']").children].filter((even, i) => !(i % 2));
for (const eachElement of resultList) {
const response = await scrapePage(eachElement);
arr.push(response);
}
return arr;
};
async function scrapePage(elem) {
// Clicks each element
const click = await elem.click(); // this is not asynchronous! click also returns undefined
// Grabs Just the Title
await delay(3);
const titleText = document.querySelector(".section-hero-header-title span").innerText;
// delay before click[ing] the back button
await delay(5);
document.querySelector(".section-back-to-list-button").click();
return titleText;
}
const final = finalData();
And, on a related note, this piece of code waits for every single result in sequence, this takes at least 8 seconds per iteration:
const finalData = async () => {
const arr = [];
const resultList = [...document.querySelector("[aria-label^='Results for']").children].filter((even, i) => !(i % 2));
for (const eachElement of resultList) {
let response = await scrapePage(eachElement);
arr.push(response);
}
return arr;
};
If you wanted to concurrently execute every iteration, you may want to consider mapping the function across the array and using Promise.all, like so:
const finalData = async () => {
const resultList = [...document.querySelector("[aria-label^='Results for']").children].filter((_, i) => !(i % 2));
return Promise.all(resultList.map(scrapePage));
};
You can try this:
async function scrapePage(elem) {
// Clicks each element
let click = await elem.click();
// Grabs Just the Title
const titleText = await new Promise((resolve) => {
setTimeout(function () {
let title = document.querySelector(".section-hero-header-title span").innerText;
resolve(title);
}, 3000);
})
// setTimeout to cause delay before click the back button
setTimeout(function () {
document.querySelector(".section-back-to-list-button").click();
}, 5000);
return titleText;
}
I have a function "promiseFunction" which returns a promise, which is resolved at a later time.
I need to call this function many times, however I only want a set number of executions of this function to happen at once.
The function calls some external single threaded c code on my computer, if I call too many instances of this code at once I crash my system, but if I call it sequentially 1 at a time it's very slow as only one thread of my cpu is doing any work.
So I came up with the code below, however it doesn't work. It will call the first 10 promises in parallel, but slowly it starts to call less and less promises at once, until it's only calling 1 promise at a time.
var totalNumberOfPromises = // total number times to run promiseFunction;
var promiseCounter = 0; // keep track of which promise call this is
for(w=0;w<10;w++){ // run 10 promises at a time
promiseFunction().then(function(resolve) {
loadNewPromise();
})
promiseCounter++;
}
function loadNewPromise(){
if(promiseCounter<totalNumberOfPromises){
promiseFunction().then(function(resolve) { loadNewPromise(); });
}else{
alert("Finished");
}
promiseCounter++;
}
Is there anything wrong with the code above that causes this behavior? And is there a standard way of doing this?
Here's a function I prepared earlier (I've used this for a few years now for just such a thing
const multiQueue = length => {
length = (isNaN(length) || length < 1) ? 1 : length;
const q = Array.from({length}, () => Promise.resolve());
let index = 0;
const add = cb => {
index = (index + 1) % length;
return (q[index] = q[index].then(() => cb()));
};
return add;
};
// demo usage
const q = multiQueue(10);
let inFlight = 0;
let maxInFlight = 0;
const promiseFunction = (i) => {
inFlight++;
maxInFlight = Math.max(inFlight, maxInFlight);
const obj = {inFlight, maxInFlight, i};
return new Promise(resolve => {
setTimeout(() => {
inFlight--;
resolve(Object.assign(obj, {t:performance.now()}));
}, 10 );
})
};
for (let i = 0; i < 40; i++) {
q(() => promiseFunction(i)).then(v => console.log(JSON.stringify(v)));
}
You can see that at most there are 10 "inFlight" requests
How about something like this? If you construct your queue which is and Array of functions that return a Promise you can splice chunks out of it and process each with a Promise.all.
const fakePromise = (id) => {
return new Promise((resolve) => {
setTimeout(() => {
console.log(`Resolving promise ${id}`)
resolve(id)
}, 100)
})
}
const queue = Array(100).fill().map((_, i) => {
return () => fakePromise(i)
})
const batchProcessPromises = (promises, batchSize) => {
if (promises && promises.length) {
return Promise.all(
promises.splice(0, batchSize)
.map(promise => promise())
)
.then(() => {
console.log('Batch complete')
return batchProcessPromises(promises, batchSize)
})
}
console.log('Batch complete')
return Promise.resolve()
}
batchProcessPromises(queue, 10)
.then(() => {
console.log('Time to get one with my day')
})
How do you plan to construct all your promises? This function effects the original queue so you would need to ensure that the array being passed into batchProcessPromises isn't shared. To get around this you could potentially use the spread operator like so
batchProcessPromises([...queue], 10)
.then(() => {
console.log('Time to get one with my day', queue)
})
Fiddle here https://jsfiddle.net/stwilz/2bpdcxo6/24/
In async, if I need to apply a asynchronousfunction to 1000 items, I can do that with:
async.mapLimit(items, 10, (item, callback) => {
foo(item, callback);
});
so that only 10 item are processed at the same time, limiting overhead and allowing control.
With ES6 promise, while I can easily do:
Promise.all(items.map((item) => {
return bar(item);
}));
that would process all 1000 items at the same time which may cause a lot of problems.
I know Bluebird have ways to handle that, but I am searching a ES6 solution.
If you don't care about the results, then it's quick to whip one up:
Promise.eachLimit = async (funcs, limit) => {
let rest = funcs.slice(limit);
await Promise.all(funcs.slice(0, limit).map(async func => {
await func();
while (rest.length) {
await rest.shift()();
}
}));
};
// Demo:
var wait = ms => new Promise(resolve => setTimeout(resolve, ms));
async function foo(s) {
await wait(Math.random() * 2000);
console.log(s);
}
(async () => {
let funcs = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".split("").map(s => () => foo(s));
await Promise.eachLimit(funcs, 5);
})();
A key performance property is running the next available function as soon as any function finishes.
Preserving results
Preserving the results in order makes it a little less elegant perhaps, but not too bad:
Promise.mapLimit = async (funcs, limit) => {
let results = [];
await Promise.all(funcs.slice(0, limit).map(async (func, i) => {
results[i] = await func();
while ((i = limit++) < funcs.length) {
results[i] = await funcs[i]();
}
}));
return results;
};
// Demo:
var wait = ms => new Promise(resolve => setTimeout(resolve, ms));
async function foo(s) {
await wait(Math.random() * 2000);
console.log(s);
return s.toLowerCase();
}
(async () => {
let funcs = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".split("").map(s => () => foo(s));
console.log((await Promise.mapLimit(funcs, 5)).join(""));
})();
There's nothing built in, but you can of course group them yourself into promise chains, and use a Promise.all on the resulting array of chains:
const items = /* ...1000 items... */;
const concurrencyLimit = 10;
const promise = Promise.all(items.reduce((promises, item, index) => {
// What chain do we add it to?
const chainNum = index % concurrencyLimit;
let chain = promises[chainNum];
if (!chain) {
// New chain
chain = promises[chainNum] = Promise.resolve();
}
// Add it
promises[chainNum] = chain.then(_ => foo(item));
return promises;
}, []));
Here's an example, showing how many concurrent promises there are any given time (and also showing when each "chain" is complete, and only doing 200 instead of 1,000):
const items = buildItems();
const concurrencyLimit = 10;
const promise = Promise.all(items.reduce((promises, item, index) => {
const chainNum = index % concurrencyLimit;
let chain = promises[chainNum];
if (!chain) {
chain = promises[chainNum] = Promise.resolve();
}
promises[chainNum] = chain.then(_ => foo(item));
return promises;
}, []).map(chain => chain.then(_ => console.log("Chain done"))));
promise.then(_ => console.log("All done"));
function buildItems() {
const items = [];
for (let n = 0; n < 200; ++n) {
items[n] = n;
}
return items;
}
var outstanding = 0;
function foo(item) {
++outstanding;
console.log("Starting " + item + " (" + outstanding + ")");
return new Promise(resolve => {
setTimeout(_ => {
--outstanding;
console.log("Resolving " + item + " (" + outstanding + ")");
resolve(item);
}, Math.random() * 500);
});
}
.as-console-wrapper {
max-height: 100% !important;
}
I should note that if you want to track the result of each of those, you'd have to modify the above; it doesn't try to track the results (!). :-)
Using Array.prototype.splice
while (funcs.length) {
await Promise.all( funcs.splice(0, 100).map(f => f()) )
}
This is the closest one to async.eachLimit
Promise.eachLimit = async (coll, limit, asyncFunc) => {
let ret = [];
const splitArr = coll.reduce((acc,item,i)=> (i%limit) ? acc :[...acc,coll.slice(i,i+limit)],[])
for(let i =0; i< splitArr.length;i++){
ret[i]=await Promise.all(splitArr[i].map(ele=>asyncFunc(ele)));
}
return ret;
}
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
async function foo(s) {
await wait(Math.random() * 2000);
console.log(s);
return s.toLowerCase();
}
(async () => {
let arr = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".split("");
console.log((await Promise.eachLimit(arr, 5, foo)));
})();
Sometimes I need to wait for a .forEach() method to finish, mostly on 'loader' functions. This is the way I do that:
$q.when(array.forEach(function(item){
//iterate on something
})).then(function(){
//continue with processing
});
I can't help but feel that this isn't the best way to wait for a .forEach() to finish. What is the best way to do this?
If there is no asynchronous code inside the forEach, forEach is not asynchronous, for example in this code:
array.forEach(function(item){
//iterate on something
});
alert("Foreach DONE !");
you will see the alert after forEach finished.
Otherwise (You have something asynchronous inside), you can wrap the forEach loop in a Promise:
var bar = new Promise((resolve, reject) => {
foo.forEach((value, index, array) => {
console.log(value);
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log('All done!');
});
Credit: #rolando-benjamin-vaz-ferreira
The quickest way to make this work using ES6 would be just to use a for..of loop.
const myAsyncLoopFunction = async (array) => {
const allAsyncResults = []
for (const item of array) {
const asyncResult = await asyncFunction(item)
allAsyncResults.push(asyncResult)
}
return allAsyncResults
}
Or you could loop over all these async requests in parallel using Promise.all() like this:
const myAsyncLoopFunction = async (array) => {
const promises = array.map(asyncFunction)
await Promise.all(promises)
console.log(`All async tasks complete!`)
}
var foo = [1,2,3,4,5,6,7,8,9,10];
If you're actually doing async stuff inside the loop, you can wrap it in a promise ...
var bar = new Promise((resolve, reject) => {
foo.forEach((value, index, array) => {
console.log(value);
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log('All done!');
});
If you have an async task inside a loop and you want to wait. you can use for await
for await (const i of images) {
let img = await uploadDoc(i);
};
let x = 10; //this executes after
Use for of instead of forEach. Like this:
for (const item of array) {
//do something
}
console.log("finished");
"finished" will be logged after finishing the loop.
forEach() doesn't return anything, so a better practice would be map() + Promise.all()
var arr = [1, 2, 3, 4, 5, 6]
var doublify = (ele) => {
return new Promise((res, rej) => {
setTimeout(() => {
res(ele * 2)
}, Math.random() ); // Math.random returns a random number from 0~1
})
}
var promises = arr.map(async (ele) => {
// do some operation on ele
// ex: var result = await some_async_function_that_return_a_promise(ele)
// In the below I use doublify() to be such an async function
var result = await doublify(ele)
return new Promise((res, rej) => {res(result)})
})
Promise.all(promises)
.then((results) => {
// do what you want on the results
console.log(results)
})
A universal solution for making sure that all forEach() elements finished execution.
const testArray = [1,2,3,4]
let count = 0
await new Promise( (resolve) => {
testArray.forEach( (num) => {
try {
//some real logic
num = num * 2
} catch (e) {
// error handling
console.log(e)
} fanally {
// most important is here
count += 1
if (count == testArray.length) {
resolve()
}
}
})
})
The idea is same with the answer using index to count. But in real case, if error happened, the index way cannot count correctly. So the solution is more robust.
Thx
const array = [1, 2, 3];
const results = [];
let done = 0;
const asyncFunction = (item, callback) =>
setTimeout(() => callback(item * 10), 100 - item * 10);
new Promise((resolve, reject) => {
array.forEach((item) => {
asyncFunction(item, (result) => {
results.push(result);
done++;
if (done === array.length) resolve();
});
});
}).then(() => {
console.log(results); // [30, 20, 10]
});
// or
// promise = new Promise(...);
// ...
// promise.then(...);
The order of results in the "results" array can be different than the order of items in the original array, depending on the time when the asyncFunction() finishes for each of the items.
Alter and check a counter at the end of every possible unique branch of code, including callbacks. Example:
const fs = require('fs');
/**
* #description Delete files older than 1 day
* #param {String} directory - The directory to purge
* #return {Promise}
*/
async function purgeFiles(directory) {
const maxAge = 24*3600000;
const now = Date.now();
const cutoff = now-maxAge;
let filesPurged = 0;
let filesProcessed = 0;
let purgedSize = 0;
await new Promise( (resolve, reject) => {
fs.readdir(directory, (err, files) => {
if (err) {
return reject(err);
}
if (!files.length) {
return resolve();
}
files.forEach( file => {
const path = `${directory}/${file}`;
fs.stat(path, (err, stats)=> {
if (err) {
console.log(err);
if (++filesProcessed === files.length) resolve();
}
else if (stats.isFile() && stats.birthtimeMs < cutoff) {
const ageSeconds = parseInt((now-stats.birthtimeMs)/1000);
fs.unlink(path, error => {
if (error) {
console.log(`Deleting file failed: ${path} ${error}`);
}
else {
++filesPurged;
purgedSize += stats.size;
console.log(`Deleted file with age ${ageSeconds} seconds: ${path}`);
}
if (++filesProcessed === files.length) resolve();
});
}
else if (++filesProcessed === files.length) resolve();
});
});
});
});
console.log(JSON.stringify({
directory,
filesProcessed,
filesPurged,
purgedSize,
}));
}
// !!DANGER!! Change this line! (intentional syntax error in ,')
const directory = ,'/tmp'; // !!DANGER!! Changeme
purgeFiles(directory).catch(error=>console.log(error));
I'm not sure of the efficiency of this version compared to others, but I used this recently when I had an asynchronous function inside of my forEach(). It does not use promises, mapping, or for-of loops:
// n'th triangular number recursion (aka factorial addition)
function triangularNumber(n) {
if (n <= 1) {
return n
} else {
return n + triangularNumber(n-1)
}
}
// Example function that waits for each forEach() iteraction to complete
function testFunction() {
// Example array with values 0 to USER_INPUT
var USER_INPUT = 100;
var EXAMPLE_ARRAY = Array.apply(null, {length: USER_INPUT}).map(Number.call, Number) // [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, n_final... ] where n_final = USER_INPUT-1
// Actual function used with whatever actual array you have
var arrayLength = EXAMPLE_ARRAY.length
var countMax = triangularNumber(arrayLength);
var counter = 0;
EXAMPLE_ARRAY.forEach(function(entry, index) {
console.log(index+1); // show index for example (which can sometimes return asynchrounous results)
counter += 1;
if (triangularNumber(counter) == countMax) {
// function called after forEach() is complete here
completionFunction();
} else {
// example just to print counting values when max not reached
// else would typically be excluded
console.log("Counter index: "+counter);
console.log("Count value: "+triangularNumber(counter));
console.log("Count max: "+countMax);
}
});
}
testFunction();
function completionFunction() {
console.log("COUNT MAX REACHED");
}
I had to deal with the same problem (forEach using multiple promises inside) and none of the solutions presented at the current date were helpful for me. So I implemented a check array, were each promise updates its complete status. We have a general promise that wraps the process. We only resolve the general promise when each promise completed. Snippet code:
function WaitForEachToResolve(fields){
var checked_fields = new Array(fields.length).fill(0);
const reducer = (accumulator, currentValue) => accumulator + currentValue;
return new Promise((resolve, reject) => {
Object.keys(fields).forEach((key, index, array) => {
SomeAsyncFunc(key)
.then((result) => {
// some result post process
checked_fields[index] = 1;
if (checked_fields.reduce(reducer) === checked_fields.length)
resolve();
})
.catch((err) => {
reject(err);
});
}
)}
}
I like to use async-await instead of .then() syntax so for asynchronous processing of data, modified the answer of #Ronaldo this way -
let finalData = [];
var bar = new Promise(resolve => {
foo.forEach((value, index) => {
const dataToGet = await abcService.getXyzData(value);
finalData[index].someKey = dataToGet.thatOtherKey;
// any other processing here
if (finalData[dataToGet.length - 1].someKey) resolve();
});
});
await Promise.all([bar]);
console.log(`finalData: ${finalData}`);
NOTE: I've modified the if condition where it resolves the promise to meet my conditions. You can do the same in your case.
You can use this, because we are using async/await inside the forEach loop. You can use your own logic inside the loop.
let bar = new Promise((resolve, reject) => {
snapshot.forEach(async (doc) => {
"""Write your own custom logic and can use async/await
"""
const result = await something()
resolve(result);
});
});
let test = []
test.push(bar)
let concepts = await Promise.all(test);
console.log(concepts);
For simple compare code i like use for statement.
doit();
function doit() {
for (var i = 0; i < $('span').length; i++) {
console.log(i,$('span').eq(i).text() );
if ( $('span').eq(i).text() == "Share a link to this question" ) { // span number 59
return;
}
}
alert('never execute');
}
If there are async (observable) method calls inside the for loop, you could use the following method:
await players.reduce(async (a, player) => {
// Wait for the previous item to finish processing
await a;
// Process this item
await givePrizeToPlayer(player);
}, Promise.resolve());
Check here: https://gist.github.com/joeytwiddle/37d2085425c049629b80956d3c618971
I've been using this and it works best .forEach()
//count
var expecting = myArray.length;
myArray.forEach(function(item){
//do logic here
var item = item
//when iteration done
if (--expecting === 0) {
console.log('all done!');
}
})