Take a look at this example code:
var arrayToSend = [1,2,3,4,5,6,7,8,9]
function send(x){ // 'sends' the value. resolves after 1 second.
return new Promise(resolve=>{
setTimeout(()=>{
resolve()
console.log('sent '+x)
},1000)
})
}
function calculate(x){return x**2} // just an example, there would be something expensive here.
async function loopAndSendArray(){
for (let val of arrayToSend){
let calculatedVal = calculate(val) //calculate this when waiting for the send promise to finish BUT don't send the value until it's resolved
await send(calculatedVal)}
}
I'd like the script to call calculate() for the next value when it's waiting for the send() promise of the former value to finish (to save time that would otherwise be spent calculating it after the promise). It would not call the next send() with the next value until the first one is resolved.
Instead of awaiting the call to send, assign the call to an outside variable, and then await it after calling calculate in the next iteration.
var arrayToSend = [1, 2, 3, 4, 5, 6, 7, 8, 9]
function send(x) {
return new Promise(resolve => {
setTimeout(() => {
resolve()
console.log('sent ' + x)
}, 1000)
})
}
function calculate(x) {
return x ** 2
}
async function loopAndSendArray() {
let lastSend;
for (const val of arrayToSend) {
const calculatedVal = calculate(val);
await lastSend;
lastSend = send(calculatedVal)
}
await lastSend;
}
loopAndSendArray();
This is optimal no matter whether the calculation takes more or less time than the network request, if the requests have to be made in serial (as your current code is doing). If the requests can be made in parallel and the calculation takes less time than the network request, you could calculate and send multiple (or all) at once.
There is no nice way to implement this, since async/await is inherently sequential and you want to do things in a different order, but this code should do it:
async function loopAndSendArray() {
let prevPromise;
for (let val of arrayToSend) {
let calculatedVal = calculate(val)
await prevPromise;
prevPromise = send(calculatedVal);
// prevent unhandled rejections crashing the app in case `calculate` throws
prevPromise.catch(e => {/* ignore */});
}
await prevPromise;
}
Related
The problem
While working with a restful API, I had to make multiple requests to retrieve data around a single search. The problem that I seem to be facing is that as the results are returned from a large database, some promises take FOREVER to resolve.
Current solution
Currently I make all the requests in a loop while adding the promises to an array and then using await Promise.all() to wait for them to be resolved but this makes loading times > 30 seconds at times even when the earliest promise resolved within a few seconds.
I am looking for a way that I can 'Lazy Load' the results in. I do have access to the restful server so any changes in either the front-end or back-end would help however I would prefer the changes to be at the front end.
Edit 1
My bad I didn't put any references to the code that I am currently using. For reference here is what I am currently doing
async function retrieve_data() {
let request_urls = [list of api endpoints to fetch]
let promises = []
for (let url of request_urls)
promises.push( fetch(url) )
await promise.all( promises )
// Do something with the returned results
}
The solution I want
async function retrieve_data() {
let request_urls = [list of api endpoints to fetch]
let promises = []
for (let url of request_urls)
promises.push( fetch(url) )
let results = Promise.some_magic_function()
// Or server side
res.write(Promise.some_magic_function()) // If results are received in chunks
// Do something with the returned results
}
Edit 2
So while I did find Promise.any() and Promise.all() to fix half of my problems, I seem unable to comprehend on how to have some function which adds the value of fulfilled promises to an array as soon as they are fulfilled. While these solutions do work for my usecase, I just cant help but think that there must be a better solution.
You can handle the individual results as you're populating the array for Promise.all, which you can use to know when they've all finished.
It's tricky without any code in the question to work with, but here's a simple example:
const promises = [1, 2, 3, 4, 5].map(async (num) => {
const result = await doRequest(num);
console.log(`Immediate processing for "${result}"`);
return result; // If you're going to use the contents of the array from `Promise.all`'s fulfillment
});
const allResults = await Promise.all(promises);
console.log(`All processing done, all results:`);
for (const result of allResults) {
console.log(result);
}
Live Example:
const rndDelay = () => new Promise((resolve) => setTimeout(resolve, Math.round(Math.random() * 1000) + 100));
async function doRequest(num) {
console.log(`Requesting ${num}`);
await rndDelay();
return `Result for ${num}`;
}
async function main() {
const promises = [1, 2, 3, 4, 5].map(async (num) => {
const result = await doRequest(num);
console.log(`Immediate processing for "${result}"`);
return result; // If you're going to use the contents of the array from `Promise.all`'s fulfillment
});
const allResults = await Promise.all(promises);
console.log(`All processing done, all results:`);
for (const result of allResults) {
console.log(result);
}
}
main()
.catch((error) => console.error(error));
.as-console-wrapper {
max-height: 100% !important;
}
Notice that the callback we give map (in this example) is an async function, so it returns a promise and we can use await within it.
If you can't use an async wrapper where you're creating the promises, that isn't a problem, you can fall back to .then:
const promises = [1, 2, 3, 4, 5].map((num) => {
return doRequest(num)
.then((result) => {
console.log(`Immediate processing for "${result}"`);
return result; // If you're going to use the contents of the array from `Promise.all`'s fulfillment
});
});
const allResults = await Promise.all(promises);
console.log(`All processing done, all results:`);
for (const result of allResults) {
console.log(result);
}
const rndDelay = () => new Promise((resolve) => setTimeout(resolve, Math.round(Math.random() * 1000) + 100));
async function doRequest(num) {
console.log(`Requesting ${num}`);
await rndDelay();
return `Result for ${num}`;
}
async function main() {
const promises = [1, 2, 3, 4, 5].map((num) => {
return doRequest(num)
.then((result) => {
console.log(`Immediate processing for "${result}"`);
return result; // If you're going to use the contents of the array from `Promise.all`'s fulfillment
});
});
const allResults = await Promise.all(promises);
console.log(`All processing done, all results:`);
for (const result of allResults) {
console.log(result);
}
}
main()
.catch((error) => console.error(error));
.as-console-wrapper {
max-height: 100% !important;
}
Extract the requests to different async functions and process them the moment they resolve:
// Request that responds in 2 seconds
async function req1() {
let response = await http.get(`${serverRoot}/api/req1`);
// Work on response from req 1. Everything below this line executes after 2'
}
//Request that responds in 30 seconds
async function req2() {
let response = await http.get(`${serverRoot}/api/req2`);
// Work on response from req 1. Everything below this line executes after 30'
}
//Request that responds in 10 seconds
async function req3() {
let response = await http.get(`${serverRoot}/api/req3`);
// Work on response from req 1. Everything below this line executes after 30'
}
Then you can go ahead and add them all in an array and wait for all of them to finish, in order to, let's say, hide a loading indicator:
loading = true;
//Notice that we have to invoke the functions inside the array
await Promise.all([req1(), req2(), req3()]);
//Everything below this line will run after the longest request is finished, 30 seconds
loading = false;
im looking for a way to asynchronous iterate over an array and update a variable and in the end return this variable.
export const test = async(
...
): Promise<number> => {
let highestAmount = 0;
for (const entry of entries) {
const check = async () => {
let amount = 0
try {
newAmount = await getAmount(entry);
} catch (e) {
return;
}
if (newAmount > amount ) {
highestAmount = newAmount;
}
}
check()
}
return highestAmount;
}
In this current state i only get 0's back because the function doesnt wait for its finish.
Is there a way that the function only returns if all processes inside the for are finished ?
Lets say the getAmount(entry) function takes 1 second to finish then i have to wait around entries.length seconds. Im trying to find a way to execute this in 1 second so getAmount is called for every entry asynchronously => function returns highest number
Lets say the getAmount(entry) function takes 1 second to finish then i have to wait around entries.length seconds. I'm trying to find a way to execute this in 1 second
If you have five calls that take one second you can't execute and return a value from the function in one second.
Is there a way that the function only returns if all processes inside the for are finished?
Yes. This, however, is possible.
If you map over your array to produce an array of promises that you can await with Promise.all, you can then use Math.max to get the highest number from that array.
// Generate a random number
function rnd() {
return Math.floor(Math.random() * (100 - 0) + 0);
}
// Mock API call which returns the number passed into
// multiplied by an erratic method of
// creating a new random number
function getAmount(el) {
return new Promise(res => {
setTimeout(() => res((el - rnd()) + rnd()), 500);
});
}
// Create an array of promises, await them to resolve
// and then return the highest number
async function getHighest(entries) {
const promises = entries.map(el => getAmount(el));
const data = await Promise.all(promises);
console.log(data);
return Math.max(...data);
}
// Await the promise that `getData` returns
// and log the result
async function main(entries) {
console.log(await getHighest(entries));
}
const entries = [1, 2, 3, 4, 5];
main(entries);
There are a couple of things missing to make this wait:
Put an async in the parent function
Put an await in the check function call
For example:
export const test = async (
...
): Promise<number> => {
//...
await check();
};
There might also be ways to make these async calls run in parallel.
I am looking at https://www.promisejs.org/patterns/ and it mentions it can be used if you need a value in the form of a promise like:
var value = 10;
var promiseForValue = Promise.resolve(value);
What would be the use of a value in promise form though since it would run synchronously anyway?
If I had:
var value = 10;
var promiseForValue = Promise.resolve(value);
promiseForValue.then(resp => {
myFunction(resp)
})
wouldn't just using value without it being a Promise achieve the same thing:
var value = 10;
myFunction(10);
Say if you write a function that sometimes fetches something from a server, but other times immediately returns, you will probably want that function to always return a promise:
function myThingy() {
if (someCondition) {
return fetch('https://foo');
} else {
return Promise.resolve(true);
}
}
It's also useful if you receive some value that may or may not be a promise. You can wrap it in other promise, and now you are sure it's a promise:
const myValue = someStrangeFunction();
// Guarantee that myValue is a promise
Promise.resolve(myValue).then( ... );
In your examples, yes, there's no point in calling Promise.resolve(value). The use case is when you do want to wrap your already existing value in a Promise, for example to maintain the same API from a function. Let's say I have a function that conditionally does something that would return a promise — the caller of that function shouldn't be the one figuring out what the function returned, the function itself should just make that uniform. For example:
const conditionallyDoAsyncWork = (something) => {
if (something == somethingElse) {
return Promise.resolve(false)
}
return fetch(`/foo/${something}`)
.then((res) => res.json())
}
Then users of this function don't need to check if what they got back was a Promise or not:
const doSomethingWithData = () => {
conditionallyDoAsyncWork(someValue)
.then((result) => result && processData(result))
}
As a side node, using async/await syntax both hides that and makes it a bit easier to read, because any value you return from an async function is automatically wrapped in a Promise:
const conditionallyDoAsyncWork = async (something) => {
if (something == somethingElse) {
return false
}
const res = await fetch(`/foo/${something}`)
return res.json()
}
const doSomethingWithData = async () => {
const result = await conditionallyDoAsyncWork(someValue)
if (result) processData(result)
}
Another use case: dead simple async queue using Promise.resolve() as starting point.
let current = Promise.resolve();
function enqueue(fn) {
current = current.then(fn);
}
enqueue(async () => { console.log("async task") });
Edit, in response to OP's question.
Explanation
Let me break it down for you step by step.
enqueue(task) add the task function as a callback to promise.then, and replace the original current promise reference with the newly returned thenPromise.
current = Promise.resolve()
thenPromise = current.then(task)
current = thenPromise
As per promise spec, if task function in turn returns yet another promise, let's call it task() -> taskPromise, well then the thenPromise will only resolve when taskPromise resolves. thenPromise is practically equivalent to taskPromise, it's just a wrapper. Let's rewrite above code into:
current = Promise.resolve()
taskPromise = current.then(task)
current = taskPromise
So if you go like:
enqueue(task_1)
enqueue(task_2)
enqueue(task_3)
it expands into
current = Promise.resolve()
task_1_promise = current.then(task_1)
task_2_promise = task_1_promise.then(task_2)
task_3_promise = task_2_promise.then(task_3)
current = task_3_promise
effectively forms a linked-list-like struct of promises that'll execute task callbacks in sequential order.
Usage
Let's study a concrete scenario. Imaging you need to handle websocket messages in sequential order.
Let's say you need to do some heavy computation upon receiving messages, so you decide to send it off to a worker thread pool. Then you write the processed result to another message queue (MQ).
But here's the requirement, that MQ is expecting the writing order of messages to match with the order they come in from the websocket stream. What do you do?
Suppose you cannot pause the websocket stream, you can only handle them locally ASAP.
Take One:
websocket.on('message', (msg) => {
sendToWorkerThreadPool(msg).then(result => {
writeToMessageQueue(result)
})
})
This may violate the requirement, cus sendToWorkerThreadPool may not return the result in the original order since it's a pool, some threads may return faster if the workload is light.
Take Two:
websocket.on('message', (msg) => {
const task = () => sendToWorkerThreadPool(msg).then(result => {
writeToMessageQueue(result)
})
enqueue(task)
})
This time we enqueue (defer) the whole process, thus we can ensure the task execution order stays sequential. But there's a drawback, we lost the benefit of using a thread pool, cus each sendToWorkerThreadPool will only fire after last one complete. This model is equivalent to using a single worker thread.
Take Three:
websocket.on('message', (msg) => {
const promise = sendToWorkerThreadPool(msg)
const task = () => promise.then(result => {
writeToMessageQueue(result)
})
enqueue(task)
})
Improvement over take two is, we call sendToWorkerThreadPool ASAP, without deferring, but we still enqueue/defer the writeToMessageQueue part. This way we can make full use of thread pool for computation, but still ensure the sequential writing order to MQ.
I rest my case.
I have an API which is limited regarding how many requests per minute (50/minute) I can send to any endpoint provided by that API.
In the following code-section, I filter the objects orders with an URL as property, every object with an URL that provides data should be stored in successfullResponses in my app.component.ts.
Promise.all(
orders.map(order => this.api.getURL(order.resource_url).catch(() => null))
).then(responses => {
const successfulResponses = responses.filter(response => response != null)
for(let data of successfulResponses) {
// some other requests should be sent with data here
}
});
There are more than 50 orders to check, but I just can check maximum 50 orders at once, so I try to handle it in my service. I set the first date when the first request is sent. After that I compare the dates of the new request with the first one. If the difference is over 60, I set the current date to the new one and set maxReq again to 50. If it is under 60, I check if there are requests left, if yes I send the request and if not I just wait one minute :
sleep(ms){
return new Promise(resolve => setTimeout(resolve, ms));
}
async getURL(){
if(!this.date){
let date = new Date();
this.date = date;
}
if((new Date().getSeconds() - this.date.getSeconds() > 60 )){
this.maxReq = 50;
this.date = new Date();
return this.http.get(url, this.httpOptions).toPromise();
} else {
if(this.maxReq > 0){
this.maxReq -= 1;
return this.http.get(url, this.httpOptions).toPromise();
} else{
console.log("wait");
await this.sleep(60*1000);
this.maxReq = 50;
this.date = new Date();
return this.http.get(url, this.httpOptions).toPromise();
}
}
}
However the code in app.component.tsis not waiting for the function getURL() and executes further code with requests which leads to the problem that I send ´too many requests too quickly´.
What can I do about that?
I had a similar problem while trying to use promises with multiple async functions. It's an easy thing to forget, but in order to make them all wait, you have to use await on the root line that calls the function in question.
I'm not entirely certain, but my presumption is that your await this.sleep(60*1000); line is indeed waiting for a timeout to occur, but whilst it is doing this, the code that called getURL() is executing the rest of its lines, because it did not have an await (or equivalent, like .then) before getURL().
The way I discovered this in my case was by using a good debugging tool (I used Chrome DevTools's own debugging features). I advise you do the same, adding breakpoints everywhere, and see where your code is going with each line.
Here is a short, rough example to show what I mean:
// This code increments a number from 1 to 2 to 3 and returns it each time after a delay of 1 second.
async function loop() {
for (i = 1; i <= 3; i++) {
console.log('Input start');
/* The following waits for result of aSync before continuing.
Without 'await', it would execute the last line
of this function whilst aSync's own 'await'
waited for its result.
--- This is where I think your code goes wrong. --- */
await aSync(i);
console.log('Input end');
}
}
async function aSync(num) {
console.log('Return start');
/* The following waits for the 1-second delay before continuing.
Without 'await', it would return a pending promise immediately
each time. */
let result = await new Promise(
// I'm not using arrow functions to show what it's doing more clearly.
function(rs, rj) {
setTimeout(
function() {
/* For those who didn't know, the following passes the number
into the 'resolved' ('rs') parameter of the promise's executor
function. Without doing this, the promise would never be fulfilled. */
rs(num);
}, 1000
)
}
);
console.log(result);
console.log('Return end');
}
loop();
I have a certain promise chain in my code that looks like this:
myPromise()
.then(getStuffFromDb)
.then(manipulateResultSet)
.then(manipulateWithAsync)
.then(returnStuffToCaller)
Now, in my manipulateWithAsync I'm trying to enhance my result set by calling the DB again, but it's not working as I expected, since while debugging i figured out the control moves to the next function which is the returnStuffToCaller
here's an idea of what's into my manipulateWithAsync function:
function manipulateWithAsync(rs) {
return rs.map( async function whoCares(singleRecord) {
let smthUseful = await getMoreData(singleRecord.someField);
singleRecord.enhancedField = smthUseful;
return singleRecord;
})
}
I get the point of this behaviour: the map function does work as expected and the promise chain doesn't give a duck about it since it's not working with the awaits.
Is there a way to allow my returnStuffToCaller function to wait till the async function did his job?
I also use bluebird and i tried to use coo-routine, so if you thing that it's a good solution I'll post my bluebird coo-routine failing code :)
Thanks!
The problem is in using async/await with Array.map
This answer should help: https://stackoverflow.com/a/40140562/5783272
rs.map iterator jumps to the next element without waiting in each separate iteration.
You need something like asyncMap
You can use - https://github.com/caolan/async
or either implement yourself
async function asyncMap(array, cb) {
for (let index = 0; index < array.length; index++) {
return await cb(array[index], index, array);
}
}
*cb function must be async one
Wrap your map with Promise.all return the Promise then await for the results wherever you call the manipulateWithAsync.
// MOCKS FOR DEMO
// Test data used as input for manipulateWithAsync
const testData = [
{ recordNumber: 1 },
{ recordNumber: 2 },
{ recordNumber: 3 }
];
// Mock function which returns Promises which resolve after random delay ranging from 1 - 3 seconds
const getMoreData = () =>
new Promise(resolve => {
const calledAt = Date.now();
setTimeout(() => {
resolve({
usefulData: `Promise called at ${calledAt}`
});
}, Math.floor(Math.random() * 3000) + 1000);
});
// SOLUTION / ANSWER
const manipulateWithAsync = async rs =>
Promise.all(
rs.map(async singleRecord => {
const smthUseful = await getMoreData(singleRecord.someField);
// Instead of manipulating original data,
// which might cause some unwanted side effects going forward,
// instead return new objects
return { ...singleRecord, enhancedField: smthUseful };
})
);
await manipulateWithAsync(testData);