I've got a series of promises.
I never get the console log "Processed folders" printed out. Execution seems to stop once it hits the first await Promise.all call.
Not entirely sure where I've missed up?
const subfolders = [];
const exportFolder = () => {
// Other stuff happening here
const subfolder = {};
subfolder.items = [];
subfolder.items.push({ name: 'item 2.1' });
const folder = {};
folder.items = [];
folder.items.push({ name: 'item 1' });
folder.items.push({ name: 'item 2', isFolder: true, items: subfolder.items });
console.log('Folder:', folder);
console.log('Started');
exportFolderToCsv(folder).then(response => console.log('Finished', response));
};
const exportFolderToCsv = async folder => {
console.log('Processing folders');
let promises = [];
for (const folderItem of folder.items) {
if (folderItem.isFolder && folderItem.items.length > 0) {
subfolders.push(folderItem);
return;
}
promises.push(processFolderItem(folderItem));
}
await Promise.all(promises).then(response => console.log('Processed folders:', response));
if (subfolders.length > 0) {
console.log('Processing subfolders');
promises = [];
for (const folderItem of subfolders.items) {
promises.push(processFolderItem(folderItem));
}
await Promise.all(promises).then(response => console.log('Processed subfolders:', response));
}
console.log('Finished');
};
const processFolderItem = folderItem => new Promise(resolve => {
console.log('Processing folder item');
// To stuff here with folderItem, get Doc Chars, process row and resolve
getCharacters(folderItem)
.then(response => {
console.log('Processed folder item characters list:', response);
createCSVRow(folderItem, response)
.then(response => {
console.log('Processed CSV row:', response);
resolve(response);
})
});
});
const getCharacters = folderItem => new Promise(resolve => {
console.log('Processing folder item characters list');
// To stuff here with folderItem and then resolve
const characters = 'Foobar characters';
resolve(characters);
});
const createCSVRow = (folderItem, characters) => new Promise(resolve => {
console.log('Processing CSV row');
// To stuff here with folderItem and characters and then resolve
const csvRow = 'Foobar row';
resolve(csvRow);
});
exportFolder();
The function is returned before any Promise call is executed, it should be continue. So please resolve it first.
const exportFolderToCsv = async (folder) => {
// ...
for (const folderItem of folder.items) {
if (folderItem.isFolder && folderItem.items.length > 0) {
subfolders.push(folderItem)
return // here lies the problem, it should be `continue` instead
}
promises.push(processFolderItem(folderItem))
}
// ...
}
The Promise.all method, will catch errors if at least one Promise in chain throws an error or reject it.
It seems that one of your promises got an exception.
You can debug it simply adding the catch like so:
Promise.all(PromiseList).then(()=>console.log("all works done") ).catch(errors=>console.log("something wrong",errors))
If you are using await there is no point in chaining.
let response = await Promise.all(promises);
console.log('Processed subfolders', response)
Your promises will encounter errors, and you need to consider the failed situations.
At the same time, Promise.all will stop if either one promise failed, so maybe your Promise.all failed.
try {
Promise.all(promises).then(response => console.log('Processed subfolders', response));
} catch (err) {
console.log(err);
}
// or to check your promise result
const folderResult = await Promise.all(promises);
to check the exact result
Related
I am looping through ids that sends it to an async function and I want to return both success and failed data, right now I am only returning success data
const successContractSignature: LpContractSla[] = [];
for (const id of lpContractSlaIds) {
const data = await createLpSignature(context, id);
if (data) {
successContractSignature.push(data);
}
}
return successContractSignature;
If the createLpSignature throws error, do i need a try catch here? but wouldnt that exit the loop?
How can I push failed data to separate array without breaking the loop?
Unless there's a specific reason to avoid the call in parallel, it's always a good practice to avoid to start async calls in a for loop with await, since you are going to wait for each promise to resolve (or reject ) before executing the next one.
This is a better pattern which lets you also get all the results of the promises, either they resolved or rejected:
const successContractSignature: LpContractSla[] = await Promise.allSettled(lpContractSlaIds.map((id: string) => createLpSignature(context,id)))
return successContractSignature;
But if for some particular reason you need to make these calls in a sequence and not in parallel, you can wrap the single call in a try catch block ,that won't exit the loop:
for (const id of lpContractSlaIds) {
let data;
try {
data = await createLpSignature(context, id);
} catch(e) {
data = e
}
if (data) {
successContractSignature.push(data);
}
}
You can test it in this example:
const service = (id) =>
new Promise((res, rej) =>
setTimeout(
() => (id %2 === 0 ? res("ID: "+id) : rej('Error ID : '+id)),
1000
)
);
const ids = [1,2,3,4,5]
const testParallelService = async () => {
try {
const data = await Promise.allSettled(ids.map(id => service(id)))
return data.map(o => `${o.status}: ${o.reason ?? o.value}`)
} catch(e) {
console.log(e)
}
}
testParallelService().then(data => console.log("Parallel data: ", data))
const testSequentialService = async () => {
const res = [];
for (const id of ids) {
let data;
try {
data = await service(id);
} catch (e) {
data = e;
}
if (data) {
res.push(data);
}
}
return res;
};
testSequentialService().then((data) => console.log('Sequential Data: ', data));
How Should I await for bot.sendMessage() inside of loop?
Maybe I Need await Promise.all But I Don't Know How Should I add to bot.sendMessage()
Code:
const promise = query.exec();
promise.then(async (doc) => {
let count = 0;
for (const val of Object.values(doc)) {
++count;
await bot.sendMessage(msg.chat.id, `💬 ${count} and ${val.text}`, opts);
}
}).catch((err) => {
if (err) {
console.log(err);
}
});
Error:
[eslint] Unexpected `await` inside a loop. (no-await-in-loop)
If you need to send each message one-at-a-time, then what you have is fine, and according to the docs, you can just ignore the eslint error like this:
const promise = query.exec();
promise.then(async doc => {
/* eslint-disable no-await-in-loop */
for (const [index, val] of Object.values(doc).entries()) {
const count = index + 1;
await bot.sendMessage(msg.chat.id, `💬 ${count} and ${val.text}`, opts);
}
/* eslint-enable no-await-in-loop */
}).catch(err => {
console.log(err);
});
However, if there is no required order for sending the messages, you should do this instead to maximize performance and throughput:
const promise = query.exec();
promise.then(async doc => {
const promises = Object.values(doc).map((val, index) => {
const count = index + 1;
return bot.sendMessage(msg.chat.id, `💬 ${count} and ${val.text}`, opts);
});
await Promise.all(promises);
}).catch(err => {
console.log(err);
});
Performing await inside loops can be avoided once iterations doesn't have dependency in most cases, that's why eslint is warning it here
You can rewrite your code as:
const promise = query.exec();
promise.then(async (doc) => {
await Promise.all(Object.values(doc).map((val, idx) => bot.sendMessage(msg.chat.id, `💬 ${idx + 1} and ${val.text}`, opts);)
}).catch((err) => {
if (err) {
console.log(err);
}
});
If you still and to send one-after-one messages, your code is ok but eslint you keep throwing this error
I had a similar issue recently, I was getting lintting errors when trying to run an array of functions in a chain as apposed to asynchronously.
and this worked for me...
const myChainFunction = async (myArrayOfFunctions) => {
let result = Promise.resolve()
myArrayOfFunctions.forEach((myFunct) => {
result = result.then(() => myFunct()
})
return result
}
I am facing the same issue when I used await inside forEach loop. But I tried with recursive function call to iterate array.
const putDelay = (ms) =>
new Promise((resolve) => {
setTimeout(resolve, ms);
});
const myRecursiveFunction = async (events, index) => {
if (index < 0) {
return;
}
callAnotherFunction(events[index].activity, events[index].action, events[index].actionData);
await putDelay(1);
myRecursiveFunction(events, index - 1);
};
I'm trying to only make one http call at time but when I log the response from getUrl they are piling up and I start to get 409s (Too many requests)
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
fetch(fetchUrl).then(async res => {
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
cb(url);
});
}
let requests = urls.map((url, i) => {
return new Promise(resolve => {
getUrl(url, i, resolve);
});
});
const all = await requests.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult => [...chainResults, currentResult]),
);
}, Promise.resolve([]));
Basically I don't want the next http to start until the previous one has finished. Otherwise I hammer their server.
BONUS POINTS: Make this work with 5 at a time in parallel.
Since you're using await, it would be a lot easier to use that everywhere instead of using confusing .thens with reduce. It'd also be good to avoid the explicit Promise construction antipattern. This should do what you want:
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
Then your results variable will contain an array of response texts (or an error will have been thrown, and the code won't reach the bottom).
The syntax for an async function is an async keyword before the argument list, just like you're doing in your original code:
const fn = async () => {
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
// do something with results
};
To have a limited number of requests at a time, make a queue system - when a request completes, recursively call a function that makes another request, something like:
const results = [];
const queueNext = async () => {
if (!urls.length) return;
const url = urls.shift();
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
await queueNext();
}
await Promise.all(Array.from({ length: 5 }, queueNext));
// do something with results
You cannot use Array methods to sequentually run async operations because array methods are all synchronous.
The easiest way to achieve sequential async tasks is through a loop. Otherwise, you will need to write a custom function to imitate a loop and run .then after a async task ends, which is quite troublesome and unnecessary.
Also, fetch is already returning a Promise, so you don't have to create a Promise yourself to contain that promise returned by fetch.
The code below is a working example, with small changes to your original code (see comments).
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
// To imitate actual fetching
const fetch = (url) => new Promise(resolve => {
setTimeout(() => {
resolve({
ok: true,
text: () => new Promise(res => setTimeout(() => res(url), 500))
});
}, 1000);
});
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
return fetch(fetchUrl).then(async res => { // <-- changes here
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
return url; // <--- changes here
});
}
async function getAllUrls(urls){
const result = [];
for (const url of urls){
const response = await getUrl(url);
result.push(response);
}
return result;
}
getAllUrls(urls)
.then(console.log);
async/await is perfect for this.
Assuming you have an array of URLs as strings:
let urls = ["https://example.org/", "https://google.com/", "https://stackoverflow.com/"];
You simply need to do:
for (let u of urls) {
await fetch(u).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
The loop will not iterate until the current fetch() has resolved, which will serialise things.
The reason array.map doesn't work is as follows:
async function doFetch(url) {
return await fetch(url).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
let mapped = urls.map(doFetch);
is equivalent to:
let mapped;
for (u of urls) {
mapped.push(doFetch(u));
}
This will populate mapped with a bunch of Promises immediately, which is not what you want. The following is what you want:
let mapped;
for (u of urls) {
mapped.push(await doFetch(u));
}
But this is not what array.map() does. Therefore using an explicit for loop is necessary.
Many people provided answers using for loop. But in some situation await in for loop is not welcome, for example, if you are using Airbnb style guide.
Here is a solution using recursion.
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
async function serialFetch(urls) {
return await doSerialRecursion(
async (url) => {
return result = await fetch(url)
.then((response) => {
// handle response
})
.catch((err) => {
// handle error
});
},
urls,
0
);
}
async function doSerialRecursion(fn, array, startIndex) {
if (!array[startIndex]) return [];
const currResult = await fn(array[startIndex]);
return [currResult, ...(await doSerialRecursion(array, fn, startIndex + 1))];
}
const yourResult = await serialFetch(urls);
The doSerialRecursion function will serially execute the function you passed in, which is fetch(url) in this example.
How Should I await for bot.sendMessage() inside of loop?
Maybe I Need await Promise.all But I Don't Know How Should I add to bot.sendMessage()
Code:
const promise = query.exec();
promise.then(async (doc) => {
let count = 0;
for (const val of Object.values(doc)) {
++count;
await bot.sendMessage(msg.chat.id, `💬 ${count} and ${val.text}`, opts);
}
}).catch((err) => {
if (err) {
console.log(err);
}
});
Error:
[eslint] Unexpected `await` inside a loop. (no-await-in-loop)
If you need to send each message one-at-a-time, then what you have is fine, and according to the docs, you can just ignore the eslint error like this:
const promise = query.exec();
promise.then(async doc => {
/* eslint-disable no-await-in-loop */
for (const [index, val] of Object.values(doc).entries()) {
const count = index + 1;
await bot.sendMessage(msg.chat.id, `💬 ${count} and ${val.text}`, opts);
}
/* eslint-enable no-await-in-loop */
}).catch(err => {
console.log(err);
});
However, if there is no required order for sending the messages, you should do this instead to maximize performance and throughput:
const promise = query.exec();
promise.then(async doc => {
const promises = Object.values(doc).map((val, index) => {
const count = index + 1;
return bot.sendMessage(msg.chat.id, `💬 ${count} and ${val.text}`, opts);
});
await Promise.all(promises);
}).catch(err => {
console.log(err);
});
Performing await inside loops can be avoided once iterations doesn't have dependency in most cases, that's why eslint is warning it here
You can rewrite your code as:
const promise = query.exec();
promise.then(async (doc) => {
await Promise.all(Object.values(doc).map((val, idx) => bot.sendMessage(msg.chat.id, `💬 ${idx + 1} and ${val.text}`, opts);)
}).catch((err) => {
if (err) {
console.log(err);
}
});
If you still and to send one-after-one messages, your code is ok but eslint you keep throwing this error
I had a similar issue recently, I was getting lintting errors when trying to run an array of functions in a chain as apposed to asynchronously.
and this worked for me...
const myChainFunction = async (myArrayOfFunctions) => {
let result = Promise.resolve()
myArrayOfFunctions.forEach((myFunct) => {
result = result.then(() => myFunct()
})
return result
}
I am facing the same issue when I used await inside forEach loop. But I tried with recursive function call to iterate array.
const putDelay = (ms) =>
new Promise((resolve) => {
setTimeout(resolve, ms);
});
const myRecursiveFunction = async (events, index) => {
if (index < 0) {
return;
}
callAnotherFunction(events[index].activity, events[index].action, events[index].actionData);
await putDelay(1);
myRecursiveFunction(events, index - 1);
};
I have a list of urls I wish to fetch. All of these urls returns a json object with a property valid. But only one of the fetch promises has the magic valid property to true.
I have tried various combinations of url.forEach(...) and Promises.all([urls]).then(...). Currently my setup is:
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
var result;
urls.forEach(function (url) {
result = fetch(`${url}/${key}/validate`)
.then((response) => response.json())
.then((json) => {
if (json.license.valid) {
return json;
} else {
Promise.reject(json);
}
});
});
return result;
}
The above is not working because of the async promises. How can I iterate my urls and return when the first valid == true is hit?
Let me throw a nice compact entry into the mix
It uses Promise.all, however every inner Promise will catch any errors and simply resolve to false in such a case, therefore Promise.all will never reject - any fetch that completes, but does not have license.valid will also resolve to false
The array Promise.all resolves is further processed, filtering out the false values, and returning the first (which from the questions description should be the ONLY) valid JSON response
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
return Promise.all(urls.map(url =>
fetch(`${url}/${key}/validate`)
.then(response => response.json())
.then(json => json.license && json.license.valid && json)
.catch(error => false)
))
.then(results => results.filter(result => !!result)[0] || Promise.reject('no matches found'));
}
Note that it's impossible for validate to return the result (see here for why). But it can return a promise for the result.
What you want is similar to Promise.race, but not quite the same (Promise.race would reject if one of the fetch promises rejected prior to another one resolving with valid = true). So just create a promise and resolve it when you get the first resolution with valid being true:
export function validate(key) {
return new Promise((resolve, reject) => {
let completed = 0;
const total = urls.length;
urls.forEach(url => {
fetch(`${url}/${key}/validate`)
.then((response) => {
const json = response.json();
if (json.license.valid) {
resolve(json);
} else {
if (++completed === total) {
// None of them had valid = true
reject();
}
}
})
.catch(() => {
if (++completed === total) {
// None of them had valid = true
reject();
}
});
});
});
}
Note the handling for the failing case.
Note that it's possible to factor out those two completed checks if you like:
export function validate(key) {
return new Promise((resolve, reject) => {
let completed = 0;
const total = urls.length;
urls.forEach(url => {
fetch(`${url}/${key}/validate`)
.then((response) => {
const json = response.json();
if (json.license.valid) {
resolve(json);
}
})
.catch(() => {
// Do nothing, converts to a resolution with `undefined`
})
.then(() => {
// Because of the above, effectively a "finally" (which we
// may get on Promises at some point)
if (++completed === total) {
// None of them had valid = true.
// Note that we come here even if we've already
// resolved the promise -- but that's okay(ish), a
// promise's resolution can't be changed after it's
// settled, so this would be a no-op in that case
reject();
}
});
});
});
}
This can be done using SynJS. Here is a working example:
var SynJS = require('synjs');
var fetchUrl = require('fetch').fetchUrl;
function fetch(context,url) {
console.log('fetching started:', url);
var result = {};
fetchUrl(url, function(error, meta, body){
result.done = true;
result.body = body;
result.finalUrl = meta.finalUrl;
console.log('fetching finished:', url);
SynJS.resume(context);
} );
return result;
}
function myFetches(modules, urls) {
for(var i=0; i<urls.length; i++) {
var res = modules.fetch(_synjsContext, urls[i]);
SynJS.wait(res.done);
if(res.finalUrl.indexOf('github')>=0) {
console.log('found correct one!', urls[i]);
break;
}
}
};
var modules = {
SynJS: SynJS,
fetch: fetch,
};
const urls = [
'http://www.google.com',
'http://www.yahoo.com',
'http://www.github.com', // This is the valid one
'http://www.wikipedia.com'
];
SynJS.run(myFetches,null,modules,urls,function () {
console.log('done');
});
It would produce following output:
fetching started: http://www.google.com
fetching finished: http://www.google.com
fetching started: http://www.yahoo.com
fetching finished: http://www.yahoo.com
fetching started: http://www.github.com
fetching finished: http://www.github.com
found correct one! http://www.github.com
done
If you want to avoid the fact of testing each URL, you can use the following code.
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
return new Promise((resolve, reject) => {
function testUrl(url) {
fetch(`${url}/${key}/validate`)
.then((response) => response.json())
.then((json) => {
if (json.license.valid) {
resolve(json);
return;
}
if (urlIndex === urls.length) {
reject("No matches found...");
return;
}
testUrl(urls[urlIndex++]);
});
}
let urlIndex = 0;
if (!urls.length)
return reject("No urls to test...");
testUrl(urls[urlIndex++]);
});
}