I have a list of urls I wish to fetch. All of these urls returns a json object with a property valid. But only one of the fetch promises has the magic valid property to true.
I have tried various combinations of url.forEach(...) and Promises.all([urls]).then(...). Currently my setup is:
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
var result;
urls.forEach(function (url) {
result = fetch(`${url}/${key}/validate`)
.then((response) => response.json())
.then((json) => {
if (json.license.valid) {
return json;
} else {
Promise.reject(json);
}
});
});
return result;
}
The above is not working because of the async promises. How can I iterate my urls and return when the first valid == true is hit?
Let me throw a nice compact entry into the mix
It uses Promise.all, however every inner Promise will catch any errors and simply resolve to false in such a case, therefore Promise.all will never reject - any fetch that completes, but does not have license.valid will also resolve to false
The array Promise.all resolves is further processed, filtering out the false values, and returning the first (which from the questions description should be the ONLY) valid JSON response
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
return Promise.all(urls.map(url =>
fetch(`${url}/${key}/validate`)
.then(response => response.json())
.then(json => json.license && json.license.valid && json)
.catch(error => false)
))
.then(results => results.filter(result => !!result)[0] || Promise.reject('no matches found'));
}
Note that it's impossible for validate to return the result (see here for why). But it can return a promise for the result.
What you want is similar to Promise.race, but not quite the same (Promise.race would reject if one of the fetch promises rejected prior to another one resolving with valid = true). So just create a promise and resolve it when you get the first resolution with valid being true:
export function validate(key) {
return new Promise((resolve, reject) => {
let completed = 0;
const total = urls.length;
urls.forEach(url => {
fetch(`${url}/${key}/validate`)
.then((response) => {
const json = response.json();
if (json.license.valid) {
resolve(json);
} else {
if (++completed === total) {
// None of them had valid = true
reject();
}
}
})
.catch(() => {
if (++completed === total) {
// None of them had valid = true
reject();
}
});
});
});
}
Note the handling for the failing case.
Note that it's possible to factor out those two completed checks if you like:
export function validate(key) {
return new Promise((resolve, reject) => {
let completed = 0;
const total = urls.length;
urls.forEach(url => {
fetch(`${url}/${key}/validate`)
.then((response) => {
const json = response.json();
if (json.license.valid) {
resolve(json);
}
})
.catch(() => {
// Do nothing, converts to a resolution with `undefined`
})
.then(() => {
// Because of the above, effectively a "finally" (which we
// may get on Promises at some point)
if (++completed === total) {
// None of them had valid = true.
// Note that we come here even if we've already
// resolved the promise -- but that's okay(ish), a
// promise's resolution can't be changed after it's
// settled, so this would be a no-op in that case
reject();
}
});
});
});
}
This can be done using SynJS. Here is a working example:
var SynJS = require('synjs');
var fetchUrl = require('fetch').fetchUrl;
function fetch(context,url) {
console.log('fetching started:', url);
var result = {};
fetchUrl(url, function(error, meta, body){
result.done = true;
result.body = body;
result.finalUrl = meta.finalUrl;
console.log('fetching finished:', url);
SynJS.resume(context);
} );
return result;
}
function myFetches(modules, urls) {
for(var i=0; i<urls.length; i++) {
var res = modules.fetch(_synjsContext, urls[i]);
SynJS.wait(res.done);
if(res.finalUrl.indexOf('github')>=0) {
console.log('found correct one!', urls[i]);
break;
}
}
};
var modules = {
SynJS: SynJS,
fetch: fetch,
};
const urls = [
'http://www.google.com',
'http://www.yahoo.com',
'http://www.github.com', // This is the valid one
'http://www.wikipedia.com'
];
SynJS.run(myFetches,null,modules,urls,function () {
console.log('done');
});
It would produce following output:
fetching started: http://www.google.com
fetching finished: http://www.google.com
fetching started: http://www.yahoo.com
fetching finished: http://www.yahoo.com
fetching started: http://www.github.com
fetching finished: http://www.github.com
found correct one! http://www.github.com
done
If you want to avoid the fact of testing each URL, you can use the following code.
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
return new Promise((resolve, reject) => {
function testUrl(url) {
fetch(`${url}/${key}/validate`)
.then((response) => response.json())
.then((json) => {
if (json.license.valid) {
resolve(json);
return;
}
if (urlIndex === urls.length) {
reject("No matches found...");
return;
}
testUrl(urls[urlIndex++]);
});
}
let urlIndex = 0;
if (!urls.length)
return reject("No urls to test...");
testUrl(urls[urlIndex++]);
});
}
Related
I've got a series of promises.
I never get the console log "Processed folders" printed out. Execution seems to stop once it hits the first await Promise.all call.
Not entirely sure where I've missed up?
const subfolders = [];
const exportFolder = () => {
// Other stuff happening here
const subfolder = {};
subfolder.items = [];
subfolder.items.push({ name: 'item 2.1' });
const folder = {};
folder.items = [];
folder.items.push({ name: 'item 1' });
folder.items.push({ name: 'item 2', isFolder: true, items: subfolder.items });
console.log('Folder:', folder);
console.log('Started');
exportFolderToCsv(folder).then(response => console.log('Finished', response));
};
const exportFolderToCsv = async folder => {
console.log('Processing folders');
let promises = [];
for (const folderItem of folder.items) {
if (folderItem.isFolder && folderItem.items.length > 0) {
subfolders.push(folderItem);
return;
}
promises.push(processFolderItem(folderItem));
}
await Promise.all(promises).then(response => console.log('Processed folders:', response));
if (subfolders.length > 0) {
console.log('Processing subfolders');
promises = [];
for (const folderItem of subfolders.items) {
promises.push(processFolderItem(folderItem));
}
await Promise.all(promises).then(response => console.log('Processed subfolders:', response));
}
console.log('Finished');
};
const processFolderItem = folderItem => new Promise(resolve => {
console.log('Processing folder item');
// To stuff here with folderItem, get Doc Chars, process row and resolve
getCharacters(folderItem)
.then(response => {
console.log('Processed folder item characters list:', response);
createCSVRow(folderItem, response)
.then(response => {
console.log('Processed CSV row:', response);
resolve(response);
})
});
});
const getCharacters = folderItem => new Promise(resolve => {
console.log('Processing folder item characters list');
// To stuff here with folderItem and then resolve
const characters = 'Foobar characters';
resolve(characters);
});
const createCSVRow = (folderItem, characters) => new Promise(resolve => {
console.log('Processing CSV row');
// To stuff here with folderItem and characters and then resolve
const csvRow = 'Foobar row';
resolve(csvRow);
});
exportFolder();
The function is returned before any Promise call is executed, it should be continue. So please resolve it first.
const exportFolderToCsv = async (folder) => {
// ...
for (const folderItem of folder.items) {
if (folderItem.isFolder && folderItem.items.length > 0) {
subfolders.push(folderItem)
return // here lies the problem, it should be `continue` instead
}
promises.push(processFolderItem(folderItem))
}
// ...
}
The Promise.all method, will catch errors if at least one Promise in chain throws an error or reject it.
It seems that one of your promises got an exception.
You can debug it simply adding the catch like so:
Promise.all(PromiseList).then(()=>console.log("all works done") ).catch(errors=>console.log("something wrong",errors))
If you are using await there is no point in chaining.
let response = await Promise.all(promises);
console.log('Processed subfolders', response)
Your promises will encounter errors, and you need to consider the failed situations.
At the same time, Promise.all will stop if either one promise failed, so maybe your Promise.all failed.
try {
Promise.all(promises).then(response => console.log('Processed subfolders', response));
} catch (err) {
console.log(err);
}
// or to check your promise result
const folderResult = await Promise.all(promises);
to check the exact result
I'm trying to only make one http call at time but when I log the response from getUrl they are piling up and I start to get 409s (Too many requests)
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
fetch(fetchUrl).then(async res => {
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
cb(url);
});
}
let requests = urls.map((url, i) => {
return new Promise(resolve => {
getUrl(url, i, resolve);
});
});
const all = await requests.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult => [...chainResults, currentResult]),
);
}, Promise.resolve([]));
Basically I don't want the next http to start until the previous one has finished. Otherwise I hammer their server.
BONUS POINTS: Make this work with 5 at a time in parallel.
Since you're using await, it would be a lot easier to use that everywhere instead of using confusing .thens with reduce. It'd also be good to avoid the explicit Promise construction antipattern. This should do what you want:
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
Then your results variable will contain an array of response texts (or an error will have been thrown, and the code won't reach the bottom).
The syntax for an async function is an async keyword before the argument list, just like you're doing in your original code:
const fn = async () => {
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
// do something with results
};
To have a limited number of requests at a time, make a queue system - when a request completes, recursively call a function that makes another request, something like:
const results = [];
const queueNext = async () => {
if (!urls.length) return;
const url = urls.shift();
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
await queueNext();
}
await Promise.all(Array.from({ length: 5 }, queueNext));
// do something with results
You cannot use Array methods to sequentually run async operations because array methods are all synchronous.
The easiest way to achieve sequential async tasks is through a loop. Otherwise, you will need to write a custom function to imitate a loop and run .then after a async task ends, which is quite troublesome and unnecessary.
Also, fetch is already returning a Promise, so you don't have to create a Promise yourself to contain that promise returned by fetch.
The code below is a working example, with small changes to your original code (see comments).
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
// To imitate actual fetching
const fetch = (url) => new Promise(resolve => {
setTimeout(() => {
resolve({
ok: true,
text: () => new Promise(res => setTimeout(() => res(url), 500))
});
}, 1000);
});
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
return fetch(fetchUrl).then(async res => { // <-- changes here
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
return url; // <--- changes here
});
}
async function getAllUrls(urls){
const result = [];
for (const url of urls){
const response = await getUrl(url);
result.push(response);
}
return result;
}
getAllUrls(urls)
.then(console.log);
async/await is perfect for this.
Assuming you have an array of URLs as strings:
let urls = ["https://example.org/", "https://google.com/", "https://stackoverflow.com/"];
You simply need to do:
for (let u of urls) {
await fetch(u).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
The loop will not iterate until the current fetch() has resolved, which will serialise things.
The reason array.map doesn't work is as follows:
async function doFetch(url) {
return await fetch(url).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
let mapped = urls.map(doFetch);
is equivalent to:
let mapped;
for (u of urls) {
mapped.push(doFetch(u));
}
This will populate mapped with a bunch of Promises immediately, which is not what you want. The following is what you want:
let mapped;
for (u of urls) {
mapped.push(await doFetch(u));
}
But this is not what array.map() does. Therefore using an explicit for loop is necessary.
Many people provided answers using for loop. But in some situation await in for loop is not welcome, for example, if you are using Airbnb style guide.
Here is a solution using recursion.
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
async function serialFetch(urls) {
return await doSerialRecursion(
async (url) => {
return result = await fetch(url)
.then((response) => {
// handle response
})
.catch((err) => {
// handle error
});
},
urls,
0
);
}
async function doSerialRecursion(fn, array, startIndex) {
if (!array[startIndex]) return [];
const currResult = await fn(array[startIndex]);
return [currResult, ...(await doSerialRecursion(array, fn, startIndex + 1))];
}
const yourResult = await serialFetch(urls);
The doSerialRecursion function will serially execute the function you passed in, which is fetch(url) in this example.
I'm using async await inside of for loop as below.
for (let i = 0; i < result.length; i += 1) {
try{
const client = await axios.get(
`${process.env.user}/client/${result[i].id}`
);
} catch(error){
console.log(error)
}
if (client.data.success === true) {
result[i].Name = rider.data.client.Name;
result[i].PhoneNumber = rider.data.client.Number;
}
}
But I want to make this using 'new Promise' and 'promiss.all' to make it asynclously.
But I don'k know how to make this correctly doing error handle well.
Could you recommend some advice for this? Thank you for reading it.
This can be a basic solution, i think
let playList = []
for (let i = 0; i < result.length; i += 1) {
playList.push(axios.get(
`${process.env.user}/client/${result[i].id}`
).then(res => {
if (res.data.success === true) {
result[i].Name = rider.data.client.Name;
result[i].PhoneNumber = rider.data.client.Number;
}
}).catch(ex => console.log(ex)));
}
await Promise.all(playList)
This can also be done by using a foreach loop.
The for/foreach loop can be simplified by using a map function.
Js map function is equivalent of c# select linq function.
The fat arrow in js map function is not bound to return a value unlike c# select inner function which must return a value.
await Promise.all(result.map(async r => {
let client;
try {
client = await axios.get(`${process.env.user}/client/${r.id}`);
} catch (error) {
console.log(error)
}
if (client.data.success === true) {
r.Name = rider.data.client.Name;
r.PhoneNumber = rider.data.client.Number;
}
}));
Try this
var promises = result.map(r => axios.get(`${process.env.user}/client/${r.id}`);
Promise.all(promises).then(function(values) {
console.log('All promises done');
});
The idea is that if you are awaiting something, that is promise, you can await it, or call it to get promise
Example:
function Foo()
{
return new Promise(...); // Promise of int for example
}
you can do
var p = Foo(); //you will get promise
Or
var v = await Foo(); // you will get int value when promise resolved
This is how you do it with async/await + Promise.all:
const myResult = await Promise.all(result.map(({ id }) => {
return axios.get(`${process.env.user}/client/${id}`);
}));
// deal with the result of those requests
const parsed = myResult.map(data => /* your code here */);
Here is an example using Array.map to call your function along with Promise.all. I wrapped the axios request in a function so if one of your request fails, it wont stop every other requests. If you don't mind stopping when you got an issue, look at others answers to your question.
function fakeRequest() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve({
data: {
success: true,
client: {
Name: 'Todd',
Number: 5,
},
},
});
}, 300);
});
}
(async() => {
const result = [{}, {}, {}];
await Promise.all(result.map(async(x, xi) => {
try {
const client = await fakeRequest();
if (client.data.success === true) {
result[xi].Name = client.data.client.Name;
result[xi].PhoneNumber = client.data.client.Number;
}
} catch (err) {
console.log(err)
}
}));
console.log(result);
})();
So I am using Forge with View API to analyze all parts from a model which contain concrete and hide everything that is not concrete. The problem is that the properties for checking concrete are called from a DB which requires me to make it a promise. Checking for concrete is working as expected and then the problem starts. I return the Ids containing concrete, but my function which is supposed to hide it uses the Ids before the promise is resolved, so the array is empty.
console.log logs it as expected but everything else misses the timing.
My code:
getProperties(dbId) {
return new Promise((resolve, reject) => {
this.gui.getProperties(
dbId,
args => {
resolve(args.properties)
},
reject
)
})
}
async getConcreteIds() {
let wallfloorids = this.getWallIds().concat(this.getFloorIds());
let concreteIds = [];
for (let id of wallfloorids) {
let p1 = this.view.getProperties(id);
p1.then(props => {
for (let prop of props) {
if (prop.displayCategory === "Materialien und Oberflächen" && prop.displayValue.contains("Concrete")) {
concreteIds.push(id);
}
}
}).catch(() => {
});
}
return new Promise((resolve, reject) => {
try {
resolve(concreteIds)
} catch (e) {
console.log("Err", reject)
}
})
}
async onlyConcrete() {
this.getConcreteIds().then(concrete => {
debugger;
this.viewer.isolateById(concrete)
});
}
Map an array of promises for your loop and use Promise.all() to resolve after all the promises in loop resolve
Something like:
async getConcreteIds() {
let wallfloorids = this.getWallIds().concat(this.getFloorIds());
let concreteIds = [];
let promises = wallfloorids.map(id => {
let p1 = this.view.getProperties(id);
return p1.then(props => {
for (let prop of props) {
if (prop.displayCategory === "Materialien und Oberflächen" && prop.displayValue.contains("Concrete")) {
concreteIds.push(id);
}
}
});
});
return Promise.all(promises)
.then(_ => concreteIds)
.catch(err => console.log("Err", err))
}
Firstly apologies if this is a simple problem to solve but I am using node-fetch and the concept of Promises for the first time. I am calling an api endpoint which returns paginated data so i get multiple links for further data.
I am handling this by putting all the links into an array and then I want to call each endpoint in the array and just print out the data to the console for now
So my first call is
var urlArray = [];
fetch(`${BASE_URL}/leagues?api_token=${AUTH_TOKEN}`, { headers: headers })
.then(function(response){
return response.json(); // pass the data as promise to next then block
})
.then(function(json){
// Collect Paginated Results
var number_of_pages = parseInt(json.meta['pagination']['total_pages']);
for (i = 2; i < number_of_pages + 1; i++) {
urlArray.push(`${BASE_URL}?page=${i}&api_token=${AUTH_TOKEN}`)
}
})
.catch(function(error) {
console.log(error);
});
So once this done i have an array of urls i need to hit sequentially, could anyone give me some pointers please on how to then setup a promise to hit each one? or can i have something where is do it in my for loop?
Thanks
sorry if this is wrong!
At the beginning, put:
var promise_array = [];
in the for loop, replace with
for (i = 2; i < number_of_pages + 1; i++) {
var url = `${BASE_URL}?page=${i}&api_token=${AUTH_TOKEN}`;
urlArray.push(url);
promise_array.push(fetch(url));
}
At the end of the function put
Promise.all(promise_array).then(all_responses => {
// .. pluck json, then continue ...
});
You can achieve this by:
fetch(`${BASE_URL}/leagues?api_token=${AUTH_TOKEN}`, { headers: headers }).then(response => {
var json = response.json();
var numberOfPages = parseInt(json.meta['pagination']['total_pages']);
var urlArray = [];
for (i = 2; i < numberOfPages + 1; i++) {
urlArray.push(`${BASE_URL}?page=${i}&api_token=${AUTH_TOKEN}`)
}
var promises = urlArray.map(url => getSomethingByUrl(url));
return Promise.all(promises);
}).then(results => {
console.log(results);
});
You can use Array.prototype.reduce to create Promise chain so they will be executed one after another
urls.reduce((prev, item) => {
return prev.then(() => {
return someAction(item);
})
}, Promise.resolve()).then(() => {
console.log('All done');
}).catch((err) => {
console.log(err);
});
Edit:
As mentioned by Bergi, value from someAction should be returned in case it's asynchronous (return Promise). If someAction is asynchronous it must return Promise.
"use strict";
function someAction(i) {
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log(i);
resolve(i);
}, i*1000);
});
}
const urls = [1,2,3,4,5];
urls.reduce((prev, item) => {
return prev.then(() => {
return someAction(item);
})
}, Promise.resolve()).then(() => {
console.log('All done');
}).catch((err) => {
console.log(err);
});