Firstly apologies if this is a simple problem to solve but I am using node-fetch and the concept of Promises for the first time. I am calling an api endpoint which returns paginated data so i get multiple links for further data.
I am handling this by putting all the links into an array and then I want to call each endpoint in the array and just print out the data to the console for now
So my first call is
var urlArray = [];
fetch(`${BASE_URL}/leagues?api_token=${AUTH_TOKEN}`, { headers: headers })
.then(function(response){
return response.json(); // pass the data as promise to next then block
})
.then(function(json){
// Collect Paginated Results
var number_of_pages = parseInt(json.meta['pagination']['total_pages']);
for (i = 2; i < number_of_pages + 1; i++) {
urlArray.push(`${BASE_URL}?page=${i}&api_token=${AUTH_TOKEN}`)
}
})
.catch(function(error) {
console.log(error);
});
So once this done i have an array of urls i need to hit sequentially, could anyone give me some pointers please on how to then setup a promise to hit each one? or can i have something where is do it in my for loop?
Thanks
sorry if this is wrong!
At the beginning, put:
var promise_array = [];
in the for loop, replace with
for (i = 2; i < number_of_pages + 1; i++) {
var url = `${BASE_URL}?page=${i}&api_token=${AUTH_TOKEN}`;
urlArray.push(url);
promise_array.push(fetch(url));
}
At the end of the function put
Promise.all(promise_array).then(all_responses => {
// .. pluck json, then continue ...
});
You can achieve this by:
fetch(`${BASE_URL}/leagues?api_token=${AUTH_TOKEN}`, { headers: headers }).then(response => {
var json = response.json();
var numberOfPages = parseInt(json.meta['pagination']['total_pages']);
var urlArray = [];
for (i = 2; i < numberOfPages + 1; i++) {
urlArray.push(`${BASE_URL}?page=${i}&api_token=${AUTH_TOKEN}`)
}
var promises = urlArray.map(url => getSomethingByUrl(url));
return Promise.all(promises);
}).then(results => {
console.log(results);
});
You can use Array.prototype.reduce to create Promise chain so they will be executed one after another
urls.reduce((prev, item) => {
return prev.then(() => {
return someAction(item);
})
}, Promise.resolve()).then(() => {
console.log('All done');
}).catch((err) => {
console.log(err);
});
Edit:
As mentioned by Bergi, value from someAction should be returned in case it's asynchronous (return Promise). If someAction is asynchronous it must return Promise.
"use strict";
function someAction(i) {
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log(i);
resolve(i);
}, i*1000);
});
}
const urls = [1,2,3,4,5];
urls.reduce((prev, item) => {
return prev.then(() => {
return someAction(item);
})
}, Promise.resolve()).then(() => {
console.log('All done');
}).catch((err) => {
console.log(err);
});
Related
I'm using async await inside of for loop as below.
for (let i = 0; i < result.length; i += 1) {
try{
const client = await axios.get(
`${process.env.user}/client/${result[i].id}`
);
} catch(error){
console.log(error)
}
if (client.data.success === true) {
result[i].Name = rider.data.client.Name;
result[i].PhoneNumber = rider.data.client.Number;
}
}
But I want to make this using 'new Promise' and 'promiss.all' to make it asynclously.
But I don'k know how to make this correctly doing error handle well.
Could you recommend some advice for this? Thank you for reading it.
This can be a basic solution, i think
let playList = []
for (let i = 0; i < result.length; i += 1) {
playList.push(axios.get(
`${process.env.user}/client/${result[i].id}`
).then(res => {
if (res.data.success === true) {
result[i].Name = rider.data.client.Name;
result[i].PhoneNumber = rider.data.client.Number;
}
}).catch(ex => console.log(ex)));
}
await Promise.all(playList)
This can also be done by using a foreach loop.
The for/foreach loop can be simplified by using a map function.
Js map function is equivalent of c# select linq function.
The fat arrow in js map function is not bound to return a value unlike c# select inner function which must return a value.
await Promise.all(result.map(async r => {
let client;
try {
client = await axios.get(`${process.env.user}/client/${r.id}`);
} catch (error) {
console.log(error)
}
if (client.data.success === true) {
r.Name = rider.data.client.Name;
r.PhoneNumber = rider.data.client.Number;
}
}));
Try this
var promises = result.map(r => axios.get(`${process.env.user}/client/${r.id}`);
Promise.all(promises).then(function(values) {
console.log('All promises done');
});
The idea is that if you are awaiting something, that is promise, you can await it, or call it to get promise
Example:
function Foo()
{
return new Promise(...); // Promise of int for example
}
you can do
var p = Foo(); //you will get promise
Or
var v = await Foo(); // you will get int value when promise resolved
This is how you do it with async/await + Promise.all:
const myResult = await Promise.all(result.map(({ id }) => {
return axios.get(`${process.env.user}/client/${id}`);
}));
// deal with the result of those requests
const parsed = myResult.map(data => /* your code here */);
Here is an example using Array.map to call your function along with Promise.all. I wrapped the axios request in a function so if one of your request fails, it wont stop every other requests. If you don't mind stopping when you got an issue, look at others answers to your question.
function fakeRequest() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve({
data: {
success: true,
client: {
Name: 'Todd',
Number: 5,
},
},
});
}, 300);
});
}
(async() => {
const result = [{}, {}, {}];
await Promise.all(result.map(async(x, xi) => {
try {
const client = await fakeRequest();
if (client.data.success === true) {
result[xi].Name = client.data.client.Name;
result[xi].PhoneNumber = client.data.client.Number;
}
} catch (err) {
console.log(err)
}
}));
console.log(result);
})();
I'm new to node.js and javascript in general but I am having issues understanding why the writeFile function is writing a blank file. I think the for loop should be a Promise but I am not sure how to write it.
const removeProviders = function () {
readline.question('Where is the file located? ', function(filePath) {
let providerArray = fs.readFileSync(filePath).toString().split('\r\n');
console.log(providerArray);
let importFile = '';
for (let providerId in providerArray) {
getProvider(providerArray[providerId]).then((response) => {
let providerInfo = response.data;
return providerInfo;
}).then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += "entry";
})
}
fs.writeFile('C:/path/to/file.txt', importFile);
You can collect all the promises from the for-loop into an Array and then pass them into Promise.all() which will resolve only after all of them resolved. If one of the promises are rejected, it will reject as well:
const promises = providerArray.map((item) => {
return getProvider(item)
.then((response) => {
let providerInfo = response.data;
return providerInfo;
})
.then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += "entry";
});
});
Promise.all(promises).then(() => {
fs.writeFile('C:/path/to/file.txt', importFile, err => {
if (err) {
console.error(err);
}
});
});
While doing this you could also get rid of the importFile variable and collect directly the results of your promises. Promise.all().then(results => {}) will then give you an array of all results. Thus no need for an updatable variable.
The below approach may be useful.
I don't know your getProvider return Promise. you can set promise in getProvider method
const getProviderValue = async function(providerArray) {
return new Promise((resolve, reject) {
let importFile = '';
for (let providerId in providerArray) {
await getProvider(providerArray[providerId]).then((response) => {
let providerInfo = response.data;
return providerInfo;
}).then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += "entry";
})
}
resolve(importFile)
})
}
const removeProviders = async function () {
readline.question('Where is the file located? ', function(filePath) {
let providerArray = fs.readFileSync(filePath).toString().split('\r\n');
console.log(providerArray);
let importFile = await getProviderValue(providerArray)
fs.writeFile('C:/path/to/file.txt', importFile);
})
}
Your for loop does not wait for the promises to resolve. A better way to approach this problem would be to use reduce.
providerArray.reduce(
(p, _, i) => {
p.then(_ => new Promise(resolve =>
getProvider(providerArray[providerId]).then((response) => {
let providerInfo = response.data;
return providerInfo;
}).then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += entry;
resolve();
}))
);
}
, Promise.resolve() );
You can also use Promise.all but the out data may not be in the order that you expect if you append to the importFile variable.
I want to retrieve different HTML body at once and as soon as all results are available work with that content.
My callback solution which works looks like this (probably only relevant to read if the idea is not clear, otherwise skip ;)):
const request = require('request')
const argLength = process.argv.length
const result_array = []
let counter = 0
function executeRequest () {
for (start = 2; start <= argLength - 1; start++) {
const copy = start
function callback (error, res, body) {
const startCopy = copy
if (error) {
console.log('error')
return
}
result_array[startCopy - 2] = body.toString().length
counter++
if (counter === argLength - 2) {
console.log(result_array)
}
}
request(process.argv[start], callback)
}
}
executeRequest()
Now I try to make it running with Promises like this:
const httpRequest = require('request')
const argumentLength = process.argv.length
function fillURLArray () {
resultArray = []
for (start = 2; start < argumentLength; start++) {
resultArray[start - 2] = process.argv[start]
}
return resultArray
}
const urls = fillURLArray()
let counter = 0
function readHttp () {
const resultArray = []
Promise.all(urls.map(url => httpRequest(url, (error, res, body) => {
console.log(body.toString())
resultArray[counter++] = body.toString()
}))).then(value => {
console.log('promise counter: ' + counter++)
console.log(resultArray)
console.log('called')
})
}
readHttp()
I tried already several attempts with different promise chains but every time I get either not a result or just an empty array. So obviously the .then() function is called before the array is actually filled (at least I guess so since console.log(body.toString()) appears to print the content some time later)
Any idea how to solve this with promises?
Thank you
request is not returning promise object so have created a method that return promise object on which you do Promise.all.
function requestPromise(url){
return new Promise((resovle,reject) => {
httpRequest(url, (error, res, body) => {
if(err){
reject(err);
}
resolve(body.toString());
});
});
}
function readHttp () {
const resultArray = []
Promise.all(urls.map(url => requestPromise(url))).then(values => {
console.log("counter => ",values.length);
resultArray = resultArray.concat(values);
console.log("values=> ",values);
console.log("resultArray=> ",resultArray);
});
}
httpRequest does not return a promise so you have to make one yourself, also your resultArray is not necessary:
const makeRequest = url => new Promise((resolve, reject) => httpRequest(url, (error, res) => error ? reject(error) : resolve(res)));
Promise.all(urls.map(makeRequest))
.then(result => {
console.log(result.map(res => res.body.toString()));
console.log('called');
});
Assuming I am calling https a multiple times to retrieve data, and I want to call a function formatJsonToLocale at the end of the last request. Is there a way to determine when that request has ended, other than checking for the last element of the array.
let sheetsArray = []
function sheetsAsJsonById (ids) {
for (let i = 0; i < ids.length; i++) {
const queryUrl = `sheets.html`
https
.get(queryUrl, res => {
let stream = []
res
.on('data', function (data) {
stream.push(data)
})
.on('end', function () {
let data = Buffer.concat(stream)
data = JSON.parse(data)
sheetArrays.push(data['values'])
formatJsonToLocale(sheetsArray) // <----- call this one after last request
})
})
.on('error', err => {
console.error(`Error in response: ${err}`)
})
}
}
when I call formatJsonToLocale outside of the function I will have the problem that the former function might not be finished as https handles stuff asynchronously.
Any suggestions on how to handle this?
You would need to keep track of execution of async code (https.get) that is getting executed within for loop. This can be achieved using promises as below:
let sheetsArray = []
function sheetsAsJsonById (ids) {
let promises = []
for (let i = 0; i < ids.length; i++) {
const queryUrl = `sheets.html`
promises.push(makeHTTPRequest(queryUrl))
}
Promise.all(promises).then((sheetArrays) => {
formatJsonToLocale(sheetsArray)
})
}
const makeHTTPRequest = (url) => {
return new Promise((resolve, reject) => {
https
.get(url, res => {
let stream = []
res
.on('data', function (data) {
stream.push(data)
})
.on('end', function () {
let data = Buffer.concat(stream)
data = JSON.parse(data)
resolve(data)
})
.on('error', err => {
console.error(`Error in response: ${err}`)
})
})
}
If you want to stick to callbacks you could use async.each function of async node module.
Wrap https.get in a Promise, which resolves on the end event, and rejects on any error. Now you can await the promise, and call the function once the for loop is done
let sheetsArray = []
function sheetsAsJsonById(ids) {
for (let i = 0; i < ids.length; i++) {
const queryUrl = `sheets.html`
await new Promise((resolve, reject) => {
https
.get(queryUrl, res => {
let stream = []
res
.on('data', function(data) {
stream.push(data)
})
.on('end', function() {
let data = Buffer.concat(stream)
data = JSON.parse(data)
sheetsArray.push(data['values'])
resolve();
})
})
.on('error', err => {
reject(err);
})
})
}
formatJsonToLocale(sheetsArray)
}
I have a list of urls I wish to fetch. All of these urls returns a json object with a property valid. But only one of the fetch promises has the magic valid property to true.
I have tried various combinations of url.forEach(...) and Promises.all([urls]).then(...). Currently my setup is:
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
var result;
urls.forEach(function (url) {
result = fetch(`${url}/${key}/validate`)
.then((response) => response.json())
.then((json) => {
if (json.license.valid) {
return json;
} else {
Promise.reject(json);
}
});
});
return result;
}
The above is not working because of the async promises. How can I iterate my urls and return when the first valid == true is hit?
Let me throw a nice compact entry into the mix
It uses Promise.all, however every inner Promise will catch any errors and simply resolve to false in such a case, therefore Promise.all will never reject - any fetch that completes, but does not have license.valid will also resolve to false
The array Promise.all resolves is further processed, filtering out the false values, and returning the first (which from the questions description should be the ONLY) valid JSON response
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
return Promise.all(urls.map(url =>
fetch(`${url}/${key}/validate`)
.then(response => response.json())
.then(json => json.license && json.license.valid && json)
.catch(error => false)
))
.then(results => results.filter(result => !!result)[0] || Promise.reject('no matches found'));
}
Note that it's impossible for validate to return the result (see here for why). But it can return a promise for the result.
What you want is similar to Promise.race, but not quite the same (Promise.race would reject if one of the fetch promises rejected prior to another one resolving with valid = true). So just create a promise and resolve it when you get the first resolution with valid being true:
export function validate(key) {
return new Promise((resolve, reject) => {
let completed = 0;
const total = urls.length;
urls.forEach(url => {
fetch(`${url}/${key}/validate`)
.then((response) => {
const json = response.json();
if (json.license.valid) {
resolve(json);
} else {
if (++completed === total) {
// None of them had valid = true
reject();
}
}
})
.catch(() => {
if (++completed === total) {
// None of them had valid = true
reject();
}
});
});
});
}
Note the handling for the failing case.
Note that it's possible to factor out those two completed checks if you like:
export function validate(key) {
return new Promise((resolve, reject) => {
let completed = 0;
const total = urls.length;
urls.forEach(url => {
fetch(`${url}/${key}/validate`)
.then((response) => {
const json = response.json();
if (json.license.valid) {
resolve(json);
}
})
.catch(() => {
// Do nothing, converts to a resolution with `undefined`
})
.then(() => {
// Because of the above, effectively a "finally" (which we
// may get on Promises at some point)
if (++completed === total) {
// None of them had valid = true.
// Note that we come here even if we've already
// resolved the promise -- but that's okay(ish), a
// promise's resolution can't be changed after it's
// settled, so this would be a no-op in that case
reject();
}
});
});
});
}
This can be done using SynJS. Here is a working example:
var SynJS = require('synjs');
var fetchUrl = require('fetch').fetchUrl;
function fetch(context,url) {
console.log('fetching started:', url);
var result = {};
fetchUrl(url, function(error, meta, body){
result.done = true;
result.body = body;
result.finalUrl = meta.finalUrl;
console.log('fetching finished:', url);
SynJS.resume(context);
} );
return result;
}
function myFetches(modules, urls) {
for(var i=0; i<urls.length; i++) {
var res = modules.fetch(_synjsContext, urls[i]);
SynJS.wait(res.done);
if(res.finalUrl.indexOf('github')>=0) {
console.log('found correct one!', urls[i]);
break;
}
}
};
var modules = {
SynJS: SynJS,
fetch: fetch,
};
const urls = [
'http://www.google.com',
'http://www.yahoo.com',
'http://www.github.com', // This is the valid one
'http://www.wikipedia.com'
];
SynJS.run(myFetches,null,modules,urls,function () {
console.log('done');
});
It would produce following output:
fetching started: http://www.google.com
fetching finished: http://www.google.com
fetching started: http://www.yahoo.com
fetching finished: http://www.yahoo.com
fetching started: http://www.github.com
fetching finished: http://www.github.com
found correct one! http://www.github.com
done
If you want to avoid the fact of testing each URL, you can use the following code.
const urls = [
'https://testurl.com',
'https://anotherurl.com',
'https://athirdurl.com' // This is the valid one
];
export function validate(key) {
return new Promise((resolve, reject) => {
function testUrl(url) {
fetch(`${url}/${key}/validate`)
.then((response) => response.json())
.then((json) => {
if (json.license.valid) {
resolve(json);
return;
}
if (urlIndex === urls.length) {
reject("No matches found...");
return;
}
testUrl(urls[urlIndex++]);
});
}
let urlIndex = 0;
if (!urls.length)
return reject("No urls to test...");
testUrl(urls[urlIndex++]);
});
}