I need to recursively go through JSON and in some cases call remote API. I need to return the whole JSON modified at the end but I cannot figure out how to wait until all promises are fulfilled
const getObjectsOfRelated = (xmlAsJson, token) => {
if (testIfIwantCallApi()) {
const jsonToReturn = JSON.parse(JSON.stringify(xmlAsJson))
jsonToReturn.elements = callApi(xmlAsJson.text).then(result => {
return result.data
})
return jsonToReturn
}
if (xmlAsJson.elements) {
const jsonToReturn = JSON.parse(JSON.stringify(xmlAsJson))
jsonToReturn.elements = xmlAsJson.elements.map(res => getObjectsOfRelated(res, token))
return jsonToReturn
}
return xmlAsJson
}
Even if I try to hack it using setTimeout the result does not include parts that were created using external API.
This way the code returns correct structure with promises instead of values I want it either return completed promises or be able to wait until the promises are fulfilled.
Wrap plain return values in Promise's using Promise.resolve:
const getObjectsOfRelated = (xmlAsJson, token) => {
if (testIfIwantCallApi()) {
const jsonToReturn = JSON.parse(JSON.stringify(xmlAsJson))
return callApi(xmlAsJson.text).then(result => {
jsonToReturn.elements = result.data;
return jsonToReturn;
})
}
if (xmlAsJson.elements) {
const jsonToReturn = JSON.parse(JSON.stringify(xmlAsJson))
Promise.all(xmlAsJson.elements.map(res => getObjectsOfRelated(res,
token)).then((results) => {
jsonToReturn.elements = results.map(result => result.data);
return jsonToReturn;
});
}
return Promise.resolve(xmlAsJson);
}
This way you will consistently return promises and you can use your function like this:
getObjectsOfRelated(xmlAsJson, token).then(result => console.log(result))
You can use "Promise.all"...
For a simple array, you map a function over the array:
The function returns a promise for the "new value" of each element.
If you use Bluebird promises, you can even return a mixture of Promises and plain values.
( without having to wrap plain values in "Promise.resolve" )
Then you pass the array of promises to "Promise.all()", which waits for all of them to complete.
To transform a tree-shaped data structure (like JSON), you do the same sort of thing, but recursively. Each node in the tree would use "Promise.all" to wait for all its child-nodes=, and the root node would only "resolve" when every node in the tree has resolved.
Note that "Promise.all" is going to run all of your ASYNC functions at the same time. If you don't want that, you can instead use "Promise.mapSeries", which does the same thing, but it waits for each async function before starting the next. This can be better if you have large data and don't want to start too many simultaneous async functions at the same time.
Related
I'm currently fetching data from an API and I need to do multiple GET requests (using axios). After all those GET requests are completed, I return a resolved promise.
However, I need to do these GET requests automatically based on an array list:
function do_api_get_requests() {
return promise = new Promise(function(resolve, reject) {
API_IDs = [0, 1, 2];
axios.get('https://my.api.com/' + API_IDs[0])
.then(data => {
// Do something with data
axios.get('https://my.api.com/' + API_IDs[1])
.then(data => {
// Do something with data
axios.get('https://my.api.com/' + API_IDs[2])
.then(data => {
// Do something with data
// Finished, resolve
resolve("success");
}
}
}
}
}
This works but the problem is API_IDs isn't always going to be the same array, it will change. So I'm not sure how to chain these requests automatically.
Since you said it may be a variable length array and you show sequencing the requests, you can just loop through the array using async/await:
async function do_api_get_requests(API_IDS) {
for (let id of API_IDS) {
const data = await axios.get(`https://my.api.com/${id}`);
// do something with data here
}
return "success";
}
And, since you said the list of API ids would be variable, I made it a parameter that you can pass into the function.
If you wanted to run all the API requests in parallel (which might be OK for a small array, but might be trouble for a large array) and you don't need to run them in a specific order, you can do this:
function do_api_get_requests(API_IDS) {
return Promise.all(API_IDS.map(async (id) => {
const data = await axios.get(`https://my.api.com/${id}`);
// do something with data here for this request
})).then(() => {
// make resolved value be "success"
return "success";
});
}
Depending upon your circumstances, you could also use Promise.allSettled(). Since you don't show getting results back, it's not clear whether that would be useful or not.
You can use Promise.all() method to do all API requests at the same time, and resolve when all of them resolves.
function do_api_get_requests() {
const API_IDs = [0, 1, 2];
let promises = [];
for (const id of API_IDS) {
promises.push(axios.get(`https://my.api.com/${id}`));
}
return Promise.all(promises);
}
If you use Bluebird.js (a better promise library, and faster than the in-built Promise), you can use Promise.each(), Promise.mapSeries(), or Promisme.reduce() to do what you want.
http://bluebirdjs.com
I am looking at https://www.promisejs.org/patterns/ and it mentions it can be used if you need a value in the form of a promise like:
var value = 10;
var promiseForValue = Promise.resolve(value);
What would be the use of a value in promise form though since it would run synchronously anyway?
If I had:
var value = 10;
var promiseForValue = Promise.resolve(value);
promiseForValue.then(resp => {
myFunction(resp)
})
wouldn't just using value without it being a Promise achieve the same thing:
var value = 10;
myFunction(10);
Say if you write a function that sometimes fetches something from a server, but other times immediately returns, you will probably want that function to always return a promise:
function myThingy() {
if (someCondition) {
return fetch('https://foo');
} else {
return Promise.resolve(true);
}
}
It's also useful if you receive some value that may or may not be a promise. You can wrap it in other promise, and now you are sure it's a promise:
const myValue = someStrangeFunction();
// Guarantee that myValue is a promise
Promise.resolve(myValue).then( ... );
In your examples, yes, there's no point in calling Promise.resolve(value). The use case is when you do want to wrap your already existing value in a Promise, for example to maintain the same API from a function. Let's say I have a function that conditionally does something that would return a promise — the caller of that function shouldn't be the one figuring out what the function returned, the function itself should just make that uniform. For example:
const conditionallyDoAsyncWork = (something) => {
if (something == somethingElse) {
return Promise.resolve(false)
}
return fetch(`/foo/${something}`)
.then((res) => res.json())
}
Then users of this function don't need to check if what they got back was a Promise or not:
const doSomethingWithData = () => {
conditionallyDoAsyncWork(someValue)
.then((result) => result && processData(result))
}
As a side node, using async/await syntax both hides that and makes it a bit easier to read, because any value you return from an async function is automatically wrapped in a Promise:
const conditionallyDoAsyncWork = async (something) => {
if (something == somethingElse) {
return false
}
const res = await fetch(`/foo/${something}`)
return res.json()
}
const doSomethingWithData = async () => {
const result = await conditionallyDoAsyncWork(someValue)
if (result) processData(result)
}
Another use case: dead simple async queue using Promise.resolve() as starting point.
let current = Promise.resolve();
function enqueue(fn) {
current = current.then(fn);
}
enqueue(async () => { console.log("async task") });
Edit, in response to OP's question.
Explanation
Let me break it down for you step by step.
enqueue(task) add the task function as a callback to promise.then, and replace the original current promise reference with the newly returned thenPromise.
current = Promise.resolve()
thenPromise = current.then(task)
current = thenPromise
As per promise spec, if task function in turn returns yet another promise, let's call it task() -> taskPromise, well then the thenPromise will only resolve when taskPromise resolves. thenPromise is practically equivalent to taskPromise, it's just a wrapper. Let's rewrite above code into:
current = Promise.resolve()
taskPromise = current.then(task)
current = taskPromise
So if you go like:
enqueue(task_1)
enqueue(task_2)
enqueue(task_3)
it expands into
current = Promise.resolve()
task_1_promise = current.then(task_1)
task_2_promise = task_1_promise.then(task_2)
task_3_promise = task_2_promise.then(task_3)
current = task_3_promise
effectively forms a linked-list-like struct of promises that'll execute task callbacks in sequential order.
Usage
Let's study a concrete scenario. Imaging you need to handle websocket messages in sequential order.
Let's say you need to do some heavy computation upon receiving messages, so you decide to send it off to a worker thread pool. Then you write the processed result to another message queue (MQ).
But here's the requirement, that MQ is expecting the writing order of messages to match with the order they come in from the websocket stream. What do you do?
Suppose you cannot pause the websocket stream, you can only handle them locally ASAP.
Take One:
websocket.on('message', (msg) => {
sendToWorkerThreadPool(msg).then(result => {
writeToMessageQueue(result)
})
})
This may violate the requirement, cus sendToWorkerThreadPool may not return the result in the original order since it's a pool, some threads may return faster if the workload is light.
Take Two:
websocket.on('message', (msg) => {
const task = () => sendToWorkerThreadPool(msg).then(result => {
writeToMessageQueue(result)
})
enqueue(task)
})
This time we enqueue (defer) the whole process, thus we can ensure the task execution order stays sequential. But there's a drawback, we lost the benefit of using a thread pool, cus each sendToWorkerThreadPool will only fire after last one complete. This model is equivalent to using a single worker thread.
Take Three:
websocket.on('message', (msg) => {
const promise = sendToWorkerThreadPool(msg)
const task = () => promise.then(result => {
writeToMessageQueue(result)
})
enqueue(task)
})
Improvement over take two is, we call sendToWorkerThreadPool ASAP, without deferring, but we still enqueue/defer the writeToMessageQueue part. This way we can make full use of thread pool for computation, but still ensure the sequential writing order to MQ.
I rest my case.
I'm writing a then() statement for extracting the json data from an array of responses from fetch(). In the code below queries is an array of promises returned by a series of calls to fetch(). I'm using async/await for the response because otherwise the promises would be returned without resolving (I found a solution in this question).
My first attempt worked properly, when I push into jsonified I obtain an array with the promises as elements:
return Promise.all(queries)
.then(async(responses)=> {
let jsonified = [];
for (let res of responses){
jsonified.push(await(res.json()));
}
return jsonified;
}.then(data=> ...
But when I went for refactoring and tried to use Array.reduce() I realised that when I push into the accumulator instead of obtaining an array with a promise as element, acc is assigned to be a promise instead.
.then(responses=> {
return responses.reduce(async(acc, next) => {
acc.push(await(next.json()));
return acc;
}, [])
})
I can use the first version without any issue and the program works properly, but whats happening inside Array.reduce()? Why pushing a promise into the accumulator returns a promise intead of an array? How could I refactor the code with Array.reduce()?
Although it's not what you've asked, you could avoid the pain of having to use reduce, and just utilise the Promise.all() that you are already using:
return Promise.all(queries.map(q => q.then(res => res.json()))
.then(data => {...})
It's a much shorter way and less of a headache to read when you come back to it.
Have the accumulator's initial value be a Promise that resolves to an empty array, then await the accumulator on each iteration (so that all prior iterations resolve before the current iteration runs)
.then(responses=> {
return responses.reduce(async (accPromiseFromLastIter, next) => {
const arr = await accPromiseFromLastIter;
arr.push(await next.json());
return arr;
}, Promise.resolve([]))
})
(That said, your original code is a lot clearer, I'd prefer it over the .reduce version)
Live demo:
const makeProm = num => Promise.resolve(num * 2);
const result = [1, 2, 3].reduce(async(accPromiseFromLastIter, next) => {
const arr = await accPromiseFromLastIter;
arr.push(await makeProm(next));
return arr;
}, Promise.resolve([]));
result.then(console.log);
Unless you have to retrieve all data in serial, consider using Promise.all to call the .json() of each Promise in parallel instead, so that the result is produced more quickly:
return Promise.all(queries)
.then(responses => Promise.all(responses.map(response => response.json())));
If the queries are an array of Responses that were just generated from fetch, it would be even better to chain the .json() call onto the original fetch call instead, eg:
const urls = [ ... ];
const results = await Promise.all(
urls.map(url => fetch(url).then(res => res.json()))
);
This way, you can consume the responses immediately when they come back, rather than having to wait for all responses to come back before starting to process the first one.
sup guys, i'm working on this firebase project and i need to iterate trought a subcollection of all sales of all stores in the root collection and sum their values... the problem that i'm getting is that i'm getting the sum printed before the iteration. I'm new to TS and Firebase... this is what i got so far:
export const newBilling = functions.firestore.document('billings/{billId}').onCreate(event =>
{
const valueArray = []
const feeArray = []
const storesCollection = afs.collection('stores').where('active', '==', true).get().then(stores => {
stores.forEach(store => {
const salesCollection = afs.collection('stores').doc(store.id).collection('sales').get().then(sales => {
sales.forEach(sale => {
return valueArray.push(sale.data().value) + feeArray.push(sale.data().fee)
// other aproach
// valueArray.push(sale.data().value)
// feeArray.push(sale.data().fee)
})
})
})
}).catch(error => {console.log(error)})
let cashbackSum, feeSum : number
cashbackArray.forEach(value => {
cashbackSum += value
})
feeArray.forEach(value => {
feeSum += value
})
console.log(cashbackSum, feeSum)
return 0
})
TKS =)
You're not using promises correctly. You've got a lot of get() method call, each of which are asynchronous and return a promise, but you're never using them to case the entire function to wait for all the work to complete. Calling then() doesn't actually make your code wait - it just runs the next bit of code and returns another promise. Your final console.log is executing first because none of the work you kicked off ahead of it is complete yet.
Your code actually needs to be substantially different in order to work correctly, and you need to return a promise from the entire function that resolves only after all the work is complete.
You can learn better how to use promises in Cloud Functions by going through the video tutorials.
New to concept of callbacks and promises. I'm trying to read contents of a directory which is correctly returning the addresses but this code is only printing console.log(value) and the console.log in the function getAccount "gettin info..." but nowhere printing the balance the api is getting closed and the process completes, I dont understand this because the address is being passed and the first console.log is printing inside the function but not doing further job. when i remove the fs.readdir and files.foreach and pass a single value to getAccount its working perfectly fine. No errors or bugs , its a runtime error and im guessing related to callbacks
'use strict';
const RippleAPI = require('ripple-lib').RippleAPI;
const testFolder = '/home/ripple/.keystore';
const fs = require('fs');
const api = new RippleAPI({server: 'wss://s1.ripple.com'});
function getAccount(address) {
var balance = 0.0;
console.log("getting info for :"+address);
return api.getAccountInfo(address).then(info => {
var balance = info.xrpBalance;
console.log("balance of "+address+" is "+balance);
return balance;
});
}
api.connect().then(() => {
fs.readdir(testFolder, function(err, files) {
// console.log("Total no of wallets : "+files.length);
// for (var i =3000, len=3200; i < len; i++) {
// var ad = files[i];
// console.log(ad + "\t" + typeof(ad));
// if(!xwallets.includes(ad))
files.forEach(function(value) {
getAccount(value).then(balance => {console.log(balance); });
console.log(value);
});
// console.log("running for index : "+i+" filedata :"+files[i]);
// }
});
// console.log(balance);
}).then(() => {
return api.disconnect();
}).then(() => {
console.log("done and disconnected");
}).catch(console.error);
You really don't want to mix asynchronous operations that return a promise with those that take a direct callback. It's hard to write good robust code when mixing like that. So, since you are already using with promises in a number of operations, it's best to take the remaining async operations and "promisify" them so you can use them with promises. Then, you can take advantage of all the clean error propagation and chaining of promises.
Then, secondly, you have to make sure all your asynchronous operations in a loop are properly chained to the parent promise. To do that here, we will accumulate the promises from getAccount() in the loop into an array of promises. Then, after the loop, we can tell when they're all done using Promise.all() and then return that promise from readdir() so that they will all be properly chained to the parent promise.
This will then make sure everything inside the loop finishes before the parent resolves and before you call api.disconnect().
Here's what that would look like:
'use strict';
const RippleAPI = require('ripple-lib').RippleAPI;
const testFolder = '/home/ripple/.keystore';
const fs = require('fs');
const api = new RippleAPI({server: 'wss://s1.ripple.com'});
function getAccount(address) {
var balance = 0.0;
console.log("getting info for :"+address);
return api.getAccountInfo(address).then(info => {
var balance = info.xrpBalance;
console.log("balance of "+address+" is "+balance);
return balance;
});
}
// promisify readdir
const readdir = util.promisify(fs.readdir);
api.connect().then(() => {
return readdir(testFolder).then(function(files) {
const promises = [];
files.forEach(function(file) {
console.log(file);
promises.push(getAccount(file).then(balance => {
console.log(balance);
// make balance be the resolved value for this promise
return balance;
}));
});
// make sure we are waiting for all these promises to be done
// by chaining them into the parent promise
return Promise.all(promises);
});
}).then(balances => {
// process balances array here
console.log(balances);
return api.disconnect();
}, err => {
// disconnect, even in the error case, but preserve the error
console.error(err);
return api.disconnect().then(() => {throw err;})
}).then(() => {
console.log("done and disconnected");
});
Summary of changes made:
Promisify fs.readdir() so we can use promise with it
Change how we call readdir() to use promises
Collect getAccount() promises inside of .forEach() into an array of promises
Add return Promise.all(promises) so that we wait for all those promises to be done and so that it is chained into the parent promise
Make balance be the resolved value of each promise in the .forEach() loop, even after logging its value
Make sure we're calling api.disconnect() even in the error cases
After logging error, preserve the error value as the reject reason