We Keep Coding

chained await in class chained methods

context: Two javascript classes in separate files, each integrating a different external service and being called in a express.js router.
See "problematic code" below:
route
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
const data = await ( await ( await awsTranscribe.Upload(req.file, bucket)).CreateJob(transcribeParams)).GetJob()
res.send(data)
})
S3 class
class AmazonS3 {
constructor() {
this.Upload = this.Upload
}
async Upload(file, bucket) {
const uploadParams = {
Bucket: bucket,
Body: fs.createReadStream(file.path),
Key: file.filename,
}
this.data = await s3.upload(uploadParams).promise()
return this
}
}
Transcribe class
class Transcribe extends AwsS3 {
constructor() {
super()
this.CreateJob = this.CreateJob
this.GetJob = this.GetJob
}
async CreateJob(params) {
if(this.data?.Location) {
params.Media = { ...params.Media, MediaFileUri: this.data.Location }
}
this.data = await transcribeService.startTranscriptionJob(params).promise()
return this
}
async GetJob(jobName) {
if(this.data?.TranscriptionJob?.TranscriptionJobName) {
jobName = this.data.TranscriptionJob.TranscriptionJobName
}
this.data = await transcribeService.getTranscriptionJob({TranscriptionJobName: jobName}).promise()
return this
}
}
problem: the problem is with the chained awaits in the router file:
await ( await ( await awsTranscribe.Upload...
Yes, it does work, but it would be horrible for another person to maintain this code in the future.
How can i make so it would be just
awsTranscribe.Upload(req.file, bucket).CreateJob(transcribeParams).GetJob() without the .then?

The problem is with the chained awaits in the router file: await ( await ( await awsTranscribe.Upload...
No, that's fine. In particular it would be trivial to refactor it to separate lines:
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
const a = await awsTranscribe.Upload(req.file, bucket);
const b = await b.CreateJob(transcribeParams);
const c = await b.GetJob();
res.send(c);
});
Your actual problem is that a, b, and c all refer to the same object awsTranscribe. Your code would also "work" if it was written
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
await awsTranscribe.Upload(req.file, bucket);
await awsTranscribe.CreateJob(transcribeParams);
await awsTranscribe.GetJob();
res.send(awsTranscribe);
});
The horrible thing is that you are passing your data between these methods through the mutable awsTranscribe.data property - even storing different kinds of data in it at different times! One could change the order of method calls and it would completely break in non-obvious and hard-to-debug ways.
Also it seems that multiple requests share the same awsTranscribe instance. This will not work with concurrent requests. Anything is possible from just "not working" to responding with the job data from a different user (request)! You absolutely need to fix that, then look at ugly syntax later.
What you really should do is get rid of the classes. There's no reason to use stateful objects here, this is plain procedural code. Write simple functions, taking parameters and returning values:
export async function uploadFile(file, bucket) {
const uploadParams = {
Bucket: bucket,
Body: fs.createReadStream(file.path),
Key: file.filename,
};
const data = s3.upload(uploadParams).promise();
return data.Location;
}
export async function createTranscriptionJob(location, params) {
params = {
...params,
Media: {
...params.Media,
MediaFileUri: location,
},
};
const data = await transcribeService.startTranscriptionJob(params).promise();
return data.TranscriptionJob;
}
async function getTranscriptionJob(job) {
const jobName = job.TranscriptionJobName;
return transcribeService.getTranscriptionJob({TranscriptionJobName: jobName}).promise();
}
Then you can import and call them as
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
const location = await uploadFile(req.file, bucket);
const job = await createTranscriptionJob(location, transcribeParams);
const data = await getTranscriptionJob(job);
res.send(c);
});

I got interested in whether it was possible to take an object with several async methods and somehow make them automatically chainable. Well, you can:
function chain(obj, methodsArray) {
if (!methodsArray || !methodsArray.length) {
throw new Error("methodsArray argument must be array of chainable method names");
}
const methods = new Set(methodsArray);
let lastPromise = Promise.resolve();
const proxy = new Proxy(obj, {
get(target, prop, receiver) {
if (prop === "_promise") {
return function() {
return lastPromise;
}
}
const val = Reflect.get(target, prop, receiver);
if (typeof val !== "function" || !methods.has(prop)) {
// no chaining if it's not a function
// or it's not listed as a chainable method
return val;
} else {
// return a stub function
return function(...args) {
// chain a function call
lastPromise = lastPromise.then(() => {
return val.apply(obj, args);
//return Reflect.apply(val, obj, ...args);
});
return proxy;
}
}
}
});
return proxy;
}
function delay(t) {
return new Promise(resolve => {
setTimeout(resolve, t);
});
}
function log(...args) {
if (!log.start) {
log.start = Date.now();
}
const delta = Date.now() - log.start;
const deltaPad = (delta + "").padStart(6, "0");
console.log(`${deltaPad}: `, ...args)
}
class Transcribe {
constructor() {
this.greeting = "Hello";
}
async createJob(params) {
log(`createJob: ${this.greeting}`);
return delay(200);
}
async getJob(jobName) {
log(`getJob: ${this.greeting}`);
return delay(100);
}
}
const t = new Transcribe();
const obj = chain(t, ["getJob", "createJob"]);
log("begin");
obj.createJob().getJob()._promise().then(() => {
log("end");
});
There's a placeholder for your Transcribe class that has two asynchronous methods that return a promise.
Then, there's a chain() function that returns a proxy to an object that makes a set of passed in method names be chainable which allows you to then do something like this:
const t = new Transcribe();
// make chainable proxy
const obj = chain(t, ["getJob", "createJob"]);
obj.createJob().getJob()
or
await obj.createJob().getJob()._promise()
I wouldn't necessarily say this is production-ready code, but it is an interesting feasibility demonstration and (for me) a chance to learn more about a Javascript proxy object.
Here's a different approach that (instead of the proxy object) adds method stubs to a promise to make things chainable:
function chain(orig, methodsArray) {
let masterP = Promise.resolve();
function addMethods(dest) {
for (const m of methodsArray) {
dest[m] = function(...args) {
// chain onto master promise to force sequencing
masterP = masterP.then(result => {
return orig[m].apply(orig, ...args);
});
// add methods to the latest promise befor returning it
addMethods(masterP);
return masterP;
}
}
}
// add method to our returned promise
addMethods(masterP);
return masterP;
}
function delay(t) {
return new Promise(resolve => {
setTimeout(resolve, t);
});
}
function log(...args) {
if (!log.start) {
log.start = Date.now();
}
const delta = Date.now() - log.start;
const deltaPad = (delta + "").padStart(6, "0");
console.log(`${deltaPad}: `, ...args)
}
class Transcribe {
constructor() {
this.greeting = "Hello";
this.cntr = 0;
}
async createJob(params) {
log(`createJob: ${this.greeting}`);
++this.cntr;
return delay(200);
}
async getJob(jobName) {
log(`getJob: ${this.greeting}`);
++this.cntr;
return delay(100);
}
}
const t = new Transcribe();
log("begin");
chain(t, ["getJob", "createJob"]).createJob().getJob().then(() => {
log(`cntr = ${t.cntr}`);
log("end");
});
Since this returns an actual promise (with additional methods attached), you can directly use .then() or await with it without the separate ._promise() that the first implementation required.
So, you can now do something like this:
const t = new Transcribe();
chain(t, ["getJob", "createJob"]).createJob().getJob().then(() => {
log(`cntr = ${t.cntr}`);
});
or:
const t = new Transcribe();
await chain(t, ["getJob", "createJob"]).createJob().getJob();
log(`cntr = ${t.cntr}`);
And, here's a third version where it creates a thenable object (a pseudo-promise) with the added methods on it (if it bothers you to add methods to an existing promise):
function chain(orig, methodsArray) {
if (!methodsArray || !methodsArray.length) {
throw new Error("methodsArray argument must be array of chainable method names");
}
let masterP = Promise.resolve();
function makeThenable() {
let obj = {};
for (const m of methodsArray) {
obj[m] = function(...args) {
// chain onto master promise to force sequencing
masterP = masterP.then(result => {
return orig[m].apply(orig, ...args);
});
return makeThenable();
}
}
obj.then = function(onFulfill, onReject) {
return masterP.then(onFulfill, onReject);
}
obj.catch = function(onReject) {
return masterP.catch(onReject);
}
obj.finally = function(onFinally) {
return masterP.finally(onFinally);
}
return obj;
}
return makeThenable();
}
function delay(t) {
return new Promise(resolve => {
setTimeout(resolve, t);
});
}
function log(...args) {
if (!log.start) {
log.start = Date.now();
}
const delta = Date.now() - log.start;
const deltaPad = (delta + "").padStart(6, "0");
console.log(`${deltaPad}: `, ...args)
}
class Transcribe {
constructor() {
this.greeting = "Hello";
this.cntr = 0;
}
async createJob(params) {
log(`createJob: ${this.greeting}`);
++this.cntr;
return delay(200);
}
async getJob(jobName) {
log(`getJob: ${this.greeting}`);
++this.cntr;
return delay(100);
}
}
const t = new Transcribe();
log("begin");
chain(t, ["getJob", "createJob"]).createJob().getJob().then(() => {
log(`cntr = ${t.cntr}`);
log("end");
});

How to push failed promises to separate array?

I am looping through ids that sends it to an async function and I want to return both success and failed data, right now I am only returning success data
const successContractSignature: LpContractSla[] = [];
for (const id of lpContractSlaIds) {
const data = await createLpSignature(context, id);
if (data) {
successContractSignature.push(data);
}
}
return successContractSignature;
If the createLpSignature throws error, do i need a try catch here? but wouldnt that exit the loop?
How can I push failed data to separate array without breaking the loop?

Unless there's a specific reason to avoid the call in parallel, it's always a good practice to avoid to start async calls in a for loop with await, since you are going to wait for each promise to resolve (or reject ) before executing the next one.
This is a better pattern which lets you also get all the results of the promises, either they resolved or rejected:
const successContractSignature: LpContractSla[] = await Promise.allSettled(lpContractSlaIds.map((id: string) => createLpSignature(context,id)))
return successContractSignature;
But if for some particular reason you need to make these calls in a sequence and not in parallel, you can wrap the single call in a try catch block ,that won't exit the loop:
for (const id of lpContractSlaIds) {
let data;
try {
data = await createLpSignature(context, id);
} catch(e) {
data = e
}
if (data) {
successContractSignature.push(data);
}
}
You can test it in this example:
const service = (id) =>
new Promise((res, rej) =>
setTimeout(
() => (id %2 === 0 ? res("ID: "+id) : rej('Error ID : '+id)),
1000
)
);
const ids = [1,2,3,4,5]
const testParallelService = async () => {
try {
const data = await Promise.allSettled(ids.map(id => service(id)))
return data.map(o => `${o.status}: ${o.reason ?? o.value}`)
} catch(e) {
console.log(e)
}
}
testParallelService().then(data => console.log("Parallel data: ", data))
const testSequentialService = async () => {
const res = [];
for (const id of ids) {
let data;
try {
data = await service(id);
} catch (e) {
data = e;
}
if (data) {
res.push(data);
}
}
return res;
};
testSequentialService().then((data) => console.log('Sequential Data: ', data));

can i make the async.retry method retry even on successfull queries but based on a condition

I'm studying the node.js module async,I want to find out if there is a way to change the async.retry method to retry even on successfull operations but stop based on some condition or response let's say its an api call.
According to its docs ,the function will continue trying the task on failures until it succeeds.if it succeeds it will only run only that time But how can i make it work the same on successfull operations and make it stop on some condition ?
const async = require('async');
const axios = require('axios');
const api = async () => {
const uri = 'https://jsonplaceholder.typicode.com/todos/1';
try {
const results = await axios.get(uri);
return results.data;
} catch (error) {
throw error;
}
};
const retryPolicy = async (apiMethod) => {
async.retry({ times: 3, interval: 200 }, apiMethod, function (err, result) {
// should retry untill the condition is met
if (result.data.userId == 5) {
// stop retring
}
});
};
retryPolicy(api);

Yes, You can just throw a custom error if condition is not met. Would be something like that:
const async = require('async');
const axios = require('axios');
const api = async () => {
const uri = 'https://jsonplaceholder.typicode.com/todos/1';
try {
const results = await axios.get(uri);
if(typeof result.data.userId != 'undefined' && result.data.userId == 5){ // change this condition to fit your needs
return results.data;
}else{
throw {name : "BadDataError", message : "I don't like the data I got"};
}
} catch (error) {
throw error;
}
};

I don't think this is possible.
On the async.retry documentation you can find this description:
Attempts to get a successful response from task no more than times
times before returning an error. If the task is successful, the
callback will be passed the result of the successful task. If all
attempts fail, the callback will be passed the error and result (if
any) of the final attempt.
However, using the delay function given here, you can do what you want another way:
const async = require('async');
const axios = require('axios');
const delay = (t, val) => {
return new Promise((resolve) => {
setTimeout(() => { resolve(val) }, t);
});
}
const api = async () => {
const uri = 'https://jsonplaceholder.typicode.com/todos/1';
try {
const results = await axios.get(uri);
return results.data;
} catch (error) {
throw error;
}
};
const retryPolicy = async (apiMethod) => {
const times = 3
const interval = 200
let data
for (count = 0; count < 3; count++) {
try {
data = await apiMethod()
catch(e) {
console.log(e)
await delay(interval)
continue
}
if (data.userId === 5) {
break;
}
await delay(interval)
}
// do something
};
retryPolicy(api);

How do I make a long list of http calls in serial?

I'm trying to only make one http call at time but when I log the response from getUrl they are piling up and I start to get 409s (Too many requests)
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
fetch(fetchUrl).then(async res => {
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
cb(url);
});
}
let requests = urls.map((url, i) => {
return new Promise(resolve => {
getUrl(url, i, resolve);
});
});
const all = await requests.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult => [...chainResults, currentResult]),
);
}, Promise.resolve([]));
Basically I don't want the next http to start until the previous one has finished. Otherwise I hammer their server.
BONUS POINTS: Make this work with 5 at a time in parallel.

Since you're using await, it would be a lot easier to use that everywhere instead of using confusing .thens with reduce. It'd also be good to avoid the explicit Promise construction antipattern. This should do what you want:
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
Then your results variable will contain an array of response texts (or an error will have been thrown, and the code won't reach the bottom).
The syntax for an async function is an async keyword before the argument list, just like you're doing in your original code:
const fn = async () => {
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
// do something with results
};
To have a limited number of requests at a time, make a queue system - when a request completes, recursively call a function that makes another request, something like:
const results = [];
const queueNext = async () => {
if (!urls.length) return;
const url = urls.shift();
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
await queueNext();
}
await Promise.all(Array.from({ length: 5 }, queueNext));
// do something with results

You cannot use Array methods to sequentually run async operations because array methods are all synchronous.
The easiest way to achieve sequential async tasks is through a loop. Otherwise, you will need to write a custom function to imitate a loop and run .then after a async task ends, which is quite troublesome and unnecessary.
Also, fetch is already returning a Promise, so you don't have to create a Promise yourself to contain that promise returned by fetch.
The code below is a working example, with small changes to your original code (see comments).
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
// To imitate actual fetching
const fetch = (url) => new Promise(resolve => {
setTimeout(() => {
resolve({
ok: true,
text: () => new Promise(res => setTimeout(() => res(url), 500))
});
}, 1000);
});
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
return fetch(fetchUrl).then(async res => { // <-- changes here
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
return url; // <--- changes here
});
}
async function getAllUrls(urls){
const result = [];
for (const url of urls){
const response = await getUrl(url);
result.push(response);
}
return result;
}
getAllUrls(urls)
.then(console.log);

async/await is perfect for this.
Assuming you have an array of URLs as strings:
let urls = ["https://example.org/", "https://google.com/", "https://stackoverflow.com/"];
You simply need to do:
for (let u of urls) {
await fetch(u).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
The loop will not iterate until the current fetch() has resolved, which will serialise things.
The reason array.map doesn't work is as follows:
async function doFetch(url) {
return await fetch(url).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
let mapped = urls.map(doFetch);
is equivalent to:
let mapped;
for (u of urls) {
mapped.push(doFetch(u));
}
This will populate mapped with a bunch of Promises immediately, which is not what you want. The following is what you want:
let mapped;
for (u of urls) {
mapped.push(await doFetch(u));
}
But this is not what array.map() does. Therefore using an explicit for loop is necessary.

Many people provided answers using for loop. But in some situation await in for loop is not welcome, for example, if you are using Airbnb style guide.
Here is a solution using recursion.
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
async function serialFetch(urls) {
return await doSerialRecursion(
async (url) => {
return result = await fetch(url)
.then((response) => {
// handle response
})
.catch((err) => {
// handle error
});
},
urls,
0
);
}
async function doSerialRecursion(fn, array, startIndex) {
if (!array[startIndex]) return [];
const currResult = await fn(array[startIndex]);
return [currResult, ...(await doSerialRecursion(array, fn, startIndex + 1))];
}
const yourResult = await serialFetch(urls);
The doSerialRecursion function will serially execute the function you passed in, which is fetch(url) in this example.

How to make async/wait inside of for loop?

I'm using async await inside of for loop as below.
for (let i = 0; i < result.length; i += 1) {
try{
const client = await axios.get(
`${process.env.user}/client/${result[i].id}`
);
} catch(error){
console.log(error)
}
if (client.data.success === true) {
result[i].Name = rider.data.client.Name;
result[i].PhoneNumber = rider.data.client.Number;
}
}
But I want to make this using 'new Promise' and 'promiss.all' to make it asynclously.
But I don'k know how to make this correctly doing error handle well.
Could you recommend some advice for this? Thank you for reading it.

This can be a basic solution, i think
let playList = []
for (let i = 0; i < result.length; i += 1) {
playList.push(axios.get(
`${process.env.user}/client/${result[i].id}`
).then(res => {
if (res.data.success === true) {
result[i].Name = rider.data.client.Name;
result[i].PhoneNumber = rider.data.client.Number;
}
}).catch(ex => console.log(ex)));
}
await Promise.all(playList)
This can also be done by using a foreach loop.
The for/foreach loop can be simplified by using a map function.
Js map function is equivalent of c# select linq function.
The fat arrow in js map function is not bound to return a value unlike c# select inner function which must return a value.
await Promise.all(result.map(async r => {
let client;
try {
client = await axios.get(`${process.env.user}/client/${r.id}`);
} catch (error) {
console.log(error)
}
if (client.data.success === true) {
r.Name = rider.data.client.Name;
r.PhoneNumber = rider.data.client.Number;
}
}));

Try this
var promises = result.map(r => axios.get(`${process.env.user}/client/${r.id}`);
Promise.all(promises).then(function(values) {
console.log('All promises done');
});
The idea is that if you are awaiting something, that is promise, you can await it, or call it to get promise
Example:
function Foo()
{
return new Promise(...); // Promise of int for example
}
you can do
var p = Foo(); //you will get promise
Or
var v = await Foo(); // you will get int value when promise resolved

This is how you do it with async/await + Promise.all:
const myResult = await Promise.all(result.map(({ id }) => {
return axios.get(`${process.env.user}/client/${id}`);
}));
// deal with the result of those requests
const parsed = myResult.map(data => /* your code here */);

Here is an example using Array.map to call your function along with Promise.all. I wrapped the axios request in a function so if one of your request fails, it wont stop every other requests. If you don't mind stopping when you got an issue, look at others answers to your question.
function fakeRequest() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve({
data: {
success: true,
client: {
Name: 'Todd',
Number: 5,
},
},
});
}, 300);
});
}
(async() => {
const result = [{}, {}, {}];
await Promise.all(result.map(async(x, xi) => {
try {
const client = await fakeRequest();
if (client.data.success === true) {
result[xi].Name = client.data.client.Name;
result[xi].PhoneNumber = client.data.client.Number;
}
} catch (err) {
console.log(err)
}
}));
console.log(result);
})();