We Keep Coding

What are different ways to handle asynchronous calls (apis) inside a for loop or array method?

I have been trying to create an api like this where I tried different things like using array methods like map/filter/reduce where either I get pending promises or result returned before execution of api call.
So my doubt is -->
How do I get total number of drawn matches of all pages ( so I need to add data.total from all pages).
How to better understand this behaviour.
async function getNumDraws(year) {
const goals = [...Array(11).keys()];
let result = 0;
console.log(`before loop ${new Date()}`);
for(let goal of goals){
console.log(`in loop before await ${new Date()}`);
await require('https').get(`https://jsonmock.hackerrank.com/api/football_matches?year=${year}&team1goals=${goal}&team2goals=${goal}`,res=>{
let data="";
res.on('data', (chunk) => {
data += chunk;
});
// The whole res has been received. Print out the result.
res.on('end', () => {
data=JSON.parse(data);
console.log(result,data.total)
result= result + data.total;
});
})
console.log(`in loop after await ${new Date()}`);
}
console.log(`after loop ${new Date()}`);
return result;
}
console.log(getNumDraws(2011));

https.get is a callbacked function so await won't work. You should promisify it first like they did in this other SO question;
const https = require("https"); // only require this once
const getJSONAsync = url => new Promise((resolve, reject) => { // define a function getJSONAsync which returns a Promise that wraps the https.get call, getJSONAsync is awaitable
let req = https.get(url, res => { // make the https.get request
if(res.statusCode < 200 || res.statusCode >= 300) { // if the status code is an error one
return reject(new Error('statusCode=' + res.statusCode)); // reject and skip the rest
}
let body = []; // otherwise accumulate..
res.on('data', chunk => body.push(chunk)); // ..the data
res.on('end', () => { // on end
try {
body = JSON.parse(Buffer.concat(body).toString()); // try to JSON.parse the data
} catch(e) {
reject(e); // reject if an error occurs
}
resolve(body); // resolve the parsed json object otherwise
});
});
req.on("error", error => reject(error)); // reject if the request fails too (if something went wrong before the request is sent for example)
});
async function getNumDraws(year) {
let result = 0;
for(let goal = 0; goal < 11; goal++) {
let data = await getJSONAsync(`https://jsonmock.hackerrank.com/api/football_matches?year=${year}&team1goals=${goal}&team2goals=${goal}`);
result += data.total;
}
return result;
}
Note: getJSONAsync is not specific to getNumDraws, you can use it somewhere else if you need it, and since it returns a Promise you can either await it like getNumDraws does or use it with then/catch blocks like so:
getJSONAsync("url")
.then(data => {
// data is the parsed json returned by the request
})
.catch(error => {
// the error message if something fails
})

timer wrapper function for recursive async/await call

I have a recursive async function getResponse(url,attempts = 0), which polls external api for the response and resolves or exits after reaching X number of retries or on a server error.
However, it's internal "clock" is based off the number of retries (after allowing for delays to avoid rate limits), but I also want to have a flexibility in setting a time based timer, which would resolve the function and end the recursion. Ideally, I want to be able to wrap time based timer around my recursive async function, like so timed(getResponse(url),3400)
I have only managed to have both the time based and "retries" based timer working together, by packaging both timers in one async function with local variable expired serving as an exit flag and setting Promise.race conditions on both functions.
async function timedgetResponse (expiry = 3500,url) {
let expired = false;
async function timeout(expiry){
await new Promise(_=> setTimeout(_,expiry));
expired = true;
return false;
};
async function getResponse(url,attempts = 0){
try {
if(expired){ return false; };
const limit = 10;
if(attempts >= limit){ok: false, e:"MAX_ATTEMPTS"};
const rawRes = await fetch(url,
{
method: 'GET',
credentials: 'include',
headers: {
'Accept': 'application/json'
}
});
if (!rawRes.ok) { throw (Error('SERVER_ERROR')); };
const res = await rawRes.json();
if(!res || res.status === 0){ throw (Error(res.request)); };
return {ok: true, res: res.request};
} catch(e){
const err = e.message;
if(err === "RESPONSE_NOT_READY"){
await new Promise(_ => setTimeout(_, 333));
attempts +=1;
return getResponse(url,attempts);
} else
if(err === "SERVER_ERROR_ON_RESOLVER"){
await new Promise(_ => setTimeout(_, 10000));
attempts +=1;
return getResponse(url,attempts);
} else {
return {ok: false, e:"MISC_ERROR"};
};
};
};
const awaited = await Promise.race([
getResponse(url),
timeout(expiry)
]);
return awaited;
};
I sense that it is not a correct way to do it and would appreciate any help towards timed(getResponse(url),3400) solution.

I have a function that might meet your need. I have updated it based on how I have interpreted your needs. The idea is you will poll until something is true ie resolves or you exceed the max attempts. It has a built-in configurable delay.
The idea here is you'd pass in a function that wraps your fetch call which would eventually resolve/reject.
setPolling(pollFunc, freq = 1000, maxAttempts = 3)
pollFunc = function that takes no args and returns a promise that eventually resolves or rejects.
freq = how frequently to run pollFunc in milliseconds
maxAttempts = max attempts before giving up
const setPolling = async (pollFunc, freq = 1000, maxAttempts = 3, _attempts = 1) => {
const wait = (delay) => new Promise(resolve=>setTimeout(resolve, delay))
try {
return await pollFunc()
} catch (e) {
if (_attempts < maxAttempts) {
await wait(freq)
return await setPolling(pollFunc, freq, maxAttempts, ++_attempts)
}
throw (e instanceof Error) ? e : new Error((typeof e !== 'undefined') ? e : 'setPolling maxAttempts exceeded!')
}
}
async function alwaysFail() {
throw new Error(`alwaysFail, failed because that's what it does!`)
}
function passAfter(x) {
let i = 0
return async ()=> {
if (i > x) return `passAfter succeeded because i(${i}) > x(${x})`
throw new Error(`i(${i++}) < x(${x})`)
}
}
setPolling(alwaysFail)
.catch((e)=>console.error(`alwaysFail, failed!\n${e.message}\n${e.stack}`))
setPolling(passAfter(5), 500, 10)
.then((res)=>console.log(`passAfter, succeeded!\n${res}`))
.catch((e)=>console.error(`passAfter, failed!\n${e.message}\n${e.stack}`))

On the basis that you want to stop retrying when a timer expires, then you can employ a token to convey a stop signal to the recursive process.
Something like this should do it:
const poll = async (work, options, token) => {
const settings = Object.assign({ 'delay':0, 'attempts':1, 'maxAttempts':3 }, options);
// Utility function which returns a Promise-wrapped setTimeout
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
// Two mechanisms for stopping the recursion.
// Performing the tests here ensures they are applied before the first work() call.
// 1. token-borne stop signal
if(token.stop) {
throw new Error('poll(): stopped');
}
// 2. max attempts reached
if (settings.attempts >= settings.maxAttempts) {
throw new Error('poll(): max attempts reached');
}
// Do the work, and recurse on error
try {
return await work();
}
catch (e) {
await delay(settings.delay);
settings.attempts += 1; // Mutate/pass `settings`; the original `options` is not guaranteed to have an `attempts` property.
return await poll(work, settings, token);
}
}
Call as follows:
// token
const token = {}; // or {'stop':false} if you like
// Time based timer:
setTimeout(() => {
token.stop = true; // raise the 'stop' flag
}, 60000); // or whatever
let pollPromise = poll(doSomethingAsync, {'delay':1000, 'maxAttempts':100}, token)
.then((res) => console.log(res))
.catch((e) => console.error(e));
Note that on setting the stop signal:
a successful reponse from in-flight work will still come through.
further recursion will be prevented but no attempt is made to abort the in-flight work.
With a little more thought these behaviours could be changed depending on exactly what is required.

nodejs/javascript limit how many async calls [duplicate]

I am building an application that will be making about a million calls to a remote api server. Will I be able to limit amount of connections to for example 10? Do I set max sockets to 10 will do it?
I am trying to understand what do these parameters do:
keepAlive: false,
maxSockets: 999,
maxFreeSockets: 1
In node http get function, in the following code:
var inputData = [];
for(i=1; i<=5000;i++){
inputData.push('number' + i);
}
var options = {
host: "localhost",
port: 80,
path: "/text.txt",
keepAlive: false,
maxSockets: 999,
maxFreeSockets: 1
}
var limit = inputData.length;
var counter = 0;
function fetchData(number){
return new Promise(function(resolve, reject){
var http = require('http');
fetch = function(resp){
var body = '';
resp.on('data',function(chunk){
body += chunk;
})
resp.on('end',function(){
console.log(resp)
resolve()
})
resp.on('error',function(err){
console.log('error');
})
}
var req = http.request(options, fetch);
req.end();
})
}
Promise.all(inputData.map(number => fetchData(number))).then(function(results) {
console.log('finished');
connection.end();
})
.catch(function(error) {
console.log('there wa an error');
console.log(error);
});

You really don't want to fire off 1,000,000 requests and somehow hope that maxSockets manages it to 100 at a time. There are a whole bunch of reasons why that is not a great way to do things. Instead, you should use your own code that manages the number of live connections to 100 at a time.
There are a number of ways to do that:
Write your own code that fires up 100 and then each time one finishes, it fires up the next one.
Use Bluebird's Promise.map() which has a built-in concurrency feature that will manage how many are inflight at the same time.
Use Async's async.mapLimit() which has a built-in concurrency feature that will manage how many are inflight at the same time.
As for writing code yourself to do this, you could do something like this;
function fetchAll() {
var start = 1;
var end = 1000000;
var concurrentMax = 100;
var concurrentCnt = 0;
var cntr = start;
return new Promise(function(resolve, reject) {
// start up requests until the max concurrent requests are going
function run() {
while (cntr < end && concurrentCnt < concurrentMax) {
++concurrentCnt;
fetchData(cntr++).then(function() {
--concurrentCnt;
run();
}, function(err) {
--concurrentCnt;
// decide what to do with error here
// to continue processing more requests, call run() here
// to stop processing more requests, call reject(err) here
});
}
if (cntr >= end && concurrentCnt === 0) {
// all requests are done here
resolve();
}
}
run();
});
}

I decided to use the async library.
Here is my complete solution to this:
var async = require('async')
var http = require('http');
var inputData = [];
for(i=1; i<=2000;i++){
inputData.push('number' + i);
}
var options = {
host: "o2.pl",
path: "/static/desktop.css?v=0.0.417",
port: 80
}
function fetchData(number, callback){
return new Promise(function(resolve, reject){
fetch = function(resp){
var body = '';
resp.on('data',function(chunk){
body += chunk;
})
process.stdout.write('.')
callback()
resp.on('error',function(err){
console.log('error');
console.log(err);
})
}
var req = http.request(options, fetch);
req.end();
})
}
function foo(item, callback){
return callback(false, 'foo');
}
async.mapLimit(inputData,100,fetchData,function(err, result){
console.log('finished');
})
Thank you for your help.

Node.JS/Express - Why does A-Sync seem to go in a random order?

I have recently started using Node.js/Express. I understand an A-Sync call must complete before it can move on. In my code there are three different end points I need to hit.
As I have taken A-Sync into account I have tried to code it so it does them in order they are presented.
However it hits the first end point, then the third and then the second. I understand it must be an issue in my code however I've been at this for hours.
Where have I gone wrong? Why does it leave the second endpoint till last?
app.get("/start/:origin/:destination", function ( reqt, resp ) {
var origin = reqt.params.origin
var destination = reqt.params.destination
var url = 'http://localhost:5000/maps/' + origin + '/' + destination
var rate;
var overallDis;
var aRoadDis;
var data;
http.get(url, res => {
res.setEncoding('utf8')
res.on('data', function(body){
data = JSON.parse(body)
overallDis = data["distance"]
aRoadDis = data["ARoads"]
})
})
var driver;
http.get("http://localhost:4000/lowestRate/", res => {
res.setEncoding('utf8')
res.on('data', function(body){
driver = JSON.parse(body)
rate = driver.rate
console.log(rate)
})
})
var totalPrice = 0
http.get("http://localhost:6000/surge/:" + rate + "/:" + overallDis + "/:" + aRoadDis, res => {
// console.log(overallDis)
// console.log(aRoadDis)
// console.log(rate)
res.setEncoding('utf8')
res.on('data', function(body){
console.log(body)
totalPrice += parseInt(body)
})
console.log(totalPrice)
})
})

I understand an A-Sync call must complete before it can move on.
This is actually not the case. When you make your HTTP request it will make that request and move on. In your case it will continue on to make the next two HTTP requests.
After it gets a response it will fire the corresponding callback function. Your callbacks will fire in the order that you get responses to your HTTP requests.
Here's a nice link for learning about the Javascript event loop.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/EventLoop
Hope that helps!
PS: If you'd like to wait for one request to finish before moving on to the rest I would suggest Promises.
app.get("/start/:origin/:destination", function ( reqt, resp ) {
const origin = reqt.params.origin
const destination = reqt.params.destination
const url = 'http://localhost:5000/maps/' + origin + '/' + destination
const totalPrice = 0
const firstPromise = new Promise((resolve, reject) => {
http.get(url, res => {
res.setEncoding('utf8')
res.on('data', function(body){
data = JSON.parse(body)
resolve({
overallDis: data["distance"],
aRoadDis: data["ARoads"]
});
})
})
});
const secondPromise = new Promise((resolve, reject) => {
http.get("http://localhost:4000/lowestRate/", res => {
res.setEncoding('utf8')
res.on('data', function(body){
const driver = JSON.parse(body)
const rate = driver.rate
console.log(rate)
resolve(rate);
})
})
});
Promise.all([firstPromise, secondPromise]).then((values) => {
// This will fire after both promises have called resolve()
const overallDis = values[0].overallDis;
const aRoadDis = values[0].aRoadDis;
const rate = values[1];
http.get("http://localhost:6000/surge/:" + rate + "/:" + overallDis + "/:"
+ aRoadDis, res => {
// console.log(overallDis)
// console.log(aRoadDis)
// console.log(rate)
res.setEncoding('utf8')
res.on('data', function(body){
console.log(body)
totalPrice += parseInt(body)
})
console.log(totalPrice)
})
});
})

As is mentioned in the other answers, your interpretation of async is the wrong way around: synchronous calls block the execution of following code, whereas asynchronous calls do not.
If you want to go through your operations in order, but they actually are asynchronous, the easiest method is using callbacks. This is doable for smaller call stacks, but it is not called callback-hell for nothing.
The best way would be to wrap the async calls in Promises and then using the async/await structure to order them in a synchronous fashion. This could look something like this.
async function (req, res) {
let op_1_result = await new Promise(function(resolve, reject) {
... do your async operation and finally call
resolve(response);
});
... do your other operations in the same fashion
let op_n_result = await new Promise(function(resolve, reject) {
... do your async operation and finally call
resolve(response);
});
return op_n_result;
}

How to run two functions synchronously with NodeJS?

I am pretty new with NodeJS and got lost with the asynchronous mechanism.
I have a code that should send a HTTP post request to the first URL (for example - https://example.com/first), and then when it got answered with status code 200, send another request to the same server that checks if the server is done with processing the first request (for example - https://example.com/statusCheck).
The server should return a text/plain response contains "true" if it's busy, and "false" if it's ready to use.
I wrote it with a while loop that queries the server every 2 seconds, up to maximum of 10 iterates.
var request = require('request');
var firstURL = "https://example.com/first";
var serverCheck = "https://example.com/statusCheck";
// Sends up to 10 requests to the server
function checkServerStatus(){
var serverReady = false;
var count = 0;
while (!serverReady && count < 10) {
count++;
setTimeout(function(){
request.get(serverCheck, function(err, resp, body){
if (err){
console.log(err);
} else if (body == "false") {
generatorReady = true;
}
})
}, 2000);
}
return generatorReady;
}
// Sends the first request and return True if the response equals to 200
function sendFirstRequest(){
var req = request.post(firstURL, function (err, resp, body) {
if (err) {
console.log(err);
return false;
} else if (resp.statusCode === 200){
return true;
} else {
return false;
}
});
};
module.exports = function (){
// Sends the first request
var firstRequestStatus = sendFirstRequest();
if (firstRequestStatus) {
return checkServerStatus();
}
};
In other words, I want to run sendFirstRequest first, wait for the response, and in case that the response is true, I want to run the checkServerStatus and get his returned value. If it's possible to do it with a sleep between each while iterate, it will be great (because the setTimeout does not work for me as well) .
Edit: I've heard that I can use function* with yield, or async-await in order to avoid callback hell - how can I implement them in this case?

You should use a Promise to do this. Below is some code using bluebird which will do what you want. The Promise.any method will return the first successful call from the Array out of 10 tries.
const Promise = require('bluebird');
var request = Promise.promisifyAll(require('request'));
var firstURL = "https://example.com/";
var serverCheck = "https://example.com/statusCheck";
request.postAsync(firstURL).then(res => {
if (res.statusCode === 200) return true;
throw new Error('server not ready');
}).then(() =>
Promise.any(new Array(10).fill(request.getAsync(serverCheck)))
).then(res => {
console.log(res);
}).catch(err => console.log(err));

You have to understand that the asynchronous operations can not return a result right after their call. They trigger some handler when they have executed. You can/should use that entry point to initiate or continue your logic flow.
http.post(params, handler(err, resp, body){
if(err){
failFlow(err);
}else if(resp.statusCode === 200) {
successFlow(resp);
}
});
and you can chain as many such asynchronous calls as you need but you can not return a response in this manner.
Also you might be interested in the concept of a Promise.
var request = require('request');
var firstURL = "https://example.com/first";
var serverCheck = "https://example.com/statusCheck";
var count = 0;
// Sends up to 10 requests to the server
function checkServerStatus() {
if (count++ > 10) return;
request.get(serverCheck, function(err, resp, body) {
if (err) {
console.log(err);
checkServerStatus();
} else if (body == "false") {
// go further
}
});
}
// Sends the first request and return True if the response equals to 200
function sendFirstRequest(cb) {
var req = request.post(firstURL, function(err, resp, body) {
if (err) {
console.log(err);
return false;
} else if (resp.statusCode === 200) {
cb();
} else {
return false;
}
});
};
module.exports = function() {
// Sends the first request
sendFirstRequest(checkServerStatus);
};

You can use the async library.
you dont need to do a setInterval or any timer for that matter, just wait for the response.
specifically you can use async.waterfall for this, something like:
var async = require('async')
var request = require('request')
async.waterfall([
function(cb) {
// send the first request
request.post("https://example.com/first", function (err, resp) {
// send the response to the next function or break in case there was an error
cb(err, resp)
})
},
function(resp, cb) {
// check for the response
if (resp.statusCode === 200) {
// in case the response code is 200 continue to the next function
return cb()
}
// if its not 200 break with the response code as an error
return cb(resp.statusCode)
},
function(cb) {
// send the verify
request.get("https://example.com/statusCheck", function (err, resp, body) {
// send the body of the response to the next function or break in case of an error
cb(err, body)
})
}
], function (err, result) {
// check if there was an error along the way
if (err) {
console.log("there was an error", err)
} else {
// all is good print the result
console.log("result:", result)
}
})

async function main() {
console.log('First call started');
let response1 = await $.ajax({url: "https://api.stackexchange.com/2.2/questions/269754/answers/?order=desc&site=meta.stackoverflow&client_id=3519&callback=?"})
console.log('First call finished', response1);
console.log('Second call started');
let response2 = await $.ajax({url: "https://api.stackexchange.com/2.2/questions/269754/answers/?order=desc&site=meta.stackoverflow&client_id=3519&callback=?"})
console.log('Second call finished',response2);
}
main();
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
In newer versions of nodejs you can use async await like the example above
Notice that $.ajax is not a node function. It is just for demonstration
You can use await on any function that return a promise.
For the next example you need to install request package and use Node >= 8 for using promisify
const {promisify} = require('util');
const request = require('request')
async function main() {
let get = promisify(request);
let response1 = await get('https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new');
console.log('first random: ',response1.body)
let response2 = await get('https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new');
console.log('second random: ',response2.body)
}
main();
http://2ality.com/2017/05/util-promisify.html
https://github.com/request/request