We Keep Coding

How to make HTTP request with promises and retries in Node.js

I am trying to write a few wrappers around Node HTTP/S module requests, without using axios. node-fetch or any other 3rd party module.
For example, I want to have functions sendGet, sendPost, sendJSON, sendFile etc. In ideal case, these functions will be implementing core functionmakeRequest, just with different parameters.
I want each wrapper to return a promise, so the caller can do anything with the result.
I also want the wrapper to have an argument, how many times will be request retried in case of failure.
So idea is something like this. So far I am able to make a wrapper and pass promise. But I am unable to add ability to retry on failure. It should be (in ideal scenario), part of makeRequest function, but I was unable to to do so, when combined with promises. Thank you for your ideas
// intended usage of wrapper
sendGet('http://example.com', 3).then().catch()
// wrapper applies makeRequest fnc with various arguments
const sendGet = (url, retries) => {
return makeRequest('GET', url, retries)
}
const sendPost = (url, retries) => {
return makeRequest('POST', url, retries)
}
// core function
const makeRequest = async (method, url, retries = 0) => {
// returns reject on bad status
// returns resolve with body on successful request
return new Promise((resolve, reject) => {
const options = {/*method, hostname, etc */};
const request = http.request(options, (response) => {
let chunks = [];
if (response.statusCode < 200 || response.statusCode >= 300) {
return reject('bad status code')
}
// collect data
response.on('data', (chunk => {
chunks.push(chunk)
}))
// resolve on end of request
response.on('end', () => {
let body = Buffer.concat(chunks).toString();
return resolve(body)
})
})
request.end();
})
}

Try this, the original function now is called tryRequest and outside there is the for loop to do the retries
// core function
const makeRequest = async (method, url, retries = 0) => {
const tryRequest = async () => {
// returns reject on bad status
// returns resolve with body on successful request
return new Promise((resolve, reject) => {
const options = {method, };
const request = http.request(url, options, (response) => {
let chunks = [];
if (response.statusCode < 200 || response.statusCode >= 300) {
return reject('bad status code')
}
// collect data
response.on('data', (chunk => {
chunks.push(chunk)
}))
// resolve on end of request
response.on('end', () => {
let body = Buffer.concat(chunks).toString();
return resolve(body)
})
})
// reject on error of request (Service down)
request.on('error', function (error) {
reject(error);
})
request.end();
})
}
for (let i=1; i<=retries; i++) {
try {
console.log('Try No.', i);
url = url.substring(0, url.length - 1); // TODO To test, delete after it
return await tryRequest();
} catch(error) {
if (i < retries) continue;
throw error;
}
}
}
Test
await sendGet('http://example.com/abc', 3); // Will work at the 3th try retry
await sendGet('http://example.xyz/abc', 3); // Will fail server not found

You can use a custom promise & axios or any other promise-based request:
(Live Demo)
import { CPromise } from "c-promise2";
import cpAxios from "cp-axios";
const url =
"https://run.mocky.io/v3/7b038025-fc5f-4564-90eb-4373f0721822?mocky-delay=2s";
(async()=>{
const response= await CPromise.retry(() => cpAxios(url).timeout(5000));
})();
More complex:
import { CPromise } from "c-promise2";
import cpAxios from "cp-axios";
const url =
"https://run.mocky.io/v3/7b038025-fc5f-4564-90eb-4373f0721822?mocky-delay=2s";
const promise = CPromise.retry(
(attempt) => {
console.log(`Attempt [${attempt}]`);
return cpAxios(url).timeout(attempt * 1000 + 500);
},
{ retries: 3, delay: (attempt) => attempt * 1000 }
).then(
(response) => console.log(`Response: ${JSON.stringify(response.data)}`),
(err) => console.warn(`Fail: ${err}`)
);
// promise.pause()
// promise.resume()
// promise.cancel()

Promise, Async Await

setDeviceTimeout = id => timeout => {
const {onSetDevices, devices} = this.props;
var newDeviceList = devices.map(device => {
if (device.id === id) {
var newDevice = {
//...device,
timeout: timeout
};
deviceTable.oncePostDevice(newDevice).then( data => {
return newDevice = data
});
}
return device;
});
onSetDevices(newDeviceList);
}
So the issue I am having here is that the onSetDevices(newDeviceList) get's called before the devices.map() is finished. This is because the devices.map() has the call to a server oncePostDevice(newDevice), then returns the data and stores it in the newDevice variable and puts that into the newDeviceList array.
Because this happens onSetDevices doesn't include the the newDevice in the newDeviceList array of objects and when I set my redux state using onSetDevices, nothing has changed.
I am wondering how I turn this into an async, await or use a promise alone to finish the task of making onSetDevices wait for the devices.map() to finish.
Also here is the code for oncePostDevice:
export const oncePostDevice = (device) => new Promise(function(resolve, reject) {
fetch('https://url/devices/'+device.id, {
method: 'PUT',
headers: {
"Accept": "application/json",
"Content-Type": "application/json"
},
body: JSON.stringify(device)
})
.then(response => response.json())
.then(
data => {return resolve(data)},
error => {return reject(error)}
)
.catch(err => console.error(this.props.url, err.toString()));
});
As you can see I already have a promise in here working and returning the data afterwards.
I just need to know how to make my setDeviceTimeout inner mapping function finish before I hit onSetDevices.

Here's how you could do it (explanations inline in code):
// make the function async
setDeviceTimeout = id => async timeout => {
const {onSetDevices, devices} = this.props;
// make a list of promises, not of devices
// note: mapping a value via an async function will create promises
const newDeviceListPromises = devices.map(async device => {
if (device.id === id) {
const newDevice = {
...device,
timeout: timeout
};
return await deviceTable.oncePostDevice(newDevice);
}
return device;
});
// wait for all promises to finish and what they return will be the devices
const newDeviceList = await Promise.all(newDeviceListPromises);
onSetDevices(newDeviceList);
};

Async / Await, how do I force my function to wait until promise is resolved before it continues?

I would like my function to execute until the nextPageToken is null. When I execute the function the first time, it waits for the promise to be resolved. However as soon there is a nextPageToken present on the response, the function does not wait for the response and a stack overflow occurs.
It seems that f() is not suspended on the when await p.then() is called.
Am I totally misunderstanding how async/await works?
Any help would be greatly appreciated...
public apiResult2(path: string, objectName: string, params: any = { }) {
let returnArray = [];
const p = new Promise<any> ((resolve, reject) => {
gapi.load('auth2:client', () => {
gapi.client.request({
path: path,
params: params
}).then(response => {
// resolve this promise with the first key in the response object.
resolve(response.result);
}, reason => {
console.log(reason);
reject(reason.result);
});
});
});
let f = async () => {
let nextPageToken = null;
do {
let r = await p.then(result => {
if (result.hasOwnProperty(objectName)) {
for (let obj of result[objectName]) {
returnArray.push(obj);
}
}
if (result.hasOwnProperty('nextPageToken')) {
params.nextPageToken = result.nextPageToken;
return result.nextPageToken;
// nextPageToken = result.nextPageToken;
} else {
params.nextPageToken = null;
return null;
// nextPageToken = null;
}
});
nextPageToken = r;
console.log(r);
} while (nextPageToken);
};
f();
return returnArray;
}

If your function needs to "await" some async call, then it must also be async. Your function apiResult2 is not going to wait for f to be finished, in order to return returnArray.
EDIT:
The main issue here is that you are trying to reuse the promise p to make different requests, but this is not possible. The promise p will be initialized with the parameters for the first request, and all the calls to p.then will be fulfilled with the same result: the response for the first page request.
I made some small changes to your code, and got it working with a mocked interface:
const apiResult2 = async (path: string, objectName: string, params: any = { }) => {
const requestPage = async () => new Promise<any> ((resolve, reject) => {
gapi.load('auth2:client', () => {
gapi.client.request({
path: path,
params: params
}).then(response => {
// resolve this promise with the first key in the response object.
resolve(response.result);
}, reason => {
console.log(reason);
reject(reason.result);
});
});
});
let returnArray: string[] = [];
do {
const page = await requestPage();
if (page.hasOwnProperty(objectName)) {
for (let obj of page[objectName]) {
returnArray.push(obj);
}
}
if (page.hasOwnProperty('nextPageToken')) {
params.nextPageToken = page.nextPageToken;
} else {
params.nextPageToken = null;
}
} while (params.nextPageToken);
return returnArray;
}
Usage example:
apiResult2(path, objectName, params).then(
result => console.log(result),
err => console.log('error', err)
);

Angular2: Dynamic synchronous http requests

Goal: To make a series of synchronous http requests and be able to subscribe to them as one observable stream.
Sample (Not Working):
let query_arr = ['test1','test2','test3']
function make_request(query_arr){
if (query_arr.length){
let payload = JSON.stringify(query_arr[0]);
let headers = new Headers();
query_arr.splice(0,1);
this.http.post('https://endpoint/post',payload,{headers:headers})
.map((res:Response) => {make_request(query_arr)})
}
}.subscribe(
data => console.log('finished http request, moving on to next http request'),
err => console.error(err),
() => console.log('all http requests have been finished')
);
make_request(query_arr)
Goal Functionality:
Need to know when each response was returned
Must know when all responses have returned

You need to leverage the flatMap operator to execute your requests in series (one after one). For this, you need to build your data processing chain recursively. The point here is to call the operator on the previous observable (the one returned by the previous request).
This way the request will wait for the previous one to be complete before executed itself. The callback provided when subscribing will be called when all requests were executed.
Here is a sample implementation of this approach:
makeRequest(queryArr, previousObservable){
if (queryArr.length) {
let payload = JSON.stringify(queryArr[0]);
let headers = new Headers();
(...)
queryArr.splice(0,1);
var observable = null;
if (previousObservable) {
observable = previousObservable.flatMap(() => {
return this.http.post('https://testsoapi.apispark.net/v1/entities', payload,{
headers:headers
})
.map((res:Response) => res.json())
.do(() => {
console.log('request finished');
});
});
} else {
observable = this.http.post('https://testsoapi.apispark.net/v1/entities', payload, {
headers:headers
})
.map((res:Response) => res.json())
.do(() => {
console.log('request finished');
});
}
return this.makeRequest(queryArr, observable);
} else {
return previousObservable;
}
}
This method can be called initially like this:
test() {
let queryArr = [
{ val: 'test1' },
{ val: 'test2' },
{ val: 'test3' }
];
this.makeRequest(queryArr).subscribe(
() => {
console.log('all requests finished');
});
}
See this plunkr: https://plnkr.co/edit/adtWwckvhwXJgPDgCurQ?p=preview.

There were a couple syntactical errors in your code as well that would need to be addressed. But those aside you can simplify greatly by using concatMap + defer instead.
let query_arr = ['test1','test2','test3'];
let self = this;
Rx.Observable.from(query_arr).map(JSON.stringify)
.concatMap(payload => {
let headers = new Headers();
return Rx.Observable.defer(() => {
self.http.post('https://endpoint/post',payload,{headers:headers});
});
}, resp => resp.json())
.subscribe(
data => console.log('finished http request, moving on to next http request'),
err => console.error(err),
() => console.log('all http requests have been finished')
);
The basic idea of this is that it will convert the query array into an Observable then it will eagerly create a series of lazy requests that will only be executed when they are subscribed to. However, by wrapping the post in a defer each request will only get dispatched when the previous one completes.

Or a non recursive version in typescript where you give an array to forkjoin
in the return observableObj(res.json()) you know each response when it returns from the httpcall
in the subscribe you know when all responses returned and an array of values
const observableObj = (obj) => Observable.of(obj)
class Requests {
private query_arr = ['test1','test2','test3']
private url = 'https://testsoapi.apispark.net/v1/entities'
public make() {
this.processHttp().subscribe(
(d) => {
console.log(d)
},
(e) => {
console.log(e)
},
() => {
console.log("http calls are done")
})
}
private httpCall(options : RequestOptions) : Observable<Response> {
let username : string = 'xxx'
let password : string = 'yyy'
let headers = new Headers()
headers.append("Authorization", "Basic " + btoa(username + ":" + password))
headers.append("Content-Type", "application/x-www-form-urlencoded")
options.headers = headers
return this.http.get(this.url,options)
}
private createRequestOptions(option1 : string) {
let data = {'option1':option1}
let params = new URLSearchParams()
for(var key in data) {
params.set(key, data[key])
}
let options = new RequestOptions({
search: params
})
return options
}
private processHttp() {
return Observable.forkJoin(
this.query_arr.map(option => {
return this.httpCall(createRequestOption(option)).flatMap((res: Response) => {
return observableObj(res.json())
})
}))
}
}

Promises with http.get node.js

Im am doing nodeschool exercises , the
This problem is the same as the previous problem (HTTP COLLECT) in
that you need to use http.get(). However, this time you will be
provided with three URLs as the first three command-line arguments.
You must collect the complete content provided to you by each of the
URLs and print it to the console (stdout). You don't need to print out
the length, just the data as a String; one line per URL. The catch is
that you must print them out in the same order as the URLs are
provided to you as command-line arguments.
in other words , i am to register 3 http.get request , and print data recieved from it in order.
I am trying to do it with promises = another get request wont be called untill the first on didnt end.
My code looks like this
var http=require("http");
var collect=[];
var dat=[];
for( var i = 2 ; i < process.argv.length;i++){
collect.push(process.argv[i]);
}
function chainIt(array,callback){
return array.reduce(function(promise,item){
return promise.then(function(){
return callback(item)
})
},Promise.resolve())
}
function getIt(item){
return http.get(item,function(response){
response.on("data",function(data){
dat.push(data);
})
})
}
chainIt(collett,function(item){
return getIt(item)
})
}).then(function(){
collect.forEach(function(x){
console.log(x);
})
})
But i actually print no data = i fail the exercise.
I do not see any bug here , but im just starting with promises and node. Where is the mistake?

For educational purposes, I recently wrote a wrapper for the http and https modules that uses native Promises. That said, I recommend using a library, such a request; that makes things simpler, has unit test coverage, as is maintained by the open source community. Also, my wrapper does a naive string concatenation with the response chunks, which I'm not convinced is the most performant way of building up the response body.
FYI: this requires Node.js 4 or above, although the methodology is pretty much the same in Node 0.x.x.
'use strict';
const http = require('http');
const url = require('url');
module.exports = {
get(url) {
return this._makeRequest('GET', url);
},
_makeRequest(method, urlString, options) {
// create a new Promise
return new Promise((resolve, reject) => {
/* Node's URL library allows us to create a
* URL object from our request string, so we can build
* our request for http.get */
const parsedUrl = url.parse(urlString);
const requestOptions = this._createOptions(method, parsedUrl);
const request = http.get(requestOptions, res => this._onResponse(res, resolve, reject));
/* if there's an error, then reject the Promise
* (can be handled with Promise.prototype.catch) */
request.on('error', reject);
request.end();
});
},
// the options that are required by http.get
_createOptions(method, url) {
return requestOptions = {
hostname: url.hostname,
path: url.path,
port: url.port,
method
};
},
/* once http.get returns a response, build it and
* resolve or reject the Promise */
_onResponse(response, resolve, reject) {
const hasResponseFailed = response.status >= 400;
var responseBody = '';
if (hasResponseFailed) {
reject(`Request to ${response.url} failed with HTTP ${response.status}`);
}
/* the response stream's (an instance of Stream) current data. See:
* https://nodejs.org/api/stream.html#stream_event_data */
response.on('data', chunk => responseBody += chunk.toString());
// once all the data has been read, resolve the Promise
response.on('end', () => resolve(responseBody));
}
};
EDIT: I only just realised you're new to Promises. Here's an example of how to use this wrapper:
'use strict';
const httpService = require('./httpService'); // the above wrapper
// get one URL
httpService.get('https://ron-swanson-quotes.herokuapp.com/v2/quotes').then(function gotData(data) {
console.log(data);
});
// get multiple URLs
const urls = [
'https://ron-swanson-quotes.herokuapp.com/v2/quotes',
'http://api.icndb.com/jokes/random'
];
/* map the URLs to Promises. This will actually start the
* requests, but Promise.prototype.then is always called,
* even if the operation has resolved */
const promises = urls.map(url => httpService.get(url));
Promise.all(promises).then(function gotData(responses) {
/* responses is an array containing the result of each
* Promise. This is ordered by the order of the URLs in the
* urls array */
const swansonQuote = responses[0];
const chuckNorrisQuote = responses[1];
console.log(swansonQuote);
console.log(chuckNorrisQuote);
});

Using Promise.all is the most efficient solution for this. You could also use async/await like below to solve this.
const http = require('http');
const bl = require('bl');
async function httpGet(url) {
return new Promise((resolve, reject) => {
http.get(url, response => {
response.setEncoding('utf8');
response.pipe(bl((err, data) => {
if (err) {
reject(err);
}
resolve(data.toString());
}));
});
});
}
async function main() {
const data1 = await httpGet(process.argv[2]);
const data2 = await httpGet(process.argv[3]);
const data3 = await httpGet(process.argv[4]);
console.log(data1);
console.log(data2);
console.log(data3);
}
main();

I don't think this exercise was meant to be solved with promises. I found my old exercice folder, this is how I did it without promises or extra libraries:
var http = require('http');
var urls = process.argv.slice(2);
// counts the number of requests done
var done = 0;
// stores the requests result
var result = [];
// this will be called by each http.get and they will provide their index
function callback(index, data) {
result[index] = data;
done++;
// all requests are done, log everything
if (done == urls.length) {
result.forEach(console.log);
}
}
function processUrl(url, index) {
var finalData = '';
http.get(url, function(response) {
response.setEncoding('utf8');
response.on('data', function(data) {
finalData += data;
});
response.on('error', console.error);
response.on('end', function() {
// console.log(finalData);
callback(index, finalData);
})
});
}
urls.forEach(processUrl);
Don't worry, you'll have enough promises to play with in the promise-it-wont-hurt workshop.

A bit late to the party here :)
Unfortunately none of the answers here uses the builtin util module in node.
Here is how to promisify http.get using util module and it is working correctly with typescript:
import util from "util";
const httpGetPromisified = util.promisify(
(url: string, cb: (err: any, result: IncomingMessage) => void) =>
http.get(url, (res) => cb(null, res))
);
// with promise.then
httpGetPromisified("http://www.google.com").then((res) => {
// res here has type http.IncomingMessage
console.log(res.statusCode);
});
// with async/await
const res = await httpGetPromisified("http://www.google.com");
console.log(res.statusCode);

Here's my solution after going through this thread:
var http = require('http');
var bl = require('bl')
promises = [
promiseLoad(process.argv[2]),
promiseLoad(process.argv[3]),
promiseLoad(process.argv[4])
];
Promise.all(promises).then(function(res) {
for(i=0; i<promises.length; i++) {
console.log(res[i]);
}
});
function promiseLoad(url) {
var body = '';
return new Promise(function(resolve, reject) {
http.get(url, function (response) {
response.setEncoding('utf8');
response.pipe(bl(function (err, data) {
resolve(data.toString())
}))
})
});
}
Here's the official solution in case you want to compare notes:
var http = require('http')
var bl = require('bl')
var results = []
var count = 0
function printResults () {
for (var i = 0; i < 3; i++) {
console.log(results[i])
}
}
function httpGet (index) {
http.get(process.argv[2 + index], function (response) {
response.pipe(bl(function (err, data) {
if (err) {
return console.error(err)
}
results[index] = data.toString()
count++
if (count === 3) {
printResults()
}
}))
})
}
for (var i = 0; i < 3; i++) {
httpGet(i)
}

const http = require('http');
const urls = process.argv.slice(2);
let callCount = 0;
const cb = (res) => {
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk.toString();
});
res.on('end', () => {
callCount += 1;
console.log(rawData);
if (callCount < urls.length) {
getData(urls[callCount]);
}
});
res.on('error', (error) => {
console.log(error);
});
};
const getData = (url) => {
http.get(url, cb);
};
getData(urls[callCount]);

Here's how I did it:
async myFunc = function {
let url = 'http://your.data/file';
let promise = new Promise((resolve, reject) => {
var data = '';
https.get(url, res => {
res.on('data', chunk => { data += chunk })
res.on('end', () => {
resolve(data);
})
})
});
let result = await promise; // wait until the promise resolves
doStuffWithResult(result);
};

One of the method is by using 'Q' library.
First create function that will hit URL and return promise
var Q = require('q);
function getIt(item){
return http.get(item,function(response){
return Q.resolve(response); // Return response
OR
return Q.resolve(error); // Return error
})
})
}
var urls = ['url1','url2','url3']; // list of urls
Q.spread(urls.map(getIt))
.then(function(res1,res2,res3){
// res1 is response for url1 and on
//Once all calls are finished you will get results here
});