We Keep Coding

Return https.get() response body to client with Nodejs

I am trying to develop a google cloud function that will make an external https GET request and return the response body to the client.
Flow:
client makes request to mockServer function
function makes GET request to example.com
function returns "results" from response body from example.com to client
exports.mockServer = (req, res) => {
'use strict';
var https = require('https');
var options = {
host: 'example.com',
path: '/path',
headers: {
'accept': 'application/json',
'X-API-Key': 'XXXX'
}
};
if (req.method == 'GET'){
https.get(options, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
if (res.statusCode === 200) {
var res_body = JSON.parse(data);
var results = JSON.stringify(res_body.result)
console.log("results:"+results);
} else {
console.log('Status:', res.statusCode);
}
});
}).on('error', function (err) {
console.log('Error:', err);
});
} else {
console.log("Wrong Method");
}
res.end()
};
I am able to successfully log the results with console.log("results:"+results); but I cannot figure out how to get it returned to the client. I am still new to this and am learning, so thank you so much in advance to any help!

Posting #YouthDev's solution as an answer:
Thanks to #DougStevenson and #Deko in the comments, I switched to an axios library and it works like a charm. Thank you to both for pointing me in the correct direction. Below is the working axios code.
exports.mockServer = (req, res) => {
const axios = require('axios').create({
baseURL: 'https://example.com'
});
return axios.get('/path',{ headers: {'accept': 'application/json','X-API-Key': 'XXXXXX'} })
.then(response => {
console.log(response.data);
return res.status(200).json({
message: response.data
})
})
.catch(err => {
return res.status(500).json({
error: err
})
})
};

Getting response, body and error from an asynchronous method making a POST API call

I am using this example to make a POST API call to an API: https://nodejs.dev/making-http-requests-with-nodejs#perform-a-post-request. No issues there, it works well.
Next, I wanted to create a function that makes this API call by taking in dynamic connection parameters, headers and payload. Did that and I am able to return the response object from the function so I can detect the response.statusCode, response.statusMessage, etc. Here's my Node.js code:
Module Code
const https = require("https");
function postLendingApplication(connection, data, callback) {
const options = {
hostname: connection.hostname,
port: connection.port,
path: connection.path,
method: connection.method,
headers: connection.headers
};
//console.log(options)
const req = https.request(options, res => {
console.log(`statusCode: ${res.statusCode}`);
res.on("data", d => {
process.stdout.write(d);
});
callback(res);
});
req.on("error", error => {
console.error(error);
});
req.write(data);
req.end();
}
exports.postLendingApplication = postLendingApplication;
Invoking the code from another file
const bpc = require("./public-api");
const data = JSON.stringify({
title: 'foo',
body: 'bar',
userId: 1
});
const connection = {
hostname: 'jsonplaceholder.typicode.com',
port: 443,
path: '/posts',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': data.length,
}
}
var response = bpc.postLendingApplication(connection, data, function(response) {
console.log("Inside the calling function");
//console.log(response);
console.log("Status Code: " + response.statusCode);
console.log("Status Message: " + response.statusMessage);
});
Successful console response
statusCode: 201
Inside the calling function
Status Code: 201
Status Message: Created
{
"title": "foo",
"body": "bar",
"userId": 1,
"id": 101
}
Question: In my callback method, I would like to receive the response body (The JSON) as well as the error so I can run some assertions based on the response/body/error that I received. I am not able to figure out how to setup callback in the module method so it can return all 3 values. If you can please help out with that, it would be greatly appreciated.
Here's the Repl URL in case you'd like to take a stab at it online: https://repl.it/#varun_verma/POST-API

I am not 100% on the question you are asking, I personally if you are wanting to use callbacks in this way use two functions one to handle the error and one for the succsessful response
however, you can use object destruction to give you undefined or default it to null if you like for the item not passed back as shown below:
Module Code
const https = require("https");
function postLendingApplication(connection, data, callback) {
const options = {
hostname: connection.hostname,
port: connection.port,
path: connection.path,
method: connection.method,
headers: connection.headers
};
//console.log(options)
const req = https.request(options, res => {
console.log(`statusCode: ${res.statusCode}`);
let data = ''
res.on("data", d => {
data += d;
});
res.on('end', () => {
callback({response: res, data});
});
});
req.on("error", error => {
console.error(error);
callback({response: res, error});
});
req.write(data);
req.end();
}
exports.postLendingApplication = postLendingApplication;
Invoking the code from another file
const bpc = require("./public-api");
const data = JSON.stringify({
title: 'foo',
body: 'bar',
userId: 1
});
const connection = {
hostname: 'jsonplaceholder.typicode.com',
port: 443,
path: '/posts',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': data.length,
}
}
var response = bpc.postLendingApplication(connection, data, function({ response, data, error}) {
// if error is not passed back in the object it defaults to undefined
if (error) console.error(error)
console.log("Inside the calling function");
//console.log(response);
console.log("Status Code: " + response.statusCode);
console.log("Status Message: " + response.statusMessage);
});

If I understand your question correctly, you want to have the response, data, and the error passed to the callback. (where you currently only pass the response)
You can pass the data to the callback like so:
const req = https.request(options, res => {
console.log(`statusCode: ${res.statusCode}`);
let data = '';
res.on("data", d => {
data += d;
});
res.on('end', () => {
callback(res, data);
});
});
This buffers the data from the response as it comes in into a string and then only when the response ends passes both the response object and the data as a string to the callback. (you can then use JSON.parse in the callback to convert the data string to an object)
Passing the error is more difficult as the error callback is given separately from the response. I would recommend having a separate callback for the error:
function postLendingApplication(connection, data, callback, error_callback) {
...
req.on("error", error => {
console.error(error);
error_callback(error);
});
...
}
However someone else on here may be able to give a better solution for the error.

request(...).then is not a function error when making a POST request

I'm trying to create a firebase function that makes a HTTP POST request whenever a new document is created.
This is my code:
import * as functions from 'firebase-functions';
const admin = require('firebase-admin');
const request = require("request");
exports.sendMessage = functions.firestore.document('comms/{comms}').onCreate((snap, context) => {
const newValue = snap.data();
if (newValue) {
//const email = newValue.email;
const msg = newValue.msg;
return request({
uri: "url",
method: 'POST',
body: msg,
json: true,
resolveWithFullResponse: true
}).then((response: { statusCode: number; }) => {
if (response.statusCode >= 400) {
throw new Error(`HTTP Error: ${response.statusCode}`);
}
console.log('SUCCESS! Posted', msg);
});
}
return Promise
});
Error received:
TypeError: request(...).then is not a function
at exports.sendMessage.functions.firestore.document.onCreate (/srv/lib/index.js:25:12)
at cloudFunction (/srv/node_modules/firebase-functions/lib/cloud-functions.js:127:23)
at /worker/worker.js:825:24
at
at process._tickDomainCallback (internal/process/next_tick.js:229:7)

request supports callback interfaces natively but does not return a promise, which is what you must do within a Cloud Function.
This is explained in the official Firebase video series here: https://firebase.google.com/docs/functions/video-series/. In particular watch the three videos titled "Learn JavaScript Promises" (Parts 2 & 3 especially focus on background triggered Cloud Functions, but it really worth watching Part 1 before).
You could use request-promise (https://github.com/request/request-promise) and the rp() method which "returns a regular Promises/A+ compliant promise". You would then adapt your code as follows:
import * as functions from 'firebase-functions';
const admin = require('firebase-admin');
const rp = require('request-promise');
exports.sendMessage = functions.firestore.document('comms/{comms}').onCreate((snap, context) => {
const newValue = snap.data();
if (newValue) {
const msg = newValue.msg;
var options = {
method: 'POST',
uri: '....',
body: msg,
json: true // Automatically stringifies the body to JSON
};
return rp(options)
.then(parsedBody => {
// POST succeeded...
console.log('SUCCESS! Posted', msg);
return null;
})
.catch(err => {
// POST failed...
console.log(err);
return null;
});
} else {
return null;
}
});

request module doesn't return a Promise instead try using a callback function for response.
return request({
uri: "url",
method: 'POST',
body: msg,
json: true,
resolveWithFullResponse: true
}, function (error, response, body) {
})

As in the documentation already mention you need to pass the callback to your request
var request = require('request');
request('http://www.google.com', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
If you want to chain your request you can use pipe
request
.get('url/img.png')
.on('response', function(response) {
console.log(response.statusCode) // 200
console.log(response.headers['content-type']) // 'image/png'
})
.pipe(request.put('url'))
If you want to use promise you can use request-promise
var rp = require('request-promise');
rp('http://www.google.com')
.then(function (htmlString) {
// Process html...
})
.catch(function (err) {
// Crawling failed...
});

The request module work on callbacks only, If you want to make Promisify you need to do like this
const request = require('request');
const webService = {};
webService.callApi = (url, bodyObj, method) => {
return new Promise((resolve, reject) => {
const options = {
method: method || 'POST',
url: url,
headers: {
'Content-Type': 'application/json',
},
body: bodyObj,
json: true,
};
// Error Handler
const errorMessge = { code: 500, error: 'INTERNAL_SERVER_ERROR' };
request(options, (error, response, resBody) => {
if (error) {
return reject(errorMessge);
} else if (response.statusCode !== 200) {
return reject(errorMessge);
}
return resolve(resBody);
});
});
};
module.exports = webService;

Multiple http post calls in foreach with nodejs

How can I execute many request in foreach without error please ?
Currently, I send a request on each entry on my array with foreach :
users.forEach(function (user) {
request({
url : 'myurl.com/api',
method: 'POST',
auth : {
'bearer': CONFIGURATION.auth.token
},
body : {
sender_id: user.sender_id
},
json : true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
resolve(body);
} else {
console.log('Error on coreService');
console.log('############### ERROR ###############');
console.log(error);
console.log('############### BODY ###############');
console.log(body);
console.log('############### RESPONSE ###############');
console.log(response);
reject(error);
}
});
});
With some request it's ok, but with some request I have this error :
Error on coreService
############### ERROR ###############
{ Error: connect ECONNRESET 127.0.0.1:80
at Object._errnoException (util.js:1022:11)
at _exceptionWithHostPort (util.js:1044:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)
code: 'ECONNRESET',
errno: 'ECONNRESET',
syscall: 'connect',
address: '127.0.0.1',
port: 80 }
############### BODY ###############
undefined
############### RESPONSE ###############
undefined
Do you have an idea how can I fix this problem please ?
I tried with :
server.timeout = 0;
or
server.timeout = 1000;
but same problem...
If I execute the request, user by user, it's fine !
But with the foreach, it's break on some request...

In the comments colinux proposes that the connection reset errors are due to the server protecting itself from too many simultaneous requests, and I think they are probably correct. This code shows how you can use async/await to make your requests to the server one at a time. This might be too slow for you, but it could help you to confirm that the problem is as explained by colinux.
Here is another answer which does not require the use of request-promise-native and instead wraps the request calls in its own Promise.
const request = require('request');
const users = [{sender_id: 1}, {sender_id: 2}, {sender_id: 3}];
// make this driver function async so we can use await which allows waiting for
// a request to finish before starting the next one
async function runUserRequests() {
for (let i = 0; i < users.length; i++) {
const user = users[i];
try {
const response = await requestPromise(user);
console.log("response for user", user, response);
} catch (error) {
console.log("error for user", user, error);
}
};
}
// wrap the request call in a Promise so that it will run synchronously
function requestPromise(user) {
return new Promise(function(resolve, reject) {
request({
url: 'http://localhost:4000/',
method: 'GET', // method 'POST'
// auth : {
// 'bearer': CONFIGURATION.auth.token
// },
// body : {
// sender_id: user.sender_id
// },
// json : true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
resolve(body);
console.log("request successful for user", user, " at ", (new Date()).getTime());
} else {
console.log('Error on coreService');
console.log('############### ERROR ###############');
console.log(error);
console.log('############### BODY ###############');
console.log(body);
console.log('############### RESPONSE ###############');
console.log(response);
reject(error);
}
});
});
}
runUserRequests();
/*
Sample server used to test the code above:
const express = require('express')
const app = express()
const port = 4000
app.get('/', (req, res) => {
console.log("spinning for a bit");
setTimeout( () => {
console.log(" now responding");
res.send('Hello World!');
}, 1000);
});
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
*/

In the comments colinux proposes that the connection reset errors are due to the server protecting itself from too many simultaneous requests, and I think they are probably correct. This code shows how you can use async/await to make your requests to the server one at a time. This might be too slow for you, but it could help you to confirm that the problem is as explained by colinux.
To get this to work you'll need to install request-promise-native. If you can't do that let me know and I can work up an example wrapping the request api in your own Promise.
const request = require('request-promise-native');
//const users = [1, 2, 3, 4]; // dummy user array for testing
async function runUserRequests(users) {
for (let i = 0; i < users.length; i++) {
const user = users[i];
console.log("starting request for user ", user);
await request({
url: 'http://localhost:4000/',
method: 'GET',
auth : {
'bearer': CONFIGURATION.auth.token
},
body : {
sender_id: user.sender_id
},
json : true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
console.log("request successful for user", user, " at ", (new Date()).getTime());
resolve(body);
} else {
console.log('Error on coreService');
console.log('############### ERROR ###############');
console.log(error);
console.log('############### BODY ###############');
console.log(body);
console.log('############### RESPONSE ###############');
console.log(response);
reject(error);
}
});
};
}
runUserRequests();

Try-Catch not handling errors with an https.get request in Node

I have an https.get request in Node for which I need to handle the errors - preferably in a try-catch block. For instance, when the url is incorrect
I have tried wrapping the https.get block in a try catch and I have tried handling with res.on('error'). It seems as if in both instances the error doesn't reach the error handling block.
const https = require('https');
const hitApi = () => {
const options = {
"hostname": "api.kanye.rest"
};
try {
https.get(options, res => {
let body = '';
res.on('data', data => {
body += data;
});
res.on('end', () => {
body = JSON.parse(body);
console.dir(body);
});
});
} catch (error) {
throw error;
}
}
hitApi();
If I change the url to a nonexistent API (e.g. api.kaye.rest) I would expect to see a handled e.rror response. Instead I see 'Unhandled error event'

The reason why try...catch.. fails is that it is meant for handling synchronous errors. https.get() is asynchronous and cannot be handled with a usual try..catch..
Use req.on('error',function(e){}); to handle the error. Like so:
var https = require('https');
var options = {
hostname: 'encrypted.google.com',
port: 443,
path: '/',
method: 'GET'
};
var req = https.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
res.on('data', function(d) {
process.stdout.write(d);
});
});
req.end();
// Error handled here.
req.on('error', function(e) {
console.error(e);
});
You can read more about the same in the documentation over here