How can I make an HTTP request from within Node.js or Express.js? I need to connect to another service. I am hoping the call is asynchronous and that the callback contains the remote server's response.
Here is a snippet of some code from a sample of mine. It's asynchronous and returns a JSON object. It can do any form of GET request.
Note that there are more optimal ways (just a sample) - for example, instead of concatenating the chunks you put into an array and join it etc... Hopefully, it gets you started in the right direction:
const http = require('http');
const https = require('https');
/**
* getJSON: RESTful GET request returning JSON object(s)
* #param options: http options object
* #param callback: callback to pass the results JSON object(s) back
*/
module.exports.getJSON = (options, onResult) => {
console.log('rest::getJSON');
const port = options.port == 443 ? https : http;
let output = '';
const req = port.request(options, (res) => {
console.log(`${options.host} : ${res.statusCode}`);
res.setEncoding('utf8');
res.on('data', (chunk) => {
output += chunk;
});
res.on('end', () => {
let obj = JSON.parse(output);
onResult(res.statusCode, obj);
});
});
req.on('error', (err) => {
// res.send('error: ' + err.message);
});
req.end();
};
It's called by creating an options object like:
const options = {
host: 'somesite.com',
port: 443,
path: '/some/path',
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
};
And providing a callback function.
For example, in a service, I require the REST module above and then do this:
rest.getJSON(options, (statusCode, result) => {
// I could work with the resulting HTML/JSON here. I could also just return it
console.log(`onResult: (${statusCode})\n\n${JSON.stringify(result)}`);
res.statusCode = statusCode;
res.send(result);
});
UPDATE
If you're looking for async/await (linear, no callback), promises, compile time support and intellisense, we created a lightweight HTTP and REST client that fits that bill:
Microsoft typed-rest-client
Try using the simple http.get(options, callback) function in node.js:
var http = require('http');
var options = {
host: 'www.google.com',
path: '/index.html'
};
var req = http.get(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
// Buffer the body entirely for processing as a whole.
var bodyChunks = [];
res.on('data', function(chunk) {
// You can process streamed parts here...
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
console.log('BODY: ' + body);
// ...and/or process the entire body here.
})
});
req.on('error', function(e) {
console.log('ERROR: ' + e.message);
});
There is also a general http.request(options, callback) function which allows you to specify the request method and other request details.
Request and Superagent are pretty good libraries to use.
note: request is deprecated, use at your risk!
Using request:
var request=require('request');
request.get('https://someplace',options,function(err,res,body){
if(err) //TODO: handle err
if(res.statusCode === 200 ) //etc
//TODO Do something with response
});
You can also use Requestify, a really cool and very simple HTTP client I wrote for nodeJS + it supports caching.
Just do the following for GET method request:
var requestify = require('requestify');
requestify.get('http://example.com/api/resource')
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
response.getBody();
}
);
This version is based on the initially proposed by bryanmac function which uses promises, better error handling, and is rewritten in ES6.
let http = require("http"),
https = require("https");
/**
* getJSON: REST get request returning JSON object(s)
* #param options: http options object
*/
exports.getJSON = function (options) {
console.log('rest::getJSON');
let reqHandler = +options.port === 443 ? https : http;
return new Promise((resolve, reject) => {
let req = reqHandler.request(options, (res) => {
let output = '';
console.log('rest::', options.host + ':' + res.statusCode);
res.setEncoding('utf8');
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', () => {
try {
let obj = JSON.parse(output);
// console.log('rest::', obj);
resolve({
statusCode: res.statusCode,
data: obj
});
}
catch (err) {
console.error('rest::end', err);
reject(err);
}
});
});
req.on('error', (err) => {
console.error('rest::request', err);
reject(err);
});
req.end();
});
};
As a result you don't have to pass in a callback function, instead getJSON() returns a promise. In the following example the function is used inside of an ExpressJS route handler
router.get('/:id', (req, res, next) => {
rest.getJSON({
host: host,
path: `/posts/${req.params.id}`,
method: 'GET'
}).then(({ statusCode, data }) => {
res.json(data);
}, (error) => {
next(error);
});
});
On error it delegates the error to the server error handling middleware.
Unirest is the best library I've come across for making HTTP requests from Node. It's aiming at being a multiplatform framework, so learning how it works on Node will serve you well if you need to use an HTTP client on Ruby, PHP, Java, Python, Objective C, .Net or Windows 8 as well. As far as I can tell the unirest libraries are mostly backed by existing HTTP clients (e.g. on Java, the Apache HTTP client, on Node, Mikeal's Request libary) - Unirest just puts a nicer API on top.
Here are a couple of code examples for Node.js:
var unirest = require('unirest')
// GET a resource
unirest.get('http://httpbin.org/get')
.query({'foo': 'bar'})
.query({'stack': 'overflow'})
.end(function(res) {
if (res.error) {
console.log('GET error', res.error)
} else {
console.log('GET response', res.body)
}
})
// POST a form with an attached file
unirest.post('http://httpbin.org/post')
.field('foo', 'bar')
.field('stack', 'overflow')
.attach('myfile', 'examples.js')
.end(function(res) {
if (res.error) {
console.log('POST error', res.error)
} else {
console.log('POST response', res.body)
}
})
You can jump straight to the Node docs here
Check out shred. It's a node HTTP client created and maintained by spire.io that handles redirects, sessions, and JSON responses. It's great for interacting with rest APIs. See this blog post for more details.
Check out httpreq: it's a node library I created because I was frustrated there was no simple http GET or POST module out there ;-)
For anyone who looking for a library to send HTTP requests in NodeJS, axios is also a good choice. It supports Promises :)
Install (npm): npm install axios
Example GET request:
const axios = require('axios');
axios.get('https://google.com')
.then(function (response) {
// handle success
console.log(response);
})
.catch(function (error) {
// handle error
console.log(error);
})
Github page
Update 10/02/2022
Node.js integrates fetch in v17.5.0 in experimental mode. Now, you can use fetch to send requests just like you do on the client-side. For now, it is an experimental feature so be careful.
If you just need to make simple get requests and don't need support for any other HTTP methods take a look at: simple-get:
var get = require('simple-get');
get('http://example.com', function (err, res) {
if (err) throw err;
console.log(res.statusCode); // 200
res.pipe(process.stdout); // `res` is a stream
});
Use reqclient: not designed for scripting purpose
like request or many other libraries. Reqclient allows in the constructor
specify many configurations useful when you need to reuse the same
configuration again and again: base URL, headers, auth options,
logging options, caching, etc. Also has useful features like
query and URL parsing, automatic query encoding and JSON parsing, etc.
The best way to use the library is create a module to export the object
pointing to the API and the necessary configurations to connect with:
Module client.js:
let RequestClient = require("reqclient").RequestClient
let client = new RequestClient({
baseUrl: "https://myapp.com/api/v1",
cache: true,
auth: {user: "admin", pass: "secret"}
})
module.exports = client
And in the controllers where you need to consume the API use like this:
let client = require('client')
//let router = ...
router.get('/dashboard', (req, res) => {
// Simple GET with Promise handling to https://myapp.com/api/v1/reports/clients
client.get("reports/clients")
.then(response => {
console.log("Report for client", response.userId) // REST responses are parsed as JSON objects
res.render('clients/dashboard', {title: 'Customer Report', report: response})
})
.catch(err => {
console.error("Ups!", err)
res.status(400).render('error', {error: err})
})
})
router.get('/orders', (req, res, next) => {
// GET with query (https://myapp.com/api/v1/orders?state=open&limit=10)
client.get({"uri": "orders", "query": {"state": "open", "limit": 10}})
.then(orders => {
res.render('clients/orders', {title: 'Customer Orders', orders: orders})
})
.catch(err => someErrorHandler(req, res, next))
})
router.delete('/orders', (req, res, next) => {
// DELETE with params (https://myapp.com/api/v1/orders/1234/A987)
client.delete({
"uri": "orders/{client}/{id}",
"params": {"client": "A987", "id": 1234}
})
.then(resp => res.status(204))
.catch(err => someErrorHandler(req, res, next))
})
reqclient supports many features, but it has some that are not supported by other
libraries: OAuth2 integration and logger integration
with cURL syntax, and always returns native Promise objects.
If you ever need to send GET request to an IP as well as a Domain (Other answers did not mention you can specify a port variable), you can make use of this function:
function getCode(host, port, path, queryString) {
console.log("(" + host + ":" + port + path + ")" + "Running httpHelper.getCode()")
// Construct url and query string
const requestUrl = url.parse(url.format({
protocol: 'http',
hostname: host,
pathname: path,
port: port,
query: queryString
}));
console.log("(" + host + path + ")" + "Sending GET request")
// Send request
console.log(url.format(requestUrl))
http.get(url.format(requestUrl), (resp) => {
let data = '';
// A chunk of data has been received.
resp.on('data', (chunk) => {
console.log("GET chunk: " + chunk);
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
console.log("GET end of response: " + data);
});
}).on("error", (err) => {
console.log("GET Error: " + err);
});
}
Don't miss requiring modules at the top of your file:
http = require("http");
url = require('url')
Also bare in mind that you may use https module for communicating over secured network. so these two lines would change:
https = require("https");
...
https.get(url.format(requestUrl), (resp) => { ......
## you can use request module and promise in express to make any request ##
const promise = require('promise');
const requestModule = require('request');
const curlRequest =(requestOption) =>{
return new Promise((resolve, reject)=> {
requestModule(requestOption, (error, response, body) => {
try {
if (error) {
throw error;
}
if (body) {
try {
body = (body) ? JSON.parse(body) : body;
resolve(body);
}catch(error){
resolve(body);
}
} else {
throw new Error('something wrong');
}
} catch (error) {
reject(error);
}
})
})
};
const option = {
url : uri,
method : "GET",
headers : {
}
};
curlRequest(option).then((data)=>{
}).catch((err)=>{
})
Basically I'm getting an "Invalid" return when I try to verify a payment made using the IPN Listeners from Paypal and NodeJS, I'm sending back a urlencoded post back to paypal, for them to tell me whether it is valid or not, however it always returns Invalid.
I have a node app, and I'm using expressJS to manage all these server calls.
I already tried 2 different approaches:
app.post('/ipn', bodyParser.text({ type: 'urlencoded' }), function (req, res) {
res.sendStatus(200);
console.log('accedio al ipn');
//console.log(req.body);
var params = req.body;
ipn.verify(string, {'allow_sandbox': true}, function callback(err, msg) {
if (err) {
console.log(string);
console.error(err);
} else {
// Do stuff with original params here
if (params.payment_status == 'Completed') {
// Payment has been confirmed as completed
}
}
});
});
and
app.post('/ipn', bodyParser.text({ type: 'urlencoded' }), function (req, res) {
console.log('Received POST /');
console.log(req.body);
console.log('\n\n');
// STEP 1: read POST data
req.body = req.body || {};
res.status(200).send('OK');
res.end();
// read the IPN message sent from PayPal and prepend 'cmd=_notify-validate'
var postreq = 'cmd=_notify-validate';
for (var key in req.body) {
if (req.body.hasOwnProperty(key)) {
var value = querystring.escape(req.body[key]);
postreq = postreq + "&" + key + "=" + value;
}
}
// Step 2: POST IPN data back to PayPal to validate
console.log('Posting back to paypal');
console.log(postreq);
console.log('\n\n');
var options = {
url: 'https://www.sandbox.paypal.com/cgi-bin/webscr',
method: 'POST',
headers: {
'Connection': 'close'
},
body: postreq,
strictSSL: true,
rejectUnauthorized: false,
requestCert: true,
agent: false
};
request(options, function callback(error, response, body) {
if (!error && response.statusCode === 200) {
// inspect IPN validation result and act accordingly
console.log(body);
if (body.substring(0, 8) === 'VERIFIED') {
// The IPN is verified, process it
console.log('Verified IPN!');
console.log('\n\n');
// assign posted variables to local variables
var item_name = req.body['item_name'];
var item_number = req.body['item_number'];
var payment_status = req.body['payment_status'];
var payment_amount = req.body['mc_gross'];
var payment_currency = req.body['mc_currency'];
var txn_id = req.body['txn_id'];
var receiver_email = req.body['receiver_email'];
var payer_email = req.body['payer_email'];
//Lets check a variable
console.log("Checking variable");
console.log("payment_status:", payment_status)
console.log('\n\n');
// IPN message values depend upon the type of notification sent.
// To loop through the &_POST array and print the NV pairs to the screen:
console.log('Printing all key-value pairs...')
for (var key in req.body) {
if (req.body.hasOwnProperty(key)) {
var value = req.body[key];
console.log(key + "=" + value);
}
}
} else if (body.substring(0, 7) === 'INVALID') {
// IPN invalid, log for manual investigation
console.log('Invalid IPN!');
console.log('\n\n');
}
}
});
});
Both of them return Invalid.
First example is taken from:
https://github.com/andzdroid/paypal-ipn
Second one is taken from:
https://github.com/HenryGau/node-paypal-ipn
As far as I know, I receive a payment notification using IPN on my server, then I resend that request to Paypal so I can verify it it's valid or not, for security purposes, this is all done within the sandbox environment.
I also tried to send the request I get, but this time using POSTMAN, and it also returns Invalid, been stuck here for a while.
Note: I'm trying to use this for suscription payments, I don't know if this is relevant or not, just in case.
So currently the tweet bot is;
var fs = require('fs'),
path = require('path'),
Twit = require('twit'),
config = require(path.join(__dirname, 'config.js'));
var T = new Twit(config);
function pick_random_countermagic(){
var countermagic = [
'Force-of-Will.jpg',
'Cryptic-Commad.jpg',
'Counterspell.jpg',
];
return countermagic[Math.floor(Math.random() * countermagic.length)];
}
function upload_random_image(){
console.log('Opening an image...');
var image_path = path.join(__dirname, '/countermagic/' + pick_random_countermagic()),
b64content = fs.readFileSync(image_path, { encoding: 'base64' });
console.log('Uploading an image...');
T.post('media/upload', { media_data: b64content }, function (err, data, response) {
if (err){
console.log('ERROR');
console.log(err);
}
else{
console.log('Uploaded an image!');
T.post('statuses/update', {
media_ids: new Array(data.media_id_string)
},
function(err, data, response) {
if (err){
console.log('Error!');
console.log(err);
}
else{
console.log('Posted an image!');
}
}
);
}
});
}
setInterval(
upload_random_image,
10000
);
All it does currently posts, at random an image, which is what I want, but instead of just posting it I would like it to post in reply to a targeted tweet from another twitter when that bot tweets, and or when the bot is active reply to all tweets that bot has tweeted that my bot has not already replied to... if you know what I mean.
This is my first time making a twitter bot, and technically my first time really using javascript and node.js. So yeah, I am just lost. So yeah any help would be insurmountably helpful.
You can retrieve mentions using statuses/mentions_timeline API and reply them using statuses/update API by passing mention's id in in_reply_to_status_id. Here is a simple example:
function replyMentions() {
// check for mentions
// you can add parameter 'since_id' to limit returns
T.get('statuses/mentions_timeline', { count: 100 }, function(err, data, response) {
if (data) {
data.forEach(function(mention) {
// reply if mention.id_str is not yet replied
reply = '#' + mention.user.screen_name + ' thanks for reaching out!'
T.post('statuses/update', { status: reply, in_reply_to_status_id: mention.id_str }, function(err, data, response) {
console.log(data)
// mark data.id_str as replied
})
})
}
})
}
You can add some additional logic to store replied id_str in database and use them to avoid redundant replies and limiting mentions retrieval (using since_id parameter).
I'm building a twitter sentiment reader and I am running into issues when I try and send a response back to angular. Everything worked fine before I added res.send() and it logged the sentiment.
Now that I added the res.send() function, it errors out sending the data back to angular. Angular has it as an error 500 and my node console has the error saying POST /postUsername Security Handshake Failed: some library stuff Description: End of TCP Stream".
Express Route
router.post("/postUsername", function(req, res){
console.log(req.body.userName);
var client = new Twitter({
consumer_key: '',
consumer_secret: '',
access_token_key: '',
access_token_secret: ''
});// client
client.get('statuses/user_timeline', {screen_name: req.body.userName, count:20}, function(error, tweets, response) {
tweet = analyzeIt.seperateData(tweets);
var document = language.document(tweet);
if (!error) {
var parsedScore;
var parsedMagnitude;
var finalScore;
var options = {
verbose: true
}; // options
document.detectSentiment(options, function(err, score) {
if (err) {
console.log(err);
} //if err
parsedScore = score.score;
parsedMagnitude = score.magnitude;
finalScore = analyzeIt.finalScore(parsedScore, parsedMagnitude);
console.log(finalScore);
});//detect sentiment
}//if not error
});//client get statuses
res.send(finalScore);
});//router post
Angular Controller
app.controller('myController', function($scope, $http){
$scope.sendData = function() {
var data = { userName: $scope.userName };
console.log($scope.userName);
$http.post('/postUsername', data)
.success(function (data, status, headers, config) {
$scope.PostDataResponse = data;
})
.error(function (data, status, header, config) {
$scope.PostDataResponse = "Data: " + status;
});
};
});
The expected output would be something like "This user trends positive."
Any help would be appreciated!
I see a few problems. First is that you are responding immediately, without waiting for the twitter request to complete.
// Call order: #1
client.get(..., function(error, tweets, response) {
// Call order: #3
// anything in here no longer matters
});
// Call order: #2
res.send(finalScore) //because this executes before the code in the callback above
So essentially when the call is made from angular, express immediately sends back the value of finalScore which is undefined.
The other problem is you aren't really handing the error case. If there is an error with the twitter client, you should respond to the request in a meaningful way, rather than just logging to the console. This way you can see, inside angular what the problem is instead of having to scratch your head and look at your server console:
if(!error) {
//do normal stuff
}
else {
res.status(500).send("Twitter error: " + error);
}
Same goes for detectSentiment:
document.detectSentiment(options, function(err, score) {
if (err) {
console.log(err);
res.status(500).send("Error detecting sentiment: " +err);
}
});
So, to fix your issue, you need to be responding inside your callbacks, not after:
router.post("/postUsername", function(req, res){
...
client.get('statuses/user_timeline', {screen_name: req.body.userName, count:20}, function(error, tweets, response) {
...
if (!error) {
...
document.detectSentiment(options, function(err, score) {
if (err) {
console.log(err);
res.status(500).send("Error detecting sentiment: " + err);
} //if err
...
console.log(finalScore);
res.send(finalScore);
});//detect sentiment
}
else {
res.status(500).send("Twitter error: " + error);
}
});//client get statuses
});//router post
It seems a little weird, at first, that you have to nest your response so deep, but it's not at all. This is the world of javascript. There are ways to use promises and deferred objects to clean up your code, but for now it's best to write it like this so that it really sinks in how asynchronous code in javascript works.
I am new to NodeJS. I am using NodeJS on server side. I am trying to make a simple REST based GET request which takes search parameter searchvar from user and returns the matching records found in a JSON array. My server code is as follows:
var express = require('express');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var ObjectId = require('mongodb').ObjectID;
var url = 'mongodb://localhost:27017/mydb';
app.get('/search', function(req, res) {
res.set({ 'content-type': 'application/json; charset=utf-8' });
res.header("Content-Type", "application/json; charset=utf-8");
res.charset = 'utf-8';
var searchvar = req.query.searchvar;
if(searchvar != null) {
var recordsArray = new Array();
console.log("searchvar :: " + searchvar );
MongoClient.connect(url, function(err, db) {
var cursor = db.collection('myCollections').find({"name": new RegExp(searchvar)});
cursor.each(function(err, doc) {
//assert.equal(err, null);
if (doc != null) {
var tempObject = {"name": doc.name, "cat": doc.cat};
recordsArray.push(tempObject);
console.dir(doc);
} else {
db.close();
}
});
if(recordsArray.length > 0) {
console.log("if........");
res.json({"status": "SUCCESS", "message": "The request processed successfully.", "records":recordsArray});
} else {
console.log("else........");
res.json({"status": "FAILURE", "message": "No records found corresponding to given search query."});
}
});
} else {
res.json({"status": "FAILURE", "message": "The searchvar is missing."});
}
//res.send('id: ' + req.query.id);
});
app.listen(3000);
When I call it with RESTClient in a browser with following URL:
http://localhost:3000/search?searchvar=aa
I get following response:
{"status": "FAILURE", "message": "No records found corresponding to given search query."}
It is an async call. I want to know the process in server using which i can return the response when the processing is done and handle this async response on client side.
Thanks in advance.
The response occurs because the if check:
if(recordsArray.length > 0) {
console.log("if........");
is executed before the callback passed to the cursor.each method has completed its processing. As a result, the array is empty.
Proposed solution
The cursor.forEach method takes a callback function that is executed asynchronously. This means you can access the results of the callback's execution only within the callback itself (except you return a Promise object). A fix would be:
cursor.forEach(function(doc) {
if (doc != null) {
var tempObject = {"name": doc.name, "cat": doc.cat};
recordsArray.push(tempObject);
console.dir(doc);
}
}, function(err) {
if(recordsArray.length > 0) {
console.log("if........");
res.json({"status": "SUCCESS", "message": "The request processed successfully.", "records":recordsArray});
} else {
console.log("else........");
res.json({"status": "FAILURE", "message": "No records found corresponding to given search query."});
}
db.close();
});
I think you're missing the callback parameter to the MongoClient Collection find() method. You should move your handling logic into the callback.
See these examples in the MongoDB documentation.
Your code would look something like this:
db.collection('myCollections').find({"name": new RegExp(searchvar)}).toArray(function(err, docs) {
// iterate through docs; perform your logic
db.close();
res.json({...});
});
I think your best bet is to go through a Nodejs/MongoDB tutorial to understand how everything works. From a quick look at your code your response is not asynchronous. Any time the server responds right aways it's not async. Async is achieved on the server side when you implement something like Websockets or HTTP Callbacks.
I agree with Jim that the problem is probably in your client (please post the code if you still have questions). Generally a JS client achieves async by using callbacks or promises, but an async client does not require an async server. The reason it is common to make async client calls is so you don't lock up your GUI while waiting for a response from the server.