I'm having trouble extracting the response body of a POST request in Node.js.I'm expecting the RESPONSE: 'access_token=...'
Should be pretty simple, not sure what I should be doing though. (Node v0.4.3)
Here's my code snippet.
payload = 'client_id='+client_id + '&client_secret='+ client_secret
+ '&code='+ code
var options = {
host: 'github.com',
path: '/login/oauth/access_token?',
method: 'POST'
};
var access_req = https.request(options, function(response){
response.on('error', function(err){
console.log("Error: " + err);
});
// response.body is undefined
console.log(response.statusCode);
});
access_req.write(payload);
access_req.end();
console.log("Sent the payload " + payload + "\n");
res.send("(Hopefully) Posted access exchange to github");
You'll need to bind to response's data event listener. Something like this:
var access_req = https.request(options, function(response){
response.on('data', function(chunk) {
console.log("Body chunk: " + chunk);
});
});
As Miikka says, the only way to get the body of a response in Node.js is to bind an event and wait for each chunk of the body to arrive. There is no response.body property. The http/https modules are very low-level; for nearly all applications, it makes more sense to use a higher-level wrapper library.
In your case, mikeal's request library is perfectly suited to the task. It waits until the full body of the response has arrived, then calls your callback of the form (err, response, body).
For parsing request bodies, you would probably want to use Connect with the bodyParser middleware (or the popular Express, which further extends Connect).
Related
I need to perform GET requests on a server that doesn't have a CORS header, so I created a very simple node app that will do the GET for me:
var io = require('socket.io')();
var request = require('request');
io.on('connection', function(socket) {
socket.on('get', function(url) {
request(url, function(error, response, body) {
socket.emit('response', body);
});
});
});
On the client side, I'm using it with a simple get function:
var socket = io();
function get(url) {
return new Promise(function(resolve) {
socket.emit('get', url);
socket.once('response', function(data) {
resolve(data);
});
}
}
This works great when I request a URL one at a time:
get('http://some/resource/1');
But when I try multiple requests to different endpoints at the same time, they will all return the same data:
get('http://some/resource/1');
get('http://some/resource/2');
get('http://some/resource/3');
In this case, the data for /resource/1 is returned for all 3. I understand that this is because the socket.once('response') callback is getting added immediately for all 3 requests, and when the data is returned for the first one, all 3 callbacks are invoked for it.
Is there a way that I can get it so that each socket.emit() and socket.on() are paired to each other? I know that I can add a unique ID to each emit and pass it back on the response, or use the unique ID in the message itself (something like socket.emit('get:a54jk14hf'), but I'm hoping for a simpler solution if there is one.
The issue is that you are overwriting the socket.io's "response" event by issuing the "once" hook inside the "get" function. It's fine to keep it in there, but you'll need to provide a unique event for "once", that way four unique events will come back. This could be done using the "url" as key. It would look something like this:
Server:
var io = require('socket.io')();
var request = require('request');
io.on('connection', function(socket) {
socket.on('get', function(url) {
request(url, function(error, response, body) {
socket.emit(
'response-' + url, // USING THIS INSTEAD OF JUST "response"
body
);
});
});
});
Client:
var socket = io();
function get(url) {
return new Promise(function(resolve) {
socket.emit('get', url);
socket.once(
'response-' + url, // USING THIS INSTEAD OF JUST "response"
function(data) {
resolve(data);
});
}
}
I'm not sure if this was a requirement of yours, but the above approach would also enable you to have the correct data always come back to the Promise's "then" block.
get('http://some/resource/1').then(function (data) {
// the "data" is now guaranteed to be from resource "1"
// previous, it could have been from 1, 2, or 3
})
I am trying to send a single https request to an API, with a bunch of data in JSON format.
However, when running the code, I first ran into the problem that everything was undefined. I commented out my loop, trying to parse the data I needed, and right now I'm just trying to console.log all of the data.
However, it seems to still be looping through stuff somehow despite not having a loop anywhere in my code anymore.
here is the code for my request:
function getCards() {
// make a request
var options = {
host: 'omgvamp-hearthstone-v1.p.mashape.com',
path: '/cards',
method: 'GET',
};
var req = https.request(options, function(res) {
console.log('STATUS ' + res.statusCode);
console.log('HEADERS ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function(data) {
//console.log(data);
updateCardsCollection(JSON.stringify(data));
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.end();
}
The weird thing is that the console.log(data) in the above code, logs out all of the data just fine.
function updateCardsCollection(data) {
var cardsRaw = [];
console.log("DATA");
console.log("===========================================================");
console.log(data.Classic);
}
Here "Classic" is one of the arrays of objects in the API.
Before implementing node, express, mongoose and jade. The following loop worked fine to parse through the data:
for(var key in data) {
for(var i = 0; i < data[key].length; i++) {
console.log(data[key][i].cardId);
However the above beginning of a loop would print out an undefined error as well.
Another strange problem I seem to be having with this code is when I run it with the loop commented out, with just the 3 console.logs in the 'updateCardsCollection' function. It logs those 3 lines a lot of times. Despite the function only being called once.
Any idea why this code is no longer working for getting my API data?
At first, you need to do JSON.parse instead of JSON.stringify while you try to update your collection, if you want work with objects, not with string.
But if you done with first, your code still not work, because in data handler you get chunked data, not full response. See https://nodejs.org/api/http.html#http_http_request_options_callback for it.
You must accumulate data like this:
const req = https.request(options, res => {
const data = [];
res.on('data', d => data.push(d));
res.on('end', () => updateCardsCollection(JSON.parse(data.join(''))));
})
I've got a proxy set up in nodejs that goes to one of our backend servers for data; some of that data (such as session id) is stored as cookies. what I want to do is have the proxy get the remote cookies, push then into the header of the response to the original request, then send the response back. I'm close, but hit a snag:
app.get(/\/json\/(.+)/, getJson);
var getJson = function(req, response1) {
response1.setHeader('Content-Type', 'application/json; charset=utf-8');
var before1stWrite = true;
utils.setCookies(response1, ["floo=flum"]) // this works
var options = {
host : config.scraperUrl.replace('http://', ''),
path : '/rwd/' + req.params[0] + '?' + querystring.stringify(req.query),
method : "GET",
rejectUnauthorized : false
};
var request = https.request(options, function(response2) {
response2.setEncoding('utf8');
// utils.setCookies(response1, ["flib=flah"]) // this fails, too
response2.on('data', function(d) {
if (before1stWrite) {
console.log(response2.headers['set-cookie']); // remote's cookies
utils.setCookies(response1, ["flib=flah"]) // this fails
before1stWrite = false;
}
response1.write(d);
});
response2.on('end', function() {
response1.end()
});
});
request.end();
request.on('error', function(e) {
console.error("error occurred: " + e.message);
response1.end();
});
}
setCookies(response1, cookies) just loops thru the cookies and does
res.setHeader('Set-Cookie', cookie)
The problem is that it looks like the headers have been baked by the time the second setCookies is called; moving the method to the 'data' event handler does not help. The error I get is:
http.js:689
throw new Error('Can\'t set headers after they are sent.');
Any way to add headers to response1 that I receive from the response2?
UPDATE
I fixed the code to be sure that the attempt to write to headers of response1 was done before any other writes; it is not a fix, however.
Yes, you cannot send headers after data has started flowing. Did you try setting the header after this line?
response.setEncoding('utf8');
Also, did you consider using streams rather than transferring in chunks? http://nodejs.org/api/stream.html
You'll need to buffer the data.
Doing this is pretty much like piping:
response.on('data', function(d) {
res.write(d);
});
so you're sending the response straight away. Haven't tried it but this should work:
var data = "";
response.on('data', function(d) {
data += d;
});
response.on('end', function() {
console.log(response.headersSent);
console.log(response.headers['set-cookie']);
utils.setCookies(res, ["flib=flah"])
res.write(data);
res.end();
});
Just remember you're buffering all that data into memory, not recommended for large responses.
Just learning Sails.js so go easy on me.
I have queried an XML service and successfully jsonified it using xml2js
var req = https.request(options, function(res) {
var xml = '';
res.on('data', function(chunk) {
xml += chunk;
});
res.on('end', function () {
var result = parseString(xml, function (err, result) {
console.log(JSON.stringify(result)); // Position 1
});
return result;
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.write(data);
var result = req.end();
console.log('Result: ' + JSON.stringify(result)); // Position 2
res.view({ message : 'hello', result : result });
The view is loading fine, and <%= message %> outputs hello. Great.
Position1 console.log is returning the stringified json object - Great.
Position 2 consile.log is returning Result: true - Not good.
I need to be able to get that json data to my view for parsing. How do I do this?
It looks like you're assuming that calling req.end() will give you the response from the https.request you started above. There are a couple of things wrong with that:
req.end() is used to finish writing to an open request, not to get a response. According to the docs, the return value is unspecified.
The https.request call is asynchronous; even if req.end() worked like you want it to, the response wouldn't have come in by the time you call it.
The solution is to put your response code (i.e. your res.view) inside the handler for the end event that you've already written. I'd also recommend refactoring your code to use different variable names for the remote request / response so that they don't collide with the req and res variables in your controller action. The whole thing would then be something like:
myAction: function (req, res) {
// Not sure how you're setting options, so just an example
var options = {url: 'http://example.com', ...}
var request = https.request(options, function(response) {
var xml = '';
response.on('data', function(chunk) {
xml += chunk;
});
response.on('end', function () {
var result = parseString(xml, function (err, result) {
return res.view({ message : 'hello', result : JSON.stringify(result)});
});
});
});
request.on('error', function(e) {
console.log('problem with request: ' + e.message);
res.serverError(e);
});
}
You might also look into using something like the Request module to simplify your external request; it would save you from having to write event handlers for data and end.
if you want to pass json to some javascript variable:
var clientJsonVar = <%- JSON.stringify(serverSideJson)%>
I am trying to interface with an external API and I need to POST an XML document over HTTPS.
So I am using the node https interface to try to make the request but if I try to write any data (The XML document) it throws a socket hang up. If I write nothing or an empty string to the request it completes the post just fine.
I've googled and found other people with this error but I haven't been able to fix it following the solutions others have found.
I am using Meteor, which has the HTTP package for making these types of requests. The HTTP package was also throwing this error so I dug down and implemented the post using the node 'https' package thinking it would solve the issue but I get the same error with https as well.
Here is my code:
var http = Npm.require("http");
var oauthSignature = Npm.require('oauth-signature');
var URL = Npm.require('url');
var content = "XML String Here";
var postDestinationUrl = "https://example.com/api/path";
var authObject = {
oauth_consumer_key: "consumerKey",
oauth_signature_method: "HMAC-SHA1",
oauth_timestamp: (Math.round(new Date().getTime() / 1000)).toString(10),
oauth_nonce: Random.id(),
oauth_version: "1.0"
};
authObject.oauth_signature = oauthSignature.generate("POST", postDestinationUrl, authObject, "shared Secret Here");
var authString = objectToQueryString(authObject);
var headers = {
Connection: "keep-alive",
'Content-Length': Buffer.byteLength(content),
Authorization: authString,
'Content-Type': 'application/xml'
};
var parsedUrl = URL.parse(postDestinationUrl);
var requestOptions = {
hostname: parsedUrl.hostname,
path: parsedUrl.pathname,
method: "POST",
headers: headers
};
var request = https.request(requestOptions, function(response){
var body = '';
console.log("statusCode from https.request: ", response.statusCode);
console.log("headers from https.request: ", response.headers);
response.on("data", function(data){
body += data;
}); // end of data
response.on("end", function(){
console.log("body from https.request: ", body);
}); // end of end
response.on("error", function(){
console.log("error in https.request response");
}); // end of error
}); // end of request
request.write(content);
request.end();
request.on("error",function(error){
console.log("Error in https.request: " + error.message);
callback(error, undefined);
}); // end of error
var objectToQueryString = function(queryObject){
var queryString = "";
_.each(queryObject, function(value, key, list){
queryString += key + '="' + value +'",';
});
return queryString.substr(0, queryString.length -1);
};
And the error I am seeing:
"stack":"Error: socket hang up
at Object.Future.wait (/Users/dsyko/.meteor/tools/f3947a4651/lib/node_modules/fibers/future.js:326:15)
at packages/meteor/helpers.js:111
at Meteor.methods.submitAPIPost (packages/api-interface/api_server.js:412)
at packages/check/match.js:77
at _.extend.withValue (packages/meteor/dynamics_nodejs.js:35)
at Object.Match._failIfArgumentsAreNotAllChecked (packages/check/match.js:76)
at maybeAuditArgumentChecks (packages/livedata/livedata_server.js:1403)
at packages/livedata/livedata_server.js:580
at _.extend.withValue (packages/meteor/dynamics_nodejs.js:35)
at packages/livedata/livedata_server.js:579
- - - - -
at createHangUpError (http.js:1472:15)
at CleartextStream.socketCloseListener (http.js:1522:23)
at CleartextStream.EventEmitter.emit (events.js:117:20)
at tls.js:696:10
at process._tickCallback (node.js:415:13)"
I've used postman to make the same API request and it goes through just fine so it doesn't seem to be a bad endpoint. I also switched over to posting to http and used Wireshark to inspect the headers and content of the HTTP POSTs to make sure I'm not mangling something in the request but it all looked ok there. Also when I switched over to http the XML document goes through just fine and I don't see the socket hangup (Although the endpoint responds with a re-direct to the https url so I can't just use http)