I've got a proxy set up in nodejs that goes to one of our backend servers for data; some of that data (such as session id) is stored as cookies. what I want to do is have the proxy get the remote cookies, push then into the header of the response to the original request, then send the response back. I'm close, but hit a snag:
app.get(/\/json\/(.+)/, getJson);
var getJson = function(req, response1) {
response1.setHeader('Content-Type', 'application/json; charset=utf-8');
var before1stWrite = true;
utils.setCookies(response1, ["floo=flum"]) // this works
var options = {
host : config.scraperUrl.replace('http://', ''),
path : '/rwd/' + req.params[0] + '?' + querystring.stringify(req.query),
method : "GET",
rejectUnauthorized : false
};
var request = https.request(options, function(response2) {
response2.setEncoding('utf8');
// utils.setCookies(response1, ["flib=flah"]) // this fails, too
response2.on('data', function(d) {
if (before1stWrite) {
console.log(response2.headers['set-cookie']); // remote's cookies
utils.setCookies(response1, ["flib=flah"]) // this fails
before1stWrite = false;
}
response1.write(d);
});
response2.on('end', function() {
response1.end()
});
});
request.end();
request.on('error', function(e) {
console.error("error occurred: " + e.message);
response1.end();
});
}
setCookies(response1, cookies) just loops thru the cookies and does
res.setHeader('Set-Cookie', cookie)
The problem is that it looks like the headers have been baked by the time the second setCookies is called; moving the method to the 'data' event handler does not help. The error I get is:
http.js:689
throw new Error('Can\'t set headers after they are sent.');
Any way to add headers to response1 that I receive from the response2?
UPDATE
I fixed the code to be sure that the attempt to write to headers of response1 was done before any other writes; it is not a fix, however.
Yes, you cannot send headers after data has started flowing. Did you try setting the header after this line?
response.setEncoding('utf8');
Also, did you consider using streams rather than transferring in chunks? http://nodejs.org/api/stream.html
You'll need to buffer the data.
Doing this is pretty much like piping:
response.on('data', function(d) {
res.write(d);
});
so you're sending the response straight away. Haven't tried it but this should work:
var data = "";
response.on('data', function(d) {
data += d;
});
response.on('end', function() {
console.log(response.headersSent);
console.log(response.headers['set-cookie']);
utils.setCookies(res, ["flib=flah"])
res.write(data);
res.end();
});
Just remember you're buffering all that data into memory, not recommended for large responses.
Related
In my Node JS server I have this route handler that sends a request to a third party API to get a username:
app.get('/players/:player', apiLimiter, function(request, response) {
const player = request.params.player;
const api_url = `https://api.com/shards/steam/players?filter[playerNames]=${player}`;
var options = {
method: "GET",
observe: 'body',
};
let apiRequest = https.request(api_url, options, function (res) {
let data = "";
res.on("data", chunk => {
data += chunk;
})
res.on("end", () => {
let objectParsed = JSON.parse(JSON.stringify(data));
response.send(objectParsed);
})
if(!player) {
res.status(404).send("Not found.");
}
})
apiRequest.end();
})
This works fine to get a user that exists. However, if I put in a fake username to my /players page, that page still loads with a 200 status instead of getting a 404 response. The page loads and looks broken because it's not actually getting any data from the API.
I feel like this is a dumb question .. In my research I have found how to handle errors if it's just the route, and not if it's the route dependent on the path parameter as in /players/:player
I found a question that was similar to mine (How to throw a 404 error in express.js?) and I tried using an If statement: if (!player){res.status(404).send("Not found."); } but no dice. Am I using this if statement in the wrong place?
How can I get my Node JS server to respond with a 404 if the user from the database doesn't exist?
You have to check the result of the API call and see if you got valid data back and send the 404 there. I also added a check to make sure something was passed for the player name and send back a 400 (bad request) if there's no player specified at all:
app.get('/players/:player', apiLimiter, function(request, response) {
const player = request.params.player;
if (!player) {
res.status(400).send("No player specified.");
return;
}
const api_url = `https://api.com/shards/steam/players?filter[playerNames]=${player}`;
var options = {
method: "GET",
observe: 'body',
};
let apiRequest = https.request(api_url, options, function(res) {
let data = "";
res.on("data", chunk => {
data += chunk;
})
res.on("end", () => {
let objectParsed = JSON.parse(data);
// test objectParsed here
if (!some condition in objectParsed) {
res.status(404).send("No data for that player name.");
} else {
response.send(objectParsed);
}
});
});
apiRequest.end();
});
Also, you don't want JSON.parse(JSON.stringify(data)) here. Your data is already a string. Just do JSON.parse(data).
FYI, if you use a small http request library such as got(), this code gets a lot simpler as it accumulates the response and parses the JSON for you in one line of code as in:
let data = await got(options).json()
I'm creating a script that will make a request 2 times per second to a localserver of cameras network and after it gets a positive response that camera detected someone I want to log three images.
In the json config file I have the triggerURL of the server, the interval port, the dataDir where logged images should be saved and a track array which contains the url of those images and the fileName they should receive.
This is the code of the script I use after reading the JSON file:
var configGet = {
host: config.triggerURL
, port: config.interval
, method: 'GET'
};
setInterval(function () {
var request = http.request(configGet, function (response) {
var content = "";
// Handle data chunks
response.on('data', function (chunk) {
content += chunk;
});
// Once we're done streaming the response, parse it as json.
response.on('end', function () {
var data = JSON.parse(response);
if (data.track.length > 0) {
//log images
var download = function (uri, filename, callback) {
request.head(uri, function (err, res, body) {
request(uri)
.pipe(fs.createWriteStream(filename))
.on('close', callback);
});
};
for (var image in data.track) {
var path = config.dataDir + '/' + image.fileName
download(image.url, path.format(config.timestamp), function () {
console.log('done');
});
}
}
});
// Report errors
request.on('error', function (error) {
console.log("Error while calling endpoint.", error);
});
request.end();
}, 500);
});
I have the following questions:
This method produces some kind of error with the download process of the images.Can you identify it?
Is there a better way of doing this process?
Without running the code or deeper inspection; should not "data = JSON.parse(response)" rather be "data = JSON.parse(content)"? Also, if data is undefined or does not contain "track" the "if (data.track.length > 0)" will throw an error. This can be fixed with "if (data && data.track && data.track.length > 0)".
I can not think of a very much better way. I would break it up more in functions to make the code more clear though.
I need to perform GET requests on a server that doesn't have a CORS header, so I created a very simple node app that will do the GET for me:
var io = require('socket.io')();
var request = require('request');
io.on('connection', function(socket) {
socket.on('get', function(url) {
request(url, function(error, response, body) {
socket.emit('response', body);
});
});
});
On the client side, I'm using it with a simple get function:
var socket = io();
function get(url) {
return new Promise(function(resolve) {
socket.emit('get', url);
socket.once('response', function(data) {
resolve(data);
});
}
}
This works great when I request a URL one at a time:
get('http://some/resource/1');
But when I try multiple requests to different endpoints at the same time, they will all return the same data:
get('http://some/resource/1');
get('http://some/resource/2');
get('http://some/resource/3');
In this case, the data for /resource/1 is returned for all 3. I understand that this is because the socket.once('response') callback is getting added immediately for all 3 requests, and when the data is returned for the first one, all 3 callbacks are invoked for it.
Is there a way that I can get it so that each socket.emit() and socket.on() are paired to each other? I know that I can add a unique ID to each emit and pass it back on the response, or use the unique ID in the message itself (something like socket.emit('get:a54jk14hf'), but I'm hoping for a simpler solution if there is one.
The issue is that you are overwriting the socket.io's "response" event by issuing the "once" hook inside the "get" function. It's fine to keep it in there, but you'll need to provide a unique event for "once", that way four unique events will come back. This could be done using the "url" as key. It would look something like this:
Server:
var io = require('socket.io')();
var request = require('request');
io.on('connection', function(socket) {
socket.on('get', function(url) {
request(url, function(error, response, body) {
socket.emit(
'response-' + url, // USING THIS INSTEAD OF JUST "response"
body
);
});
});
});
Client:
var socket = io();
function get(url) {
return new Promise(function(resolve) {
socket.emit('get', url);
socket.once(
'response-' + url, // USING THIS INSTEAD OF JUST "response"
function(data) {
resolve(data);
});
}
}
I'm not sure if this was a requirement of yours, but the above approach would also enable you to have the correct data always come back to the Promise's "then" block.
get('http://some/resource/1').then(function (data) {
// the "data" is now guaranteed to be from resource "1"
// previous, it could have been from 1, 2, or 3
})
I have a situation where in order to get images for a site that I am building, I need to make a http request to an external server for information. Currently, the responses from the requests come in two forms, XML and images. I am doing this using Node.js.
For the XML, I'm able to parse it without issues and it can be passed into a variable and handled like everything else. With the images, I'm stuck, I have no idea how to get them "displayed" on the page after making the request for them. The farthest I have been able to get is to correctly set the request up in postman. My question is, can I pull the image from the body of the response of the request that I'm making to another server and get it to display in the web app that I'm building?
I'm very new to the back end world and am trying to learn as I go. This is an example of what I have been able to do find and use for parsing an XML response that I get from the API
var request = require("request");
var express = require("express");
var jsxml = require("node-jsxml");
var app = express();
var fs = require("fs");
app.get('/users', function(req,res) {
console.log("List of users requested.");
// We will grab the list of users from the specified site, but first we have to grab the site id
// (Same idea as when we added users. We could have checked if req.session.SiteID has been populated,
// but I chose to keep it simple instead)
request(
{
url: 'http://' + SERVERURL + '/api/2.0/sites/' + SITE + '?key=name',
headers: {
'Content-Type': 'text/xml',
'X-Tableau-Auth': req.session.authToken
}
},
function(err, response, body) {
if(err) {
req.session.err = err;
res.redirect('/');
} else {
var bodyXML = new jsxml.XML(body);
console.log("site id: " + siteID);
}
// OK. We have the site, now let's grab the list of users
// Since we're just making a GET request, we don't need to build the xml. All the is needed
// is the SiteID which is inserted in the url and the auth token which is included in the headers
request(
{
url: 'http://' + SERVERURL + '/api/2.0/sites/' + siteID + '/users/',
headers: {
'Content-Type': 'text/xml',
'X-Tableau-Auth': authToken
}
},
function(err, response, body) {
if(err) {
req.session.err = err;
} else {
// A succesful request returns xml with a <users> which contains multiple <user> elements.
// The <user> elements have name attributes and id attributes which we'll grab, store in a
// javascript object and render those in the html that loads.
var bodyXML = new jsxml.XML(body);
bodyXML.descendants('user').each(function(item, index) {
userIDs[item.attribute('name').getValue()] = item.attribute('id').getValue();
});
for(var user in userIDs) {
console.log(user + " " + userIDs[user]);
}
}
res.render("users.ejs", {
err: req.session.err,
userIDs: userIDs,
site: SITE
});
}
);
}
);
});
Any help would be hugely appreciated. Thanks!
Step 1: Fetch image and save it on node server. request module documentation on streaming for more options
request('http://google.com/doodle.png').pipe(fs.createWriteStream('doodle.png'));
Step 2: send the saved image as response.
app.get('/display', function(req, res)) {
fs.readFile('doodle.png', function(err, data) {
if (err) throw err; // Fail if the file can't be read.
else {
res.writeHead(200, {'Content-Type': 'image/jpeg'});
res.end(data); // Send the file data to the browser.
}
});
};
I'm having trouble extracting the response body of a POST request in Node.js.I'm expecting the RESPONSE: 'access_token=...'
Should be pretty simple, not sure what I should be doing though. (Node v0.4.3)
Here's my code snippet.
payload = 'client_id='+client_id + '&client_secret='+ client_secret
+ '&code='+ code
var options = {
host: 'github.com',
path: '/login/oauth/access_token?',
method: 'POST'
};
var access_req = https.request(options, function(response){
response.on('error', function(err){
console.log("Error: " + err);
});
// response.body is undefined
console.log(response.statusCode);
});
access_req.write(payload);
access_req.end();
console.log("Sent the payload " + payload + "\n");
res.send("(Hopefully) Posted access exchange to github");
You'll need to bind to response's data event listener. Something like this:
var access_req = https.request(options, function(response){
response.on('data', function(chunk) {
console.log("Body chunk: " + chunk);
});
});
As Miikka says, the only way to get the body of a response in Node.js is to bind an event and wait for each chunk of the body to arrive. There is no response.body property. The http/https modules are very low-level; for nearly all applications, it makes more sense to use a higher-level wrapper library.
In your case, mikeal's request library is perfectly suited to the task. It waits until the full body of the response has arrived, then calls your callback of the form (err, response, body).
For parsing request bodies, you would probably want to use Connect with the bodyParser middleware (or the popular Express, which further extends Connect).