When trying to load simple JSON APIs with the following code, I noticed the speeds were extremely slow:
function get(url, callback) {
http.get(url, function(r) {
var body = "";
r.on('data', function(d) {
body+=d;
});
r.on('end', function() {
callback(body, false)
});
}).on('error', function (err) {
callback(null, err.message.code)
});
}
Or posting with the following code:
function post(url, post_data, callback) {
if (!post_data) post_data = "";
var Proto = url.split("://")[0]+":";
var Host = url.split("://")[1].split("/")[0];
var Path = "/"+url.split("://")[1].replace("/", "#CHECK").split("#CHECK")[1];
var req = http.request({
host: Host,
port: '80',
path: Path,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(post_data)
}
}, function(r) {
var body = '';
r.on('data', function(d) {
body += d;
});
r.on('end', function() {
callback(body);
});
}).on('error', function (err) {
callback(null, err.message.code)
});
req.write(querystring.stringify(post_data));
req.end();
}
Both take an extremely long time to go through, usually averaging me about 150-300 milliseconds per request on small files that would take me under 80ms to load in Google Chrome. To test, I used Google Chrome to benchmark and sent simple get and post requests, but it took 2 to 3 times longer to receive a response in node for each request than it did in Google Chrome. After doing some research, I found out it could possibly be due to Chrome, like most browsers, using the Connection:"keep-alive" header, but adding this in the headers seemed to have no impact. I'm already setting http.globalAgent.maxSockets = 100000; so I know that isn't the issue. How I can speed up my requests?
Still have yet to find anything, if anyone knows if anything to speed things up please post a reply
Related
Traditionally I use jQuery for all my JS code, but I'm tasked to launch a simple API with node.js. Today is my first day with Node but I know enough about JS and closures to do OK. One of the tasks of the API is to authenticate across a third party service and being a python guy, I wanted to abstract all my outbound request calls like so:
EDIT
var http = require('http');
var init = function(nconf) {
var methods = {
/*
Helper method to create the request header
*/
headers: function(params) {
var content = JSON.stringify(params);
return {
'Content-Type': 'application/json',
'Content-Length': content.length
}
},
/*
Helper method to create the options object
which is used in making any type of
outbound http request
*/
options: function(host, path, method, params) {
return {
host: host,
port: 80,
path: path,
method: method,
headers: methods.headers(params)
}
},
/*
Helper method to abstract the making of
outbound http requests
*/
call: function(options, params, success, err) {
var req = http.request(options, success);
req.on('error', err);
req.write(params);
req.end();
},
/*
Helper method to parse the response
and return a json object
*/
parse: function(res, result) {
var responseString = '';
res.on('data', function(data) {
responseString += data;
});
res.on('end', function() {
result = JSON.parse(responseString);
});
},
/*
API method to return the latest
release and tag names
*/
latest: function(req, res, next){
// // var url = nconf.get('prod:authenticate');
//authenticate the test user
msg = methods.authenticate(nconf.get('test:user'), nconf.get("test:password"));
res.send(msg);
next();
},
/*
Method used by this API to authenticate users.
It is used to de-couple this API from the Database
Schema by calling out to the TTCPAS App and requesting it
to handle the authentication
*/
authenticate: function(username, password){
// create post parameters with API key
var params = {"username": username, "password": password, "api_key": nconf.get('api_key')};
//construct options object with params and header
var options = methods.options(nconf.get('ttcpas:host'), nconf.get('ttcpas:auth_url'), 'POST', params);
var result;
var success = function(res) {
res.setEncoding('utf-8');
methods.parse(res, result);
};
methods.call(options, params, success, function(err){});
while (typeof(result.statusCode) == 'undefined') {
//wait 1 second;
setTimeout(function(){
console.log("waiting on request at " + nconf.get('ttcpas:host') + nconf.get('ttcpas:auth_url'));
}, 1000);
}
//then down here
if (result.statusCode == 200) {return result};//success
if (result.statusCode == 403) {return "forbidden"};//forbidden
}
}
return methods;
};
module.exports.init = init;
#jfriend00 As I said I don't know how node.js is supposed to be styled. I wanted to just abstract as much as possible to make the code clean and reusable
Now when I do http://localhost:9000/latest/
I get:
{"code":"InternalError","message":"first argument must be a string or Buffer"}
Uhhh, this part will simply not work:
while (typeof(result.statusCode) == 'undefined') {
//wait 1 second;
setTimeout(function(){
console.log("waiting on request at " + nconf.get('ttcpas:host') + nconf.get('ttcpas:auth_url'));
}, 1000);
}
If result.statusCode is ever undefined, this will spin forever piling up setTimeout() calls in the event queue until eventually something fills up or you run out of memory.
Because node.js is primarily single threaded, you can't loop waiting for something to change. Because you never finish this while loop, no other node.js code gets to run so result.statusCode can never change. Thus, you have an infinite loop here.
All of your nodejs code needs to be event driven, not spin/wait loops. FYI, this is similar to browser-based Javascript.
I've got a proxy set up in nodejs that goes to one of our backend servers for data; some of that data (such as session id) is stored as cookies. what I want to do is have the proxy get the remote cookies, push then into the header of the response to the original request, then send the response back. I'm close, but hit a snag:
app.get(/\/json\/(.+)/, getJson);
var getJson = function(req, response1) {
response1.setHeader('Content-Type', 'application/json; charset=utf-8');
var before1stWrite = true;
utils.setCookies(response1, ["floo=flum"]) // this works
var options = {
host : config.scraperUrl.replace('http://', ''),
path : '/rwd/' + req.params[0] + '?' + querystring.stringify(req.query),
method : "GET",
rejectUnauthorized : false
};
var request = https.request(options, function(response2) {
response2.setEncoding('utf8');
// utils.setCookies(response1, ["flib=flah"]) // this fails, too
response2.on('data', function(d) {
if (before1stWrite) {
console.log(response2.headers['set-cookie']); // remote's cookies
utils.setCookies(response1, ["flib=flah"]) // this fails
before1stWrite = false;
}
response1.write(d);
});
response2.on('end', function() {
response1.end()
});
});
request.end();
request.on('error', function(e) {
console.error("error occurred: " + e.message);
response1.end();
});
}
setCookies(response1, cookies) just loops thru the cookies and does
res.setHeader('Set-Cookie', cookie)
The problem is that it looks like the headers have been baked by the time the second setCookies is called; moving the method to the 'data' event handler does not help. The error I get is:
http.js:689
throw new Error('Can\'t set headers after they are sent.');
Any way to add headers to response1 that I receive from the response2?
UPDATE
I fixed the code to be sure that the attempt to write to headers of response1 was done before any other writes; it is not a fix, however.
Yes, you cannot send headers after data has started flowing. Did you try setting the header after this line?
response.setEncoding('utf8');
Also, did you consider using streams rather than transferring in chunks? http://nodejs.org/api/stream.html
You'll need to buffer the data.
Doing this is pretty much like piping:
response.on('data', function(d) {
res.write(d);
});
so you're sending the response straight away. Haven't tried it but this should work:
var data = "";
response.on('data', function(d) {
data += d;
});
response.on('end', function() {
console.log(response.headersSent);
console.log(response.headers['set-cookie']);
utils.setCookies(res, ["flib=flah"])
res.write(data);
res.end();
});
Just remember you're buffering all that data into memory, not recommended for large responses.
I'm a long time PHP (CodeIgniter & WordPress) developer that only recently wanted to learn a few other languages. I've set out to learn Ruby (on Rails, and Sinatra), Python (w/ Flask framework) and Javascript with node.js.
I decided to create the most basic application I can think of, a URL expander, using each of these languages. I have managed to create a working version in every language, except node.js and Javascript.
I kinda know my problem, I know it is related to callbacks. I know I'm not doing it right. I get the basic idea of callbacks, but I just cannot figure out how to fix this mess I have created.
This is my whole code:
var http = require('http');
var url = require('url');
function expand() {
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL(response) {
console.log(response.headers.location);
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {
"Content-Type": "text/plain"
});
response.write("Hello World");
expand();
response.end();
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
start();
The server starts, and when a request is made, it calls the expand function which returns the expanded URL in the terminal. I'm trying to get it to print in the browser.
Any help is appreciated. Thanks in advance.
You've made a few flaws.
You should rewrite expand to pass the url in and pass a callback in. Any function that does anything asynchronous generally has the signature (data, callback) in node. This basically allows you to say I want this function to do something then tell me when it's done.
function expand(urlToParse, callback) {
// note we pass in the url this time
var short = url.parse(urlToParse);
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
// note we store the clientRequest object temporarily
var clientRequest = http.get(options, extractRealURL);
// Always attach the error handler and forward any errors
clientRequest.on("error", forwardError);
function extractRealURL(res) {
callback(null, res.headers.location);
}
function forwardError(error) {
callback(err);
}
}
Here the callback is expected to have the signature of (err, data) which almost all callbacks in node have. We've also added error handling which is a must.
We now change onRequest to actually call expand properly
function onRequest(request, response) {
// parse the incoming url. true flag unpacks the query string
var parsedUrl = url.parse(request.url, true),
// extract the querystring url.
// http://localhost:8888/?url=http://t.co/wbDrgquZ
urlToExpand = parsedUrl.query.url;
// call expand with the url and a callback
expand(urlToExpand, writeResponse);
function writeResponse(error, newUrl) {
// handle the error case properly
if (error) {
response.writeHead(500, { 'Content-Type': 'text/plain'});
// early return to avoid an else block
return response.end(error.message);
}
response.writeHead(200, { 'Content-Type': 'text/plain'});
// write the new url to the response
response.end(newUrl);
}
}
Here we have added error handling logic and also unpacked the actual url to expand from the query string.
Generally the pattern of doSomething<data, callback<err, result>> works very well in node.js.
It's the exact same as let result = doSomething<data> mayThrow err that you expect in your normal blocking languages except it's asynchronous.
Note that the alternative option of passing the ServerResponse object into the function is frowned upon, by doing so your creating unnecessary hard coupling between the expand function and the server response.
The expand function should only expand an url and return the expanded url, it has no business doing IO itself.
Full code
A callback is just a word to describe a function that we pass to some other code for that other code to invoke.
In your example, onRequest is a callback function that gets passed to createServer to be invoked whenever a request is received.
I think the issue you're having is that you're expecting expand() to have access to all the same variables/parameters that the onRequest function has access to. This isn't the case.
You need pass the response object to expand(). Because the call to expand creates a new callback longURL for the http.get call, it will have access to the response object that you passed in.
function expand( resp ) {
// receive the original response object, and end the response when ready
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL( response ) {
console.log(response.headers.location);
resp.end( response.headers.location ); // end the original response
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {
"Content-Type": "text/plain"
});
response.write("Hello World");
expand( response ); // pass this response object to expand
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
You weren't sending the response as a parameter to the expand function and also you were calling response.end() before the expand() function could write anything, here's the corrected version:
var http = require('http');
var url = require('url');
function expand(res) {
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL(response){
console.log(response.headers.location);
res.end("<br />" + response.headers.location);
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {"Content-Type": "text/html"});
response.write("Hello World");
expand(response);
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
start();
I am making a request to the facebook api to get a list of friends. When I make the request through node.js, my request is always truncated. Does anyone understand why the response is being truncated?
Here is the code of my function:
var loadFriends;
loadFriends = function(access_token, callback) {
var https, options, start;
https = require('https');
start = new Date();
options = {
host: 'graph.facebook.com',
port: 443,
path: '/me/friends?access_token=' + access_token
};
return https.get(options, function(res) {
console.log("Request took:", new Date() - start, "ms");
res.setEncoding("utf8");
return res.on("data", function(responseData) {
var data;
console.log(responseData);
data = JSON.parse(responseData);
return callback(data);
});
});
};
The res.on('data') event will happen multiple times as chunks of data arrives; you need to concatenate this together to get the whole response.
http://nodejs.org/docs/v0.4.0/api/http.html#event_data_
I'm having a really frustrating problem I hope someone can help me with. Here is a piece of my Greasemonkey script, I can't figure out why the asynchronous requests are always sent to the same URL.
function parse(details) {
var element = $(details);
var coll = element.find("#my valid selector");
$.each(coll, function(index, href) {
SendData(href);
});
}
function SendData(url) {
GM_xmlhttpRequest ({
method: 'GET',
url: url,
headers: {
'User-agent': 'Mozilla/4.0 (compatible) Greasemonkey',
'Accept': 'application/atom+xml,application/xml,text/xml',
},
onload: function(responseDetails) {
doSomething(responseDetails.responseText);
}
});
}
When I fire up Fiddler, I can see that it makes the same request no matter how many items are in my collection. Whatever the first link is, all requests are made to the that link. I have verified that the parse method successfully passes a different link to the SendData function every time, but the requests are always made to the first URL in the collection.
I thought what I had was similar to what I found here, but maybe I'm missing something. Any help would be appreciated.
It seems as though url is not getting captured in a closure, so it's undefined for all but the first GM_xmlhttpRequest run.
Modifying SendData(), like so:
function SendData(url)
{
var moreSubstantial = url + " ";
GM_xmlhttpRequest(
{
method: 'GET',
url: moreSubstantial,
should be enough.
Or, you can get the pages sequentially. Change parse() to something like:
function parse (details)
{
var element = $(details);
var coll = element.find("#my valid selector");
var TargetPages = coll.map (function() {return this.href;} );
(function getNextPage (J)
{
var PageURL = TargetPages[J];
GM_xmlhttpRequest
( {
method: "GET",
url: PageURL,
headers: {
'User-agent': 'Mozilla/4.0 (compatible) Greasemonkey',
'Accept': 'application/atom+xml,application/xml,text/xml',
},
onload: function (responseDetails)
{
doSomething (responseDetails.responseText);
if (--J >= 0)
getNextPage (J);
}
} );
} ) (TargetPages.length - 1);
}