We Keep Coding

http.get and http.request are both being extremely slow?

When trying to load simple JSON APIs with the following code, I noticed the speeds were extremely slow:
function get(url, callback) {
http.get(url, function(r) {
var body = "";
r.on('data', function(d) {
body+=d;
});
r.on('end', function() {
callback(body, false)
});
}).on('error', function (err) {
callback(null, err.message.code)
});
}
Or posting with the following code:
function post(url, post_data, callback) {
if (!post_data) post_data = "";
var Proto = url.split("://")[0]+":";
var Host = url.split("://")[1].split("/")[0];
var Path = "/"+url.split("://")[1].replace("/", "#CHECK").split("#CHECK")[1];
var req = http.request({
host: Host,
port: '80',
path: Path,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(post_data)
}
}, function(r) {
var body = '';
r.on('data', function(d) {
body += d;
});
r.on('end', function() {
callback(body);
});
}).on('error', function (err) {
callback(null, err.message.code)
});
req.write(querystring.stringify(post_data));
req.end();
}
Both take an extremely long time to go through, usually averaging me about 150-300 milliseconds per request on small files that would take me under 80ms to load in Google Chrome. To test, I used Google Chrome to benchmark and sent simple get and post requests, but it took 2 to 3 times longer to receive a response in node for each request than it did in Google Chrome. After doing some research, I found out it could possibly be due to Chrome, like most browsers, using the Connection:"keep-alive" header, but adding this in the headers seemed to have no impact. I'm already setting http.globalAgent.maxSockets = 100000; so I know that isn't the issue. How I can speed up my requests?
Still have yet to find anything, if anyone knows if anything to speed things up please post a reply

Is there a way to get the response for a specific emit using socket.io?

I need to perform GET requests on a server that doesn't have a CORS header, so I created a very simple node app that will do the GET for me:
var io = require('socket.io')();
var request = require('request');
io.on('connection', function(socket) {
socket.on('get', function(url) {
request(url, function(error, response, body) {
socket.emit('response', body);
});
});
});
On the client side, I'm using it with a simple get function:
var socket = io();
function get(url) {
return new Promise(function(resolve) {
socket.emit('get', url);
socket.once('response', function(data) {
resolve(data);
});
}
}
This works great when I request a URL one at a time:
get('http://some/resource/1');
But when I try multiple requests to different endpoints at the same time, they will all return the same data:
get('http://some/resource/1');
get('http://some/resource/2');
get('http://some/resource/3');
In this case, the data for /resource/1 is returned for all 3. I understand that this is because the socket.once('response') callback is getting added immediately for all 3 requests, and when the data is returned for the first one, all 3 callbacks are invoked for it.
Is there a way that I can get it so that each socket.emit() and socket.on() are paired to each other? I know that I can add a unique ID to each emit and pass it back on the response, or use the unique ID in the message itself (something like socket.emit('get:a54jk14hf'), but I'm hoping for a simpler solution if there is one.

The issue is that you are overwriting the socket.io's "response" event by issuing the "once" hook inside the "get" function. It's fine to keep it in there, but you'll need to provide a unique event for "once", that way four unique events will come back. This could be done using the "url" as key. It would look something like this:
Server:
var io = require('socket.io')();
var request = require('request');
io.on('connection', function(socket) {
socket.on('get', function(url) {
request(url, function(error, response, body) {
socket.emit(
'response-' + url, // USING THIS INSTEAD OF JUST "response"
body
);
});
});
});
Client:
var socket = io();
function get(url) {
return new Promise(function(resolve) {
socket.emit('get', url);
socket.once(
'response-' + url, // USING THIS INSTEAD OF JUST "response"
function(data) {
resolve(data);
});
}
}
I'm not sure if this was a requirement of yours, but the above approach would also enable you to have the correct data always come back to the Promise's "then" block.
get('http://some/resource/1').then(function (data) {
// the "data" is now guaranteed to be from resource "1"
// previous, it could have been from 1, 2, or 3
})

Angular with hapi js server jsonp

I have an endpoint defined at /api/profile which accepts post parameters.
var http = require('http');
var serverConfig = require('../server.config.js');
var request = require('request');
module.exports = function(server){
server.route({
method: 'POST',
path: '/api/profile',
handler: getProfileData
});
function getProfileData(request, reply){
var battleTag = request.payload.battleTag;
getProfileDataHttp(battleTag, function(err, data){
if(err){
reply(new Error(err));
}
reply(data);
});
}
function getProfileDataHttp(battleTag, callback){
var key = serverConfig.battleNet.apiKey;
var tag = encodeURIComponent(battleTag);
var url = 'https://eu.api.battle.net/d3/profile/'+ tag + '/?locale=en_GB&callback=JSON_CALLBACK&apikey=' + key;
console.log(url);
request(url,function(error, response, body){
if(error){
callback(err);
}
if(!error && response.statusCode ==200){
callback(null, body);
}
});
}
};
it is calling an api with a json callback, when I am receiving the data it is in format:
JSON_CALLBACK({ json data here})
how can I get this endpoint to return just the json data, I have tried JSON.parse() but it causes errors in the server.
the angular service that calls this endpoint is like below:
function getProfileData(battleTag){
var defer = $q.defer();
var tag = validTag(battleTag);
if(!tag){
defer.reject('Invalid Tag please use format 1[a-z]11[a-z0-9]#4[0-9]');
return defer.promise;
}
$http.post('/api/profile', {
battleTag: battleTag
})
.success(function(data){
if(data.reason){
defer.resolve(data.reason);
}
defer.resolve(data);
})
.error(function(err){
defer.reject(err);
});
return defer.promise;
}
the call would work when using $http.jsonp in angular however I had to create the server to hide the secret key from the client

Your question is a bit confusing. You are talking about JSONP, but you want to fetch the data directly.
The whole point of JSONP is to return the data encapsulated inside a function that you choose. You then simply have to execute it.
If you want the data in a direct way, don't use JSONP. Simply do a "normal" call.
After having a quick look at the Battle.net API, it seems that to get the data directly, you should simply omit the 'callback' parameter in the URL of your request.
Thus, your request URL would looks like that:
var url = 'https://eu.api.battle.net/d3/profile/'+ tag + '/?locale=en_GB&apikey=' + key;

Fetch data on different server with backbone.js

I can't see what the problem with this is.
I'm trying to fetch data on a different server, the url within the collection is correct but returns a 404 error. When trying to fetch the data the error function is triggered and no data is returned. The php script that returns the data works and gives me the output as expected. Can anyone see what's wrong with my code?
Thanks in advance :)
// function within view to fetch data
fetchData: function()
{
console.log('fetchData')
// Assign scope.
var $this = this;
// Set the colletion.
this.collection = new BookmarkCollection();
console.log(this.collection)
// Call server to get data.
this.collection.fetch(
{
cache: false,
success: function(collection, response)
{
console.log(collection)
// If there are no errors.
if (!collection.errors)
{
// Set JSON of collection to global variable.
app.userBookmarks = collection.toJSON();
// $this.loaded=true;
// Call function to render view.
$this.render();
}
// END if.
},
error: function(collection, response)
{
console.log('fetchData error')
console.log(collection)
console.log(response)
}
});
},
// end of function
Model and collection:
BookmarkModel = Backbone.Model.extend(
{
idAttribute: 'lineNavRef'
});
BookmarkCollection = Backbone.Collection.extend(
{
model: BookmarkModel,
//urlRoot: 'data/getBookmarks.php',
urlRoot: 'http://' + app.Domain + ':' + app.serverPort + '/data/getBookmarks.php?fromCrm=true',
url: function()
{
console.log(this.urlRoot)
return this.urlRoot;
},
parse: function (data, xhr)
{
console.log(data)
// Default error status.
this.errors = false;
if (data.responseCode < 1 || data.errorCode < 1)
{
this.errors = true;
}
return data;
}
});

You can make the requests using JSONP (read about here: http://en.wikipedia.org/wiki/JSONP).
To achive it using Backbone, simply do this:
var collection = new MyCollection();
collection.fetch({ dataType: 'jsonp' });
You backend must ready to do this. The server will receive a callback name generated by jQuery, passed on the query string. So the server must respond:
name_of_callback_fuction_generated({ YOUR DATA HERE });
Hope I've helped.

This is a cross domain request - no can do. Will need to use a local script and use curl to access the one on the other domain.

Understanding callbacks in Javascript and node.js

I'm a long time PHP (CodeIgniter & WordPress) developer that only recently wanted to learn a few other languages. I've set out to learn Ruby (on Rails, and Sinatra), Python (w/ Flask framework) and Javascript with node.js.
I decided to create the most basic application I can think of, a URL expander, using each of these languages. I have managed to create a working version in every language, except node.js and Javascript.
I kinda know my problem, I know it is related to callbacks. I know I'm not doing it right. I get the basic idea of callbacks, but I just cannot figure out how to fix this mess I have created.
This is my whole code:
var http = require('http');
var url = require('url');
function expand() {
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL(response) {
console.log(response.headers.location);
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {
"Content-Type": "text/plain"
});
response.write("Hello World");
expand();
response.end();
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
start();
The server starts, and when a request is made, it calls the expand function which returns the expanded URL in the terminal. I'm trying to get it to print in the browser.
Any help is appreciated. Thanks in advance.

You've made a few flaws.
You should rewrite expand to pass the url in and pass a callback in. Any function that does anything asynchronous generally has the signature (data, callback) in node. This basically allows you to say I want this function to do something then tell me when it's done.
function expand(urlToParse, callback) {
// note we pass in the url this time
var short = url.parse(urlToParse);
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
// note we store the clientRequest object temporarily
var clientRequest = http.get(options, extractRealURL);
// Always attach the error handler and forward any errors
clientRequest.on("error", forwardError);
function extractRealURL(res) {
callback(null, res.headers.location);
}
function forwardError(error) {
callback(err);
}
}
Here the callback is expected to have the signature of (err, data) which almost all callbacks in node have. We've also added error handling which is a must.
We now change onRequest to actually call expand properly
function onRequest(request, response) {
// parse the incoming url. true flag unpacks the query string
var parsedUrl = url.parse(request.url, true),
// extract the querystring url.
// http://localhost:8888/?url=http://t.co/wbDrgquZ
urlToExpand = parsedUrl.query.url;
// call expand with the url and a callback
expand(urlToExpand, writeResponse);
function writeResponse(error, newUrl) {
// handle the error case properly
if (error) {
response.writeHead(500, { 'Content-Type': 'text/plain'});
// early return to avoid an else block
return response.end(error.message);
}
response.writeHead(200, { 'Content-Type': 'text/plain'});
// write the new url to the response
response.end(newUrl);
}
}
Here we have added error handling logic and also unpacked the actual url to expand from the query string.
Generally the pattern of doSomething<data, callback<err, result>> works very well in node.js.
It's the exact same as let result = doSomething<data> mayThrow err that you expect in your normal blocking languages except it's asynchronous.
Note that the alternative option of passing the ServerResponse object into the function is frowned upon, by doing so your creating unnecessary hard coupling between the expand function and the server response.
The expand function should only expand an url and return the expanded url, it has no business doing IO itself.
Full code

A callback is just a word to describe a function that we pass to some other code for that other code to invoke.
In your example, onRequest is a callback function that gets passed to createServer to be invoked whenever a request is received.
I think the issue you're having is that you're expecting expand() to have access to all the same variables/parameters that the onRequest function has access to. This isn't the case.
You need pass the response object to expand(). Because the call to expand creates a new callback longURL for the http.get call, it will have access to the response object that you passed in.
function expand( resp ) {
// receive the original response object, and end the response when ready
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL( response ) {
console.log(response.headers.location);
resp.end( response.headers.location ); // end the original response
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {
"Content-Type": "text/plain"
});
response.write("Hello World");
expand( response ); // pass this response object to expand
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}

You weren't sending the response as a parameter to the expand function and also you were calling response.end() before the expand() function could write anything, here's the corrected version:
var http = require('http');
var url = require('url');
function expand(res) {
var short = url.parse('http://t.co/wbDrgquZ');
var options = {
host: short.hostname,
port: 80,
path: short.pathname
};
function longURL(response){
console.log(response.headers.location);
res.end("<br />" + response.headers.location);
}
http.get(options, longURL);
}
function start() {
function onRequest(request, response) {
console.log("Request received.");
response.writeHead(200, {"Content-Type": "text/html"});
response.write("Hello World");
expand(response);
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
start();