I stumbled on a very strange issue recently concerning my Node.JS REST API. It's aim is to convert HTTP requests to SQL requests and the SQL response to HTTP response so I can then transfer everything over SSL/TLS and it's encoded.
The problem is that when the response size is small, it all works fine but when it exceed a certain size (about 255.37 KB), the response body is cutted right in the middle of the JSON. After multiple tests it appears that the issue is related to the HTTP response total size (including the headers) because when some custom headers are removed, more of the body is sent. I wrote a similar code in PHP and the response in JSON from the PHP API is fine so I assumed that the issue was originating from a bug in the Node.JS web server. Also, the Content-Lenght header is fine and has the right value (like the PHP API response does).
I'm using Node.JS v6.11.0 with the last release of Express. All my dependencies are up to date thanks to npm.
Here is the code of the function that handle the GET HTTP requests and proceed in doing SELECT SQL requests, then parse to json string the answer
function SelectData(connection){
return function (req, res) {
let tableName = req.params.table;
let id = req.params.id;
console.log("GET request received with tableName = " + tableName + " and id = " + id);
// Should match this regex
if (!tableName.match(/^[\-\_\s0-9A-Za-z]+$/)){
console.log("Error: Invalid table name provided : " + tableName);
badRequestError(req, res);
return;
}
if (id !== undefined){
if (!id.match(/^[0-9]+$/)){
console.log("Error: Invalid id provided : " + id + " (table name was valid : " + tableName + ")");
badRequestError(req, res);
return;
}
}
// if id is empty, then don't use any condition, else, add SQL condition
idPart = (id == undefined) ? "" : ('WHERE id="' + id + '"');
// TODO: try to replace " + var + " by question marks and bind params, find a way to do it w/ id
connection.query("SELECT * FROM " + tableName + " " + idPart + ";", function(err, result){
res.setHeader('Content-Type', 'application/json');
console.log("Request executed successfully !");
// Works too but I prefer the .send() method
// res.status(200).write(JSON.stringify(result)).end();
res.status(200).send(JSON.stringify(result));
req.connection.destroy();
return;
});
}
}
function badRequestError(req, res){
res.setHeader('Content-Type', 'text/plain');
res.status(400).send("Bad Request");
req.connection.destroy();
}
exports.SelectData = SelectData;
Edit: After researching yesterday I found similar issue resulting of the use of the HTTPS module so I removed it but it still happens.
Related
I am using a query to a postgres db to store the result in a variable in a GET request on the server-side of an Express app.
My query returns the issueID that i am looking for when i run it in psql, but I am having a hard time getting the response to store in a variable so I can send the response to the client side.
My code looks like this:
const pg = require('pg');
const connection = 'postgres://aa2:aa2#localhost:5432/bugtrackingdb';
const client = new pg.Client(connection) ;
client.connect();
app.get('/tracking', function(req, res) {
var sql = "SELECT bugtracking.issues.issueID FROM bugtracking.issues WHERE bugtracking.issues.issueID = (SELECT MAX(bugtracking.issues.issueID) FROM bugtracking.issues);"
var issueID;
client.query(sql, function(err, result) {
if (err){
console.log("error in tracking");
}
issueID = JSON.stringify(result.rows[0]);
console.log("Tracking Result" + issueID);
//GET results
res.json(res.statusCode);
// res.json(issueID);
console.log("Selected Bug Tracking IssueID. Status Code: " + res.statusCode);
});
});
How do I get the data from the query to store the data in the issueID variable to send it to the client-side of my app?
I am using body-parser also.
EDIT:
It's almost as if the query isn't even executing because I am not able to get my console.log statements to print. This is my front-end code to help understand better:
$(window).load(function() {
$.ajax({
type: 'GET',
url: 'http://139.169.63.170:' + port + '/tracking',
dataType: 'json',
cache: true,
success: function (data) {
console.log("DEBUG DONE WITH CAPTURING tacking#: " + data);
if (data === Number(200)) {
issueID = issueID + 1;
$(".trackingID").val(issueID);
}
}
});
console.log("Window loads & Ajax runs???");
});
2nd EDIT:
Here are the results of running my query in psql. I am trying to get the number '33' to return from my get request:
bugtrackingdb=# SELECT bugtracking.issues.issueID FROM bugtracking.issues WHERE bugtracking.issues.issueID = (SELECT MAX(bugtracking.issues.issueID) FROM bugtracking.issues);
issueid
---------
33
(1 row)
AFter a few changes this is what I am getting to print from the back end console.log statements:
Tracking Result: [object Object]
Selected Bug Tracking IssueID. Status Code: 304
EDIT (Getting closer):
I stringified the result.rows[0] from the object returned from my query. I am now able to get the result to print to the server-side console.log which gives me the following results:
Tracking Result: {"issueid":"37"}
Selected Bug Tracking IssueID. Status Code: 304
status: 200
However, when I uncomment the line res.json(issueID) I get an error.
How do I send my JSON object back to the client side so that I can display it as needed?
app.get('/tracking', function(req, res) {
var sql = "SELECT bugtracking.issues.issueID FROM bugtracking.issues WHERE bugtracking.issues.issueID = (SELECT MAX(bugtracking.issues.issueID) FROM bugtracking.issues);"
var issueID;
client.query(sql, function(err, result) {
if (err){
console.log("error in tracking");
}
issueID = JSON.stringify(result.rows[0]);
console.log("Tracking Result" + issueID);
// TRY THIS
res.send(issueID);
});
I want to make a select on the database. But I get following error (only the first rows of the error):
An unhandled exception was generated during the execution of the current web request. Information regarding the origin and location of the exception can be identified using the exception stack trace below
[HttpException (0x80004005): Server cannot set content type after HTTP headers have been sent.]
System.Web.HttpResponse.set_ContentType(String value) +9752569
System.Web.HttpResponseWrapper.set_ContentType(String value) +14
System.Web.Mvc.JsonResult.ExecuteResult(ControllerContext context) +177
System.Web.Mvc.ControllerActionInvoker.InvokeActionResult(ControllerContext controllerContext, ActionResult actionResult) +27
A part of my DBcontroller:
string sql = #"exec [results].[GetNewResults] '" + Sources + #"', '" + Searchstring + #"', '" + username + #"', '" + sessionid + #"'";
var result = dbcontext_hke.Database.SqlQuery<UnifiedResultset>(sql);
if (results == null)
{
Debug.WriteLine("results is null");
return Json(null);
}
Debug.WriteLine("results isn't null");
Debug.WriteLine(results);
return Json(results, JsonRequestBehavior.AllowGet);
A part of my js:
var url = $('#initialrequest').val();
$.ajax({
url: url,
data: {sources, searchstring},
type: 'POST',
})
.success(function (result) {
for (var i = 0; i <= (9); i++) {
try {
console.log("lenght of result is :" + result.length);
add_error(result);
console.log("result is : " + result);
add_result(result[i].header, result[i].source, result[i].subsource, result[i].description);
} catch (err) {
add_error();
}
})
.error(function (xhr, status) {
$('#error').append(xhr.responseText);
console.log(xhr.responseText);
});
Basically after some research I found this with that code as a solution for a (maybe) similar problem:
this.Context.Response.Write(response);
this.Context.Response.Flush();
May anyone explain to me what is going on there and how I can fix it (or modify the code of the other solution)?
EDIT1: I have ensured that the controller get's the information. So the error might be at the communication between controller and js. Any ideas?
EDIT2: Also adding
Response.BufferOutput = true;
at the Index() of the HomeController and the DBController doesn't change anything.
EDIT3: I identified that the Problem is with the JSON in the return. If I return a different type, it works fine. Any ideas why this happens and how to fix it?
EDIT4: The reason my code doesn't work is because the command
var result = dbcontext_hke.Database.SqlQuery<UnifiedResultset>(sql);
doesn't execute immediately I think. If I want to see what's inside result with Debug.WriteLine(result) I only get the plain sql command. Ideas?
Andrei was right - there was another line that caused this error.
As stephbu mentioned in his comment as item 3: "the stream was unbuffered forcing the response headers to get written before the markup writing could begin."
This scenario was caused by the line
Response.Flush();
that was above the DB controller code I've mentioned
I'm working on a Gnome-shell extension, and I'm stuck in using Soup to contact a server.
Basically, I want to send a POST request which can be performed after authentication. If authenticated, the response if a JSON content, and if not authenticated, the response content is HTML, the welcome page of the site.
Here is my code so far:
let session = new Soup.SessionSync();
session.user_agent = Me.metadata.uuid;
let authParams = {'login': 'xxx', 'password': 'xxx'};
let message = Soup.form_request_new_from_hash('POST', authURL, authParams);
session.queue_message(message, Lang.bind(this, function(session, response) {
global.log('AUTHENTICATE: ' + response.status_code + ' - ' + response.reason_phrase);
global.log('AUTHENTICATE: ' + response.response_body.data);
let msg = Soup.form_request_new_from_hash('POST', url, params);
session.queue_message(msg, Lang.bind(this, function(session, response) {
global.log(response.status_code + ' - ' + response.reason_phrase);
global.log(response.response_headers.get_one('content-type'));
}));
}));
I can see that the authentication request works well, according to the response, but the second requests returns me the HTML content.
I checked in SoapUI by sending these 2 requests, and it works well (I get some JSON content) when I set the option "Maintain HTTP Session".
Therefore, I'm thinking that my session doesn't store the authentication when the second request is sent. Do you know what I am missing in here, in order to have the authentication saved in my session?
I alternatively tried to use some tips from the following link, but without success: Consume a webservice with basic authentication using Soup
Thanks in advance for your help.
let session = new Soup.SessionSync();
session.user_agent = Me.metadata.uuid;
let authParams = {'login': 'xxx', 'password': 'xxx'};
let message = Soup.form_request_new_from_hash('POST', authURL, authParams);
session.queue_message(message, Lang.bind(this, function(session, response) {
global.log('AUTHENTICATE: ' + response.status_code + ' - ' + response.reason_phrase);
global.log('AUTHENTICATE: ' + response.response_body.data);
let msg = Soup.form_request_new_from_hash('POST', url, params);
session.queue_message(msg, Lang.bind(this, function(session, response) {
global.log(response.status_code + ' - ' + response.reason_phrase);
global.log(response.response_headers.get_one('content-type'));
}));
}));
I am trying to make an API request to fitbit, using the oauth debugger from fitbit (https://dev.fitbit.com/apps/oauthtutorialpage) i am trying to figure out what i am doing wrong. I have added comments to my code below to help you understand what i am trying to achieve. What i am quite sure of is that i am either signing my request wrong, or using the wrong data to sign it. This is echoed by the API response.
I know there are more fitbit api questions here on stackoverflow, however did not find my answer there.
Is there anyone with more experience in Oauth signatures that knows what i could be doing wrong? Or could help me find a different approach to this?
var request = require('request');
var crypto = require('crypto');
var params = {
'oauth_consumer_key' : 'key12345',
'oauth_nonce' : Math.random().toString(36).substring(3), //random string
'oauth_signature_method' : 'HMAC-SHA1',
'oauth_timestamp' : Date.now().toString().substring(0,10), //timestamp with the same length as in the tutorial
'oauth_version' : '1.0'
}
var oauth_consumer_secret = 'secret123';
var post_string = 'POST&https://api.fitbit.com/oauth/request_token';
for(var key in params){
post_string += '&' + key + '=' + params[key];
}
/*At this point we have made a post string that we have to hash with hmac-sha1
the post string looks like this:
POST&https://api.fitbit.com/oauth/request_token&oauth_consumer_key=key12345&oauth_nonce=az6r8cqlzyqfr&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1439147378&oauth_version=1.0
The post_string from the tutorial looks like this:
POST&%2Foauth%2Frequest_token&oauth_consumer_key%3D%26oauth_nonce%3D%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1439145944%26oauth_version%3D1.0
*/
var hmac = crypto.createHmac('sha1', oauth_consumer_secret + "&");
// The tutorial page shows me the signature was 'signed with secret&'. I have tried with and without the & at the end, but without luck.
hmac.setEncoding('base64'); //i'm not sure if this is correct
hmac.write(post_string);
hmac.end();
var hash = hmac.read();
//and finally adding the hash to the parameters.
params.oauth_signature = hash;
//now, making the request with an authorization header.
var header='';
for (var key in params){
if(header.length === 0){
header = ' OAuth ' + key + '="' + params[key] + '"';
}
else{
header += ', ' + key + '="' + params[key] + '"';
}
}
/*
At this point the header parameter looks like this
OAuth oauth_consumer_key="key12345", oauth_nonce="jnr97ppvjs2lnmi", oauth_signature_method="HMAC-SHA1", oauth_timestamp="1439148049", oauth_version="1.0", oauth_signature="random_signature"
The tutorial tells me to use the headers:
OAuth oauth_consumer_key="key12345", oauth_nonce="jnr97ppvjs2lnmi", oauth_signature="different_signature", oauth_signature_method="HMAC-SHA1", oauth_timestamp="1439145944", oauth_version="1.0"
*/
var headers ={
'Authorization' : header
}
var url="https://api.fitbit.com/oauth/request_token";
var requestTimeout = 5000;
var opts = {
url: url,
timeout: requestTimeout,
headers : headers
}
request(opts, function (err, res, body) {
if (err) {
console.dir(err);
return;
}
var statusCode = res.statusCode;
if(res.statusCode === 200){
console.log(body);
}
else{
console.log("http-error-code: " + res.statusCode);
console.log(body);
}
})
/*
The response:
http-error-code: 401
{"errors":[{"errorType":"oauth","fieldName":"oauth_signature","message":"Invalid signature: 9fXI85C7GvZqMyW1AK1EkOSWZCY="}],"success":false}
*/
To get access token and secret use Grant (you can test FitBit in the playground).
Once you have access token and secret use Purest to make subsequent request to the FitBit API.
Here is an example on how to get the user's profile:
var Purest = require('purest')
var fitbit = new Purest({provider:'fitbit',
key:'[CONSUMER_KEY]', secret:'[CONSUMER_SECRET]'})
fitbit.get('user/-/profile', {
oauth:{token:'[ACCESS_TOKEN]', secret:'[ACCESS_SECRET]'}
}, function (err, res, body) {})
Alternatively you can use request for that:
var request = require('request')
request.get('https://api.fitbit.com/1/user/-/profile.json', {
oauth:{
consumer_key:'..',
consumer_secret:'..',
token:'..',
token_secret:'..'
}
}, function (err, res, body) {})
In short - don't try to implement the web server OAuth flow by yourself - use Grant, then use either Purest or request, just keep in mind that you don't have to pass all of the OAuth parameters by yourself, just pass the credentials.
I have recently set up node.js using Express and I created a simple HTML form using Jade. The form is to insert the data in a PostgreSQL database. The problem is that when I press submit on the form, everything is inserted on the database, but the HTML form is just hanging/lingering, and at some point it stops with No data received, ERR_EMPTY_RESPONSE. Sometimes it also inserts the data twice. I guess this is because the server side does not return a response, but I cannot see how (I am new to node.js).
The form has action="add_device" which is routed to routes/add_device.js. add_device.js looks like this:
var express = require('express');
var router = express.Router();
router.get('/', function(request, response, next) {
res.send('Nothing to see here. Move along.');
});
router.post('/', function(request, response, next) {
var db = require('../public/javascripts/db/insert');
var result = db.insertDevice(request, response);
return result;
});
module.exports = router;
The insertDevice function in my db module looks like this (it is exported with module.exports):
// Insert new QA device. Data arriving as a request from a HTML form.
insertDevice: function (request, response) {
// Input that is verified in the HTML form.
// Convert to proper format for PostgreSQL query.
var name = '\'' + request.body.name + '\'';
var ip_address = '\'' + request.body.ip_address + '\'';
var os = '\'' + request.body.os + '\'';
// Input that needs to be verified. Prepare for PostgreSQL query.
var mac_address;
var os_version;
request.body.mac_address == "" ? mac_address = 'NULL' : mac_address = '\'' + request.body.mac_address + '\'';
request.body.os_version == "" ? os_version = 'NULL' : os_version = '\'' + request.body.os_version + '\'';
var pg = require('pg'); // PostgreSQL module.
var td = require('./table_data') // Database constants.
var client = new pg.Client(request.app.get('postgreConnection'));
client.connect(function(err) {
if (err) {
return console.error('Could not connect to postgres', err);
}
var QUERY = "INSERT INTO " + td.QA_DEVICES.TABLE_NAME + "(" +
td.QA_DEVICES.COLUMN_NAME + ", " +
td.QA_DEVICES.COLUMN_MAC_ADDRESS + ", " +
td.QA_DEVICES.COLUMN_IP_ADDRESS + ", " +
td.QA_DEVICES.COLUMN_OS + ", " +
td.QA_DEVICES.COLUMN_OS_VERSION + ") VALUES(" +
name + ", " +
mac_address + ", " +
ip_address + ", " +
os + ", " +
os_version + ");";
client.query(QUERY, function (err, result) {
if (err) {
return console.error('Error running query: ' + QUERY, err);
}
console.log('Query performed: ' + QUERY);
client.end();
});
});
}
The 'Query performed' is always logged to console and data inserted into the database, but the form is still hanging. My questions are:
Is it the lack of response from the server that makes the form hang?
How can I "send a response back" to the front end?
Is it possible to route the front end to another page after insertion into the database? What is the best practice?
Yes, your request is receiving no response, so it is hanging.
In order to send a response, you can either send a blind acknowledgement right when the request is received (that is not dependent upon the success of the query and may be bad practice), or you can send it in the callback.
client.query(QUERY, function (err, result) {
if (err) {
// response.json({status: 'error'});
response.write('Error');
return console.error('Error running query: ' + QUERY, err);
} else {
// You can send json here too
// response.json({status: 'success'});
response.write('Success');
}
console.log('Query performed: ' + QUERY);
client.end();
});
If you want to go to another page, simply parse the incoming response on the client side and do a redirect. Using json is a good way to carry this out. You can also do a response.redirect(url) on the server side too, instead of sending back data. Have fun