We Keep Coding

NodeJS HTTP response max size reached?

I stumbled on a very strange issue recently concerning my Node.JS REST API. It's aim is to convert HTTP requests to SQL requests and the SQL response to HTTP response so I can then transfer everything over SSL/TLS and it's encoded.
The problem is that when the response size is small, it all works fine but when it exceed a certain size (about 255.37 KB), the response body is cutted right in the middle of the JSON. After multiple tests it appears that the issue is related to the HTTP response total size (including the headers) because when some custom headers are removed, more of the body is sent. I wrote a similar code in PHP and the response in JSON from the PHP API is fine so I assumed that the issue was originating from a bug in the Node.JS web server. Also, the Content-Lenght header is fine and has the right value (like the PHP API response does).
I'm using Node.JS v6.11.0 with the last release of Express. All my dependencies are up to date thanks to npm.
Here is the code of the function that handle the GET HTTP requests and proceed in doing SELECT SQL requests, then parse to json string the answer
function SelectData(connection){
return function (req, res) {
let tableName = req.params.table;
let id = req.params.id;
console.log("GET request received with tableName = " + tableName + " and id = " + id);
// Should match this regex
if (!tableName.match(/^[\-\_\s0-9A-Za-z]+$/)){
console.log("Error: Invalid table name provided : " + tableName);
badRequestError(req, res);
return;
}
if (id !== undefined){
if (!id.match(/^[0-9]+$/)){
console.log("Error: Invalid id provided : " + id + " (table name was valid : " + tableName + ")");
badRequestError(req, res);
return;
}
}
// if id is empty, then don't use any condition, else, add SQL condition
idPart = (id == undefined) ? "" : ('WHERE id="' + id + '"');
// TODO: try to replace " + var + " by question marks and bind params, find a way to do it w/ id
connection.query("SELECT * FROM " + tableName + " " + idPart + ";", function(err, result){
res.setHeader('Content-Type', 'application/json');
console.log("Request executed successfully !");
// Works too but I prefer the .send() method
// res.status(200).write(JSON.stringify(result)).end();
res.status(200).send(JSON.stringify(result));
req.connection.destroy();
return;
});
}
}
function badRequestError(req, res){
res.setHeader('Content-Type', 'text/plain');
res.status(400).send("Bad Request");
req.connection.destroy();
}
exports.SelectData = SelectData;
Edit: After researching yesterday I found similar issue resulting of the use of the HTTPS module so I removed it but it still happens.

Two if statements in node js

In the following code, I want to read the parameters from the link and display them. I used two separate if statements to check each parameter. The problem is it only reads the first if statement and discards the second one (does not check the second parameter):
var http = require('http');
var url = require('url');
var server = http.createServer(function (request, response) {
if (request.method == 'GET') {
var queryData = url.parse(request.url, true).query;
response.writeHead(200, {"Content-Type": "text/plain"});
// if parameter is provided
if (queryData.name) {
response.end('Hello ' + queryData.name + '\n');
} else {
response.end("Hello World\n");
}
if (queryData.age) {
response.end('Your age is ' + queryData.age + '\n');
} else {
response.end("No age provided\n");
}
}
});
// Listen on port 3000, IP defaults to 127.0.0.1 (localhost)
server.listen(8081);
The link is:
http://127.0.0.1:8081/start?name=john&age=25

Do not use response.end. Instead use response.write and add an response.end() after both if-statements.
See docs: https://nodejs.org/api/http.html#http_response_write_chunk_encoding_callback

Try response.write in the first if block.

Two factor authentication doesn't work

So, I want to request something from external api that is using TOTP. I have everything set up the right way and! On my localhost everything works fine, but on remote server - VPS - it crashes and gives following error :
{
"status" : "success",
"data" : {
"error_message" : "API access enabled, but unable to verify two-factor authentication code. If you need help with this, please contact support#bitskins.com."
}
}
HTML :
// Secret and keys for Bitskins API
var totp = new TOTP('MPKXR23QBRQVW7SZ');
var codeFromSecret = totp.now();
var url = 'https://bitskins.com/api/v1/get_item_price/?api_key=' + bitskins.apikey + '&code=' + totp.now() + '&names=' + namesHashed + '&delimiter=!END!';
console.log("url " + url);
request(url, function (error, response, body) {
if (!error) {
var json = JSON.parse(body);
console.log("body " + body);
console.log("json " + json);
for (var a = 0; a < items.length; a++) {
itemPrices.push(json.data.prices[a].price);
}
console.log('about to emit pricesFetched # steambot.js');
lastStep(itemPrices);
}
});

node.js: HTML form hangs after submit when inserting data into PostgreSQL database

I have recently set up node.js using Express and I created a simple HTML form using Jade. The form is to insert the data in a PostgreSQL database. The problem is that when I press submit on the form, everything is inserted on the database, but the HTML form is just hanging/lingering, and at some point it stops with No data received, ERR_EMPTY_RESPONSE. Sometimes it also inserts the data twice. I guess this is because the server side does not return a response, but I cannot see how (I am new to node.js).
The form has action="add_device" which is routed to routes/add_device.js. add_device.js looks like this:
var express = require('express');
var router = express.Router();
router.get('/', function(request, response, next) {
res.send('Nothing to see here. Move along.');
});
router.post('/', function(request, response, next) {
var db = require('../public/javascripts/db/insert');
var result = db.insertDevice(request, response);
return result;
});
module.exports = router;
The insertDevice function in my db module looks like this (it is exported with module.exports):
// Insert new QA device. Data arriving as a request from a HTML form.
insertDevice: function (request, response) {
// Input that is verified in the HTML form.
// Convert to proper format for PostgreSQL query.
var name = '\'' + request.body.name + '\'';
var ip_address = '\'' + request.body.ip_address + '\'';
var os = '\'' + request.body.os + '\'';
// Input that needs to be verified. Prepare for PostgreSQL query.
var mac_address;
var os_version;
request.body.mac_address == "" ? mac_address = 'NULL' : mac_address = '\'' + request.body.mac_address + '\'';
request.body.os_version == "" ? os_version = 'NULL' : os_version = '\'' + request.body.os_version + '\'';
var pg = require('pg'); // PostgreSQL module.
var td = require('./table_data') // Database constants.
var client = new pg.Client(request.app.get('postgreConnection'));
client.connect(function(err) {
if (err) {
return console.error('Could not connect to postgres', err);
}
var QUERY = "INSERT INTO " + td.QA_DEVICES.TABLE_NAME + "(" +
td.QA_DEVICES.COLUMN_NAME + ", " +
td.QA_DEVICES.COLUMN_MAC_ADDRESS + ", " +
td.QA_DEVICES.COLUMN_IP_ADDRESS + ", " +
td.QA_DEVICES.COLUMN_OS + ", " +
td.QA_DEVICES.COLUMN_OS_VERSION + ") VALUES(" +
name + ", " +
mac_address + ", " +
ip_address + ", " +
os + ", " +
os_version + ");";
client.query(QUERY, function (err, result) {
if (err) {
return console.error('Error running query: ' + QUERY, err);
}
console.log('Query performed: ' + QUERY);
client.end();
});
});
}
The 'Query performed' is always logged to console and data inserted into the database, but the form is still hanging. My questions are:
Is it the lack of response from the server that makes the form hang?
How can I "send a response back" to the front end?
Is it possible to route the front end to another page after insertion into the database? What is the best practice?

Yes, your request is receiving no response, so it is hanging.
In order to send a response, you can either send a blind acknowledgement right when the request is received (that is not dependent upon the success of the query and may be bad practice), or you can send it in the callback.
client.query(QUERY, function (err, result) {
if (err) {
// response.json({status: 'error'});
response.write('Error');
return console.error('Error running query: ' + QUERY, err);
} else {
// You can send json here too
// response.json({status: 'success'});
response.write('Success');
}
console.log('Query performed: ' + QUERY);
client.end();
});
If you want to go to another page, simply parse the incoming response on the client side and do a redirect. Using json is a good way to carry this out. You can also do a response.redirect(url) on the server side too, instead of sending back data. Have fun

Node.js extract value from callback function

I have a server file with a switch using the URL to display appropriate content. One of the cases is /users which should display a JSON string of a certain table. This is returned from a mysql file.
server.js
var http = require('http')
var url = require('url')
var port = 8080
function onRequest(request, response) {
var pathname = url.parse(request.url).pathname
console.log('Request for ' + pathname + ' received.')
response.writeHead(200, {'Content-Type': 'text/html'})
response.write(run(pathname))
response.end()
}
function run(pathname) {
switch(pathname) {
case '/':
response = 'Welcome to my little test'
break
case '/time':
response = 'The time is ' + new Date().toLocaleTimeString()
break
case '/users':
var response
require('./mysql').getUsers(function(users) {
console.log(users)
response = users
})
return response
break
default:
response = 'Unable to locate the requested page'
}
return response
}
http.createServer(onRequest).listen(port)
console.log('Server started on port ' + port + '.')
mysql.js
var mysql = require('mysql')
var connection = mysql.createConnection({
user: "root",
password: "password",
database: "main"
})
exports.getUsers = function(callback) {
connection.query('SELECT * FROM users;', function (error, rows, fields) {
callback(JSON.stringify(rows));
});
};
The console.log(users) in server.js displays the JSON string fine, but I cannot figure out how to get the value out of the callback and into the response variable.
Any ideas will be greatly appreciated.

The way you could extract the value out of the callback is to assign that value to a variable out of the callback's scope, but I don't recommend you to do that since you would end up with lots of global variables, besides you don't know when the variable will be assigned. Try this and see what happens so you get some insight with how callbacks and node.js works:
function run(pathname) {
switch(pathname) {
case '/':
response = 'Welcome to my little test'
break
case '/time':
response = 'The time is ' + new Date().toLocaleTimeString()
break
case '/users':
var response
var out_of_callback_users
require('./mysql').getUsers(function(users) {
out_of_callback_users = users
console.log("In the callback")
console.log(users)
response = users
})
console.log("After require");
console.log(out_of_callback_users) //Users have not been assigned yet
setTimeout(function(){
console.log("In the timeout")
console.log(out_of_callback_users)
},5000) //After 5 secs the query has been completed and users have been assigned.
return response
break
default:
response = 'Unable to locate the requested page'
}
return response
}
The way I would go is something like this:
function onRequest(request, response) {
var pathname = url.parse(request.url).pathname
console.log('Request for ' + pathname + ' received.')
response.writeHead(200, {'Content-Type': 'text/html'})
run(pathname, function(response){
response.write(response)
response.end()
})
}
function run(pathname,cb) {
switch(pathname) {
case '/':
cb('Welcome to my little test');
break;
case '/time':
cb('The time is ' + new Date().toLocaleTimeString());
break;
case '/users':
require('./mysql').getUsers(function(users) {
console.log(users);
cb(users);
})
break;
default:
cb('Unable to locate the requested page');
}
return;
}
http.createServer(onRequest).listen(port)
console.log('Server started on port ' + port + '.')

you can't do it like this. the problem is easy. let's talk about it:
function getUsers is an asynchronous. so the code follow runs like this:
case '/users':
var response
require('./mysql').getUsers(function(users) {
console.log(users)
response = users
})
return response
break
first, run require('./mysql').getUser() , then it will do return response directly, then break . when the getUser function is finished, it will run
function(users) {
console.log(users)
response = users
})
so, a rule you need to follow: once you use asynchronous, the other function have to be asynchronous.
i wonder you can modify like follow:
function onRequest(request, response) {
var pathname = url.parse(request.url).pathname
console.log('Request for ' + pathname + ' received.')
response.writeHead(200, {'Content-Type': 'text/html'})
run(pathname, function(res){ response.write(res)}) //changed
response.end()
}
function run(pathname, callback) {
switch(pathname) {
case '/':
callback('Welcome to my little test')
break
case '/time':
callback('The time is ' + new Date().toLocaleTimeString())
break
case '/users':
var response
require('./mysql').getUsers(function(users) {
console.log(users)
callback(users) # changed
})
break
default:
callback('Unable to locate the requested page')
}
}
http.createServer(onRequest).listen(port)
console.log('Server started on port ' + port + '.')

You do not need to serialize the mysql returned rows to use it. Either you can process it within getUsers, or return it back to the controller. If you return it, change code to:
exports.getUsers = function(callback) {
connection.query('SELECT * FROM users;', function (error, rows, fields) {
callback(rows);
});
};
Now within the server.js file, you can process the returned rows, like:
case '/users':
var response = ''
require('./mysql').getUsers(function(users) {
for (var i in users) {
var user = users[i];
var userId = user.id;
var userName = user.user_name;
response += "User - ID: "+userId+" Name: "+userName+"\n";
}
})
return response;
break;
You can process