We Keep Coding

Server shutting down itself using twitter api with nodejs and expressjs

I have an web app with using Twitter API. Main focus is to get the users that are not following back based on given twitter user name. It works fine until this point but when I get an error because of the fact that user name does not exist on Twitter, Server shuts down itself. Using Nodejs, expressjs.
The Error Message :
App is running on port : 3000
[ { code: 34, message: 'Sorry, that page does not exist.' } ]
_http_outgoing.js:494
throw new Error('Can\'t set headers after they are sent.');
^
Error: Can't set headers after they are sent.
at validateHeader (_http_outgoing.js:494:11)
at ServerResponse.setHeader (_http_outgoing.js:501:3)
at ServerResponse.header (/home/ugurcan/dev/Projects/Twitter-App/twitter-api-app/node_modules/express/lib/response.js:767:10)
at ServerResponse.send (/home/ugurcan/dev/Projects/Twitter-App/twitter-api-app/node_modules/express/lib/response.js:170:12)
at /home/ugurcan/dev/Projects/Twitter-App/twitter-api-app/twitter-api.js:30:21
at Request._callback (/home/ugurcan/dev/Projects/Twitter-App/twitter-api-app/node_modules/twitter/lib/twitter.js:215:14)
at Request.self.callback (/home/ugurcan/dev/Projects/Twitter-App/twitter-api-app/node_modules/request/request.js:186:22)
at emitTwo (events.js:126:13)
at Request.emit (events.js:214:7)
at Request.<anonymous> (/home/ugurcan/dev/Projects/Twitter-App/twitter-api-app/node_modules/request/request.js:1163:10)
Problematic part of the code is below. The question is : How can I avoid this situation ? Or is it even possible ?
client.get('followers/ids', params, function(error, followers_results, response) {
if (error) {
res.send("error");
console.log(error);
}
let followers = followers_results.ids;
client.get('friends/ids', params, function(error, following_results, response ) {
if (error) {
res.send("error");
console.log(error);
}
let following = following_results.ids;
following.forEach(function(person){
if(followers.indexOf(person) === -1){
one_way_following.push(person);
}
});
// console.log(one_way_following);
one_way_following = one_way_following.slice(0, 100);
one_way_following_string = one_way_following.join();
// console.log("----------------------------------------------------");
// console.log(one_way_following_string);
// console.log("----------------------------------------------------");
client.get('users/lookup', {user_id: one_way_following_string}, function(error, user_results, response, next) {
if (error) {
res.send("error");
console.log(error);
}
user_results.forEach(function(user){
let userObject = {
name : user.name,
screen_name : user.screen_name,
avatar: user.profile_image_url
}
// console.log(user.name);
users.push(userObject);
})
res.render("results.ejs",{users:users});
// console.log(users);
});
});
});

This error is actually happening because, you can't call res.render("results.ejs",{users:users}); or res.send("error"); multiple times, because headers can't be changed once set.
Easiest solution would be to break out of the function once you've caught an error (if (error) { on line 2), or handle it appropriately (render something blank, or render your friendly error view).
For more information, about this, check out the answer to this question, Error: Can't set headers after they are sent to the client
Hope that helps :)

https.get() works with address directly but not using object with host, path etc

I have a code like this:
var options = {
host: "https://basic:authentication#website.com",
path: "/api/address"
};
var request = https.get(options, function(response){
var str = "";
response.on('data', function(chunk){
str+=chunk;
});
response.on('end', function(){
console.log(str);
res.json(str);
});
});
request.end();
request.on('error', function(err){
console.log(err);
});
This gives me
{ [Error: getaddrinfo ENOTFOUND] code: 'ENOTFOUND', errno: 'ENOTFOUND', syscall: 'getaddrinfo' }
I don't know what's wrong because if I change the request to look like this:
var request = https.get("https://basic:authentication#website.com/api/address", function(response){
It works and gets an answer from the api. The problem is that I can't input parameters into the call if I do it this way. Does anyone have tips?

The problem is that your host value is not correct, it should really just be the hostname. For the HTTP basic auth, you can use the auth setting. For example:
var options = {
host: "website.com",
path: "/api/address",
auth: "basic:authentication"
};
Also, explicitly calling request.end() is unnecessary since https.get() internally does that for you automatically.
I should also note that since it seems like you're responding to an existing request with a new, external request, you can simplify it further by simply piping the external response to the existing response:
https.get(options, function(response) {
// You should probably check `response.statusCode` first ...
res.set('Content-Type', 'application/json');
response.pipe(res);
}).on('error', function(err) {
// ...
});

Error: Can't set headers after they are sent. RabbitMQ interfering with NodeJS response

After a http response, I am sending a messing using RabbitMQ (creating a channel etc) and however, the server is complaining that "Error: Can't set headers after they are sent."
Here is the code:
var amqp = require('amqplib');
var when = require('when');
var rabbitmq_conn = amqp.connect('amqp://localhost' );
function push_keystroke_data(session_id, data) {
var QUEUE_NAME = 'hello';
var msg = session_id;
when(rabbitmq_conn.createChannel()).then(function(ch) {
var ok = ch.assertQueue(QUEUE_NAME, {durable: false});
ok.then(function(_qok) {
ch.sendToQueue(QUEUE_NAME, new Buffer(msg));
console.log(" [x] Sent '%s'", msg);
ch.close();
});
}).ensure(function() {
conn.close();
});
}
router.post('/', function(req, res, next) {
// current session id
var sid;
if (req.cookies.papi) {
sid = req.cookies.papi.session_id;
} else {
sid = generate_session_id();
res.cookie('papi', {session_id: sid}, {maxAge: COOKIE_MAX_AGE});
}
res.send(JSON.stringify({ user_id: get_user_id(sid)}));
var data = process_keystroke_data(req.body);
push_keystroke_data(sid, data);
});
I assuming RabbitMQ is setting the headers after the response (I have also tried sending the RabbitMQ message before the response but that also didn't solve anything).
Here is the stack trace:
POST /api 500 220.100 ms - 16
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:335:11)
at ServerResponse.header (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/response.js:700:10)
at ServerResponse.send (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/response.js:154:12)
at fn (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/response.js:934:10)
at View.exports.renderFile [as engine] (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/jade/lib/index.js:374:12)
at View.render (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/view.js:93:8)
at EventEmitter.app.render (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/application.js:566:10)
at ServerResponse.res.render (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/response.js:938:7)
at /Users/mikeecb/Documents/KeyNA/jsbackend/app.js:62:7
at Layer.handle_error (/Users/mikeecb/Documents/KeyNA/jsbackend/node_modules/express/lib/router/layer.js:58:5)
Any solutions or ideas would be much appreciated.

It turns out that the issue was not that RabbitMQ was setting the headers (which would be weird, why would it be doing any http stuff?) but that after responding res.send(JSON.stringify({ user_id: get_user_id(sid)}));, I tried to send another respond which obviously the issue.

Secure POST request from NodeJS to Node/Express hangs

I'm using the following to send POST data to a secure nodejs server:
File: main.js
var strdata = JSON.stringify({"data":"thisdata"});
var options = {
host: '192.168.1.63',
port: 3001,
path: '/saveconfig',
method: 'POST',
rejectUnauthorized: false,
requestCert: true,
agent: false,
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(strdata)
}
};
var req = https.request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
console.log(req.write(strdata));
console.log(req.end());
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.on('finish', function() {
console.log("finished request");
});
In an otherwise functional expressjs server, these are the appropriate snippets:
File: app.js
app.post('/saveconfig', function() {
data.saveconfig; console.log("received request"); } );
app.get('/getconfig', data.getconfig);
File: data.js
exports.saveconfig = function(req, res) {
console.log("saveing config");
res.send(200);
res.end();
};
exports.getconfig = function(req, res) {
res.send("get OK");
}
With app.js running on the server (Ubuntu), I run main.js from the client (Windows 7). req.write and req.end execute and "finished request" logs to the console, but the request callback never fires.
On the server in app.js, the app.post event fires and logs "received request" to the console. But "saving config" never logs to the console. Only after I kill (^C) main.js, express then logs to the console "POST /saveconfig".
I know I'm missing something simple, but I've read dozens of coding examples and you can likely gather from my snippet, I've tried everything I can find or think of. I'd guess the request isn't finishing, but I don't know why. What is missing to get "exports.saveconfig" to fire?
additional information
The answer posted below fixed my problem. Because I'm new to stackoverflow, I can't post my own answer, but here's the rest of the story...
I appreciate your help. Being still new to JavaScript, I found I can learn a lot about an object by converting it to string. I was originally attempting to convert the req parameter to a string using a custom function. I just discovered it was apparently running into an endless loop after using JSON.stringify instead.
The code looked something like this:
exports.saveconfig = function (db) {
return function(req, res) {
console.log("saving config");
console.log(mymodule.serialize(req));
res.end("OK");
console.log(req.body);
};
};
I would have thought the above code should have logged the following to the console- even if the serialize method was in an endless loop:
POST /saveconfig
saving config
[nothing because of the endless loop]
Instead I got:
saving config
connections property is deprecated. Use getConnections() method
Being new to JavaScript, I assumed something was wrong with the request, the server, or some plumbing in-between. Of course, then the debugging code I added (along with my ignorance of JS) compounded the problem.
Changing
app.post('/saveconfig', function() {
data.saveconfig; console.log("received request"); } );
to
app.post('/saveconfig', datarts.saveconfig);
and removing the endless loop fixed the problem.

The problem is in you're app.js. You use data.saveConfig inside your callback without calling it. The right way would be
app.post('/saveconfig', function(req,res) {
data.saveconfig(req, res);
console.log("received request");
});
app.get('/getconfig', data.getconfig);
or (I assume the console.log is just for debugging purposes):
app.post('/saveconfig', data.saveconfig);
app.get('/getconfig', data.getconfig);
You could do your console.log() inside your data.saveconfig method if you want to go with the second example.

Error handling on request piping

I wrote simple proxy on nodejs and it looks like
var request = require( 'request' );
app.all( '/proxy/*', function( req, res ){
req.pipe( request({
url: config.backendUrl + req.params[0],
qs: req.query,
method: req.method
})).pipe( res );
});
It works fine if remote host is available, but if remote host is unavailable the whole node server crashes with unhandled exception
stream.js:94
throw er; // Unhandled stream error in pipe.
^
Error: connect ECONNREFUSED
at errnoException (net.js:901:11)
at Object.afterConnect [as oncomplete] (net.js:892:19)
How can I handle such errors?

Looking at the docs (https://github.com/mikeal/request) you should be able to do something along the following lines:
You can use the optional callback argument on request, for example:
app.all( '/proxy/*', function( req, res ){
req.pipe( request({
url: config.backendUrl + req.params[0],
qs: req.query,
method: req.method
}, function(error, response, body){
if (error.code === 'ECONNREFUSED'){
console.error('Refused connection');
} else {
throw error;
}
})).pipe( res );
});
Alternatively, you can catch an uncaught exception, with something like the following:
process.on('uncaughtException', function(err){
console.error('uncaughtException: ' + err.message);
console.error(err.stack);
process.exit(1); // exit with error
});

If you catch the uncaught exception for ECONNREFUSED make sure to restart your process. I saw in testing that the socket becomes unstable if you ignore the exception and simply try to re-connect.
Here's a great overview: http://shapeshed.com/uncaught-exceptions-in-node/
I ended up using the "forever" tool to restart my node process, with the following code:
process.on('uncaughtException', function(err){
//Is this our connection refused exception?
if( err.message.indexOf("ECONNREFUSED") > -1 )
{
//Safer to shut down instead of ignoring
//See: http://shapeshed.com/uncaught-exceptions-in-node/
console.error("Waiting for CLI connection to come up. Restarting in 2 second...");
setTimeout(shutdownProcess, 2000);
}
else
{
//This is some other exception..
console.error('uncaughtException: ' + err.message);
console.error(err.stack);
shutdownProcess();
}
});
//Desc: Restarts the process. Since forever is managing this process it's safe to shut down
// it will be restarted. If we ignore exceptions it could lead to unstable behavior.
// Exit and let the forever utility restart everything
function shutdownProcess()
{
process.exit(1); //exit with error
}

You should actually try to prevent the ECONNREFUSED exception from becoming uncaught:
var request = require( 'request' );
app.all( '/proxy/*', function( req, res ){
req.pipe( request({
url: config.backendUrl + req.params[0],
qs: req.query,
method: req.method
}))
.on('error', err => {
const msg = 'Error on connecting to the webservice.';
console.error(msg, err);
res.status(500).send(msg);
})
.pipe( res );
});
If you get an actual uncaught exception, then you should just let the application die.