We've got a Node.js script that is run once a minute to check the status of our apps. Usually, it works just fine. If the service is up, it exits with 0. If it's down, it exits with 1. All is well.
But every once in a while, it just kinda stops. The console reports "Calling status API..." and stops there indefinitely. It doesn't even timeout at Node's built-in two-minute timeout. No errors, nothing. It just sits there, waiting, forever. This is a problem, because it blocks following status check jobs from running.
At this point, my whole team has looked at it and none of us can figure out what circumstance could make it hang. We've built in a start-to-finish timeout, so that we can move on to the next job, but that essentially skips a status check and creates blind spots. So, I open the question to you fine folks.
Here's the script (with names/urls removed):
#!/usr/bin/env node
// SETTINGS: -------------------------------------------------------------------------------------------------
/** URL to contact for status information. */
const STATUS_API = process.env.STATUS_API;
/** Number of attempts to make before reporting as a failure. */
const ATTEMPT_LIMIT = 3;
/** Amount of time to wait before starting another attempt, in milliseconds. */
const ATTEMPT_DELAY = 5000;
// RUNTIME: --------------------------------------------------------------------------------------------------
const URL = require('url');
const https = require('https');
// Make the first attempt.
make_attempt(1, STATUS_API);
// FUNCTIONS: ------------------------------------------------------------------------------------------------
function make_attempt(attempt_number, url) {
console.log('\n\nCONNECTION ATTEMPT:', attempt_number);
check_status(url, function (success) {
console.log('\nAttempt', success ? 'PASSED' : 'FAILED');
// If this attempt succeeded, report success.
if (success) {
console.log('\nSTATUS CHECK PASSED after', attempt_number, 'attempt(s).');
process.exit(0);
}
// Otherwise, if we have additional attempts, try again.
else if (attempt_number < ATTEMPT_LIMIT) {
setTimeout(make_attempt.bind(null, attempt_number + 1, url), ATTEMPT_DELAY);
}
// Otherwise, we're out of attempts. Report failure.
else {
console.log("\nSTATUS CHECK FAILED");
process.exit(1);
}
})
}
function check_status(url, callback) {
var handle_error = function (error) {
console.log("\tFailed.\n");
console.log('\t' + error.toString().replace(/\n\r?/g, '\n\t'));
callback(false);
};
console.log("\tCalling status API...");
try {
var options = URL.parse(url);
options.timeout = 20000;
https.get(options, function (response) {
var body = '';
response.setEncoding('utf8');
response.on('data', function (data) {body += data;});
response.on('end', function () {
console.log("\tConnected.\n");
try {
var parsed = JSON.parse(body);
if ((!parsed.started || !parsed.uptime)) {
console.log('\tReceived unexpected JSON response:');
console.log('\t\t' + JSON.stringify(parsed, null, 1).replace(/\n\r?/g, '\n\t\t'));
callback(false);
}
else {
console.log('\tReceived status details from API:');
console.log('\t\tServer started:', parsed.started);
console.log('\t\tServer uptime:', parsed.uptime);
callback(true);
}
}
catch (error) {
console.log('\tReceived unexpected non-JSON response:');
console.log('\t\t' + body.trim().replace(/\n\r?/g, '\n\t\t'));
callback(false);
}
});
}).on('error', handle_error);
}
catch (error) {
handle_error(error);
}
}
If any of you can see any places where this could possibly hang without output or timeout, that'd be very helpful!
Thank you,
James Tanner
EDIT: p.s. We use https directly, instead of request so that we don't need to do any installation when the script runs. This is because the script can run on any build machine assigned to Jenkins without a custom installation.
Aren't you missing the .end()?
http.request(options, callback).end()
Something like explained here.
Inside your response callback your not checking the status..
The .on('error', handle_error); is for errors that occur connecting to the server, status code errors are those that the server responds with after a successful connection.
Normally a 200 status response is what you would expect from a successful request..
So a small mod to your http.get to handle this should do..
eg.
https.get(options, function (response) {
if (response.statusCode != 200) {
console.log('\tHTTP statusCode not 200:');
callback(false);
return; //no point going any further
}
....
Related
I'm creating a REST API to store setting from a specific camera sending data through TCP.
Camera communicate through TCP with a request/response pattern.
e.g : You can send "Get rate\n" to the camera and it responds "100 fps", in order to get the current framerate of the camera.
How to get data and store it in API?
var command = "rate"; //Array with setting I want to store in my API
function getDataFromCamera(command) { // This function will be iterate in another function
async.series([
console.log('Test1'); // console log to test output
function(callback) {
CAMERA_TCP_SOCKET.send("get "+command+"\n");
callback(null,command);
},
function(callback) {
console.log('Test2'); // console log to test output
CAMERA_TCP_SOCKET.onmessage = function(event) {
callback(null, event.data); // THIS line is the problem. I can't retrieve event.data because i don't know how to callback this variable
}
}],
function(err, results) {
if (err) {
//Handle the error in some way. Here we simply throw it
throw err;
}
if (results) {
console.log(results); // expected output : // ['rate','100']
}
});
}
At the moment, I get this error:
Test1
Test2
/Users/maximecongi/Desktop/Hologram/node_modules/async/dist/async.js:966
if (fn === null) throw new Error("Callback was already called.");
^
Error: Callback was already called.
at /Users/maximecongi/Desktop/Hologram/node_modules/async/dist/async.js:966:32
at /Users/maximecongi/Desktop/Hologram/node_modules/async/dist/async.js:3885:13
at W3CWebSocket.CAMERA_TCP_SOCKET.onmessage (/Users/maximecongi/Desktop/Hologram/core/api.js:91:13)
at W3CWebSocket._dispatchEvent [as dispatchEvent] (/Users/maximecongi/Desktop/Hologram/node_modules/yaeti/lib/EventTarget.js:107:17)
at W3CWebSocket.onMessage (/Users/maximecongi/Desktop/Hologram/node_modules/websocket/lib/W3CWebSocket.js:234:14)
at WebSocketConnection.<anonymous> (/Users/maximecongi/Desktop/Hologram/node_modules/websocket/lib/W3CWebSocket.js:205:19)
at WebSocketConnection.emit (events.js:188:13)
at WebSocketConnection.processFrame (/Users/maximecongi/Desktop/Hologram/node_modules/websocket/lib/WebSocketConnection.js:554:26)
at /Users/maximecongi/Desktop/Hologram/node_modules/websocket/lib/WebSocketConnection.js:323:40
at process.internalTickCallback (internal/process/next_tick.js:70:11)
[nodemon] app crashed - waiting for file changes before starting...
How to solve my problem?
It's because you are listening on every message, and looks like you're sending 3 commands, and I guess, every command get a response. This callback after first serie is not destroyed, its alive. For me, you should check if the response is for this specific command, then execute the problematic callback.
the following lamdba code works perfectly fine when testing locally using Alex-app-server but when published and tested on AWS Lambda, it gets within the else statement and prints the console.log('OUT PUBLISH') But it doesn't publish the 'lambda/channelnumber' nor does it send the correct response back to me or print 'IN PUBLISH'
Any ideas why its just completing the bottom half of the else statement and not touching the publish function?
Code Snippet where I believe the problem lies
function (request, response) {
var channelNumber = request.slot('CHANNELNUMBER');
if (_.isEmpty(channelNumber)) {
var prompt = 'I didn\'t hear a channel code. Tell me a channel code.';
response.say(prompt).shouldEndSession(true);
return true;
} else {
//Doesn't publish any of this?????
thingShadows.publish('lambda/channelNumber', channelNumber, function () {
var prompt1 = 'Okay.';
response.say(prompt1).shouldEndSession(true);
console.log('in publish');
});
////But prints this??
console.log('out publish');
return true;
}
}
Full Code
'use strict';
module.change_code = 1;
var Alexa = require('alexa-app');
var skill = new Alexa.app('smartmote');
var awsIot = require('aws-iot-device-sdk');
var deviceName = "tv";
var _ = require('lodash');
var path = require('path');
var host = "XXXXXXXXXXXXXXXXXXXX.iot.us-east-1.amazonaws.com";
//App id is the skill being used.
var app_id = "amzn1.ask.skill.YYYYYYYYYYYYYYYYYYYYY";
var thingShadows = awsIot.thingShadow({
keyPath: path.join(__dirname, '/Raspi.private.key'),
certPath: path.join(__dirname, '/Raspi.cert.pem'),
caPath: path.join(__dirname, '/root-CA.crt'),
clientId: deviceName,
region: "us-east-1",
});
var reprompt = 'I didn\'t hear a channel, tell me a channel number or name to change to that channel';
skill.launch(function (request, response) {
var prompt = 'To change channel, tell me a channel number.';
response.say(prompt).reprompt(reprompt).shouldEndSession(true);
});
skill.intent('ChannelNumberIntent', {
'slots': {
'CHANNELNUMBER': 'CHANNELID'
},
'utterances': ['{|Change|put} {|the|on} {|channel} {|to} {-|CHANNELNUMBER}']
},
function (request, response) {
var channelNumber = request.slot('CHANNELNUMBER');
if (_.isEmpty(channelNumber)) {
var prompt = 'I didn\'t hear a channel code. Tell me a channel code.';
response.say(prompt).shouldEndSession(true);
return true;
} else {
thingShadows.publish('lambda/channelNumber', channelNumber, function () {
console.log('in pub');
var prompt1 = 'Okay.';
response.say(prompt1).shouldEndSession(true);
callback();
});
console.log('out pub');
return true;
}
}
);
module.exports = skill;
This is most likely because of the asynchronous nature of your code.
You haven't told us what thingShadows.publish() does, but it appears to take a callback function as its second argument. Presumably this function will be called when publish() has finished doing whatever it does.
When running locally I would imagine that the output you see is (in this order):
out publish
in publish
Notice that out publish gets called before in publish. This is because the publish method is asynchronous, so execution will continue as soon as it is called. In your case, you are calling return immediately after calling publish, which probably means your lambda job is ending before it has a chance to log in publish.
You haven't provided enough information about the rest of your lambda code/setup to provide a full answer, but you need to make sure that you are waiting for your publish method to have finished before continuing. One way to achieve this is to use the callback object that is passed to your lambda handler:
exports.myHandler = function(event, context, callback) {
// Other code
thingShadows.publish('lambda/channelNumber', channelNumber, function () {
var prompt1 = 'Okay.';
response.say(prompt1).shouldEndSession(true);
console.log('in publish');
// When the publish method is complete, we can call `callback`
// to tell lambda we are done
callback();
});
}
So I want to fetch a large amount of data on Parse, a good solution I found is to make a recursive function: when the data are successfully found, launch another request. The way I'm doing it is pretty simple:
var containedLimit = 1000, // Also tried with 500, 300, and 100
parseUsersWaiting = {
// A lot of Users
},
parseUsers = {}, // Recipt for Users
getPlayers = function (containedIn) {
var count = 0;
if (containedIn == undefined) {
containedIn = [];
for (var i in parseUsersWaiting) {
count++;
if (count > containedLimit) {
break;
}
containedIn.push(parseUsersWaiting[i].id);
delete parseUsersWaiting[i];
}
}
var UserObject = Parse.Object.extend('User'),
UserQuery = new Parse.Query(UserObject);
UserQuery.limit(containedLimit);
UserQuery.containedIn('objectId', containedIn);
UserQuery.find({
success: function (results) {
if (results) {
for (var i in results) {
if (parseUsers[results[i].id] == undefined) {
parseUsers[results[i].id] = results[i];
}
// Other stuff
if (results.length < containedLimit) {
// End of process
} else {
getPlayers();
}
}
} else {
// Looks like an end of process too
}
}, error: function (error) {
console.log(error.message);
getPlayers(containedIn);
}
});
};
Now, here is what I would like to call the "issue": it happen, very frequently, that the "error" callback is launched with this:
Received an error with invalid JSON from Parse: Error: getaddrinfo ENOTFOUND api.parse.com
at errnoException (dns.js:44:10)
at GetAddrInfoReqWrap.onlookup [as oncomplete] (dns.js:94:26)
(With code 107, of course) So I searched on the Parse Documentation, and it says the exact same thing: "Received an error with invalid JSON". Yeah.
I'm using the Parse SDK provided by Parse.com (npm install parse)
I also tried to modify a bit the Parse code with replacing the host key by 'hostname' on line 361 of the file parse/node_modules/xmlhttprequest/lib/XMLHttpRequest.js (Parse package version: 1.5.0), and it didn't worked, so I removed my changes.
(By the way, I found a solution talking about using ulimit to change memory usage limit that could solve the problem, but actually, I haven't the necessary rights to execute this command)
This error occurs when it can not connect to the API (technically when it cannot lookup the IP address of the API server). Perhaps your internet connection was lost for a brief moment or their SDK server were unavailable (or maybe the server is denying your request due to rate limits).
Either way it is good to code some resilience into your application. I see your on error function retries the API call, but perhaps it would be worth adding a timeout before you do that to give the problem a chance to recover?
error: function (error) {
console.log(error.message);
setTimeout(getPlayers, 10000, containedIn);
}
If the sever is rate limiting your requests, this will also help.
I am new to node.js. How to detect client is disconnected from node.js server .
Here is my code:
var net = require('net');
var http = require('http');
var host = '192.168.1.77';
var port = 12345;//
var server = net.createServer(function (stream) {
stream.setEncoding('utf8');
stream.on('data', function (data) {
var comm = JSON.parse(data);
if (comm.action == "Join_Request" && comm.gameId =="game1") // join request getting from client
{
var reply0 = new Object();
reply0.message = "WaitRoom";
stream.write(JSON.stringify(reply0) + "\0");
}
});
stream.on('disconnect', function() {
});
stream.on('close', function () {
console.log("Close");
});
stream.on('error', function () {
console.log("Error");
});
});
server.listen(port,host);
How to know client side internet disconnection.
The best way to detect "dead sockets" is to send periodic application-level ping/keepalive messages. What that message looks like depends on the protocol you're using for communicating over the socket. Then it's just a matter of using a timer or other means of checking if you've received a "ping response" within a certain period of time after you sent the ping/keepalive message to the client.
On a semi-related note, it looks like you're using JSON messages for communication, but you're assuming a complete JSON string on every data event which is a bad assumption. Try using a delimiter (a newline is pretty common for something like this, and it makes debugging the communication more human-readable) instead.
Here is a simple example of how to achieve this:
var PING_TIMEOUT = 5000, // how long to wait for client to respond
WAIT_TIMEOUT = 5000; // duration of "silence" from client until a ping is sent
var server = net.createServer(function(stream) {
stream.setEncoding('utf8');
var buffer = '',
pingTimeout,
waitTimeout;
function send(obj) {
stream.write(JSON.stringify(obj) + '\n');
}
stream.on('data', function(data) {
// stop our timers if we've gotten any kind of data
// from the client, whether it's a ping response or
// not, we know their connection is still good.
clearTimeout(waitTimeout);
clearTimeout(pingTimeout);
buffer += data;
var idx;
// because `data` can be a chunk of any size, we could
// have multiple messages in our buffer, so we check
// for that here ...
while (~(idx = buffer.indexOf('\n'))) {
try {
var comm = JSON.parse(buffer.substring(0, idx));
// join request getting from client
if (comm.action === "Join_Request" && comm.gameId === "game1") {
send({ message: 'WaitRoom' });
}
} catch (ex) {
// some error occurred, probably from trying to parse invalid JSON
}
// update our buffer
buffer = buffer.substring(idx + 1);
}
// we wait for more data, if we don't see anything in
// WAIT_TIMEOUT milliseconds, we send a ping message
waitTimeout = setTimeout(function() {
send({ message: 'Ping' });
// we sent a ping, now we wait for a ping response
pingTimeout = setTimeout(function() {
// if we've gotten here, we are assuming the
// connection is dead because the client did not
// at least respond to our ping message
stream.destroy(); // or stream.end();
}, PING_TIMEOUT);
}, WAIT_TIMEOUT);
});
// other event handlers and logic ...
});
You could also just have one interval instead of two timers that checks a "last data received" timestamp against the current timestamp and if it exceeds some length of time and we have sent a ping message recently, then you assume the socket/connection is dead. You could also instead send more than one ping message and if after n ping messages are sent and no response is received, close the connection at that point (this is basically what OpenSSH does).
There are many ways to go about it. However you may also think about doing the same on the client side, so that you know the server didn't lose its connection either.
There are two examples in between these pages 16 and 18.
Example 1.3 is a server app.
Example 1.4 is a client app doing GET requests to the server.
When I run the two examples (at the same time) I notice some quite weird behavior
in the client. All requests are executed (i.e. the for loop in the client completes)
but the callbacks of only 5 of them get called. The client doesn't exit and also
doesn't error out. And just no more callbacks are called.
Any ideas what might be happening or how I can troubleshoot this further?
Note: I am running Node.js v0.10.20 on Windows 7.
Server:
var http = require('http');
var fs = require('fs');
// write out numbers
function writeNumbers(res) {
var counter = 0;
// increment, write to client
for (var i = 0; i<100; i++) {
counter++;
res.write(counter.toString() + '\n');
}
}
// create http server
http.createServer(function (req, res) {
var query = require('url').parse(req.url).query;
var app = require('querystring').parse(query).file;
// content header
res.writeHead(200, {'Content-Type': 'text/plain'});
if (!app){
res.end();
console.log('No file argument found in query string.');
return;
}else{
app = app + ".txt";
}
// write out numbers
writeNumbers(res);
// timer/timeout to open file and read contents
setTimeout(function() {
console.log('Opening file: ' + app + '.');
// open and read in file contents
fs.readFile(app, 'utf8', function(err, data) {
res.write('\r\n');
if (err)
res.write('Could not find or open file ' + app + ' for reading.\r\n');
else {
res.write(data);
}
// response is done
res.end();
});
},2000);
}).listen(8124);
console.log('Server running at 8124');
Client:
var http = require('http');
var N = 200;
// The URL we want, plus the path and options we need
var options = {
host: 'localhost',
port: 8124,
path: '/?file=automatic',
method: 'GET'
};
var callback_function = function(response) {
// finished? ok, write the data to a file
console.log('got response back');
};
for (var i = 1; i <= N; i++) {
// make the request, and then end it, to close the connection
http.request(options, callback_function).end();
console.log('done with call # ' + i);
}
--- Experiment Done ---
If I lower N to 10 and also if I do a
global "var i = 1" and then do this thing:
function schedule(){
http.request(options, callback_function).end();
console.log('done with call ' + i);
i++;
if (i<=N){
setTimeout(function(){
schedule();
}, 1000);
}
}
schedule();
instead of the loop in the client, I get similar behavior.
I guess that's what Milimetric meant by "sleep" i.e. just
to make sure I don't hit the server too quickly with too
many simultaneous requests.
But the behavior is not fully identical, it takes several mins
to print 'got response back' on the second set of 5 requests
and then another maybe 5-6 mins for the client to exit.
Still, all that does look weird to me.
C:\PERSONAL\NODE_TEST>node test004.js
done with call 1
got response back
done with call 2
got response back
done with call 3
got response back
done with call 4
got response back
done with call 5
got response back
done with call 6
done with call 7
done with call 8
done with call 9
done with call 10
got response back
got response back
got response back
got response back
got response back
C:\PERSONAL\NODE_TEST>
The problem is that the client doesn't consume the response body sent by the server, so the connection remains (half) open and the http agent only allows 5 concurrent requests per client by default, causing it to hang after 5 requests. The connection will eventually timeout, causing the next 5 requests to be processed.
node.js http.get hangs after 5 requests to remote site
Change your callback function to consume any data sent down the response.
var callback_function = function(response) {
// finished? ok, write the data to a file
console.log('got response back');
response.on('data', function () {});
};