We Keep Coding

Running node-rdkafka code in server

I'm running the below node-rdkafka code in Eclipse as Node.js application. This is the sample code from https://blizzard.github.io/node-rdkafka/current/tutorial-producer_.html
I want to run this in a test server and call from iOS Mobile application.
I knew about running node.js app in AWS.
Question I: Is there any other options to run in a free test server environment like Tomcat?
Question II: Even If I am able to run this node.js app in a server, how do i call from a mobile application? Do I need to call producer.on('ready', function(arg) (or) What function i need to call from Mobile app?
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function() {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
/*
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});*/
//starting the producer
producer.connect();

First of all, you need an HTTP server. ExpressJS can be used. Then, just tack on the Express code basically at the end, but move the producer loop into the request route.
So, start with what you had
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
});
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
Then, you can add this in the same file.
var express = require('express')
var app = express()
app.get('/', (req, res) => res.send('Ready to send messages!'))
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
} // end if
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'))
I don't think the poll loop is necessary since you don't care about the counter anymore.
Now, connect your mobile app to http://<your server IP>:3000/ and send test messages with a POST request to http://<your server IP>:3000/10, for example, and adjust to change the number of messages to send

I might be late on this but this is how I did using promises and found it better than have a time out etc.
const postMessageToPublisher = (req, res) => {
return new Promise((resolve, reject) => {
producer.connect();
producer.setPollInterval(globalConfigs.producerPollingTime);
const actualBody = requestBody.data;
const requestBody = req.body;
const topicName = req.body.topicName;
const key = requestBody.key || uuid();
const partition = requestBody.partition || undefined;
const data = Buffer.from(JSON.stringify(udpatedBody));
/**
* Actual messages are sent here when the producer is ready
*/
producer.on(kafkaEvents.READY, () => {
try {
producer.produce(
topic,
partition,
message,
key // setting key user provided or UUID
);
} catch (error) {
reject(error);
}
});
// Register listener for debug information; only invoked if debug option set in driver_options
producer.on(kafkaEvents.LOG, log => {
logger.info('Producer event log notification for debugging:', log);
});
// Register error listener
producer.on(kafkaEvents.ERROR, err => {
logger.error('Error from producer:' + JSON.stringify(err));
reject(err);
});
// Register delivery report listener
producer.on(kafkaEvents.PUBLISH_ACKNOWLEDGMENT, (err, ackMessage) => {
if (err) {
logger.error(
'Delivery report: Failed sending message ' + ackMessage.value
);
logger.error('and the error is :', err);
reject({ value: ackMessage.value, error: err });
} else {
resolve({
teamName: globalConfigs.TeamNameService,
topicName: ackMessage.topic,
key: ackMessage.key.toString()
});
}
});
});
};
Please note that kafkaEvents contains my constants for the events we listen to and it is just a reference such as kafkaEvents.LOG is same as event.log
and also the calling function is expecting this to a promise and accordingly we user .then(data => 'send your response to user from here') and .catch(error => 'send error response to user
this is how I achieved it using promises

Promise.all() not resolving when running server - otherwise works fine

I've written a small tool that returns a promise after calling several other promises. This tool works great when I test it solo, it takes about 10 seconds in the example below. However, when I try to run it along with a http server instance it, takes in the order of several minutes to return, if at all!
I'm fairly sure I'm just misunderstanding something here, as I'm not extremely proficient in Node. If anyone can spot an issue, or suggest an alternative to using promises for handling asynchronous methods, please let me know!
Just to clarify, it's the Promise.all returned by the traceRoute function which is hanging. The sub-promises are all resolving as expected.
Edit: As suggested in the comments, I have also tried a recursive version with no call to Promise.all; same issue.
This is a working standalone version being called without any http server instance running:
const dns = require('dns');
const ping = require('net-ping');
var traceRoute = (host, ttl, interval, duration) => {
var session = ping.createSession({
ttl:ttl,
timeout: 5000
});
var times = new Array(ttl);
for (var i=0; i<ttl; i++){
times[i] = {'ttl': null, 'ipv4': null, 'hostnames': [], 'times': []}
};
var feedCb = (error, target, ttl, sent, rcvd) => {
var ms = rcvd - sent;
if (error) {
if (error instanceof ping.TimeExceededError) {
times[ttl-1].ttl = ttl;
times[ttl-1].ipv4 = error.source;
times[ttl-1].times.push(ms)
} else {
console.log(target + ": " +
error.toString () +
" (ttl=" + ttl + " ms=" + ms +")");
}
} else {
console.log(target + ": " +
target + " (ttl=" + ttl + " ms=" + ms +")");
}
}
var proms = new Array();
var complete = 0
while(complete < duration){
proms.push(
new Promise((res, rej) => {
setTimeout(function(){
session.traceRoute(
host,
{ maxHopTimeouts: 5 },
feedCb,
function(e,t){
console.log('traceroute done: resolving promise')
res(); // resolve inner promise
}
);
}, complete);
})
)
complete += interval;
}
return Promise.all(proms)
.then(() => {
console.log('resolving traceroute');
return times.filter((t)=> t.ttl != null);
});
}
traceRoute('195.146.144.8', 20, 500, 5000)
.then( (times) => console.log(times) )
Below, is the same logic being called from inside the server instance, this is not working as it should. See the inline comment for where exactly it hangs.
const express = require('express');
const http = require('http');
const WebSocket = require('ws');
const app = express();
const server = http.createServer(app);
const wss = new WebSocket.Server({server: server, path: "/wss"});
const dns = require('dns');
const ping = require('net-ping');
var traceRoute = (host, ttl, interval, duration) => {
var session = ping.createSession({
ttl:ttl,
timeout: 5000
});
var times = new Array(ttl);
for (var i=0; i<ttl; i++){
times[i] = {'ttl': null, 'ipv4': null, 'hostnames': [], 'times': []}
};
var feedCb = (error, target, ttl, sent, rcvd) => {
var ms = rcvd - sent;
if (error) {
if (error instanceof ping.TimeExceededError) {
times[ttl-1].ttl = ttl;
times[ttl-1].ipv4 = error.source;
times[ttl-1].times.push(ms)
} else {
console.log(target + ": " +
error.toString () + " (ttl=" + ttl + " ms=" + ms +")");
}
} else {
console.log(target + ": " + target +
" (ttl=" + ttl + " ms=" + ms +")");
}
}
var proms = new Array();
var complete = 0
while(complete < duration){
proms.push(
new Promise((res, rej) => {
setTimeout(function(){
session.traceRoute(
host,
{ maxHopTimeouts: 5 },
feedCb,
function(e,t){
console.log('traceroute done: resolving promise')
res(); // resolve inner promise
}
);
}, complete);
})
)
complete += interval;
}
console.log('Promise all:', proms);
// #####################
// Hangs on this promise
// i.e. console.log('resolving traceroute') is not called for several minutes.
// #####################
return Promise.all(proms)
.then(() => {
console.log('resolving traceroute')
return times.filter((t)=> t.ttl != null)
});
}
wss.on('connection', function connection(ws, req) {
traceRoute('195.146.144.8', 20, 500, 5000)
.then((data) => ws.send(data));
});
app.use('/tools/static', express.static('./public/static'));
app.use('/tools/templates', express.static('./public/templates'));
app.get('*', function (req, res) {
res.sendFile(__dirname + '/public/templates/index.html');
});
server.listen(8081);
Note: I have tried calling it before the server.listen, after server.listen, from inside wss.on('connection', .... None of which makes a difference. Calling it anywhere, while the server is listening, causes it to behave in a non-deterministic manner.

I'm not going to accept this answer as it's only a workaround; it was just too long to put in the comments...
None of the promises, including the Promise.all, are throwing exceptions. However, Node seems to be parking the call to Promise.all. I accidentally discovered that if I keep a timeout loop running while waiting for the promise.all to resolve, then it will in fact resolve as and when expected.
I'd love if someone could explain exactly what is happening here as I don't really understand.
var holdDoor = true
var ps = () => {
setTimeout(function(){
console.log('status:', proms);
if (holdDoor) ps();
}, 500);
}
ps();
return Promise.all(proms)
.then(() => {
holdDoor = false
console.log('Resolving all!')
return times.filter((t)=> t.ttl != null)
});

Your code is working perfectly fine!
To reproduce this I've created a Dockerfile with a working version. You can find it in this git repository, or you can pull it with docker pull luxferresum/promise-all-problem.
You can run the docker image with docker run -ti -p 8081:8081 luxferresum/promise-all-problem. This will expose the webserver on localhost:8081.
You can also just run the problematic.js with node problematic.js and then opening localhost:8081 in the web browser.
The web socket will be opened by const ws = new WebSocket('ws://localhost:8081/wss'); which then triggers the code to run.
Its just very important to actually open the web socket, without that the code will not run.

I would suggest replacing the trace route with something else, like a DNS lookup, and see of the issue remains. At this point you cannot be sure it relates to raw-socket, since that uses libuv handles directly and does not effect other parts of the Node.js event loop.

Using jquery.go.js to retrieve raw HTTP responses

I am quite new to Jquery, node.js, Phantom.js and JavaScript in general and having a few problems wrapping my head around all of it.
I'm working on a simple script using Jquery.go.js (https://github.com/travist/jquery.go.js) that should log me in to a site, then access a specific resource and parse the response (JSON) that it returns.
So far I managed to get the login working, but am really lost now for the second part.
What I have so far is based on https://github.com/travist/makemeasandwich.js
var async = require('async'),
$ = require('jquerygo'),
path = require('path'),
fs = require('fs');
// Add some default configs.
$.config.site = 'https://host.com';
$.config.addJQuery = false;
var login = function(done) {
async.series([
$.go(false, 'visit', '/login'),
$.go(false, 'waitForElement', '#email'),
debugCapture('login1'),
$('#email').go('val', 'email#example.com'),
$('#password').go('val', 'securepassword'),
debugCapture('login2'),
$("form").go('attr','id','validForm'),
$('#validForm').go('submit'),
sleep(3000),
debugCapture('login3'),
print('Successfully logged in.'),
sleep(3000),
], done);
};
var viewJournalEntries = function(done){
$.config.addJQuery = true;
async.series([
$.go(false, 'visit', '/api/journals/show/546'),
$.getPage(function(page) {
// Is this right, what do I need to do here??
}),
debugCapture('step2'),
],done);
}
var debugCapture = function(fileName) {
// if (config.get('debug')) {
return capture(fileName);
// }
return function(done) { done(); }
};
/**
* Method to capture and ensure the screenshots directory exists.
*/
var capture = function(fileName) {
// Return the async function.
return function(done) {
// The directory to store the screenshots.
var dir = __dirname + '/screenshots-ls';
// Check that the directory exists.
fs.exists(dir, function(exists) {
if (exists) {
$.capture(dir + '/' + fileName + '.png', done);
}
else {
fs.mkdir(dir, function(err) {
if (err) return done(err);
$.capture(dir + '/' + fileName + '.png', done);
});
}
});
}
}
/**
* Helper to print something when it is executed.
*
* #param {type} text
* #returns {unresolved}
*/
var print = function(text) {
return function(done) {
console.log(text);
done();
};
};
/**
* Helper function to take a pause...
*
* #param {type} time
* #returns {unresolved}
*/
var sleep = function(time) {
return function(done) {
setTimeout(done, time);
};
};
async.series([
login,
viewJournalEntries,
], function() {
$.close();
});
The login works like a charm and I can call the viewJournalEntries function which calls the url /api/journals/show/546 that returns a typical JSON string like this:
{"data":{"id":546,"user_id":1,[etc...]
The screen capture proofs that it works, but I am just not sure how to proceed to get the JSON into a variable that I can parse.
Any help/directions are appreciated.
Thanks,
Ben

Nodejs server stop accept connections after a few hours

I made an application to the control of simultaneous logins, in starts everything works perfectly, however after a few hours i can not longer connect to the server, my client returns me the following error: net :: ERR_CONNECTION_TIMED_OUT and on the server side does not happen any error like it was running correctly... code below:
CLIENT SIDE:
var socket;
function connect(id) {
socket = io.connect('http://IP:4200');
socket.on('connect', function (data) {
socket.emit('join', id);
});
socket.on('messages', function (data) {
console.log('MSG: ' + data.toString());
switch (data.toString()) {
case "kick":
socket.close();
console.log("KICK!");
break;
case "duplicate_entry":
socket.close();
console.log("Another user connection!");
break;
}
});
}
SERVER SIDE:
var express = require('express');
var app = express();
var server = require('http').createServer(app);
var io = require('socket.io')(server);
var clients = [];
app.use(express.static(__dirname + '/bower_components'));
function logtimestamp() {
var log_date = new Date();
log_date = '[' + log_date.getFullYear() + '/' + (log_date.getMonth() + 1) + '/' + log_date.getDate() + ' ' + log_date.getHours() + ':' + log_date.getMinutes() + ':' + log_date.getSeconds() + ']';
return log_date;
}// FUNCTION logtimestamp
console.log("Start time: " + logtimestamp());
console.log("Server port 4200")
console.log("websocket server created!");
try {
io.on('connection', function (client) {
try {
var id;
var conexao;
client.on('join', function (data) {
try {
console.log('Client connected...'+logtimestamp()+' ID:' + data);
id = data;
conexao = {
ws: client,
id_user: data
};
clients.push(conexao);
for (var x = 0; x < clients.length; x++) {
//desconect previous user
try {
if (clients[x].id_user == id) {
if (clients[x].ws != conexao.ws) {
clients[x].ws.emit('messages', 'duplicate_entry');
clients.splice(x, 1);
}
}
} catch (err) {
console.log("ERROR 1: " + err.message);
}
}
} catch (err) {
console.log("ERROR 2: " + err.message);
}
});
} catch (err) {
console.log("ERROR 3: " + err.message);
}
});
} catch (err) {
console.log("ERROR 4: " + err.message);
}
server.listen(4200);

I see a couple possible issues. It is hard for us to know by just inspecting code which issues actually are the cause of your issue. In any case, you should clean up these issues and see if it improves the situation:
You should respond the the disconnect event and immediately remove any socket from your clients array when it disconnects.
In your loop where you are looking to removing any prior instances of a given user, your for loop will not work properly when you are doing .splice(x, 1) in the middle of the for loop. This will move all items after it does one in the array causing you to skip the comparison of the next element in the array. One simple way to get around this is to iterate the array backwards: for (var x = clients.length - 1; x >= 0; x--) because then the elements who's position are affected after the .splice() are elements you have already looked at. None will be missed.
Beyond this, you should examine the memory usage of the nodejs process, the open sockets by the nodejs process and the CPU usage of the nodejs process to see if any of those point to any possible issues.
And, what is your deployment environment? Is there a proxy server in front of your web server?

Socket.io Multiple messages emitted from one event

I've been looking for quite a while for a solution but haven't found anything yet.
I'm trying to emit a message from a server every time the server sees that a file has changed in a specified directory. However, instead of only emitting one message, it insists on emitting the same message three times. I am using chokidar to watch the directory, and inside of the 'change' event I emit the message.
Server side code:
var express = require('express')
, app = express()
, http = require('http')
, server = http.Server(app)
, io =require('socket.io')(server)
, chokidar = require('chokidar');
server.listen(1234);
app.use('/public', express.static( __dirname + '/public'));
app.get('/', function(request, response){
var ipAddress = request.socket.remoteAddress;
console.log("New express connection from: " + ipAddress);
response.sendfile(__dirname + '/public/index.html'); //Server client
});
var watcher = chokidar.watch("temp", {ignored: /[\/\\]\./, persistent: true});
watcher.on('change', function(path){
console.log(path + " has changed.");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = JSON.stringify(compiled)
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("Sent message");
}
});
});
watcher.on('unlink', function(path){
console.log('File: ', path, ' has been removed');
});
watcher.on('add', function(path){
console.log("hi");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = compiled;
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("message sent");
}
//fs.unlinkSync(path);
});
});
Client Side:
var socket = io.connect('http://localhost');
socket.on('message', function(data){
console.log(data);
var parsed = JSON.parse(data);
recPsecNew = parsed.recPsec;
typeNew = parsed.type;
analyze(recPsecNew, typeNew);
});
I am using socket.io in conjunction with express 4.
Chokidar is found here: https://github.com/paulmillr/chokidar
Logs from the console if I change the name of a file twice are shown here: http://s000.tinyupload.com/?file_id=95726281991906625675

Have you tried lodash's Function?
Probably you can use lodash.debounce function
According to its docs:
_.debounce(func, [wait=0], [options])
Creates a debounced function that delays invoking func until after wait milliseconds have elapsed since the last time the debounced function was invoked. The debounced function comes with a cancel method to cancel delayed invocations. Provide an options object to indicate that func should be invoked on the leading and/or trailing edge of the wait timeout. Subsequent calls to the debounced function return the result of the last func invocation.