We Keep Coding

Running node-rdkafka code in server

I'm running the below node-rdkafka code in Eclipse as Node.js application. This is the sample code from https://blizzard.github.io/node-rdkafka/current/tutorial-producer_.html
I want to run this in a test server and call from iOS Mobile application.
I knew about running node.js app in AWS.
Question I: Is there any other options to run in a free test server environment like Tomcat?
Question II: Even If I am able to run this node.js app in a server, how do i call from a mobile application? Do I need to call producer.on('ready', function(arg) (or) What function i need to call from Mobile app?
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function() {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
/*
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});*/
//starting the producer
producer.connect();

First of all, you need an HTTP server. ExpressJS can be used. Then, just tack on the Express code basically at the end, but move the producer loop into the request route.
So, start with what you had
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
});
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
Then, you can add this in the same file.
var express = require('express')
var app = express()
app.get('/', (req, res) => res.send('Ready to send messages!'))
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
} // end if
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'))
I don't think the poll loop is necessary since you don't care about the counter anymore.
Now, connect your mobile app to http://<your server IP>:3000/ and send test messages with a POST request to http://<your server IP>:3000/10, for example, and adjust to change the number of messages to send

I might be late on this but this is how I did using promises and found it better than have a time out etc.
const postMessageToPublisher = (req, res) => {
return new Promise((resolve, reject) => {
producer.connect();
producer.setPollInterval(globalConfigs.producerPollingTime);
const actualBody = requestBody.data;
const requestBody = req.body;
const topicName = req.body.topicName;
const key = requestBody.key || uuid();
const partition = requestBody.partition || undefined;
const data = Buffer.from(JSON.stringify(udpatedBody));
/**
* Actual messages are sent here when the producer is ready
*/
producer.on(kafkaEvents.READY, () => {
try {
producer.produce(
topic,
partition,
message,
key // setting key user provided or UUID
);
} catch (error) {
reject(error);
}
});
// Register listener for debug information; only invoked if debug option set in driver_options
producer.on(kafkaEvents.LOG, log => {
logger.info('Producer event log notification for debugging:', log);
});
// Register error listener
producer.on(kafkaEvents.ERROR, err => {
logger.error('Error from producer:' + JSON.stringify(err));
reject(err);
});
// Register delivery report listener
producer.on(kafkaEvents.PUBLISH_ACKNOWLEDGMENT, (err, ackMessage) => {
if (err) {
logger.error(
'Delivery report: Failed sending message ' + ackMessage.value
);
logger.error('and the error is :', err);
reject({ value: ackMessage.value, error: err });
} else {
resolve({
teamName: globalConfigs.TeamNameService,
topicName: ackMessage.topic,
key: ackMessage.key.toString()
});
}
});
});
};
Please note that kafkaEvents contains my constants for the events we listen to and it is just a reference such as kafkaEvents.LOG is same as event.log
and also the calling function is expecting this to a promise and accordingly we user .then(data => 'send your response to user from here') and .catch(error => 'send error response to user
this is how I achieved it using promises

Node.js broadcasting twice

I'm giving my first steps with node.js. I'm trying to implement a simple chat room to get the basics, but can't figure out why after every broadcast, a second (empty) message is automatically sent. This behaviour happens most of the time but not always. I'm using Netbeans in Windows, with Putty to simulate the client connections.
Console output:
Server running
::1:60616 joined
::1:60617 joined
-----------
Broadcasting:
hello
-----------
Broadcasting:
-----------
Broadcasting:
goodbye
-----------
Broadcasting:
-----------
Client 1:
Hi ::1:60616!
hello
goodbye
Client 2:
Hi ::1:60617!
::1:60616 says hello::1:60616 says
::1:60616 says goodbye::1:60616 says
Code
var net = require('net');
var chatServer = net.createServer();
var clientList = [];
console.log('Server running');
chatServer.on('connection', function (client) {
client.name = client.remoteAddress + ':' + client.remotePort;
client.write('Hi ' + client.name + '!\n');
console.log(client.name + ' joined');
clientList.push(client);
client.on('data', function (data) {
console.log('Broadcasting: ');
console.log(data.toString());
console.log('-----------');
broadcast(data, client);
});
client.on('end', function () {
console.log(client.name + ' quit');
clientList.splice(clientList.indexOf(client), 1);
});
client.on('error', function (e) {
console.log(e);
});
});
function broadcast(message, sender) {
var text = sender.name + " says " + message;
var cleanup = [];
clientList.forEach(function (client) {
if (client !== sender) {
if (client.writable) {
client.write(text);
} else {
cleanup.push(client);
client.destroy();
}
}
});
cleanup.forEach(function (client) {
clientList.splice(clientList.indexOf(client), 1);
});
}
chatServer.listen(9000);

You can't rely on the raw data event to present you with "well-rounded" chunks of data. It may come in pieces, and you can't necessarily control how large those pieces are, or that they get split on particular boundaries.
However, there are modules that can help you, for instance split, which will split the data into separate (full) lines:
const split = require('split');
...
client.pipe(split()).on('data', function (data) {
// `data` is now a separate line
console.log('Broadcasting: ');
console.log(data.toString());
console.log('-----------');
broadcast(data, client);
});

Socket.io Multiple messages emitted from one event

I've been looking for quite a while for a solution but haven't found anything yet.
I'm trying to emit a message from a server every time the server sees that a file has changed in a specified directory. However, instead of only emitting one message, it insists on emitting the same message three times. I am using chokidar to watch the directory, and inside of the 'change' event I emit the message.
Server side code:
var express = require('express')
, app = express()
, http = require('http')
, server = http.Server(app)
, io =require('socket.io')(server)
, chokidar = require('chokidar');
server.listen(1234);
app.use('/public', express.static( __dirname + '/public'));
app.get('/', function(request, response){
var ipAddress = request.socket.remoteAddress;
console.log("New express connection from: " + ipAddress);
response.sendfile(__dirname + '/public/index.html'); //Server client
});
var watcher = chokidar.watch("temp", {ignored: /[\/\\]\./, persistent: true});
watcher.on('change', function(path){
console.log(path + " has changed.");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = JSON.stringify(compiled)
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("Sent message");
}
});
});
watcher.on('unlink', function(path){
console.log('File: ', path, ' has been removed');
});
watcher.on('add', function(path){
console.log("hi");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = compiled;
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("message sent");
}
//fs.unlinkSync(path);
});
});
Client Side:
var socket = io.connect('http://localhost');
socket.on('message', function(data){
console.log(data);
var parsed = JSON.parse(data);
recPsecNew = parsed.recPsec;
typeNew = parsed.type;
analyze(recPsecNew, typeNew);
});
I am using socket.io in conjunction with express 4.
Chokidar is found here: https://github.com/paulmillr/chokidar
Logs from the console if I change the name of a file twice are shown here: http://s000.tinyupload.com/?file_id=95726281991906625675

Have you tried lodash's Function?
Probably you can use lodash.debounce function
According to its docs:
_.debounce(func, [wait=0], [options])
Creates a debounced function that delays invoking func until after wait milliseconds have elapsed since the last time the debounced function was invoked. The debounced function comes with a cancel method to cancel delayed invocations. Provide an options object to indicate that func should be invoked on the leading and/or trailing edge of the wait timeout. Subsequent calls to the debounced function return the result of the last func invocation.

Duplicate Events Socket.io and Node.js over STOMP

I need some help about my node.js+socket.io implementation.
This service expose a server that connects to an ActiveMQ broker over the STOMP protocol, using the stomp-js node.js module to receive events; that then are displayed in a web front end through websockets using socket.io.
So, everything was fine until I started use the Filters feature of ActiveMQ, but this was not the failure point because of my and my team researching, we found the way to ensure the implementation was fine, the problem comes with the connections: So here's the thing, I receive the filters to subscribe, I successfully subscribe to but when I receive a new set of filters is when comes the duplicated, triplicated and more and more messages depending the number of times that I subscribe-unsubscribe to.
So making some debug, I cannot see what's the problem but I'm almost sure that is some bad implementation of the callbacks or the program flow, I'll attach my code to read your comments about it.
Thanks a lot!
var sys = require('util');
var stomp = require('stomp');
var io = require('socket.io').listen(3000);
var socket = io.sockets.on('connection', function (socket) {
var stomp_args = {
port: 61616,
host: 'IP.ADDRESS',
debug: true,
};
var headers;
var client = new stomp.Stomp(stomp_args);
var setFilters = false;
socket.on('filtros', function (message) {
console.log('DEBUG: Getting filters');
if(setFilters){
client.unsubscribe(headers);
}
else{
client.connect();
}
var selector = '';
headers = '';
for(var attributename in message){
console.log(attributename+" : " + message[attributename]);
if(message[attributename] != ''){
selector += ' ' + attributename + '=\'' + message[attributename] + '\' AND ';
}
}
selector = selector.substring(0, selector.length - 4)
console.log('DEBUG: Selector String: ' + selector);
headers = {
destination: '/topic/virtualtopic',
ack: 'client',
selector: selector
};
if(setFilters)
client.subscribe(headers);
client.on('connected', function() {
client.subscribe(headers);
console.log('DEBUG: Client Connected');
setFilters = true;
});
});
var bufferMessage;
client.on('message', function(message) {
console.log("Got message: " + message.headers['message-id']);
var jsonMessage = JSON.parse(message.body);
if(bufferMessage === jsonMessage){
console.log('DEBUG: recibo un mensaje repetido');
return 0;
}
else{
console.log('DEBUG: Cool');
socket.emit('eventoCajero', jsonMessage);
}
client.ack(message.headers['message-id']);
bufferMessage = jsonMessage;
});
socket.on('disconnect', function(){
console.log('DEBUG: Client disconnected');
if(setFilters){
console.log('DEBUG: Consumer disconnected');
client.disconnect();
}
});
client.on('error', function(error_frame) {
console.log(error_frame.body);
});
});

Looking in the Socket.IO documentation, I've found that this is a known issue (I think critical known issue) and they have not fixed it yet. So, to correct this is necessary to reconnect to the socket in the client side to avoid duplicate messages, using:
socket.socket.reconnect();
function to force reconnection explicitly.

Memory Leak with socket.io + node.js

I appear to have a memory leak with my Node.js application. I built it quickly, and my JavaScript isn't too strong, so this might be easy.
I've done some heap dumps on it, and it's the String object? leaking memory, at the rate of about 1MB every 5 minutes. I expanded String, and it's actually String.Array?
Heap stack:
#!/usr/local/bin/node
var port = 8081;
var io = require('socket.io').listen(port),
sys = require('sys'),
daemon = require('daemon'),
mysql = require('mysql-libmysqlclient');
var updateq = "SELECT 1=1";
var countq = "SELECT 2=2";
io.set('log level', 2);
process.on('uncaughtException', function(err) {
console.log(err);
});
var connections = 0;
var conn = mysql.createConnectionSync();
dbconnect();
io.sockets.on('connection', function(client){
connections++;
client.on('disconnect', function(){ connections--; })
});
process.on('exit', function () {
console.log('Exiting');
dbdisconnect();
});
function dbdisconnect() {
conn.closeSync();
}
function dbconnect() {
conn.connectSync('leet.hacker.org','user','password');
}
function update() {
if (connections == 0)
return;
conn.query(updateq, function (err, res) {
if (err) {
dbdisconnect();
dbconnect();
return;
}
res.fetchAll(function (err, rows) {
if (err) {
throw err;
}
io.sockets.json.send(rows);
});
});
}
function totals() {
if (connections == 0)
return;
conn.query(countq, function (err, res) {
if (err) {
// Chances are that the server has just disconnected, lets try reconnecting
dbdisconnect();
dbconnect();
throw err;
}
res.fetchAll(function (err, rows) {
if (err) {
throw err;
}
io.sockets.json.send(rows);
});
});
}
setInterval(update, 250);
setInterval(totals,1000);
setInterval(function() {
console.log("Number of connections: " + connections);
},1800000);
daemon.daemonize('/var/log/epiclog.log', '/var/run/mything.pid', function (err, pid) {
// We are now in the daemon process
if (err) return sys.puts('Error starting daemon: ' + err);
sys.puts('Daemon started successfully with pid: ' + pid);
});
Current version
function totals() {
if (connections > 0)
{
var q = "SELECT query FROM table";
db.query(q, function (err, results, fields) {
if (err) {
console.error(err);
return false;
}
for (var row in results)
{
io.sockets.send("{ ID: '" + results[row].ID + "', event: '" + results[row].event + "', free: '" + results[row].free + "', total: '" + results[row].total + "', state: '" + results[row]$
row = null;
}
results = null;
fields = null;
err = null;
q = null;
});
}
}
Still leaking memory, but it seems only on these conditions:
From startup, with no clients -> Fine
1st client connection -> Fine
2nd client (even with the 1st client disconnecting and reconnecting) -> Leaking memory
Stop all connections -> Fine
1 new connection (connections = 1) -> Leaking memory

Do yourself a favour and use node-mysql, it's a pure javascript mysql client and it's fast. Other than that, you should be using asynchronous code to stop IO being blocked whilst you're working. Using the async library will help you here. It has code for waterfall callback passing among other things.
As for your memory leaking, it probably isn't socket.io, although I haven't used it in a few months, I have had many thousands of concurrent connections and not leaked memory, and my code wasn't the best either.
Two things, however. Firstly your code is faily unreadable. I suggest looking into properly formatting your code (I use two spaces for every indentation but some people use four). Secondly, printing the number of connections every half an hour seems a little silly, when you could do something like:
setInterval(function() {
process.stdout.write('Current connections: ' + connections + ' \r');
}, 1000);
The \r will cause the line to be read back to the start of the line and overwrite the characters there, which will replace the line and not create a huge amount of scrollback. This will help with debugging if you choose to put debugging details in your logging.
You can also use process.memoryUsage() for quickly checking the memory usage (or how much node thinks you're using).

Could this be related to the connected clients array not clearing properly when a client disconnects? The array value gets set to NULL rather than being dropped from the array.