We Keep Coding

Running node-rdkafka code in server

I'm running the below node-rdkafka code in Eclipse as Node.js application. This is the sample code from https://blizzard.github.io/node-rdkafka/current/tutorial-producer_.html
I want to run this in a test server and call from iOS Mobile application.
I knew about running node.js app in AWS.
Question I: Is there any other options to run in a free test server environment like Tomcat?
Question II: Even If I am able to run this node.js app in a server, how do i call from a mobile application? Do I need to call producer.on('ready', function(arg) (or) What function i need to call from Mobile app?
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function() {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
/*
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});*/
//starting the producer
producer.connect();

First of all, you need an HTTP server. ExpressJS can be used. Then, just tack on the Express code basically at the end, but move the producer loop into the request route.
So, start with what you had
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
});
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
Then, you can add this in the same file.
var express = require('express')
var app = express()
app.get('/', (req, res) => res.send('Ready to send messages!'))
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
} // end if
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'))
I don't think the poll loop is necessary since you don't care about the counter anymore.
Now, connect your mobile app to http://<your server IP>:3000/ and send test messages with a POST request to http://<your server IP>:3000/10, for example, and adjust to change the number of messages to send

I might be late on this but this is how I did using promises and found it better than have a time out etc.
const postMessageToPublisher = (req, res) => {
return new Promise((resolve, reject) => {
producer.connect();
producer.setPollInterval(globalConfigs.producerPollingTime);
const actualBody = requestBody.data;
const requestBody = req.body;
const topicName = req.body.topicName;
const key = requestBody.key || uuid();
const partition = requestBody.partition || undefined;
const data = Buffer.from(JSON.stringify(udpatedBody));
/**
* Actual messages are sent here when the producer is ready
*/
producer.on(kafkaEvents.READY, () => {
try {
producer.produce(
topic,
partition,
message,
key // setting key user provided or UUID
);
} catch (error) {
reject(error);
}
});
// Register listener for debug information; only invoked if debug option set in driver_options
producer.on(kafkaEvents.LOG, log => {
logger.info('Producer event log notification for debugging:', log);
});
// Register error listener
producer.on(kafkaEvents.ERROR, err => {
logger.error('Error from producer:' + JSON.stringify(err));
reject(err);
});
// Register delivery report listener
producer.on(kafkaEvents.PUBLISH_ACKNOWLEDGMENT, (err, ackMessage) => {
if (err) {
logger.error(
'Delivery report: Failed sending message ' + ackMessage.value
);
logger.error('and the error is :', err);
reject({ value: ackMessage.value, error: err });
} else {
resolve({
teamName: globalConfigs.TeamNameService,
topicName: ackMessage.topic,
key: ackMessage.key.toString()
});
}
});
});
};
Please note that kafkaEvents contains my constants for the events we listen to and it is just a reference such as kafkaEvents.LOG is same as event.log
and also the calling function is expecting this to a promise and accordingly we user .then(data => 'send your response to user from here') and .catch(error => 'send error response to user
this is how I achieved it using promises

Working with multiple tabs with Socket.io

I've the following code working in my server-side, it's all ok. But, I want to keep the same connection between n tabs, because when I open a new tab, looks like I've disconnected from the first tab... So, how can I keep the same connection?
client.js
socket.emit("connected", {user: inputUser.val()};
app.js
var express = require("express"),
app = express(),
http = require("http").Server(app),
io = require("socket.io")(http),
users = {};
io.on("connection", function(socket) {
socket.on("connected", function(data) {
socket.user = data.user;
users[socket.user] = socket;
updateUsers();
});
function updateUsers() {
io.emit("users", Object.keys(users));
}
socket.on("typing", function(data) {
var userMsg = data.user;
if(userMsg in users) {
users[userMsg].emit("typing", {user: socket.user});
}
});
socket.on("disconnect", function(data) {
if(!socket.user) {
return;
}
delete users[socket.user];
updateUsers();
});
});
var port = Number(process.env.PORT || 8000);
http.listen(port, function() {
console.log("Server running on 8000!");
});
Update:
The typing event above works fine... So I tried the typing event according to the answer:
var express = require("express"),
app = express(),
http = require("http").Server(app),
io = require("socket.io")(http),
users = {};
io.on("connection", function(socket) {
socket.on("connected", function(data) {
socket.user = data.user;
// add this socket to the Set of sockets for this user
if (!users[socket.user]) {
users[socket.user] = new Set();
}
users[socket.user].add(socket);
updateUsers();
});
function updateUsers() {
io.emit("users", Object.keys(users));
}
socket.on("typing", function(data) {
var userMsg = data.user;
if(userMsg in users) {
users[userMsg].emit("typing", {user: socket.user});
}
});
socket.on("disconnect", function(data) {
if(!socket.user) {
return;
}
// remove socket for this user
// and remove user if socket count hits zero
if (users[socket.user]) {
users[socket.user].delete(socket);
if (users[socket.user].size === 0) {
delete users[socket.user];
}
}
updateUsers();
});
});
var port = Number(process.env.PORT || 8000);
http.listen(port, function() {
console.log("Server running on 8000!");
});
But it is giving the following error:
users[userMsg].emit("typing", {user: socket.user});
^
TypeError: users[userMsg].emit is not a function
Update²:
To fix the typing event error, I just changed to:
socket.on("typing", function(data) {
var userMsg = data.user;
if(userMsg in users) {
for(let userSet of users[userMsg]) {
userSet.emit("typing", {user: socket.user});
}
}
});

There is no simple way to share a single socket.io connection among multiple tabs in the same browser. The usual model for multiple tabs would be that each tab just has its own socket.io connection.
The opening of a new tab and a new socket.io connection should not, on its own, cause your server to think anything was disconnected. If your code is doing that, then that is a fault in your code and it is probably easier to fix that particular fault.
In fact, if you want to explicitly support multiple tabs and be able to recognize that multiple tabs may all be used by the same user, then you may want to change your server side code so that it can keep track of multiple sockets for a single user, rather than how it is currently coded to only keep track of one socket per user.
If your server code is really just trying to keep track of which users online, then there's probably an easier way to do that by referencing counting each user. I will post a code example in a bit.
var express = require("express"),
app = express(),
http = require("http").Server(app),
io = require("socket.io")(http),
users = {};
io.on("connection", function(socket) {
socket.on("connected", function(data) {
socket.user = data.user;
// increment reference count for this user
if (!users[socket.user]) {
users[socket.user] = 0;
}
++users[socket.user];
updateUsers();
});
function updateUsers() {
io.emit("users", Object.keys(users));
}
socket.on("disconnect", function(data) {
if(!socket.user) {
return;
}
// decrement reference count for this user
// and remove user if reference count hits zero
if (users.hasOwnProperty(socket.user)) {
--users[socket.user];
if (users[socket.user] === 0) {
delete users[socket.user];
}
}
updateUsers();
});
});
var port = Number(process.env.PORT || 8000);
http.listen(port, function() {
console.log("Server running on 8000!");
});
If you need the users object to have the socket object in it, then you can change what is stored in the users object to be a Set of sockets like this:
var express = require("express"),
app = express(),
http = require("http").Server(app),
io = require("socket.io")(http),
users = {};
io.on("connection", function(socket) {
socket.on("connected", function(data) {
socket.user = data.user;
// add this socket to the Set of sockets for this user
if (!users[socket.user]) {
users[socket.user] = new Set();
}
users[socket.user].add(socket);
updateUsers();
});
function updateUsers() {
io.emit("users", Object.keys(users));
}
socket.on("disconnect", function(data) {
if(!socket.user) {
return;
}
// remove socket for this user
// and remove user if socket count hits zero
if (users[socket.user]) {
users[socket.user].delete(socket);
if (users[socket.user].size === 0) {
delete users[socket.user];
}
}
updateUsers();
});
});
var port = Number(process.env.PORT || 8000);
http.listen(port, function() {
console.log("Server running on 8000!");
});

For anyone still having this issue. here is how i fixed it.
let me explain.
once the page refreshes or a new tab is opened, socket dosen't really care so it opens a new connection every time . this is more of a advantage than disadvantage. the best way to tackle the issue is on the server side, once a user logs in with his or her user name , you can send that name along with the query options on the client so it can be used as a unique identifier. in my case i used a token
this.socket = io.connect(`${environment.domain}` , {
query: {token: this.authservice.authToken}
});
then on the server side you can create an empty array to a key and an array of values. the username of the user will be used as a key and the corresponding array of socket as the value. in my own case like i said i used a token
const users = [ ]
socket.nickname = (decoded token username);
users[socket.nickname] = [socket];
then you can perform a simple logic to check if a user already exists in an array, if it does, push the new socket to the array of the user
if ( user.username in users) {
console.log('already exists')
users[user.username].push(socket);
}
if it dosent, just create a new key and add the socket as the key.(make sure its an array because a user can always refresh or open a new tab with the same account and you dont want the chat message to deliver in one tab and not deliver in another)
else {
socket.nickname = username;
users[socket.nickname] = [socket];
}
then to emit a message you simply loop through the array and emit the message accordingly. this way each tab gets the message
socket.on('chat', (data) => {
if (data.to in users) {
for(let i = 0; i < users[data.to].length; i++) {
users[data.to][i].emit('chat', data)
}
for(let i = 0; i < users[data.user].length; i++) {
users[data.user][i].emit('chat', data)
}
}
})
you can add a disconnect logic to remove the socket from the users array too to save memory, so only currently open tabs acre active and closed tabs are removed. i hope it solved your problem

My solution is joining socket to a room with specific user Id.
io.on('connection', async (socket) => {
socket.join('user:' + socket.handshake.headers.uid) // The right way is getting `uid` from cookie/token and verifying user
})
One advantage is sending data to specific user (sending to all tabs)
io.to('user:' + uid).emit('hello');
Hope it's helpful!

I belive the best way is create a channel for the user and unique it by their ID, so, when you need to receive or send something you use the channel and every socket connected to it will receive.

Another solution is to save the flag to localStorage and use eventListener to change localStorage.
Do not connect when another connection exists.
and save message in local storage for send with master tab.

Disconnect event fired after cilent reconnect - socket.io

I'm using socket.io for realtime communication. When inetrnet is not available for few seconds, then it will connect to server when the connection is establish and then it will immediately trigger event which will log in my clinet. After establishing connection, (i think) my socket.io on server side fire disconnect event after ~one minute (in callback function I log off my client), and that is my problem.
This is my server side. (a have server.js and he run other.js files, in object 'files' a have that files).
server.js :
io = require('socket.io').listen(server);//server listening on 8080 port
io.configure(function () {
io.set('transports', ['websocket', 'xhr-polling']);
io.set('log level', 1);
io.disable('browser client');
});
var files = {
control: null,
terminal: null,
messages: null
};
for (var game in games) {
games[game] = require('./game_modules/' + game + '.js').listen(io, create_waiter(game));
}
require('http').createServer(function (req, res) {
var resultCode = 500;
var resultText = '';
var parts = require('url').parse(req.url, true);
var matches = parts.pathname.match('^/([a-z_-]+)/([a-z_-]+)/$');
if (matches) {
var game = matches[1];
var command = matches[2];
resultText = games[game][command](parts.query);
resultCode = 200;
res.writeHead(resultCode, {'Content-Type': 'text/plain'});
res.end(resultText);
}).listen(8081, "127.0.0.1");
control.js :
var control = module.exports = function() {
};
control.listen = function(io) {
this.server = io.of('/control');
this.onTerminate = onTerminate;
//********************************************************************************
this.server.on('connection', function(socket) {
socket.on('disconnect', function(data) {
console.log(data); //after my client reconnect, disconnect event is triggered, and
//console.log write 'close timeout'
After client reconnect, I do not want trigger disconnect event, or if disconnect event was triggered I want check in callback function: is my client connected and prevent "log off" for my client.
Thanks :)

Socket.io Multiple messages emitted from one event

I've been looking for quite a while for a solution but haven't found anything yet.
I'm trying to emit a message from a server every time the server sees that a file has changed in a specified directory. However, instead of only emitting one message, it insists on emitting the same message three times. I am using chokidar to watch the directory, and inside of the 'change' event I emit the message.
Server side code:
var express = require('express')
, app = express()
, http = require('http')
, server = http.Server(app)
, io =require('socket.io')(server)
, chokidar = require('chokidar');
server.listen(1234);
app.use('/public', express.static( __dirname + '/public'));
app.get('/', function(request, response){
var ipAddress = request.socket.remoteAddress;
console.log("New express connection from: " + ipAddress);
response.sendfile(__dirname + '/public/index.html'); //Server client
});
var watcher = chokidar.watch("temp", {ignored: /[\/\\]\./, persistent: true});
watcher.on('change', function(path){
console.log(path + " has changed.");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = JSON.stringify(compiled)
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("Sent message");
}
});
});
watcher.on('unlink', function(path){
console.log('File: ', path, ' has been removed');
});
watcher.on('add', function(path){
console.log("hi");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = compiled;
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("message sent");
}
//fs.unlinkSync(path);
});
});
Client Side:
var socket = io.connect('http://localhost');
socket.on('message', function(data){
console.log(data);
var parsed = JSON.parse(data);
recPsecNew = parsed.recPsec;
typeNew = parsed.type;
analyze(recPsecNew, typeNew);
});
I am using socket.io in conjunction with express 4.
Chokidar is found here: https://github.com/paulmillr/chokidar
Logs from the console if I change the name of a file twice are shown here: http://s000.tinyupload.com/?file_id=95726281991906625675

Have you tried lodash's Function?
Probably you can use lodash.debounce function
According to its docs:
_.debounce(func, [wait=0], [options])
Creates a debounced function that delays invoking func until after wait milliseconds have elapsed since the last time the debounced function was invoked. The debounced function comes with a cancel method to cancel delayed invocations. Provide an options object to indicate that func should be invoked on the leading and/or trailing edge of the wait timeout. Subsequent calls to the debounced function return the result of the last func invocation.

Using Socket.IO to show object count within Compound.js

I'm testing out Socket.IO for a new project that uses Compound.js (built on node.js/Express.js), but I've come across a problem. I've managed to get the basics working and the following message works fine:
Server-side JS (applicationname/config/initializers/socketio.js):
var sio = require('socket.io');
var http = require('http');
var activeClients = 0;
module.exports = function (compound) {
var app = compound.app;
var server = http.createServer(app);
compound.server = server;
var io = compound.io = sio.listen(server);
io.sockets.on('connection', function (socket) {
activeClients +=1;
var connections = setInterval(function () {
socket.emit('news', { clients: activeClients });
}, 1000);
socket.on('disconnect', function () {
activeClients -= 1;
io.sockets.emit('user disconnected');
clearInterval(connections);
});
});
}
Front-end JS (applicationname/public/index.html):
<script src="/socket.io/socket.io.js"></script>
<script>
var socket = io.connect('http://localhost');
function msgReceived(msg){
$clientCounter.html(msg.posts);
}
$(document).ready(function () {
$clientCounter = $("#client_count");
socket.on('news', function (data) {
msgReceived(data);
socket.emit('my other event', { my: 'data' });
});
});
</script>
What I'm trying to do now is get the number of posts that have been made on the server. I used a default scaffold to generate the Posts method, controller and views.
How do I now find the total number of posts and pass that value to the server-side JS?

Simply count the number of emits on the server side and send that?
...
var emits = 0;
var connections = setInterval(function () {
emits++;
socket.emit('news', { clients: activeClients , num_messages: emits});
}, 1000);
...
Or create a specific message that can be polled from the client