We Keep Coding

Running node-rdkafka code in server

I'm running the below node-rdkafka code in Eclipse as Node.js application. This is the sample code from https://blizzard.github.io/node-rdkafka/current/tutorial-producer_.html
I want to run this in a test server and call from iOS Mobile application.
I knew about running node.js app in AWS.
Question I: Is there any other options to run in a free test server environment like Tomcat?
Question II: Even If I am able to run this node.js app in a server, how do i call from a mobile application? Do I need to call producer.on('ready', function(arg) (or) What function i need to call from Mobile app?
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function() {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
/*
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});*/
//starting the producer
producer.connect();

First of all, you need an HTTP server. ExpressJS can be used. Then, just tack on the Express code basically at the end, but move the producer loop into the request route.
So, start with what you had
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
});
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
Then, you can add this in the same file.
var express = require('express')
var app = express()
app.get('/', (req, res) => res.send('Ready to send messages!'))
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
} // end if
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'))
I don't think the poll loop is necessary since you don't care about the counter anymore.
Now, connect your mobile app to http://<your server IP>:3000/ and send test messages with a POST request to http://<your server IP>:3000/10, for example, and adjust to change the number of messages to send

I might be late on this but this is how I did using promises and found it better than have a time out etc.
const postMessageToPublisher = (req, res) => {
return new Promise((resolve, reject) => {
producer.connect();
producer.setPollInterval(globalConfigs.producerPollingTime);
const actualBody = requestBody.data;
const requestBody = req.body;
const topicName = req.body.topicName;
const key = requestBody.key || uuid();
const partition = requestBody.partition || undefined;
const data = Buffer.from(JSON.stringify(udpatedBody));
/**
* Actual messages are sent here when the producer is ready
*/
producer.on(kafkaEvents.READY, () => {
try {
producer.produce(
topic,
partition,
message,
key // setting key user provided or UUID
);
} catch (error) {
reject(error);
}
});
// Register listener for debug information; only invoked if debug option set in driver_options
producer.on(kafkaEvents.LOG, log => {
logger.info('Producer event log notification for debugging:', log);
});
// Register error listener
producer.on(kafkaEvents.ERROR, err => {
logger.error('Error from producer:' + JSON.stringify(err));
reject(err);
});
// Register delivery report listener
producer.on(kafkaEvents.PUBLISH_ACKNOWLEDGMENT, (err, ackMessage) => {
if (err) {
logger.error(
'Delivery report: Failed sending message ' + ackMessage.value
);
logger.error('and the error is :', err);
reject({ value: ackMessage.value, error: err });
} else {
resolve({
teamName: globalConfigs.TeamNameService,
topicName: ackMessage.topic,
key: ackMessage.key.toString()
});
}
});
});
};
Please note that kafkaEvents contains my constants for the events we listen to and it is just a reference such as kafkaEvents.LOG is same as event.log
and also the calling function is expecting this to a promise and accordingly we user .then(data => 'send your response to user from here') and .catch(error => 'send error response to user
this is how I achieved it using promises

NodeJS restify API caching best practice

I am very new to NodeJS and I am building my first API using restify.
I want to find out what is best practice for caching the response data - each API call must have its own cache time.
I have looked at res.cache() but that seems to be only per user request and not a global application cache.
I then looked at restify-cache but the documentation did not clearly tell me how to use it.
My application works like this:
server.js code:
var restify = require('restify');
var mysqlDB = require('./config/connection');
// REST server declaration and configuration
var server = restify.createServer({
name: 'test-api',
version: '0.0.1'
});
server.pre(restify.pre.sanitizePath());
server.use(restify.queryParser());
server.use(restify.acceptParser(server.acceptable));
server.use(restify.queryParser());
server.use(restify.bodyParser());
server.listen(9007, function() {
console.log('%s listening at %', server.name, server.url);
mysqlDB.handleDisconnect();
console.log(new Date() +': Started Cricket API on port 9007');
});
var routes = require('./routes')(server);
routes.js code:
module.exports = function(app) {
app.get('/', function(req, res, next) {
return res.send("You have reached the test API");
});
var fixtures = require('./controllers/fixtures');
app.get('/getfixtures', fixtures.getFixtures); // Get All Fixtures
};
fixtures.js code snippet:
this.getFixtures = function (req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "X-Requested-With");
console.log("Get All Fixtures");
var mysql = mysqlDB.getConnection();
var query = "SELECT * FROM fixtures WHERE fixture_date >= CURDATE() ORDER BY fixture_date, fixture_time";
mysql.query(query,function(err,rows){
if(err) {
var status = mysqlDB.getErrorStatus(err.code);
return res.status(status.code).send("Error : "+ status.Message);
} else {
var data = [];
for (i in rows){
var item = rows[i];
var output = util.formatDate(item.fixture_date);
item.fixture_date = output;
data.push(item);
};
return res.send(data);
}
});
};
Can someone please send me in the right direction? I don't know where to add the caching part?

From the library file:
server.use(cache.before); is a middleware that will be triggered to load before the request is handled, going to Redis and checking if the header_{url} key and payload_{url} exits, and at that case the value is returned.
You could put it as mentioned in this gist:
https://gist.github.com/jeffstieler/3d84fa5468c7eadb7685
var server = restify.createServer({
name: 'test-api',
version: '0.0.1'
});
server.pre(restify.pre.sanitizePath());
server.use(cache.before);
server.use(restify.queryParser());
server.use(restify.acceptParser(server.acceptable));
server.use(restify.queryParser());
server.use(restify.bodyParser());
server.on('after', cache.after);
In your code I would add the cache.before after you sanitize the path as this will be saved in Redis. also a next() should be included in every route cached.

I ended up using node-cache
It was easy to use since I come from a Java/Play Framework background - hopefully it helps someone else in future.
Example usage:
var nodeCache = require( "node-cache" );
var myCache = new nodeCache();
var cachedValue = myCache.get("alltests", true);
if (cachedValue != undefined) {
return res.send(cachedValue);
} else {
// Do work here and then:
success = myCache.set("alltests", valueHere, cacheTime);
}

Mocha calling callback after done()

I am aiming for ease of use in my unit tests, I have created a master_test.js file that I will include below. In it, I simply require other test.js files as a function and run them. An issue I had is that Mocha does not detect any unit tests to be run unless I encase my startup in a describe() block, so I am forced to have a master describe encase my application to resolve that. My problem now is that in order to run the other tests, I have to call a callback() after a done(). Is there any easy way to fix my problem?
function app_setup() {
options.database = 'testing';
it('app launched successfully', function(done) {
require('../app.js').initialize(0, options, function(err, app) {
remove_all_entities(function() {
app.set('port', port);
var server = app.listen(app.get('port'), function() {
console.log('Express server listening on port ' + server.address().port);
//ISSUE LOCATION, NEED TO CALL run_tests() callback after done()//
done();
run_tests();
});
});
});
});
}
function run_tests() {
var database = require('../database.js');
var entity_controller_test = require('./controllers/entity_controller_test.js').entity_controller_test;
var login_test = require('./controllers/login_test.js').login_test;
var token_access_test = require('./controllers/token_access_test.js').token_access_test;
var token_auth_test = require('./controllers/auth_token_test.js').token_auth_test;
var business_rules_insert = require('./business_rules/basic_database_rules.js').business_rules_insert_test;
var logout_test = require('./controllers/logout_test.js').logout_test;
var schema_override = require('./business_rules/schema_overrides').schema_overrides;
var aggregation_test = require('./entity_specific/aggregation').aggregation_test;
var tests = [login_test, aggregation_test, logout_test];
async.series(tests, function() {
test_teardown(done);
});
}
function test_teardown(done) {
remove_all_entities(done);
};

Socket.io Multiple messages emitted from one event

I've been looking for quite a while for a solution but haven't found anything yet.
I'm trying to emit a message from a server every time the server sees that a file has changed in a specified directory. However, instead of only emitting one message, it insists on emitting the same message three times. I am using chokidar to watch the directory, and inside of the 'change' event I emit the message.
Server side code:
var express = require('express')
, app = express()
, http = require('http')
, server = http.Server(app)
, io =require('socket.io')(server)
, chokidar = require('chokidar');
server.listen(1234);
app.use('/public', express.static( __dirname + '/public'));
app.get('/', function(request, response){
var ipAddress = request.socket.remoteAddress;
console.log("New express connection from: " + ipAddress);
response.sendfile(__dirname + '/public/index.html'); //Server client
});
var watcher = chokidar.watch("temp", {ignored: /[\/\\]\./, persistent: true});
watcher.on('change', function(path){
console.log(path + " has changed.");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = JSON.stringify(compiled)
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("Sent message");
}
});
});
watcher.on('unlink', function(path){
console.log('File: ', path, ' has been removed');
});
watcher.on('add', function(path){
console.log("hi");
fs.readFile(path,'utf8', function(err, data){
if(err) {
return console.log(err);
}
else
{
var json = JSON.parse(data), recPsec, type;
recPsec = json.data[0].values[0];
type = json.data[0].values[16];
var compiled = {
"recPsec" : recPsec,
"type" : type
}
var jsonMessage = compiled;
io.sockets.emit('message', JSON.stringify(jsonMessage));
console.log("message sent");
}
//fs.unlinkSync(path);
});
});
Client Side:
var socket = io.connect('http://localhost');
socket.on('message', function(data){
console.log(data);
var parsed = JSON.parse(data);
recPsecNew = parsed.recPsec;
typeNew = parsed.type;
analyze(recPsecNew, typeNew);
});
I am using socket.io in conjunction with express 4.
Chokidar is found here: https://github.com/paulmillr/chokidar
Logs from the console if I change the name of a file twice are shown here: http://s000.tinyupload.com/?file_id=95726281991906625675

Have you tried lodash's Function?
Probably you can use lodash.debounce function
According to its docs:
_.debounce(func, [wait=0], [options])
Creates a debounced function that delays invoking func until after wait milliseconds have elapsed since the last time the debounced function was invoked. The debounced function comes with a cancel method to cancel delayed invocations. Provide an options object to indicate that func should be invoked on the leading and/or trailing edge of the wait timeout. Subsequent calls to the debounced function return the result of the last func invocation.

Dynamic Namespaces Socket.IO

How can I use dynamic namespaces in socket.io.
I'm looking in the (poor) documentation, and it says that namespaces must be used like this:
io.of('/news')
io.of('/akfda')
To use a namespace you do io.of("/namespace").
Do I need to register every single namespace in the server? Maybe I want a namespace for dynamic content.
How can I do something like :
io.of('/:somethign/:id')

Socket.IO supports 'rooms' (https://github.com/LearnBoost/socket.io/wiki/Rooms), you can use it instead of namespaces. Also when you need dynamic in routes (and you using express in your app) - best way is to use use route-engine from express box.
Best way to do dynamic routing with Express.js (node.js)
Using routes in Express-js
http://expressjs.com/api.html#app.routes
http://shtylman.com/post/expressjs-re-routing/
http://jordanhoff.com/post/22602013678/dynamic-express-routing
However, if you still think that you need dynamic in namespaces in socket.io, here is small example how it can be implemented:
User-side:
var connect = function (ns) {
return io.connect(ns, {
query: 'ns='+ns,
resource: "socket.io"
});
}
var socket = connect('/user/12');
Server-side:
var url = require('url');
, ev = new events.EventEmitter()
// <ns name>: <ns regexp>
var routes = {
// /user/:id
'user': '^\\/user\\/(\\d+)$',
// /:something/:id
'default': '^\\/(\\\w+)\\/(\\d+)$'
};
// global entry point for new connections
io.sockets.on('connection', function (socket) {
// extract namespace from connected url query param 'ns'
var ns = url.parse(socket.handshake.url, true).query.ns;
console.log('connected ns: '+ns)
//
for (var k in routes) {
var routeName = k;
var routeRegexp = new RegExp(routes[k]);
// if connected ns matched with route regexp
if (ns.match(routeRegexp)) {
console.log('matched: '+routeName)
// create new namespace (or use previously created)
io.of(ns).on('connection', function (socket) {
// fire event when socket connecting
ev.emit('socket.connection route.'+routeName, socket);
// #todo: add more if needed
// on('message') -> ev.emit(...)
});
break;
}
}
// when nothing matched
// ...
});
// event when socket connected in 'user' namespace
ev.on('socket.connection route.user', function () {
console.log('route[user] connecting..');
});
// event when socket connected in 'default' namespace
ev.on('socket.connection route.default', function () {
console.log('route[default] connecting..');
});
I hope this will help you!

I would use "rooms" to support your dynamic content.
Server Side
var server = require('http').createServer(),
io = require('socket.io')(server);
io.on('connection', function(socket){
var room = socket.handshake['query']['r_var'];
socket.join(room);
console.log('user joined room #'+room);
socket.on('disconnect', function() {
socket.leave(room)
console.log('user disconnected');
});
socket.on('chat message', function(msg){
io.to(room).emit('chat message', msg);
});
});
server.listen(3000);
Client Side
var socket_connect = function (room) {
return io('localhost:3000', {
query: 'r_var='+room
});
}
var random_room = Math.floor((Math.random() * 2) + 1);
var socket = socket_connect(random_room);
socket.emit('chat message', 'hello room #'+random_room);
....

As of version 2.1.1 I was able to make it work with this:
wss.of((nsp, query, next) => {
const { token } = query;
// Do your authentication or whatever here...
// If success
next(null, true);
}).on('connect', (socket) => {
// socket connected to your namespace
});

Server
var MAX_CLIENTS = 5;
var namespace_queue = [];
function searchObjectOnArray(nameKey, myArray) {
for (var i = 0; i < myArray.length; i++) {
if (myArray[i].id === nameKey) {
return myArray[i];
}
}
}
function createNamespace(data){
var ns = {
//id: require('node-uuid')(),
id : data.name,
clients: 0,
};
namespace_queue.push(ns);
return ns;
}
createNamespace({name: 'primer'});
io.of('').on('connection', function(socket){
console.log('-' + socket.id);
/// Welcome to the new client
socket.emit('Welcome', {SocketId : socket.id});
socket.on('JoinToApp', function (data, callback) {
var namespaceToConnect = searchObjectOnArray(data.namespace, namespace_queue)
if(namespaceToConnect.clients <= MAX_CLIENTS){
var dynamicNamespace = io.of('/' + namespaceToConnect.id);
dynamicNamespace.on('connection', function(ns_socket){
console.log('user connected to ' + namespaceToConnect.id);
dynamicNamespace.emit('hi', 'everyone!');
});
namespaceToConnect.clients++;
}
callback({namespaces:namespace_queue});
})
socket.on('createNamespace',function(data,join_cb){
createNamespace(data);
join_cb({message:'Namespace created'});
});
});
Client
<input id="namespaceInput" type="text" placeholder="New namespace name">
<input id="namespaceToConnect" type="text" placeholder="namespace to connect">
<button onclick="javascript: createNamespace()">Create Namespace</button>
<button onclick="javascript: joinToNamespace()">Connect to Namespace</button>
<script src="https://cdn.socket.io/socket.io-1.4.5.js"></script>
<script>
var socket = null;
(function(){
socket = io.connect('http://localhost:3000/');
})()
function createNamespace(){
var namespaceName = document.getElementById("namespaceInput").value;
socket.emit('createNamespace', {name : namespaceName}, function(data){
alert(data.message);
})
}
function joinToNamespace(){
var name = document.getElementById("namespaceToConnect").value;
socket.emit('JoinToApp', {namespace: name}, function(data){
console.log('Namespaces created:');
console.log(data)
var ns_socket = io.connect('http://localhost:3000/' + name);
ns_socket.on('connect',function(){
console.log('joined namespace ' + name);
});
ns_socket.on('hi', function(data){
console.log('hi ' + data)
})
});
}
</script>
More details on: https://ingcamilorodriguez.wordpress.com/2016/06/21/como-hacer-namespaces-dinamicos-en-socket-io/

Here is one way. Here is a socket.io subclass I created to solve the problem:
https://github.com/PencilCode/dynamic.io
That subclass adds dynamic namespaces as well as virtual hostname support (each host can go into its own namespace tree if you like). That repo has some examples.
Here is a universal socket.io listener that listens to every namespace requested, and logs a message for every socket that connects. You could listen to a different regexp to listen to any subset of namespaces.
It works with the standard socket.io client library without any modification.
var DynamicServer = require('dynamic.io');
io = DynamicServer({
host: true, // Enable virtual host handling
publicStatus: true // Enable /socket.io/status page.
});
// Any number of namespace patterns can be set up.
// This is an example of a single catch-all pattern.
io.setupNamespace(/.*/, function(nsp) {
nsp.on('connect', function(socket) {
console.log('a socket connected on', nsp.fullname());
});
nsp.expire(function() {
console.log(nsp.fullname(), 'is expiring');
});
});
io.listen(8888);

On newer versions you can use something like io.of(/^\/\w+$/).on('connection', (socket) where /^\/\w+$/ is a regular expression that will allow connection if it is a match.
Here is a full example on how to use this to setup many namespaces, here I suppose the only concerne is preventing emit diffusion from reaching other namespaces.
const workspaces = {}
io.of(/^\/\w+$/).on('connection', (socket) => {
const workspace = socket.nsp;
const namespace = workspace.name;
console.log("New Connection NameSpace", namespace);
// you can test here if "namespace" is allowed to be used
// if event handlers are set no need to got further
if (workspaces[namespace]) return;
// save workspace to prevent setting event handlers on each connection
workspaces[namespace] = workspace;
workspace.on("connection", (socket) => {
console.log(`${namespace} > connection from ${socket.id}`);
// set the event handlers same as normal socket
socket.on('event-1', (msg) => {
console.log("event-1", msg);
})
})
})