We Keep Coding

Node.js and Kafka: Messages not publishing to the kafka topic

Comparitively new to Node.js, can somebody tell me why I am not able to send the events to the Kafka topic? Code is not producing any errors though.
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var config = require('./config.js');
var nforce = require('nforce');
var dateTime = require('node-datetime');
var kafka = require('kafka-node');
var routes = require('./routes/index');
var dt = dateTime.create();
var org = nforce.createConnection({
clientId: config.CLIENT_ID,
clientSecret: config.CLIENT_SECRET,
redirectUri: config.CALLBACK_URL + '/oauth/_callback',
mode: 'multi',
environment: config.ENVIRONMENT // optional, sandbox or production, production default
});
HighLevelProducer = kafka.HighLevelProducer,
client = new kafka.Client('localhost:9092'),
producer = new HighLevelProducer(client),
org.authenticate({ username: config.USERNAME, password: config.PASSWORD }, function(err, oauth) {
if(err) return console.log(err);
if(!err) {
console.log('*** Successfully connected to Salesforce ***');
// add any logic to perform after login
}
// subscribe to a pushtopic
var str = org.stream({ topic: config.PUSH_TOPIC, oauth: oauth });
str.on('connect', function(){
console.log('Connected to pushtopic: ' + config.PUSH_TOPIC);
});
str.on('error', function(error) {
console.log('Error received from pushtopic: ' + error);
});
str.on('data', function(data) {
console.log('Received the following from pushtopic ---');
var dataStream1 = data['sobject'];
dataStream1['timestamp'] = dt.format('Y-m-d H:M:S');
console.log(dataStream1);
var dataStreamFinal = '[' + JSON.stringify(dataStream1) + ']';
payloads = [
{ topic: 'testing1', messages: dataStreamFinal }
];
console.log(payloads);
producer.on('ready', function () {
producer.send(payloads, function (err, data) {
console.log(data);
});
});
});
});
module.exports = {app: app, server: server};
I am using the following module of node https://github.com/SOHU-Co/kafka-node
And before that I just added my code to extract the events from the salesforce pushTopic and publishing it to the kafka is my aim.

Finally able to push the data to Kafka. Though nforce does not support replay messages feature yet.
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var config = require('./config.js');
var nforce = require('nforce');
var dateTime = require('node-datetime');
var routes = require('./routes/index');
var app = express();
var server = require('http').Server(app);
var dt = dateTime.create();
var io = require('socket.io')(server);
// get a reference to the socket once a client connects
var socket = io.sockets.on('connection', function (socket) { });
var org = nforce.createConnection({
clientId: config.CLIENT_ID,
clientSecret: config.CLIENT_SECRET,
redirectUri: config.CALLBACK_URL + '/oauth/_callback',
mode: 'multi',
environment: config.ENVIRONMENT, // optional, sandbox or production, production default
apiVersion: 'v43.0'
});
org.authenticate({ username: config.USERNAME, password: config.PASSWORD }, function(err, oauth) {
if(err) return console.log(err);
if(!err) {
console.log('*** Successfully connected to Salesforce ***');
// add any logic to perform after login
}
// subscribe to a pushtopic
var str = org.stream({ topic: config.PUSH_TOPIC, oauth: oauth });
str.on('connect', function(){
console.log('Connected to pushtopic: ' + config.PUSH_TOPIC);
});
str.on('error', function(error) {
console.log('Error received from pushtopic: ' + error);
});
str.on('data', function(data) {
console.log('Received the following from pushtopic ---');
console.log(data);
var dataStream1 = data['sobject'];
dataStream1['timestamp'] = dateTime.create().format('Y-m-d H:M:S');
var dataStreamFinal = '[' + JSON.stringify(dataStream1) + ']';
var kafka = require('kafka-node'),
HighLevelProducer = kafka.HighLevelProducer,
client = new kafka.Client(),
producer = new HighLevelProducer(client),
payloads = [
{ topic: 'testing1', messages: dataStreamFinal}
];
producer.on('ready', function () {
producer.send(payloads, function (err, data) {
console.log(data);
});
});
// emit the record to be displayed on the page
//Send to kafka topic
socket.emit('record-processed', JSON.stringify(dataStreamFinal));
});
});
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'hbs');
app.use(function(req, res, next){
res.io = io;
next();
});
// uncomment after placing your favicon in /public
//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
module.exports = {app: app, server: server};

socket.io not working with node server

I'm trying to set up my node server to update all the connected clients with new information in real-time. When I run the code below, the io.sockets.on('connection') callback is fired constantly, flooding the console with the message Client connected!, and the front-end is not being updated from socket.emit(). What am I doing wrong?
app.js
var express = require('express');
var path = require('path');
var favicon = require('static-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var mongoose = require('mongoose');
var models = require('./models.js');
var routes = require('./routes/index');
var users = require('./routes/users');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
app.use(favicon());
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded());
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
/// catch 404 and forwarding to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
/// error handlers
// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
module.exports = app;
// Connect to the Mongo database
mongoose.connect('mongodb://localhost/test');
bin/www.js
#!/usr/bin/nodejs
var debug = require('debug')('my-application');
var app = require('../app');
app.set('port', process.env.PORT || 3000);
var io = require('socket.io').listen(app.listen(app.get('port')));
io.sockets.on('connection', function(socket) {
console.log('Client connected!');
socket.on('message', function(data) {
console.log('Sending update!');
socket.emit('update', 'Working!');
});
});
public/javascripts/update.js
var socket = io.connect('http://' + document.location.hostname + ':3000/');
socket.on('connect', function() {
console.log('Socket connected!');
socket.emit('message', 'Message from client');
});
socket.on('update', function(data) {
alert(data);
});
And when I end the npm process, the client begins to log
http://<ip>:3000/socket.io/?EIO=2&transport=polling&t=1498772846992-691 net::ERR_CONNECTION_REFUSED
I've read posts about the express router messing with the socket requests but I can't seem to get it working no matter what I try.

Can you try this setup?
EDITED:
app.js:
var express = require('express');
var app = express();
app.set('views', __dirname + '/views');
app.set('view engine', 'pug');
app.get('/', function (req, res) {
res.render('index');
});
module.exports = app;
bin/www.js:
var app = require('../app');
var http = require('http');
var server = http.createServer(app);
server.listen(process.env.PORT || '3000', function () {
console.log('server listens on port 3000');
});
var io = require('socket.io')(http);
io.listen(server);
io.on('connection', function(socket) {
console.log('Client connected!');
socket.on('message', function (data) {
console.log('Sending update!');
socket.emit('update', 'Working!');
});
});
index.pug (or jade, they say jade is obsolete):
doctype html
html
body
h1 Testing socket.io
h3#status not connected
br
p#update update:&nbsp
script(src="/socket.io/socket.io.js")
script.
var socket = io();
socket.on('connect', function() {
document.getElementById("status").innerHTML = "connected";
socket.emit('message', 'Hello!');
});
socket.on('update', function (data) {
document.getElementById("update").innerHTML += data;
});

Calling app.listen (where app is the result of invoking express()) returns an HTTPServer, which can be used to setup the socket connection:
var server = app.listen(port);
var io = socket(server);
without manually running your solution, I'd guess that because you're creating a new HTTPServer to be feed into the socket, it's somehow getting hung in a loop.

Express Js: Router.use() Requires Middleware function

I am developing an application in Express Js. When I try to run the application I get this error:
My app.js file is like this:
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var users = require('./routes/users');
var app = express();
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
var server = app.listen(3000, function() {
var host = server.address().address;
var port = server.address().port;
console.log('Example app listening at http://%s:%s', host, port);
});
module.exports = router;
My index.js is like this:
var express = require('express');
var router = express.Router();
/* GET home page. */
router.get('/', function (req, res, next) {
res.render('index', {title: 'Polls'});
});
router.get("/list", function (req, res, next) {
Poll.find({}, 'question', function (error, polls) {
res.json(polls);
});
});
router.get("/poll", function (req, res, next) {
var pollId = req.params.id;
// Find the poll by its ID, use lean as we won't be changing it
Poll.findById(pollId, '', {lean: true}, function (err, poll) {
if (poll) {
var userVoted = false,
userChoice,
totalVotes = 0;
// Loop through poll choices to determine if user has voted
// on this poll, and if so, what they selected
for (c in poll.choices) {
var choice = poll.choices[c];
for (v in choice.votes) {
var vote = choice.votes[v];
totalVotes++;
if (vote.ip === (req.header('x-forwarded-for') || req.ip)) {
userVoted = true;
userChoice = {_id: choice._id, text: choice.text};
}
}
}
// Attach info about user's past voting on this poll
poll.userVoted = userVoted;
poll.userChoice = userChoice;
poll.totalVotes = totalVotes;
res.json(poll);
} else {
res.json({error: true});
}
});
});
router.get("/create", function (req, res, next) {
var reqBody = req.body,
// Filter out choices with empty text
choices = reqBody.choices.filter(function (v) {
return v.text != '';
}),
// Build up poll object to save
pollObj = {question: reqBody.question, choices: choices};
// Create poll model from built up poll object
var poll = new Poll(pollObj);
// Save poll to DB
poll.save(function (err, doc) {
if (err || !doc) {
throw 'Error';
} else {
res.json(doc);
}
});
});
module.exports = router;
And user.js is this:
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
res.send('respond with a resource');
});
module.exports = router;
I tried to find my solution on SO, but couldn't. Feel free to tell me if i need to provide any other file. Any help?

You should define routes in your index.js like you do in user.js.
app.use('/', routes) in your code expects routes to be an instance of a Router, but you're exporting an object with functions instead of that.
So your index.js file should have the following structure:
var express = require('express');
var router = express.Router();
router.get("/", function (req, res) {
res.render('index');
});
router.get("/list", function(req, res) {/*list implementation*/});
....
module.exports = router;

Extremely Long TTFB on localhost (Node/Express)

I'm experiencing extremely long TTFB on my localhost node/express server.
On average each request's(including static resource) TTFB is around 1s, occasionally up to 7s(I guess this is NOT normal)
What could cause this long TTFB and how to improve?
Could it be my fault for configing express wrong?
Here is my app.js:
var express = require('express');
var path = require('path');
var bodyParser = require('body-parser');
var methodOverride = require('method-override');
var favicon = require('serve-favicon');
var morgan = require('morgan')
//Router files
var all = require('./api/all');
var admin = require('./api/admin');
var student = require('./api/student');
var teacher = require('./api/teacher');
var course = require('./api/course');
var forum = require('./api/forum');
var group = require('./api/group');
var board = require('./api/board');
// App Setup
var app = express();
// View engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
app.use(morgan(':remote-addr [:date[clf]] :user :method :url :status :response-time ms',{
skip: function (req, res) {
//var noUser = (req.AV.user === undefined);
//return noUser;
return true;
}
}));
app.use(favicon(__dirname + '/public/favicon.ico'));
// Load Cloud Code
//app.use(cloud);
//Express Middleware
app.use(methodOverride('_method'))
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(express.static(__dirname+'/public'));
//The Route
app.get('/', function(req, res) {
res.send('./public/index.html');
});
//Routers
app.use('/api',all);
app.use('/api/admin',admin);
app.use('/api/student',student);
app.use('/api/teacher',teacher);
app.use('/api/course',course);
app.use('/api/forum',forum);
app.use('/api/group',group);
app.use('/api/board',board);
//404 Handler
app.use(function(req, res, next) {
//console.log("NOT FOUND: "+req.url);
var err = new Error('Not Found');
err.status = 404;
res.render('notfound.jade');
//next(err);
});
// error handlers for different env
if (app.get('env') === 'development') {
console.log("dev env");
app.use(function(err, req, res, next) {
res.status(err.status || 500);
if(res.status===500){
console.log(err);
//console.log(err.stack.split("\n"));
}
res.send({
message: err.message,
error: err
});
});
}
app.use(function(err, req, res, next) {
res.status(err.status || 500);
console.log(err.message);
res.send({
message: err.message,
error: {}
});
});
module.exports = app;
ps:I'm using node 4.0.0 and latest chrome

Duplicates when I use socket.io in express.js route files

I have a problem with Node.JS/Socket.IO/Express.JS and i didn't find any solution.
I use:
"express": "~4.10.6"
"socket.io": "1.2.1"
on Ubuntu Linux
My problem is that when I run the server and access localhost:3000/chat at the first run all works great but after a refresh/2/3/4 the messages inserted in chat duplicates. For one refresh, i have to identical message on one submit. For two refreshes, I have 3 identical message and so on. This behaviour began when i put the socket functionality in express.js routes folder.
P.S. I'm new to node.js/socket.io/express.js
app.js
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var users = require('./routes/users');
var chat = require('./routes/chat');
var app = express();
// Socket IO Things
var server = require('http').createServer(app);
var io = require('socket.io').listen(server);
// END - Socket IO Things
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'ejs');
// uncomment after placing your favicon in /public
//app.use(favicon(__dirname + '/public/favicon.ico'));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
app.use('/chat', chat.random_function(io));
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
module.exports = app;
server.listen('3000');
/routes/chat.js
var express = require('express');
var router = express.Router();
/* GET home page. */
router.get('/', function(req, res) {
res.sendFile(__dirname + '/index.html');
});
module.exports = {
router: router,
random_function: function(io) {
return function(req, res){
res.sendFile(__dirname + '/index.html');
io.sockets.on('connection', function (socket) {
socket.on('send message', function (data) {
io.sockets.emit('new message', data);
});
});
}
}
};
Client side sockets (routes/index.html)
<html>
<head>
<title>Chat with socket.io and node.js</title>
<style>
#chat {
height: 500px;
}
</style>
</head>
<body>
<div id="chat"></div>
<form id="chat-message" action="">
<input size="35" id="message"></input>
<input type="submit"></input>
</form>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
<script src="/socket.io/socket.io.js"></script>
<script>
var socket = io.connect();
var $messageForm = $('#chat-message');
var $messageBox = $('#message');
var $chat = $('#chat');
$messageForm.submit(function(e){
e.preventDefault();
socket.emit('send message', $messageBox.val());
$messageBox.val("");
});
socket.on('new message', function(data){
$chat.append(data + "<br>");
});
</script>
</body>
</html>
How can I handle this problem? Thanks in advance! I also checked the debugger and on server come 1/2/3/4/5 different packages each time.

You're creating & attaching new connection listener on each /chat request. Try moving it outside of random_function and perhaps creating separate socket instance for each connection/client.
(Edit) code added:
Try this (should work, but I didn't test it)
app.js
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var users = require('./routes/users');
var chat = require('./routes/chat');
var app = express();
// Socket IO Things
var server = require('http').createServer(app);
var io = require('socket.io').listen(server);
io.sockets.on('connection', function (socket) {
socket.on('send message', function (data) {
io.sockets.emit('new message', data);
});
});
// END - Socket IO Things
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'ejs');
// uncomment after placing your favicon in /public
//app.use(favicon(__dirname + '/public/favicon.ico'));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use('/users', users);
app.use('/chat', chat.random_function);
routes/chat.js
var express = require('express');
var router = express.Router();
/* GET home page. */
router.get('/', function(req, res) {
res.sendFile(__dirname + '/index.html');
});
module.exports = {
router: router,
random_function: function(req, res){
res.sendFile(__dirname + '/index.html');
}
};