Here is a simple TCP-Server stress test. As long as we send only one message per client everything works as expected. But when two messages per client are send,
the server suddenly stops without an exception or error.
So is this a bug or a feature?
var net = require("net");
var async = require("async");
var cluster = require("cluster");
// `ulimit -n` tells us that we can open max. 1024 files per process.
// Creating a socket means opening a file so we are limited.
var CLIENTS = 1000;
// Here is the weird part:
// - sending one message per client works fine
// - sending multiple message per client sucks
var MESSAGES = 2;
var TOTAL = CLIENTS * MESSAGES;
var PORT = 1234;
var HOST = "127.0.0.1";
if (cluster.isMaster) {
var count = 0;
var start = new Date;
var server = net.createServer(function(socket) {
socket.on("data", function(data) {
var t;
count++;
console.log("server received " + count + " messages");
socket.write(data, function(err) { if (err) console.error(err); });
if (count === TOTAL) {
t = (new Date) - start;
console.log("server received and sent " + count + " messages within " + t + "ms");
}
});
});
server.listen(PORT, HOST, function() { cluster.fork(); });
} else {
var run = function(i) {
var c = net.connect({ port: PORT, host: HOST }, function() {
var tasks = (function() {
var results = [];
for (var x = 1; x <= MESSAGES; ++x) {
results.push((function(x) {
return function(next) { c.write("Hello server!", next); };
})(x));
}
return results;
})();
async.series(tasks, function(err) {
if (err) { console.error(err); }
});
});
};
for (var i = 1; i <= CLIENTS; ++i) { run(i); }
}
Tested on Linux 3.11, Node.js 0.10.21
You're assuming that calling .write twice from a client triggers the data event twice on the server, not taking into account any buffering that might be going on which will coalecse multiple writes.
When the callback to .write is called, it doesn't mean the message it actually sent, it means that the message is put in some kernel buffer (which might contain more than one message when it's sent to the server).
Related
I have been developing a home automation system
Scenario:
ESP8266 sending keep alive to Rasp pi MQTT server
webpage on Chrome using IBM utility javascript Paho client
ESP sends alive every 10 secs and shows when it receives its own message including received time, this proves the server is sending the messages back on time.
Webpage(also has timer) displays the time messages are received with the message
Messages are received on the webpage in a random manner, by this I mean that no messages arrive for a long time then a glut arrives all at the same time
from the esp I can see that the messages are being received when the are published. there seems to be a problem in the webpage code
I am just wondering if anyone has seen this before and can point me in the right direction, my code is below:
thanks in advance
Dave
/*
Original Copyright (c) 2015 IBM Corp.
Eclipse Paho MQTT-JS Utility
This utility can be used to test MQTT brokers
its been bent by me to fit my needs
*/
// global variables
var MyLib = {}; // space for my bits
MyLib.theMessage = ""; // received message
MyLib.theTopic = ""; // received topic
MyLib.clientId = 'webRunpage2' + (Math.random() * 10);
MyLib.connected = false;
MyLib.ticker = setInterval(function(){MyLib.tickTime ++;}, 1000); // 5secs
MyLib.tickTime = 0;
MyLib.aliveTimer = setInterval(function(){keepMeAlive()}, 90000); // 1.5 mins
//=============================================================
function keepMeAlive() {
var topic='webpage', message='KeepAlive';
publish(topic, 1, message, false); // keeps the conection open
console.info("Alive Time: ", MyLib.tickTime);
}
//=============================================================
function onMessageArrived(message) {
MyLib.theTopic = message.destinationName; MyLib.theMessage = message.payloadString;
var topic = MyLib.theTopic;
var messg = MyLib.theMessage;
var id = messg.substring(0,6);
console.log('Message Recieved: Topic: ', message.destinationName, "Time ", MyLib.tickTime);
}
//=============================================================
function onFail(context) {
console.log("Failed to connect");
MyLib.connected = false;
}
//=============================================================
function connectionToggle(){
if(MyLib.connected){ disconnect();
} else { connect(); }
}
//=============================================================
function onConnect(context) {
// Once a connection has been made
console.log("Client Connected");
document.getElementById("status").value = "Connected";
MyLib.connected = true;
client.subscribe('LIGHTS/#'); // subscribe to all 'LIGHTS' members
}
//=================================================
// called when the client loses its connection
function onConnectionLost() {
console.log("Connection Lost: ");
document.getElementById("status").value = "Connection Lost";
MyLib.connected = false;
connectionToggle();
}
//=============================================================
function connect(){
var hostname = "192.168.1.19";
var port = 1884;
var clientId = MyLib.clientId;
var path = "/ws";
var user = "dave";
var pass = "vineyard01";
var keepAlive = 120; // 2 mins stay alive
var timeout = 3;
var ssl = false;
var cleanSession = true; //*******************
var lastWillTopic = "lastwill";
var lastWillQos = 0;
var lastWillRetain = false;
var lastWillMessage = "its Broken";
if(path.length > 0){
client = new Paho.MQTT.Client(hostname, Number(port), path, clientId);
} else {
client = new Paho.MQTT.Client(hostname, Number(port), clientId);
}
// console.info('Connecting to Server: Hostname: ', hostname, '. Port: ', port, '. Path: ', client.path, '. Client ID: ', clientId);
// set callback handlers
client.onConnectionLost = onConnectionLost;
client.onMessageArrived = onMessageArrived;
var options = {
invocationContext: {host : hostname, port: port, path: client.path, clientId: clientId},
timeout: timeout,
keepAliveInterval:keepAlive,
cleanSession: cleanSession,
useSSL: ssl,
onSuccess: onConnect,
onFailure: onFail
};
if(user.length > 0){ options.userName = user; }
if(pass.length > 0){ options.password = pass; }
if(lastWillTopic.length > 0){
var lastWillMessage = new Paho.MQTT.Message(lastWillMessage);
lastWillMessage.destinationName = lastWillTopic;
lastWillMessage.qos = lastWillQos;
lastWillMessage.retained = lastWillRetain;
options.willMessage = lastWillMessage;
}
// connect the client
client.connect(options);
}
//=============================================================
function disconnect(){
// console.info('Disconnecting from Server');
client.disconnect();
MyLib.connected = false;
}
//=============================================================
function publish(topic, qos, message, retain){
// console.info('Publishing Message: Topic: ', topic, '. QoS: ' + qos + '. Message: ', message, '. Retain: ', retain);
message = new Paho.MQTT.Message(message);
message.destinationName = topic;
message.qos = Number(qos);
message.retained = retain;
client.send(message);
}
//=============================================================
function subscribe(topic){
var qos = 1;
client.subscribe(topic, {qos: Number(qos)});
}
//=============================================================
// left next 4 functions but not used
function unsubscribe(topic){
client.unsubscribe(topic, {
onSuccess: unsubscribeSuccess,
onFailure: unsubscribeFailure,
invocationContext: {topic : topic}
});
}
//=============================================================
function unsubscribeSuccess(context){
// console.info('Successfully unsubscribed from ', context.invocationContext.topic);
}
//=============================================================
function unsubscribeFailure(context){
// console.info('Failed to unsubscribe from ', context.invocationContext.topic);
}
//=============================================================
// Just in case someone sends html
function safe_tags_regex(str) {
return str.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
}
//=============================================================
i'm having a serious issue with an app i'm building with node.js and express.js.
the app converts videos to mp3. when the video is small upto 5min length everything work as expected, the http server respond with a download button to the client.
but when the video is too big the server prematurely closes connection, and because i'm using http protocol, the client retry the request and this time receives ERR_EMPTY_RESPONSE after a certain amount of time of waiting.
app.post('/', function(req, res) {
var obj_dlConvert = apart_dl_cv.dlConvert(req.body.yt_url,140,apart_dl_cv.generateDir()); //the function that download from youtube and convert
var lien = obj_dlConvert.link;
var dossier = obj_dlConvert.dossier;
var video_stream = obj_dlConvert.streame;
obj_dlConvert.processus.on('end', () =>{
fs.rename(path.join(__dirname,'uploads',dossier,dossier+'.mp3'), path.join(__dirname,'uploads',dossier,'video.mp3'), function(err) {
if (err) {
res.render('dlpage.hbs',{
renameError: true
});
}else res.render('dlpage.hbs',{
dossier: dossier,
fullLink: lien
});
});
}
}
req.on("close", function() {
obj_dlConvert.processus.kill();
obj_dlConvert.processus.on('error', () => {
if (fs.existsSync(path.join(__dirname,'uploads',dossier))){
fse.removeSync(path.join(__dirname,'uploads',dossier));
}
});
});
});
Serving video is not a one time deal. There is a hand-shake between the browser and server. The server needs to be able to provide the 'next' chunk when asked by the browser. Following may by used as an inspiration:
var fs = require("fs"),
http = require("http"),
url = require("url");
exports.serveVideo = function(req, res, file) {
var range = req.headers.range;
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
fs.stat(file, function(err, stats) {
var total = stats.size;
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
var stream = fs.createReadStream(file, { start: start, end: end })
.on("open", function() {
stream.pipe(res);
}).on("error", function(err) {
res.end(err);
});
});
}
After struggling a lot (more then a week Google chrome - chrome.serial connection failed), i have found that on OSX 10.11 Chrome.serial does not work because OSX refuse Chrome.serial if its not using native OSX way of connection requests .
Then i moved to Windows OS with following code and its working. But i am failing to read the data in readable format, can anyone please help?
var serial_port = "COM3";
var connectionId = -1;
var csv = null;
var serial = chrome.serial;
// 1 - Device Query
serial.getDevices(function(ports) {
for (var i=0; i<ports.length; i++) {
console.log('OK: DEVICES', ports[i].path);
csv = csv + ports[i].path;
}
});
// 2 - Connect and listen
serial.connect(serial_port, {bitrate: 9600}, function(connectionInfo) {
console.log('OK: CONNECTED', connectionInfo.connectionId);
connectionId = connectionInfo.connectionId;
serial.onReceive.addListener(function(receiveInfo) {
if (receiveInfo.connectionId !== connectionId) {
console.log("FAIL: connectionId mismatch");
return;
}
console.log(receiveInfo.data);
});
});
// 3 - Sync - WebPage
chrome.runtime.onMessageExternal.addListener(function(request, sender, sendResponse) {
console.log('OK: WEB-PAGE ARRIVED');
});
Output:
How do i get the value as readable format instead of ArrayBuffer {} ?
Is this OK? (please improve the code if you can)
var serial_port = "COM3";
var connectionId = -1;
var csv = null;
var serial = chrome.serial;
var lineBuffer = "";
var ab2str = function(buf) {
var bufView = new Uint8Array(buf);
var encodedString = String.fromCharCode.apply(null, bufView);
return decodeURIComponent(escape(encodedString));
};
// 1 - Device Query
serial.getDevices(function(ports) {
for (var i=0; i<ports.length; i++) {
console.log('OK: DEVICES', ports[i].path);
csv = csv + ports[i].path;
}
});
// 2 - Connect and listen
serial.connect(serial_port, {bitrate: 9600}, function(connectionInfo) {
console.log('OK: CONNECTED', connectionInfo.connectionId);
connectionId = connectionInfo.connectionId;
serial.onReceive.addListener(function(receiveInfo) {
if (receiveInfo.connectionId !== connectionId) {
console.log("FAIL: connectionId mismatch");
return;
}
lineBuffer += ab2str(receiveInfo.data);
console.log(lineBuffer);
});
});
// 3 - Sync - WebPage
chrome.runtime.onMessageExternal.addListener(function(request, sender, sendResponse) {
console.log('OK: WEB-PAGE ARRIVED');
});
I am currently working on an auction script using node.js and socket.io.But site was developed by using PHP & MySQL. Here I'm using node.js + socket.io for auction bidding process only. The site will have 500-1000 logged in users viewing a single page during the auction. Only one item will be on auction and it will be sold at one day once.
I will be broadcasting(emitting) a countdown timer to all of the users from the server to the client. On the server side I will be using setInterval(),recursive setTimeout() of 1 second to countdown to the auction end time. Apart from this the only other message being sent across will be the current bid being passed from a single client to the server then broadcast to all. This way to do be a reliable? And will it be able to handle the usage on the server?. Here I've tested with 500 users means in browsers getting hanging the timer.
Server.js
var cluster = require('cluster');
var app = require('express')();
//var http = require('http');
var https = require('https');
var socket = require('socket.io');
var redis = require('redis');
var redisAdapter = require('socket.io-redis');
var request = require('request');
var fs = require('fs');
var options = {
key: fs.readFileSync('keys/e1317_0f2c9_71565598d419e37e376ccef5c2827113.key'),
cert: fs.readFileSync('certs/e1317_0f2c9_1468152279_2dc46c1f2cc135a.crt'),
ca: fs.readFileSync('cabundles/90490a5c829d2aca24f22b5820864c6e_1935558000.cabundle')
};
//var server = http.createServer( app );
var server = https.createServer(options, app);
var io = socket.listen(server);
var port = process.env.PORT || 8080;
var workers = process.env.WORKERS || require('os').cpus().length;
var redisUrl = process.env.REDISTOGO_URL || 'redis://127.0.0.1:6379';
var redisOptions = require('parse-redis-url')(redis).parse(redisUrl);
var pub = redis.createClient(redisOptions.port, redisOptions.host, {
detect_buffers: true,
return_buffers: true,
auth_pass: redisOptions.password
});
var sub = redis.createClient(redisOptions.port, redisOptions.host, {
detect_buffers: true,
return_buffers: true,
auth_pass: redisOptions.password
});
io.adapter(redisAdapter({
pubClient: pub,
subClient: sub
}));
console.log('Redis adapter started with url: ' + redisUrl);
io.sockets.on('connection', function(client) {
//console.log('first');
client.on('nauction', function(data) {
io.sockets.emit('nauction', data);
});
});
io.on('connection', function(socket) {
//console.log('in');
console.log('connected client count:' + io.sockets.sockets.length);
var recursive = function() {
//console.log("It has been one second!");
if (io.sockets.sockets.length > 0) {
request('https://www.example.com/file.php', function(error, response, body) {
if (!error && response.statusCode == 200) {
data = JSON.parse(body);
socket.volatile.emit('auction_data', {
'auction_data': data
});
//console.log(data);
} else {
//console.log('else');
console.log(error);
}
});
} //else{
//console.log('No clients connected now');
//}
setTimeout(recursive, 1000);
}
recursive();
socket.on("disconnect", function() {
console.log('clear interval')
//clearInterval(interval);
clearTimeout(recursive);
});
});
if (cluster.isMaster) {
console.log('start cluster with %s workers', workers - 1);
workers--;
for (var i = 0; i < workers; ++i) {
var worker = cluster.fork();
console.log('worker %s started.', worker.process.pid);
}
cluster.on('death', function(worker) {
console.log('worker %s died. restart...', worker.process.pid);
});
} else {
start();
}
function start() {
server.listen(port, function() {
console.log('listening on *:' + port);
});
}
Client.js
socket.on('auction_data', function(auction_details) {
//console.log(auction_details);
$.each(auction_details, function(keys, values) {
//countdwon formation
var tm, days, hrs, mins, secs;
days = value.auction_data.time.days;
if (value.auction_data.time.hours < 10) {
hrs = ("0" + value.auction_data.time.hours);
} else {
hrs = value.auction_data.time.hours;
}
if (value.auction_data.time.mins < 10) {
mins = ("0" + value.auction_data.time.mins);
} else {
mins = value.auction_data.time.mins;
}
if (value.auction_data.time.secs < 10) {
secs = ("0" + value.auction_data.time.secs);
} else {
secs = value.auction_data.time.secs;
}
if (days == 0) {
tm = '' + hrs + '' + '' + mins + '' + '' + secs + '';
} else {
tm = '' + days + '' + '' + hrs + '' + '' + mins + '' + '' + secs + '';
}
$('#auction_' + value.auction_data.product_id + " .countdown").html(tm);
});
});
I'm waiting for your answers to fix the browser hanging problem.
First Question: Is This way to do be a reliable?
Sending the time every Second to EVERY client is not necessary. Simply send them the time at their first visit and use a local timer (at their local page) to reduce the time every second.
You also need to check for server-time on every bid (more secure).
If this is not "secure" enough for you, send the time with the changing bid. You only have to send the actual Bid, when it changed (using Broadcast) or when the user joins the site (just send it to him).
Second Question: And will it be able to handle the usage on the server?
Yes and No.
If your Sever is good enough (every 5$ server with endless traffic would fit),
you should not get in trouble. Only, if your script is very very bad and seeded with Memory Leaks.
Now a few tips:
Never trust the user input - parse it before you use it!
Recalculate everything you get from the client on the Server.
Send the Client only what he needs. He does not need information about stuff that he does not use.
If this was the answer you hoped for, please select the green arrow on the left.
If not, write a comment here and I will give more tips.
I've tried to search through stackoverflow for a similar question but most people are asking about the client-side of the NTLMv2 protocol.
I'm implementing a proxy that is performing the server-side of the protocol to authenticate users connecting to the proxy.
I've coded a lot of the protocol but I'm now stuck because the documentation that should take me further is difficult to understand.
This is the best documentation I've found so far: http://www.innovation.ch/personal/ronald/ntlm.html, but how to deal with the LM and NT responses is oblivious to me.
The proxy is located on an application server. The domain server is a different machine.
Example code for the node proxy:
var http = require('http')
, request = require('request')
, ProxyAuth = require('./proxyAuth');
function handlerProxy(req, res) {
ProxyAuth.authorize(req, res);
var options = {
url: req.url,
method: req.method,
headers: req.headers
}
req.pipe(request(options)).pipe(res)
}
var server = http.createServer(handlerProxy);
server.listen(3000, function(){
console.log('Express server listening on port ' + 3000);
});
ProxyAuth.js code:
ProxyAuth = {
parseType3Msg: function(buf) {
var lmlen = buf.readUInt16LE(12);
var lmoff = buf.readUInt16LE(16);
var ntlen = buf.readUInt16LE(20);
var ntoff = buf.readUInt16LE(24);
var dlen = buf.readUInt16LE(28);
var doff = buf.readUInt16LE(32);
var ulen = buf.readUInt16LE(36);
var uoff = buf.readUInt16LE(40);
var hlen = buf.readUInt16LE(44);
var hoff = buf.readUInt16LE(48);
var domain = buf.slice(doff, doff+dlen).toString('utf8');
var user = buf.slice(uoff, uoff+ulen).toString('utf8');
var host = buf.slice(hoff, hoff+hlen).toString('utf8');
var lmresp = buf.slice(lmoff, lmoff+lmlen).toString('utf8');
var ntresp = buf.slice(ntoff, ntoff+ntlen).toString('utf8');
console.log(user, lmresp, ntresp);
/* NOW WHAT DO I DO? */
},
authorize: function(req, res) {
var auth = req.headers['authorization'];
if (!auth) {
res.writeHead(401, {
'WWW-Authenticate': 'NTLM',
});
res.end('<html><body>Proxy Authentication Required</body></html>');
}
else if(auth) {
var header = auth.split(' ');
var buf = new Buffer(header[1], 'base64');
var msg = buf.toString('utf8');
console.log("Decoded", msg);
if (header[0] == "NTLM") {
if (msg.substring(0,8) != "NTLMSSP\x00") {
res.writeHead(401, {
'WWW-Authenticate': 'NTLM',
});
res.end('<html><body>Header not recognized</body></html>');
}
// Type 1 message
if (msg[8] == "\x01") {
console.log(buf.toString('hex'));
var challenge = require('crypto').randomBytes(8);
var type2msg = "NTLMSSP\x00"+
"\x02\x00\x00\x00"+ // 8 message type
"\x00\x00\x00\x00"+ // 12 target name len/alloc
"\x00\x00\x00\x00"+ // 16 target name offset
"\x01\x82\x00\x00"+ // 20 flags
challenge.toString('utf8')+ // 24 challenge
"\x00\x00\x00\x00\x00\x00\x00\x00"+ // 32 context
"\x00\x00\x00\x00\x00\x00\x00\x00"; // 40 target info len/alloc/offset
type2msg = new Buffer(type2msg).toString('base64');
res.writeHead(401, {
'WWW-Authenticate': 'NTLM '+type2msg.trim(),
});
res.end();
}
else if (msg[8] == "\x03") {
console.log(buf.toString('hex'));
ProxyAuth.parseType3Msg(buf);
/* NOW WHAT DO I DO? */
}
}
else if (header[0] == "Basic") {
}
}
}
};
module.exports = ProxyAuth;
The /* NOW WHAT DO I DO? */ comment specifies where I am stuck.
I hope I put enough information there, but let me know if anything else is needed.