I'm trying to inflate messages from a third party websocket using javascript, but receiving a pako.js
"invalid stored block lengths" error.
The sample code I'm using to implement the web socket can be found here:
https://github.com/okcoin-okex/API-docs-OKEx.com/tree/master/demo
A full working version of the code can be found implemented here:
https://github.com/Tucsky/SignificantTrades/blob/master/src/exchanges/okex.js
(https://aggr.trade)
I've tried using zlib.js instead of Pako.js, but running into more difficult problems with that library.
I've tried other alternate solutions found on stackoverflow but they don't seem to do what I need.
This is the code I'm running which shows the error.
https://jsfiddle.net/fwkgmvjr/
let myAPI = new WebSocket('wss://real.okex.com:10442/ws/v3');
//When the connection is first opened
myAPI.onopen = event => {
console.log('Socket opened')
myAPI.send('{"op": "subscribe", "args":["futures/ticker:ETH-USD-190927"]}')
}
myAPI.onmessage = event => {
if (event.data instanceof String) {
console.log(event.data)
} else {
try {
console.log(JSON.parse(pako.inflateRaw(event.data, {
to: 'string'
})))
} catch (err) {
console.log(err)
}
}
}
myAPI.onclose = event => {
console.log('Socket closed')
}
myAPI.onerror = event => {
console.log(`[error] ${error.message}`)
}
I expect the message from the websocket to be easily inflatable somehow.
The solution was to add
this.api.binaryType = 'arraybuffer'
when declaring the websocket.
I'm trying to develop a Node.js server that acts as a metronome (sends a repeated timing message) for everybody connected. I have socket.io rooms working so clients can "subscribe" to different metronomes, but sync between them drifts wildly. I've been using multiple setInterval() calls to schedule timing messages, and I understand that it isn't a very reliable clock source:
Client:
const socket = io('localhost:8000', {});
socket
.on('connect', function() {
console.log('connected');
})
.on('message', function(payload) {
console.log(payload);
});
// window.max is specific to the Max/MSP web browser object.
// This basically just subscribes to room 'inst0' or 'inst1'.
window.max.bindInlet('subscribe', function(inst) {
socket.emit('subscribe', inst);
});
Server:
io.sockets
.on('connection', (socket) => {
console.log('max connected!');
socket
.on('subscribe', (payload) => {
console.log('subscription: ' + payload);
Object.keys(socket.rooms).forEach(key => socket.leave(key));
socket.join('inst'+payload);
});
});
setInterval(() => {
io.to('inst0').emit('message', 'bang');
}, 500);
setInterval(() => {
io.to('inst1').emit('message', 'bang');
}, 250);
Are there other strategies for event scheduling like this? In a latency-free world the messages would have sample-accurate timing (44.1kHz resolution), but I'd be ecstatic to get something even around 30Hz.
I've started learning Golang after writing in Node.js for a long time and I'm a bit curious as to how am I to implement a handler - I've opted to use Gorilla Websocket since I understood it's the most reliable package out there.
In socket.io for example you have the simple socket.on function that allows me to call a function based on the "name" parameter passed in JSON.
Gorilla websocket doesn't implement such a thing, so my question is am I to sort of implement the logic behind socket.io in order to achieve what I want ?
As in do a certain procedure based on the value transferred in the websocket ?
If so - I need to implement it both client (I'm using AngularJS on the front-end) and server side separately by my own - make a switch case statement based on a value I get in JSON both in AngularJS for the front-end and in Go for the back-end, and also - is that the most efficient way ?
Thanks !
If you've been using Javascript for a while it is really easy to implement your own version of socket.on and socket.emit here is one I made for my own projects but you can have it if you need,
// e.g.
// let socket = new Socket("ws://w/e");
// socket.on('connected', () => { console.log('Connected'); });
// socket.emit('lobby join', { data: { username: 'Boo' } });
// Using ES2015 with Babel
import {EventEmitter} from 'events';
class Socket {
constructor(wsurl, ee = new EventEmitter()) {
let ws = new WebSocket(wsurl);
this.ee = ee;
this.ws = ws;
ws.onmessage = this.message.bind(this);
ws.onopen = this.open.bind(this);
ws.onclose = this.close.bind(this);
}
on(name, fn) {
this.ee.on(name, fn);
}
off(name, fn) {
this.ee.removeListener(name, fn);
}
emit(name, data) {
const message = JSON.stringify({name, data});
this.ws.send(message);
}
message(e) {
try {
const msgData = JSON.parse(e.data);
this.ee.emit(msgData.name, msgData.data);
}
catch(err) {
let error = {
message: err
}
console.log(err)
this.ee.emit(error.message)
}
}
open() {
this.ee.emit('connected');
}
close() {
this.ee.emit('disconnected');
}
}
export default Socket
This will let you use your common socket.on('event', fn); and what not
As for handling it on the servers end:
For receiving messages, I personally just make a switch statement that will match an incoming string to a function, e.g.:
// readPump pumps messages from the websocket connection to the hub.
func (c *connection) readPump() {
defer c.ws.Close()
for {
_, message, err := c.ws.ReadMessage()
if err != nil {
break
}
var incMessage interface{}
err = json.Unmarshal(message, &incMessage)
if err != nil {
log.Println(err)
}
incMessageMap := incMessage.(map[string]interface{})
switch incMessageMap["name"] {
case "lobby join":
// Do something to handle joining
case "lobby leave":
// Do something to handle leaving
}
}
}
For sending them I have a send channel on my connections that is stored in a map and when I need to emit I have a simple struct that takes a message name, and data, e.g.:
type wsMsg struct {
Name string `json:"name"`
Data map[string]interface{} `json:"data"`
}
c.send <- wsMsg{
"user joined",
map[string]interface{}{
"username": "Booh",
},
}
Then on the client side it would come as
socket.on('user joined', (msg) => {
console.log(msg) // { username: "Booh" }
});
I suggest looking at the examples on gorillas git: https://github.com/gorilla/websocket/tree/master/examples/chat
Here is an working example of golang websocket to stream video
https://github.com/interviewparrot/OpenAVStream
Let me know if it is good enough
By default all tokens generated with the Twilio helper libraries expire after one hour. But you should configure this expiration to be as short as possible for your application.
I am trying to generate a new token each time a user attempts a new connection and try to setup Twilio device. But it creates new device each time. So all Twilio device get incoming call and i can see multiple notification for that. Multiple connections created an dmultiple dtmf sent. I want only one twilio device with fresh token.
Twilio.Device.destroy() method is there but it is not working. What are other option do I have?
How to release/destroy/stop/delete Twilio.Device?
After saving credentials globalTwilioSagaSetup() called and after 58min of that again token is generated and Twilio.Device setup is done.
function globalTwilioSagaSetup()
{
// Get Twilio credentials
// Get Twilio Token
// Setup Twilio Device
// For token re-generation before expire. 58min
setInterval(function(){globalTwilioSagaSetup();},3480000);
}
I've had similar issue although in current version of twilio lib (i.e. 1.2). The thing is that once setup is called ready event is fired but only after first call to the setup method. It means that even if one will initialize device with new token there will be problems with establishing new connection. Therefore calling Twilio.Device.destroy() then setup and then connect (via ready event) solved that issue for me. Here is an example:
srv.connectToTwilio = () => $q((resolve, reject) => {
var connection;
try {
connection = Twilio.Device.connect();
} catch (err) {
$log.debug('Device.connect(): throw', err);
}
if (connection) {
try {
connection.accept(() => {
$log.debug(`Twilio connection.accept.`);
resolve();
});
} catch (err) {
$log.debug('connection.accept(): throw', err);
}
} else {
reject(`Device.connect() did not return connection`);
}
});
srv.connect = (token) => {
return srv.setToken(token).then(() => srv.connectToTwilio());
};
srv.disconnect = () => {
shouldBeConnected = false;
try {
Twilio.Device.activeConnection().disconnect();
} catch (error) {
$log.debug(error);
} finally {
Twilio.Device.destroy();
}
$log.debug(`Twilio disconnect.`);
};
I am currently testing how some code stands up against the following scenario:
Node.js application is started and successfully establishes a connection to mongodb
After it has successfully setup a connection, the mongodb server dies and all subsequent requests fail
To do this I have got the following code which makes use of the official driver (found here: https://github.com/mongodb/node-mongodb-native) :
MongoClient.connect('mongodb://localhost:27017/testdb', function(err, db) {
app.get('/test', function(req, res) {
db.collection('users', function (err, collection) {
console.log(err);
if (err) {
// ## POINT 1 ##
// Handle the error
}
else {
collection.find({ 'username': username }, { timeout: true }).toArray(function(err, items) {
console.log(err);
if (err) {
// ## POINT 2 ##
// Handle the error
}
else {
if (items.length > 0) {
// Do some stuff with the document that was found
}
else {
// Handle not finding the document
}
}
});
}
});
});
});
As the mongodb server is no longer running when the request is being handled, I'd made the assumption that at either the points which I have labelled ## POINT 1 ## or ## POINT 2 ##, it would return an error indicating a timeout; this however, isn't the case.
I have tried a number of different settings (including one you can see here that explicitly allows the cursor to timeout), however I cannot seem to enable it in any way. In every configuration I've tried Node.js will simply keep waiting for the find() operation to callback and it never does.
If I start the Node.js application before running mongodb, it catches the error in the connect callback fine, but if the connection dies after that it doesn't seem to handle it in any way.
Is there a setting I am missing or is there no way to detect connections being terminated after they've been established?
Edit: just to be clear, the username variable used in the find method is actually declared in my full code, the code I've put in this post is a cut down version to illustrate the structure and error checking.
UPD:
Based on this post, looks like they've deployed fix that will do the same as what we do here. Not sure if this is already within npm (15.10.13). https://github.com/mongodb/node-mongodb-native/issues/1092#ref-commit-2667d13
After some investigation I've managed to understand what is going on there:
Every time you call any method to deal with database (find, update, insert, etc.) it creates cursor, that has own ID and registers itself to EventEmitter of Db for being called back later. In meantime it registers itself to _notReplied object within same CallBackStore.
But once connection is closed, I couldn't locate anything that would iterate through _notReplied cursors and would trigger them with errors or any logic with timers (it still might be somewhere there). So I've managed to write small work around, that does force triggers cursors with error when DB emits close event:
new mongodb.Db('testdb', new mongodb.Server('localhost', 27017, { }), { safe: true }).open(function (err, db) {
if (!err) {
db.on('close', function() {
if (this._callBackStore) {
for(var key in this._callBackStore._notReplied) {
this._callHandler(key, null, 'Connection Closed!');
}
}
});
// ...
} else {
console.log(err)
}
});
I recommend using first approach instead of MongoClient. Reasons are few: for example when you close connection and then call .find it will properly trigger error in callback, while with MongoClient it won't.
If you are using MongoClient:
MongoClient.connect('mongodb://localhost:27017/testdb', function(err, db) {
if (!err) {
db.on('close', function() {
if (this._callBackStore) {
for(var key in this._callBackStore._notReplied) {
this._callHandler(key, null, 'Connection Closed!');
}
}
});
// ...
} else {
console.log(err);
}
});
What this will do? Once connection is closed, it will iterate through All _notReplied cursors and trigger events for them with error Connection Closed!.
Test case:
items.find({ }).toArray(function(err, data) {
if (!err) {
console.log('Items found successfully');
} else {
console.log(err);
}
});
db.close();
That will force close database connection and trigger close event that you handle earlier and will make sure that cursor will be closed.
UPD:
I've added Issue on GitHub: https://github.com/mongodb/node-mongodb-native/issues/1092 we'll see what they say regarding this.
I had the same problem, and found this page from google.
But your choosed answer didn't resolve the problem and it is as same as you, this._callBackStore can't use
but i tried to wrap the Mongo, and it seems work fine
var MongoClient = require('mongodb').MongoClient;
var mongo = {};
mongo.init = function() {
MongoClient.connect('mongodb://localhost:27017/testdb', function(err, db) {
if (err) {
mongo.DB = '';
} else {
mongo.DB = db;
}
db.on('close', function() {
mongo.DB = '';
});
db.on('reconnect', function() {
mongo.DB = db;
});
}
}
mongo.getdb = function(callback) {
if (mongo.DB) {
callback(null, mongo.DB);
} else {
callback('can not connect to db', null);
}
}
module.exports = mongo;
firstly start server and init() it
and then you can require it and use
mongo.getdb(function(err, db) {
if (err) {
console.log(err);
} else {
db.collection('user').find({'xxx':'xxx'}).toArray(function(err, items) {
console.log(items);
});
}
});
After some further investigation, it seems that you can't specify "offline" timeouts such as in the scenario outlined above. The only timeout that can be specified is one which informs the server to timeout the cursor after 10 minutes of inactivity, however as in the scenario above the connection to the server is down this does not work.
For reference, I found the information here: https://github.com/mongodb/node-mongodb-native/issues/987#issuecomment-18915263 by who I believed to be one of the main contributors to the project.
I'm making api with Hapi and Mongodb (w/o mongoose). Features:
Start responding to API request only if mongo db is available
Stop responding if mongo dies during cycle
Re-start when mongo available again
Keep single connection for all requests
Combining some ideas from other answers and this post https://productbuilder.wordpress.com/2013/09/06/using-a-single-global-db-connection-in-node-js/ my approach is this:
server.js
Utilities.initializeDb(() => {
server.start((err) => {
if (err) throw err;
console.log('Server running at:', server.info.uri);
});
}, () => {
server.stop((err) => {
if (err) throw err;
console.log('Server stopped');
});
});
Utilities.js
"use strict";
const MongoClient = require('mongodb').MongoClient;
const MongoUrl = 'mongodb://localhost:27017/db';
export const Utilities = {
initializeDb: (next, onCrash) => {
const ConnectToDatabase = (params) => {
MongoClient.connect(MongoUrl, (err, db) => {
if (err !== null) {
console.log('#t4y4542te Can not connect to mongo db service. Retry in 2 seconds. Try #' + params.retry);
console.error(err);
setTimeout(() => {
ConnectToDatabase({retry: params.retry + 1});
}, 2000);
} else {
db.on('close', () => {
onCrash();
console.log('#21df24sf db crashed!');
ConnectToDatabase({retry: 0});
});
global.db = global.db || db;
next();
}
});
};
ConnectToDatabase({retry: 0});
}
};
I'm exporting db connection to global space. It feels like not best solution, but I had projects where db connection was passed as param to all modules and that sucked more. Maybe there should be some modular approach where you import db connection where you need it, but in my situation i need it almost everywhere, I would have to write that include statement in most files. This API is pointless w/o connection to db, so I think it might be best solution even if I'm against having something flying magically in global space..