main.js
var http = require('http');
var UserModel = require('./models/user.js');
var server = http.createServer(function(req, res){
UserModel.create({
}), function(e, o){
if(e) { console.log(e); } else {
} console.log(o); }
});
}).listen(3000);
connections.js
var mongo = require('mongodb');
module.exports = {
dbMain: new mongo.Db('main', new mongo.Server('127.0.0.1', 27017, { auto_reconnect: true }, {})),
dbLog: new mongo.Db('log', new mongo.Server('127.0.0.1', 27017, { auto_reconnect: true }, {}))
};
/models/user.js
var mongodb = require('mongodb');
var db = require('./connections.js').dbMain;
module.exports = {
create: function(newData, callback){
db.open(function(e, db){
db.collection('users', function(e, collection){
collection.insert(newData, callback);
});
});
}
}
When I use the code above, the server crashes with the problem that, the SECOND time a request comes in, we still have the database connection opened, so lets add db.close to our Users.create function.
create: function(newData, callback){
db.open(function(e, db){
db.collection('users', function(e, collection){
collection.insert(newData, function(e, o){
db.close(); // Voila.
callback(e, o);
});
});
});
}
At this stage the server CAN still crash, because of multiple connections open, I don't understand why or how this can happen but it does.
How do I organize my project into models (I don't want to use Mongoose, my validation is done in a different layer not the model, so Mongoose would be an overkill for me)? Also how do I handle connections in the project?
you could have a library that wraps all this up nicely - it means that only one connection to the database will be opened and the same (open) connection will be returned for the second request - if you are getting 1000+ per second, this is a make-or-break issue (i.e. not re-opening the connection each request)...
users.js:
var connections = require('./connections.js');
var serverCache = connections('127.0.0.1', 27017);
module.exports = {
create: function(newData, callback){
serverCache('main', 'users', function(e, collection){
collection.insert(newData, callback);
})
}
}
connections.js
var mongo = require('mongodb');
// a mongo connection cache
// pass in host & port
// it returns a function accepting dbName, collectionName & callback
var mongoCache = function(host, port){
// keep our open connections
var mongoDatabases = {};
var ensureDatabase = function(dbName, readyCallback){
// check if we already have this db connection open
if(mongoDatabases[dbName]){
readyCallback(null, mongoDatabases[dbName]);
return;
}
// get the connection
var server = new mongo.Server(host, port, {auto_reconnect: true});
// get a handle on the database
var db = new mongo.Db(dbName, server);
db.open(function(error, databaseConnection){
if(error) throw error;
// add the database to the cache
mongoDatabases[dbName] = databaseConnection;
// remove the database from the cache if it closes
databaseConnection.on('close', function(){
delete(mongoDatabases[dbName]);
})
// return the database connection
readyCallback(error, databaseConnection);
})
}
var ensureCollection = function(dbName, collectionName, readyCallback){
ensureDatabase(dbName, function(error, databaseConnection){
if(error) throw error;
databaseConnection.createCollection(collectionName, function(error, collection) {
if(error) throw error;
// return the collection finally
readyCallback(error, collection);
})
})
}
return ensureCollection;
}
module.exports = mongoCache;
I'm currently using a global connection with multiple http requests. In the past I created a complex library that was creating several connections to MongoDB and picking up one randomly for each connection.
Later I found that the native driver can do that for me, which is pretty neat. Currently I'm using a single object, and the driver chooses to which connection send the query.
var srvOpts = {
auto_reconnect: true,
poolSize: 10,
};
var conn = new Mongo.Server("localhost", 27017, srvOpts),
db = new Mongo.Db("dbname", conn, {});
db.open(function (){});
As you can this is a great idea, I'm thinking to copy that idea into the Redis driver that I'm using, but I'm short on time so I doubt that I will do any time soon.
Related
I have a problem with the approach I use to connect to Mondo DB.
I use the following method:
import { Db, MongoClient } from "mongodb";
let cachedConnection: { client: MongoClient; db: Db } | null = null;
export async function connectToDatabase(mongoUri?: string, database?: string) {
if (!mongoUri) {
throw new Error(
"Please define the MONGO_URI environment variable inside .env.local"
);
}
if (!database) {
throw new Error(
"Please define the DATABASE environment variable inside .env.local"
);
}
if (cachedConnection) return cachedConnection;
cachedConnection = await MongoClient.connect(mongoUri, {
useNewUrlParser: true,
useUnifiedTopology: true,
}).then((client) => ({
client,
db: client.db(database),
}));
return cachedConnection!;
}
Everytime I need to connect to MongoDB I do as follows:
const { db } = await connectToDatabase(config.URI, config.USERS_DATABASE);
const myUniversity = await db
.collection(config.MY_COLLECTION)
.findOne({})
Everything seems ok, so what is the problem?
The problem is that the connections to my DB don't close after I use them. In fact I thought that my server is stateless so after every time i use my DB, the connections end. But it is not true! They stay alive, and after few hours of using my app mongo atlas sends me an email saying that the limit is exceeded.
As you can see in this screenshot, this chart is ever growing. That means that connections stay on and they accumulate. How do you think I can solve this problem?
Keep in mind that it uses cachedConnection only if I use the same connection. If I call a different API from the first one it creates another connection and it doesn't enter in if (cachedConnection) block, but it goes forward till the end.
You can try this simple demo which will allow you to use the same connection throughout the application in different modules. There are three modules: the index.js is the starter program, the dbaccess.js is where you have code to create and maintain a connection which can be used again and again, and a apis.js module where you use the database connection to retrieve data.
index.js:
const express = require('express');
const mongo = require('./dbaccess');
const apis = require('./apis');
const app = express();
const init = async () => {
await mongo.connect();
app.listen(3000);
apis(app, mongo);
};
init();
dbaccess.js:
const { MongoClient } = require('mongodb');
class Mongo {
constructor() {
this.client = new MongoClient("mongodb://127.0.0.1:27017/", {
useNewUrlParser: true,
useUnifiedTopology: true
});
}
async connect() {
await this.client.connect();
console.log('Connected to MongoDB server.');
this.db = this.client.db('test');
console.log('Database:', this.db.databaseName);
}
}
module.exports = new Mongo();
apis.js:
module.exports = function(app, mongo) {
app.get('/', function(req, res) {
mongo.db.collection('users').find().limit(1).toArray(function(err, result) {
res.send('Doc: ' + JSON.stringify(result));
});
});
}
Change the appropriate values in the url, database name and collection name before trying.
I am using xterm.js in my web project to have a terminal on the web page. Every time I refresh my page or reconnect socket when a socket connection is broken due to internet fluctuation from the client. The current PWD directory is lost and it falls to specified CWD directory which is user home in my case. So again I have to do cd where I was working.
How can I connect and remain at same PWD where I was last time before page refreshing or socket disconnect?
One of the things I tried is to store term object and connect through the same object when reconnecting if it is already present. Not deleting process and object in on WebSocket disconnect.
var http = require('http');
var express = require('express');
var app = express();
var expressWs = require('express-ws')(app);
var pty = require('node-pty');
var cors = require('cors');
app.use(cors());
app.options('*', cors());
var terminals = {}; //global terminals
function getUser(token) {
return new Promise((resolve, reject) => {
try {
return http.get({
host: '',
path: '',
headers: {'token': token}
}, function(response) {
// Continuously update stream with data
var body = '';
response.on('data', function(d) {
body += d;
});
response.on('end', function() {
return resolve(JSON.parse(body));
});
});
} catch (err) {
console.log('Api failed');
console.log(err);
reject;
}
})
}
app.ws('/terminals/:user_id', function (ws, req) {
try {
getUser(req.params.user_id) /* cheking with api if user exist in my database*/
.then(user_info => {
if(terminals[parseInt(req.params.user_id)]){
var term = terminals[parseInt(req.params.user_id)];
}else {
var term = pty.spawn(process.platform === 'win32' ? 'cmd.exe' : 'bash', [], {
name: 'xterm-color',
cwd: cwd,
env: process.env
});
terminals[parseInt(req.params.user_id)] = term;
}
term.on('data', function(data) {
ws.send(data);
});
ws.on('message', function(msg) {
term.write(msg);
});
ws.on('close', function () {
// process.kill(term.pid);
// delete terminals[parseInt(req.params.pid)];
// delete logs[req.params.pid];
});
})
.catch(err => {
})
} catch (err) {
console.log('Terminal webSocket failed');
console.log(err);
}
});
app.listen(3000);
This is not working for me. This gets me connect only first time but when I refresh my page terminal does not connect with existing store object.
Also, this has a problem if the spawned process is killed by the system but it still remains in javascript object and script try to reconnect with same term object it will fail.
Any guidelines how to achieve reconnect with same PWD.
Details
OS version: Mac OS ,
xterm.js version: 2.2.3
This can be solved very easily by just updating the ~/.bashrc on server
Putting below two line in ~/.bashrc file worked for me
PROMPT_COMMAND+='printf %s "$PWD" > ~/.storepwd'
[ -s ~/.lastdirectory ] && cd `cat ~/.lastdirectory`
Ref Save last working directory on Bash logout
At the moment, I develop a node.js REST webservice with express. I used MongoDB + Mongoose to establish a database.
Now, I have the problem, that I can only use the db connection in the file where I established the connection. I found a solution to use the connection also in other files by "module.exports" the _db variable. But I don't know, if this is the best practise. Here is my code:
databaseManager.js
// Establish a connection to the database.
mongoose.Promise = global.Promise
mongoose.connect('mongodb://'+cfg.db.ip+':'+cfg.db.port+'/'+cfg.db.name)
var _db = mongoose.connection
_db.on('error', console.error.bind(console, 'DB connection error'))
_db.once('open', function()
{
console.log("DatabaseM: Connected to the database")
})
[...]
module.exports =
{
db : _db,
}
otherFile.js
var database = require('./databaseManagement')
[...]
database.db.collection('users').findOne({ name: "ashton"}, function(err, user)
{
if (err) return callback(consts.ERROR_DB, null)
if (!user) return callback(consts.WARN_DB_NO_CLIENT)
callback(null, user)
})
It works great. But there may be a risk that I do not see?
Thanks a lot :-)
In your app.js file :
var url="mongdb:\\localhost:27017\dbname";
mongoose.connect(url); //it open default connection for mongodb and is handled by mongoose
Now perform all your task whatever you want :
mongoose.connection.on('connected', function () {
console.log('Mongoose default connection open to ' + dbURI);
});
Bring all your database model in app.js file like as such:
var model1 = require('./models/model1');
model1.js
var mongoose = require('mongoose');
var data = new mongoose.Schema({
name:{type:String, required:true}
});
module.exports = mongoose.model('collectionName', data);
Now, when all your tasks are over. Simply close default connection like this :
mongoose.connection.on('disconnected', function () {
console.log('Mongoose default connection disconnected');
});
If any error occurs in connection handle it like this :
mongoose.connection.on('error',function (err) {
console.log('Mongoose default connection error: ' + err);
});
If node service exits then close connection usig this code
process.on('SIGINT', function() {
mongoose.connection.close(function () {
console.log('Mongoose default connection disconnected through app termination');
process.exit(0);
});
});
First question here, so be kind ;)
I am configuring a Node.js server to connect to a MongoDB database in Modulus.io node.js hosting (really good stuff, worth checking it out), but I can't seem to properly stablish connection. Per the getting-started guide I get a connection uri in the format:
mongodb://user:pass#mongo.onmodulus.net:27017/3xam913
But that doesn't seem to work with the structure of the code I was trying to port to the server (had it running locally) because of the Server class argument structure with only host and port to define...
This is the code I am trying to adapt to the connection:
// server setup
var mongo = require('mongodb'),
mdbServer = mongo.Server,
mdbDb = mongo.Db,
mdbObjectID = mongo.ObjectID;
// open a connection to the mongoDB server
var mdbserver = new mdbServer('localhost', 27017, {auto_reconnect: true});
// request or create a database called "spots03"
var db = new mdbDb('spots03', mdbserver, {safe: true});
// global var that will hold the spots collection
var spotsCol = null;
// open the database
db.open(function(err, db) {
if(!err) {
// if all cool
console.log("Database connection successful");
// open (get/create) a collection named spotsCollection, and if 200,
// point it to the global spotsCol
db.createCollection(
'spotsCollection',
{safe: false}, // if col exists, get the existing one
function(err, collection) {spotsCol = collection;}
);
}
});
Any help would be much appreciated, thanks!
Looks like a couple of things:
The connection URL should be mongo.onmodulus.net
var mdbserver = new mdbServer('mongo.onmodulus.net', 27017, {auto_reconnect: true});
rounce is correct, the database name is auto-generated by Modulus.
var db = new mdbDb('3xam913', mdbserver, {safe: true});
Modulus databases will need authentication. Before you call createCollection, you'll have to call auth and pass it the user credentials that are setup on the project dashboard.
I'm a Modulus developer, and I know the DB name thing is not ideal.
Edit: here's full source for a working example. It records every HTTP request and then sends all requests back to the user.
var express = require('express'),
mongo = require('mongodb'),
Server = mongo.Server,
Db = mongo.Db;
var app = express();
var server = new Server('mongo.onmodulus.net', 27017, { auto_reconnect: true });
var client = new Db('piri3niR', server, { w: 0 });
client.open(function(err, result) {
client.authenticate('MyUser', 'MyPass', function(err, result) {
if(!err) {
console.log('Mongo Authenticated. Starting Server on port ' + (process.env.PORT || 8080));
app.listen(process.env.PORT || 8080);
}
else {
console.log(err);
}
});
});
app.get('/*', function(req, res) {
client.collection('hits', function(err, collection) {
collection.save({ hit: req.url });
// Wait a second then print all hits.
setTimeout(function() {
collection.find(function(err, cursor) {
cursor.toArray(function(err, results) {
res.send(results);
});
});
}, 1000)
});
});
Wrong database name perhaps?
From the MongoDB docs on the subject '3xam913' is your database name, not 'spots03'.
var db = new mdbDb('3xam913', mdbserver, {safe: true});
I am looking for help with a design pattern for creating a database connection in my node.js application.
It seems obvious to do:
module1:
var db;
exports.get_db = function(callback) {
if (db == null) {
dblibrary.create(connection_params, function(error, conn) {
if (error == null) {
db = conn;
callback(null, db);
}
});
} else {
callback(null, db);
}
};
module2:
exports.do_something = function () {
module1.get_db(function (err, conn) {
if (err == null) {
// continue using query
}
});
};
It seems painful to have to penalize every single person who wants to get the db connection with the requirement of using a callback.
I could do this:
module1:
var db;
dblibrary.create_connection(connection_params, function (err, conn) {
if (err != null) {
console.log("can't create connection");
console.log(err);
process.exit();
} else {
db = conn;
}
});
exports.get_db = function() {
return db;
};
This makes it so that getting the db connection is simple and fast, but means we have to "wait" at node startup time for the connection to be established.
Which is the better design? Is there a better way of doing things?
mydb.js module:
var db
exports.db = function() {
if (db === null) {
db = dblibrary.createClient()
}
return db
}
Other modules:
var db = require('mydb').db()
...
db.query(...)
This creates the DB client instance once at startup. I like this solution because the creation code is encapsulated in a separate module and the other modules can get access to the client with one require() statement.
Best answer I've seen for this is:
in start.js:
function init_done() {
app.listen(8080);
}
init_databases(init_done);
in databases.js:
init_databases(init_done_cb) {
db.create_async(/* connect data */ , function (err, res) {
if (err == null) init_done_cb();
});
}
This way you can do the async startup of the database server without that awkward / dangerous waiting period.
I wrote connect-once just for solving this kind of problems. There are two main goals, that are achived by this module:
Connection should be initialized before request arrives
Connection should be initialized once, even there are multiple requests coming in at the same time
You can look at express-mongo-db and express-mongoose-db as examples of usage.