I'm trying to create a mongo connection pool factory that checks if a connection to mongo already exists and returns a connection. If it doesn't create the connection pool and return a connection.
I want to be able to require this from multiple files that are going to query mongo. Each file should require mongo like this:
var fooMongoFactory = require('../../lib/mongoFactory').init(mongodb://localhost:27017/foo);
and then you use it in your file like this:
fooMongoFactory.getConnection().then(function(db) {
// do stuff
});
The problem i'm having is that I want to be able to specify multiple different mongo instances in a file, but when doing that, my second init overrides the first one. Example:
var fooMongoFactory = require('../../lib/mongoFactory').init(mongodb://localhost:27017/foo);
var barMongoFactory = require('../../lib/mongoFactory').init(mongodb://localhost:27017/bar);
fooMongoFactory.getConnection().then(function(db) {
// querying in here is actually pointing at the mongo db "bar"
});
How can I tweak my factory so that I can connect to multiple different instances of mongo, as well as use this same factory across multiple files without having to instantiate it every time? I thought of using a constructor, but that would create a new connection pool in every single file that uses the mongoFactory.
/**
* Creates and manages the Mongo connection pool
*
* #type {exports}
*/
var Q = require('q');
var MongoClient = require('mongodb').MongoClient;
var dbPromise = null;
var db = null;
module.exports = function() {
return {
init: function init(connectionString) {
db = connectionString;
return module.exports;
},
/**
* Gets a connection to Mongo from the pool. If the pool has not been instantiated it,
* instantiates it and returns a connection. Else it just returns a connection from the pool
*
* #returns {*} - A promise object that will resolve to a mongo db object
*/
getConnection: function getConnection() {
// get a connection to mongo using the db string and return dbPromise
}
}
}();
The mongodb node module already has built-in connection pooling functionality that is used automatically when you call connect(). The default max connection pool size is 5, however you can change this value in the connection url (e.g. 'mongodb://localhost:27017/foo?maxPoolSize=15').
You'll also want to change the number of actual connections created by setting poolSize in the server config options to some value less than or equal to maxPoolSize. You may also want to set auto_reconnect to true.
Now what you could do is keep an object keyed on host:port that contains the database object for that server. If someone passes in a connection string that contains a host:port in your object, then return the pool. Otherwise create, cache, and return the new database object.
I ended up making the module so that you have to pass in the connection string for the mongodb that you want to connect to. This module ultimately keeps track of all the connections that have been made to mongo and whether there is a current connection to the mongodb passed in or not.
/**
* Creates and manages the Mongo connection pool
*
* #type {exports}
*/
var Q = require('q');
var MongoClient = require('mongodb').MongoClient;
var _ = require('underscore');
var connections = [];
var dbPromise = null;
module.exports = function() {
return {
/**
* Gets a connection to Mongo from the pool. If the pool has not been instantiated it,
* instantiates it and returns a connection. Else it just returns a connection from the pool
*
* #returns {*} - A promise object that will resolve to a mongo db object
*/
getConnection: function getConnection(connectionString) {
var def = Q.defer();
// If connectionString is null or undefined, return an error
if(_.isEmpty(connectionString)) {
def.reject('getConnection must contain a first parameter');
return dbPromise = def.promise;
}
// Check if connections contains an object with connectionString equal to the connectionString passed in and set the var to it
var pool = _.findWhere(connections, {connectionString: connectionString});
// If no conneciton pool has been instantiated, instantiate it, else return a connection from the pool
if(_.isUndefined(pool)) {
// Initialize connection once
MongoClient.connect(connectionString, function(err, database) {
if (err) {
def.reject(err);
}
// Add the connection to the array
connections.push({connectionString: connectionString, db: database});
def.resolve(database);
});
} else { // Else we have not instantiated the pool yet and we need to
def.resolve(pool.db);
}
return dbPromise = def.promise;
}
};
}();
https://github.com/toymachiner62/mongo-factory
Related
I have a bit of an issue with the asynchronous implementation of the MongoDB driver for Node.js.
In the documentation examples, connection takes place as follows:
const client = new MongoClient(uri, ...);
async function run() {
try {
await client.connect();
const coll = client.db('locations').collection('streets');
coll.find({...});
} catch {
...
} finally {
client.close();
}
}
run().catch(console.dir);
But let's say I want to use the connection in an object rather than create a function for every case when I need the connection. For example, I want to create an object which would allow me to insert comments into a database:
const Comments = {
connection: /* how would I put a MongoDB connection here when it's async? */,
commentsCollectionRef: /* how would I put a collection reference here? */
add: function(user, comment) {
collectionRef.insertOne({user, comment});
}
};
/* And to use the object like this to insert comments: */
Comment.add("Martin", "hello");
Comment.add("Julie", "hi");
Comment.add("Mary", "hello");
Presumably, it is impossible to do something like this:
async function connect() {
await client.connect();
}
const Comments = {
connection: connect() /* this returns a promise, but you can't store a reference to its value like this */
...
}
Is having a function which connects each time and closes the connection each time really the only option with MongoDB?
Thank you
You don't have to open a new connection every time you call the database. You can simply keep a separate variable with the status of the connection.
Something like:
const client = new MongoClient(uri, ...);
let connected = false;
async function connnect() {
if(!connected) {
await client.connect();
connected = true;
}
}
async function disconnect() {
if(connected) {
await client.close();
connected = false;
}
}
// All other comment specific code next...
Then you can build your library around this.
For each of your methods that interact with the database, call connect first.
Or simply call connect when you boot up the app and disconnect when you exit.
However, if you're going to have objects that represent database collections I would recommend having a look at Mongoose. You can easily define models and it can make your life a bit easier.
I am creating the nodejs application which uses the mongodb.
I am connecting to mongodb only once. I want to use db in all other api's so as to achieve the connection pooling.
I have following code for mongodb connectivity:
var mongodb = require('mongodb');
var MongoClient = require('mongodb').MongoClient;
var db;
var mongoUrl = "mongodb://localhost:27017/testDB";
/**
* Connects to the MongoDB Database with the provided URL
*/
exports.connect = function(callback){
if(!db){
MongoClient.connect(mongoUrl, function(err, _db){
if (err) { throw new Error('Could not connect: '+err); }
db = _db;
console.log(db);
connected = true;
console.log(connected +" is connected?");
callback(db);
});
}else{
console.log("Not connected tis time as I am already connected");
callback(db);
}
};
exports.db = db;
I am calling connect only once when server starts from app.js. Whenever other api such as signin, register get called, they should simply use db that is exported.
So my api calls will something like(please ignore the syntax error in api call :D):
var mongo = require('./mongo');
collection = mongo.db.collection("testCollection");
// Here mongo.db id undefined
collection.findOne({"name":"John"}, function(err, result){
// Do your stuff with result here
});
From other stackoverflow posts, I tried something like in mongo.js as
module.export{
db: db,
connect : function(callback){
//function code goes here
}
}
But still I am getting the undefined for mongo.db
How would I access mongo.db in my other files?
Thanks
The reason this happens is, because connect overwrites db in the module. The exports.db=db; is not executed after calling your connect function, but on execution of the module import.
So, when you call connect, db is set to another variable, but that is not exposed outside.
Didn't do much JS lately, but this should do it:
module.exports = new mongobj();
function mongobj() {
this.db = null;
this.connect = function(callback) { /* your connect code set mongobj.db */
this.db = /* new value */ ;
}
}
When you import the module, you get the object. Accessing the objects db property will always expose the latest db value set by the connect function of the module.
var mongo = require('yourmodule');
// mongo.db is null
mongo.connect(some callback);
// mongo.db is set
This connection add in main script file...
var mongodb = require('mongodb');
var mongodb = require('mongodb');
var MongoClient = require('mongodb').MongoClient;
var mongoUrl = "mongodb://localhost:27017/testDB";
ObjectId = module.exports = require("mongojs").ObjectId;
MongoClient.connect(mongoUrl, function(err, database){
if(err){
console.log("mongodb error >>"+err);
} else {
db = module.exports = database;
}});
db.collection('game_users').findOne({_id:ObjectId("123456789")},function(err, data) {});
define an object:
var db = {__db: undefined}
and then:
exports.db = db
const db = require('./mongo').db.__db
I made node.js app that includes some REST services. Those services connect to a database (for example Oracle or DB2) to execute some query.
Since I'm a beginner in node.js programming, I have a question about my case:
What's the right way to access to a database? Is it better to have one connection reference while the app is running and use the same connection instance when REST services are called?
I found some examples that includes database connection in a separate module and use that module in app, something like that:
db2.js:
var db2 = require('ibm_db');
var db2ConnSettings = "DRIVER={DB2};DATABASE=mydb;HOSTNAME=localhost;UID=db2test;PWD=db2test;PORT=50000;PROTOCOL=TCPIP";
var db2Conn = db2.open(db2ConnSettings, function(err, conn) {
if (err)
return console.log(err);
});
module.exports = db2Conn;
server.js:
var express = require('express');
var app = express();
var db2Connection = require('./db2.js');
app.get('/data', function(req, res) {
console.log(db2Connection );
// make some query
});
When this service is called, db2connection is undefined. How come? How should I retrieve a db2 connection from db2.js file?
As said by #Sirko:
db2.js
var db2 = require('ibm_db');
var db2ConnSettings = "DRIVER={DB2};DATABASE=mydb;HOSTNAME=localhost;UID=db2test;PWD=db2test;PORT=50000;PROTOCOL=TCPIP";
var err, conn;
var callbacks = [];
module.exports = function(callback) {
// db2 module is called
if (err || conn) {
// connection has already been established
// (results of db2.open have been stored)
// callback immediately
callback(err, conn);
}
else {
// connection has not been established
// store the callback for when db connects
callbacks.push(callback);
}
};
db2.open(db2ConnSettings, function(_err, _conn){
// db has connected
err = _err; conn = _conn; // store results
var next_callback;
// array.pop() removed the last item from the array
// and returns it. if no items are left, returns null.
// so this loops through all stored callbacks.
while(next_callback = callbacks.pop()) {
// the removed item is stored in next_callback
next_callback(err, conn); // send connection results to callback
}
// no more items in callbacks to trigger
});
server.js
var express = require('express');
var app = express();
var db2Connection = require('./db2.js')(function(err, conn) {
// triggered if the connection has already been established
// or as soon as it HAS been established
app.get('/data', function(req, res) {
console.log(conn);
// ...
});
});
For Oracle with node-oracledb it's simple to create and use a connection pool. Your app would just get a free connection from the pool whenever it handles an HTTP REST request. Look at webapp.js and webapppromises.js in the examples. Node-oracledb has a 'connection pool queue' (see doc) which handles connection load spikes. It also has a 'connection pool cache' (also see the doc) which makes it easy to access a pool created in a different module.
I'm currently in the process of reorganizing the routes in my web application (I stupidly defined all the routes in index.js) and for some reason in several of these files I'm having this inexplicable problem: I'm getting errors saying the "globals" variable is undefined when it is, in fact, defined.
This is one of the offending files:
http://pastebin.com/7Q5ExZDa
At line 37 I log the contents of globals.DB_URL, and it exists. The very next line I get an error that globals isn't defined. What am I doing wrong?
mongodb://localhost:27017/[redacted_db_name] // console log output
--- Error: ReferenceError: globals is not defined ---
Location: function (err){
utilities.logError(err, arguments.callee.toString());
res.redirect("/");
return;
}
UPDATE:
First problem was solved: I wasn't importing globals.js in utilities.js, and was trying to call a function that needed data from globals to function.
Unfortunately, now I get this error:
--- Error: TypeError: Cannot call method 'connect' of undefined ---
Location: function (err){
utilities.logError(err, arguments.callee.toString());
res.redirect("/");
return;
}
This error happens at the second promise. I think it may have something to do with the code in utilities, specifically the identifyUserByToken function.
/**
* identifyUserByToken
* Compares a given session token against the session tokens collection
* executes a given callback function if a match is found
* #param {String} userToken The session token to search for
* #param {function(Object, String)} The callback function to be called upon success, failure, or error
*/
function identifyUserByToken(userToken, callback){
var user_tokens;
var users
var promise = new Promise(function(resolve, reject){
mongoClient.connect(globals.DB_URL)
.then(function(db){ // Search user_tokens for userToken
user_tokens = db.collection("user_tokens");
users = db.collection("users");
return user_tokens.find({token : userToken}).toArray();
})
.then(function(result){ // Search users for the returned userID
var userID = result[0].userid;
return users.find({ userid : userID }).toArray();
})
.then(function(matchingUsers){ // Pass returned user object to callback
var user = matchingUsers[0];
if(callback != undefined) callback(undefined, user);
resolve(user);
})
.catch(function(err){
if(callback != undefined) callback(err, undefined);
reject(err);
});
});
return promise;
}
I know this means mongodb is undefined, but I'm importing it in the file
var globals = require("./globals");
/* == Third Party Libraries == */
var chalk = require("chalk"); /* usage: console output coloring */
var crypto = require("crypto"); /* usage: cryptograpgic primitives (password hashing, etc...) */
var mongodb = require("mongodb"); /* usage: data storage schema */
var mongoClient = mongodb.mongoClient;
EDIT: Solved TypeError
Simple typo. In utilities I was assigning the mongoClient variable incorrectly
How it was being defined: var mongoClient = mongodb.mongoClient;
How it needed to be defined: var mongoClient = mongodb.MongoClient;
Sorry! My bad.
The issue is with var mongoClient = mongodb.mongoClient; it should be with a capital M:
var mongoClient = mongodb.MongoClient;
I'd like to create a model to handle everything related to users, starting with a findOne() function.
app.js:
var u = new User(client);
u.findOne(function(error, user) {
console.log(error, user);
});
models/User.js:
var User = function (client) {
this.client = client
};
User.prototype.findOne = function (id, callback) {
client.connect();
client.get('testkey', function(error, result) {
var test = "hello#world.com";
callback(null, test);
client.close();
});
};
module.exports = User;
node.js complains findOne() would be undefined.
What's the correct way of creating such models and providing them with objects, like database pools etc.?
Your code contains various errors:
You do not use new when creating the instance
You mixed a function with the object literal syntax:
var User = function (client) {
client: client
};
You want this.client = client; instead. Right now the function body does nothing as it just defines a label called client does nothing with the variable client.
I would suggest you to search for an existing ORM for node.js instead of trying to write one on your own.