I'm connecting and making an insert with Node/MongoDB, but due to a scope issue, I can't access the connection from the function. Any idea how to make the 'db' variable global scope?
mongodb.connect("mongodb://localhost:27017/userDB", function(err, db) {
if(!err) {
console.log("We are connected");
} else {
console.log(err);
}
});
function RegisterUser(user, pass) {
var collection = db.collection('users');
var docs = [{username:user}, {password: pass}];
collection.insert(docs, {w:1}, function(err, result) {
collection.find().toArray(function(err, items) {});
socket.emit('message', items);
});
}
/var/www/baseball/app.js:80
var collection = db.collection('users'); <--db is not defined
^
ReferenceError: db is not defined
at RegisterUser (/var/www/baseball/app.js:80:20)
at ParseData (/var/www/baseball/app.js:63:6)
In general, only make the connection once, probably as you app starts up. The mongo driver will handle pooling etc., at least, so I was told by experts from MongoLabs. Note that this is very different than what you might do in Java, etc... Save the db value returned by connect() somewhere, perhaps a global or in app, or in a commonly used one of your modules. Then use it as needed in RegisterUser, DeleteUser, etc.
More of a node question really, but probably worth the tag since it gets asked a bit. So you say there is a scoping issue and you are right as the variable is local to the callback function on the .connect() method and is not visible anywhere else. One way is to dump all your logic inside that callback, so there is no scoping issue, but you probably don't want to do that.
Asking "how do I set a global", is also not really the right approach. Well not directly as there are general funny things about breaking up the "async" pattern of node. So a better approach is with some kind of "singleton" instance where you set the connection only once, but as that is global or can otherwise be "required" in for use in other areas of your application.
Here is one "trivial" approach to demonstrate, but there are many ways to do the same thing:
var async = require('async'),
mongodb = require('mongodb'),
MongoClient = mongodb.MongoClient;
var Model = (function() {
var _db;
var conlock;
return {
getDb: function(callback) {
var err = null;
if ( _db == null && conlock == null ) {
conlock = 1;
MongoClient.connect('mongodb://localhost/test',function(err,db) {
_db = db;
conlock == null;
if (!err) {
console.log("Connected")
}
callback(err,_db);
});
} else if ( conlock != null ) {
var count = 0;
async.whilst(
function() { return ( _db == null ) && (count < 5) },
function(callback) {
count++
setTimeout(callback,500);
},
function(err) {
if ( count == 5 )
err = new Error("connect wait exceeded");
callback(err,_db);
}
);
} else {
callback(err,_db);
}
}
};
})();
async.parallel(
[
function(callback) {
console.log("call model");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined");
callback();
});
},
function(callback) {
console.log("call model again");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined here as well");
callback();
})
}
],
function(err) {
Model.getDb(function(err,db) {
db.close();
});
}
);
So out little "Model" object here has a single method in .getDb(), and it also maintains a private variable holding the _db connection once it has been established. The basic logic on that method is to see if _db is defined and where it not then establish a connection with the driver. On the connection callback the _db variable is then set.
The other thing here is the method itself accepts a "callback" so this is how you use it later, where either an error or the current connection will be returned.
The last part is just a demonstration of two functions to be implemented in code. Where in the first call the call to connect to the database is made before following into the callback function provided.
The next time we call though, the connection is already set in the private variable, so that data is merely returned and you don't establish a connection again.
There are various ways to implement this sort of thing, but that is the basic logic pattern to follow. There are many other "helper" implementations that wrap the MongoDB driver to make these sort of things simple, as well as managing connection pools and ensuring the connection is up as well for you, so it may be well worth looking at these even if you are still insistent on doing all the work yourself from the lower level driver base.
First of all, you are only going to be registering users once you have a connection, so do what ever work you need to do in there... so call RegisterUser from within the connection scope. If you want to use the db object within that function you will need to pass in the parameters as db
RegisterUsers(db, user, pass)
you may then use db within the function
Related
I have a MEAN app that works well with single requests, let's say calling /api/products?pid=500. But I recently discovered that at a "burst" of requests (i'm updating bulk around 50 products = 50 requests /api/products?pid=500 *** 550 with post data), the req.body sometimes gets a value of a new upcoming request.
The front app makes the calls in a foreach of selected products:
ds.forEach((d, key) => {
this.ApiCall.setData('products', { action: 'send-product', data: d })
.subscribe((result) => {
//we have results
});
});
//setData makes a http.post().map
Back app / mean analyses the post, tried to synthesize the code:
router.route('/')
.post(function (req, response) {
if(req.body.data){
var obj = { id: req.body.data.product_id }
if(req.body.data.linked_products){
req.body.data.linked_products.forEach(function(entry) {
obj.linked = entry; //more ifs
});
}
var async = require('async');
async.series({
q2: function(cb){
queryProducts.findOne({id: req.body.data.product_id, null).exec(cb);
},
q3: function(cb){
queryCategories.findOne({id: req.body.data.category_id, null).exec(cb);
}
}, function(err, qResults){
var alreadysent = false;
if (qResults.q3) qResults.q3.logs.forEach(function(entry) {
if(entry.sent){
alreadysent = true;
}
});
//more ifs
qResults.q3.external_codes.forEach(function(entry) {
obj.external_code = entry;//more ifs
});
if(req.body.data.price < 0){
response.json({message: "Negative price didn't sent"});
return;
}
if(qResults.q2.status=="inactive"){
response.json({message: "Inactive didn't sent"});
return;
}
req.body.data.campaigns(function(entry) {
obj.price_offers = entry;//more ifs
});
//more ifs and foreach similar
queryProducts.update({id: req.body.data.id}, {$push: { synced_products: obj }}, function (err, result) {
//HERE I found req.body.data with values of a future request
if(!err)
response.json({message: "Sent"});
return;
});
});
}
});
module.exports = router;
I understand that making requests
/api/products?pid=500
/api/products?pid=501
/api/products?pid=502
/api/products?pid=503
...
have different timings, but how is possible that a request (pid=501), calling the last req.body to have the value of req.body of new req (pid=503)?
Any ideas how to avoid? putting async first right after the post or making a
var reqbody = req.body
Thanks!
I believe this is due to the async module initialization. To quote from the node docs:
Caching
Modules are cached after the first time they are loaded. This means (among other things) that every call to require('foo') will get exactly the same object returned, if it would resolve to the same file.
Multiple calls to require('foo') may not cause the module code to be executed multiple times. This is an important feature. With it, "partially done" objects can be returned, thus allowing transitive dependencies to be loaded even when they would cause cycles.
To have a module execute code multiple times, export a function, and call that function.
When a burst of requests causes overlapping execution, you will have two (or more) uses of the async variable being modified "concurrently". I would suggest using some sort of mutex to control access to the async variable.
I have only recently started developing for node.js, so forgive me if this is a stupid question - I come from Javaland, where objects still live happily sequentially and synchronous. ;)
I have a key generator object that issues keys for database inserts using a variant of the high-low algorithm. Here's my code:
function KeyGenerator() {
var nextKey;
var upperBound;
this.generateKey = function(table, done) {
if (nextKey > upperBound) {
require("../sync/key-series-request").requestKeys(function(err,nextKey,upperBound) {
if (err) { return done(err); }
this.nextKey = nextKey;
this.upperBound = upperBound;
done(nextKey++);
});
} else {
done(nextKey++);
}
}
}
Obviously, when I ask it for a key, I must ensure that it never, ever issues the same key twice. In Java, if I wanted to enable concurrent access, I would make make this synchronized.
In node.js, is there any similar concept, or is it unnecessary? I intend to ask the generator for a bunch of keys for a bulk insert using async.parallel. My expectation is that since node is single-threaded, I need not worry about the same key ever being issued more than once, can someone please confirm this is correct?
Obtaining a new series involves an asynchronous database operation, so if I do 20 simultaneous key requests, but the series has only two keys left, won't I end up with 18 requests for a new series? What can I do to avoid that?
UPDATE
This is the code for requestKeys:
exports.requestKeys = function (done) {
var db = require("../storage/db");
db.query("select next_key, upper_bound from key_generation where type='issue'", function(err,results) {
if (err) { done(err); } else {
if (results.length === 0) {
// Somehow we lost the "issue" row - this should never have happened
done (new Error("Could not find 'issue' row in key generation table"));
} else {
var nextKey = results[0].next_key;
var upperBound = results[0].upper_bound;
db.query("update key_generation set next_key=?, upper_bound=? where type='issue'",
[ nextKey + KEY_SERIES_WIDTH, upperBound + KEY_SERIES_WIDTH],
function (err,results) {
if (err) { done(err); } else {
done(null, nextKey, upperBound);
}
});
}
}
});
}
UPDATE 2
I should probably mention that consuming a key requires db access even if a new series doesn't have to be requested, because the consumed key will have to be marked as used in the database. The code doesn't reflect this because I ran into trouble before I got around to implementing that part.
UPDATE 3
I think I got it using event emitting:
function KeyGenerator() {
var nextKey;
var upperBound;
var emitter = new events.EventEmitter();
var requesting = true;
// Initialize the generator with the stored values
db.query("select * from key_generation where type='use'", function(err, results)
if (err) { throw err; }
if (results.length === 0) {
throw new Error("Could not get key generation parameters: Row is missing");
}
nextKey = results[0].next_key;
upperBound = results[0].upper_bound;
console.log("Setting requesting = false, emitting event");
requesting = false;
emitter.emit("KeysAvailable");
});
this.generateKey = function(table, done) {
console.log("generateKey, state is:\n nextKey: " + nextKey + "\n upperBound:" + upperBound + "\n requesting:" + requesting + " ");
if (nextKey > upperBound) {
if (!requesting) {
requesting = true;
console.log("Requesting new series");
require("../sync/key-series-request").requestSeries(function(err,newNextKey,newUpperBound) {
if (err) { return done(err); }
console.log("New series available:\n nextKey: " + newNextKey + "\n upperBound: " + newUpperBound);
nextKey = newNextKey;
upperBound = newUpperBound;
requesting = false;
emitter.emit("KeysAvailable");
done(null,nextKey++);
});
} else {
console.log("Key request is already underway, deferring");
var that = this;
emitter.once("KeysAvailable", function() { console.log("Executing deferred call"); that.generateKey(table,done); });
}
} else {
done(null,nextKey++);
}
}
}
I've peppered it with logging outputs, and it does do what I want it to.
As another answer mentions, you will potentially end up with results different from what you want. Taking things in order:
function KeyGenerator() {
// at first I was thinking you wanted these as 'class' properties
// and thus would want to proceed them with this. rather than as vars
// but I think you want them as 'private' members variables of the
// class instance. That's dandy, you'll just want to do things differently
// down below
var nextKey;
var upperBound;
this.generateKey = function (table, done) {
if (nextKey > upperBound) {
// truncated the require path below for readability.
// more importantly, renamed parameters to function
require("key-series-request").requestKeys(function(err,nKey,uBound) {
if (err) { return done(err); }
// note that thanks to the miracle of closures, you have access to
// the nextKey and upperBound variables from the enclosing scope
// but I needed to rename the parameters or else they would shadow/
// obscure the variables with the same name.
nextKey = nKey;
upperBound = uBound;
done(nextKey++);
});
} else {
done(nextKey++);
}
}
}
Regarding the .requestKeys function, you will need to somehow introduce some kind of synchronization. This isn't actually terrible in one way because with only one thread of execution, you don't need to sweat the challenge of setting your semaphore in a single operation, but it is challenging to deal with the multiple callers because you will want other callers to effectively (but not really) block waiting for the first call to requestKeys() which is going to the DB to return.
I need to think about this part a bit more. I had a basic solution in mind which involved setting a simple semaphore and queuing the callbacks, but when I was typing it up I realized I was actually introducing a more subtle potential synchronization bug when processing the queued callbacks.
UPDATE:
I was just finishing up one approach as you were writing about your EventEmitter approach, which seems reasonable. See this gist which illustrates the approach. I took. Just run it and you'll see the behavior. It has some console logging to see which calls are getting deferred for a new key block or which can be handled immediately. The primary moving part of the solution is (note that the keyManager provides the stubbed out implementation of your require('key-series-request'):
function KeyGenerator(km) {
this.nextKey = undefined;
this.upperBound = undefined;
this.imWorkingOnIt = false;
this.queuedCallbacks = [];
this.keyManager = km;
this.generateKey = function(table, done) {
if (this.imWorkingOnIt){
this.queuedCallbacks.push(done);
console.log('KG deferred call. Pending CBs: '+this.queuedCallbacks.length);
return;
};
var self=this;
if ((typeof(this.nextKey) ==='undefined') || (this.nextKey > this.upperBound) ){
// set a semaphore & add the callback to the queued callback list
this.imWorkingOnIt = true;
this.queuedCallbacks.push(done);
this.keyManager.requestKeys(function(err,nKey,uBound) {
if (err) { return done(err); }
self.nextKey = nKey;
self.upperBound = uBound;
var theCallbackList = self.queuedCallbacks;
self.queuedCallbacks = [];
self.imWorkingOnIt = false;
theCallbackList.forEach(function(f){
// rather than making the final callback directly,
// call KeyGenerator.generateKey() with the original
// callback
setImmediate(function(){self.generateKey(table,f);});
});
});
} else {
console.log('KG immediate call',self.nextKey);
var z= self.nextKey++;
setImmediate(function(){done(z);});
}
}
};
If your Node.js code to calculate the next key didn't need to execute an async operation then you wouldn't run into synchronization issues because there is only one JavaScript thread executing code. Access to the nextKey/upperBound variables will be done in sequence by only one thread (i.e. request 1 will access first, then request 2, then request 3 et cetera.) In the Java-world you will always need synchronization because multiple threads will be executing even if you didn't make a DB call.
However, in your Node.js code since you are making an async call to get the nextKey you could get strange results. There is still only one JavaScript thread executing your code, but it would be possible for request 1 to make the call to the DB, then Node.js might accept request 2 (while request 1 is getting data from the DB) and this second request will also make a request to the DB to get keys. Let's say that request 2 gets data from the DB quicker than request 1 and update nextKey/upperBound variables with values 100/150. Once request 1 gets its data (say values 50/100) then it will update nextKey/upperBound. This scenario wouldn't result in duplicate keys, but you might see gaps in your keys (for example, not all keys 100 to 150 will be used because request 1 eventually reset the values to 50/100)
This makes me think that you will need a way to sync access, but I am not exactly sure what will be the best way to achieve this.
I feel like I'm making an obvious mistake here. I'm just trying to initialize a variable in the constructor of an object:
var Lobby = function(io, dbConnection, lobbyName) {
this.connectedPlayers = {};
this.seatedPlayers = [];
for (var i=0;i<this.NUMBER_OF_TABLES;i++) {
this.seatedPlayers[i] = [];
for (var j=0;j<this.PLAYERS_PER_TABLE;j++) {
this.seatedPlayers[i][j] = null;
}
}
if (lobbyName == "lobbya") {
dbConnection.query("SELECT id, username, rating FROM players WHERE id < 0 order by id desc;",
function (error, results, fields) {
for (var i=0;i<results.length;i++) {
var computerPlayer = new Player(results[i]["id"],results[i]["username"],results[i]["rating"]);
this.connectedPlayers[results[i]["id"]] = computerPlayer;
this.seatedPlayers[i+4][1] = computerPlayer;
this.connectedPlayers[results[i]["id"]].setSeatedAt(i+4);
}
});
}
}
When I run this, connectedPlayers is undefined. (I assume because it's out of scope, since I'm within the function processing the results. So how do I initialize a variable outside of the scope?
You can't initialize async operations in a Javascript constructor, because constructors are, by definition, synchronous.
There are three ways to solve this problem that are common:
1) Pass in an initialized DB.
2) Lazy-load the DB when something needs it.
Lobby.prototype.connectPlayers = function(cb) {
self.getConnection(err, db) {
// getConnection will either init connection or return already inited connection
// run stuff requiring DB connection
cb(foo);
});
}
3) Add an async initialize method which is called after it is new'd.
var lobby = new Lobby();
lobby.init(function(err) {
// now we have it
});
In my eyes, lazy-load is the most efficient, since it means you aren't running anything more than you need. Some Lobby may not need a DB connection (especially for unit tests).
I am wondering why trying to run the following test suite fails when I try to delete the table I have stored entities in. The error I get is the following
1) Azure Storage cloud storage operations "after all" hook:
Error: The specified resource does not exist. RequestId:3745d709-fa5e-4a2b-b517-89edad3efdd2
Time:2013-12-03T22:26:39.5532356Z
If I comment out the actual insertion of data it fails every other time, and if I try to do the insertion of data it fails every time with an additional "The table specified does not exist.".
For the first case this seems to indicate that there is some kind of delay in the table creation, so in every other test it is successful, and for the second case it seems to indicate that even though my callbacks are being called after table creation, the table(s) still aren't ready for data insertion.
The test suite and associated code looks like this:
describe('cloud storage operations', function () {
var storage;
before(function (done) {
this.timeout(5000);
storage = AzureStorage.usingTable('TEST', done);
});
after(function (done) {
storage.deleteTable(done);
});
it('should store without trouble', function (done) {
storage.save(factory.createChangeSet()).then(done, done);
});
});
... // snipped from azure.js
var AzureStorage = function (storageClient, tableName, callback) {
assert(storageClient && tableName && partitionKey, "Missing parameters");
this.storageClient = storageClient;
this.tableName = tableName;
var defaultCallback = function (err) { if (err) { throw error; } };
this.storageClient.createTableIfNotExists(this.tableName, function () {
callback();
} || defaultCallback);
};
AzureStorage.usingTable = function (tableName, callback) {
return new AzureStorage(
azure.createTableService(accountName, accountKey)
, tableName
, callback
);
};
AzureStorage.prototype.deleteTable = function (callback) {
this.storageClient.deleteTable(this.tableName, callback);
};
I've hit this using the c# library as well but I'm pretty sure the error message indicated the table could not be created because an operation was still in process for a table of the same name. Thinking of the backend supporting storage, it makes sense that it would not be instant. The table needs to be removed from the 3 local replicas as well as the replicas in the paired data center.
With that kind of async operation, it is going to be challenging to build up an tear them down fast enough for tests.
A workaround might be to increment a value appended to the "TEST" table name that would be unique to that test run.
I'm writing a multiplayer game(mongojs, nodejs) and trying to figure out how to update user stats based on the outcome of the game. I already have the code written to compute all the post game stats. The problem comes when I try to update the users' stats in a for loop. Here's what i got:
//Game Stats
var tempgame = {
gameid: 1234,
stats: [
{
score: 25,
user: 'user1'
},
{
score: 25,
user: 'user2'
}
]
}
for(i = 0; i < tempgame.stats.length; i++){
db.users.find({ username: tempgame.stats[i].user }, function(err, res){
if( err != null){
//handle errors here.
} else {
var userstats = res[0].stats;
if( tempgame.stats[i].score > userstats.bestscore ){ //this is where it chokes
userstats.bestscore = tempgame.stats[i].score;
}
//code here to pass back new manipulated stats
}
});
}
Everything works fine until i try to use the tempgame object within the callback function. It says "cannot read property 'score' of undefined". Is this just a scoping issue?
Also i was thinking it could be an issue with the callback function itself. Maybe the loop would increment before the callback is even run. But even in that case, the score should be be there it would just be pulling from the wrong array index... that's what lead me to believe it may just be a scope issue.
Any help would be greatly appreciated.
You've been tripped up by the notorious "defining functions inside a loop" problem.
Use "forEach" instead:
tempgame.stats.forEach(function (stat) {
db.users.find({ username: stat.user }, function(err, res){
if( err != null){
//handle errors here.
} else {
var userstats = res[0].stats;
if( stat.score > userstats.bestscore ){ //this is where it chokes
userstats.bestscore = stat.score;
}
//code here to pass back new manipulated stats
}
});
});
Part of your problem is as mjhm stated in his answer to your question, and is as you have suspected. The i variable is changing before the callback is invoked.
The other half of your problem is because your database calls have not returned yet. Due to the asynchronous nature of NodeJS, your loop will finish before your database calls complete. Additionally, your database calls are not necessarily coming back in the same order you called them. What you need is some sort of flow control like async.js. Using async.map will allow you to make all calls to the DB in parallel and return them as an array of values you can use, after all db calls have been completed.
async.map(tempgame.stats, function(stat, callback){
db.users.find({ username: stat.user }, function(err, res){
if( err != null){
callback(err);
} else {
callback(null, res[0].stats);
}
});
}, function(err, stats){
if(err){
//handle errors
} else{
stats.forEach(function(stat){
//do something with your array of stats
//this wont be called until all database calls have been completed
});
}
});
In addition to the above, if you want to return results back to the application,
http://nodeblog.tumblr.com/post/60922749945/nodejs-async-db-query-inside-for-loop