I am using nodejs to query data from Mongodb throught Mongoose.
After get the data, I want do something on that data before responding it to client. But I can not get the return-value. After looking on Google, I have learned Node.js functions is asynchronous javascript function (non I/O blocking). I try this tut (http://www.youtube.com/watch?v=xDW9bK-9pNY) but it is not work. Below is my code. The myObject is valued inside "find()" function and undefined outside "find()" function. So what should I do to get the data? Thanks!
var Person = mongoose.model('Person', PersonSchema);
var Product = mongoose.model('Product', ProductSchema);
var myObject = new Object();
Person.find().exec(function (err, docs) {
for (var i=0;i<docs.length;i++)
{
Product.find({ user: docs[i]._id},function (err, pers) {
myObject[i] = pers;
console.log(myObject[i]); //return the value is ok
});
console.log(myObject[i]); //return undefined value
}
console.log(myObject); //return undefined value
});
console.log(myObject); //return undefined value
app.listen(3000);
console.log('Listening on port 3000');
The reason you're getting undefined values is because the find function is asynchronous, and can finish at any time. In your case, it is finishing after you're using console.log(), so the values are undefined when you're accessing them.
To fix this problem, you can only use the values inside the find function's callback. It would look something like this:
var Person = mongoose.model('Person', PersonSchema);
var Product = mongoose.model('Product', ProductSchema);
var myObject = new Object();
function getData(docs, callback) {
function loop(i) {
Product.find({ user: docs[i]._id}, function (err, pers) {
myObject[i] = pers;
if (i < docs.length) {
loop(i + 1);
} else {
callback();
}
});
};
loop(0);
};
Person.find().exec(function(err, docs) {
getData(docs, function() {
// myObject has been populated at this point
});
});
The data processing has been moved to a loop that waits for the previous iteration to complete. This way, we can determine when the last callback has fired in order to fire the callback in the wrapper function.
Keep in mind that by the time the console.log functions are executed, the query has not yet finished, thus will display "undefined". That's the essence of nodeJS's asynchronicity.
For example,
Person.find().exec(function (err, docs) {
for (var i=0;i<docs.length;i++)
{
Product.find({ user: docs[i]._id},function (err, pers) {
myObject[i] = pers;
console.log(myObject[i]); //return the value is ok
});
console.log(myObject[i]); //return undefined value
}
console.log(myObject); //return undefined value
});
console.log(myObject); // <-- Initially, this value will be undefined. After some miliseconds (Or the duration of the .exec function, myObject will contain the results.
If you want to actually wait until the query is finished so you can use the values, I would recommend moving the app.listen(3000); and console.log('Listening on port 3000'); into the function's final callback.
I'd also recommend you to check out this node module. It will help you build asynchronous / synchronous functions more easily, and allow you to execute a callback when all the asynchronous functions are finished.
Related
I'm ok with javascript and callbacks, but I'm getting really annoyed at this and need to call on the the world of stackoverflow for help.
I have written a function, to be used in the following way:
var meth = lib.funcTion(a,b); // meth should hold an array of properties { c, d } once executed
So now inside lib.js, we have a structure like:
exports.funcTion = function (a,b) {
database.connect(params, function(err,get){
get.query(querylang, function(err, results){
var varsIwantToReturn = { var1: results[i].foo, var2: results[i].bar };
});
});
// Now how do i return 'varsIwantToReturn'?
};
I have seen some things about incorporating callback() into the function, but I'm not exactly sure how this works. I've also seen some people use exec() - again, im not sure on how or why to use it.
Please help :) thanks in advance.
Well, it's all asynchronous so if you attempt to return it - it'll return undefined. In JavaScript (Sans the new yield keyword) functions execute from top to bottom synchronously. When you make an IO call like a database call - it still executes synchronously. In fact- when varIwantToReturn gets population the function has long run and terminated.
What is left is to do the same thing async functions like database.connect and get.query do and have the function take a callback:
exports.function = function (a,b, callback) {
database.connect(params, function(err,get){
if(err) return callback(err, null); // don't suppress errors
get.query(querylang, function(err, results){
if(err) return callback(err, null); // don't suppress errors
var varsIwantToReturn = { var1: results[i].foo, var2: results[i].bar };
callback(null, varsIwantToReturn);
});
});
};
Then you'd call it like
myExportedFunction(myA,myB, function(err, resp){
if(err) recoverFromError(err);
// varsIWantToReturn are contained in `resp`
});
I have a function that returns an array of items from MongoDB:
var getBooks = function(callback){
Comment.distinct("doc", function(err, docs){
callback(docs);
}
});
};
Now, for each of the items returned in docs, I'd like to execute another mongoose query, gather the count for specific fields, gather them all in a counts object, and finally pass that on to res.render:
getBooks(function(docs){
var counts = {};
docs.forEach(function(entry){
getAllCount(entry, ...){};
});
});
If I put res.render after the forEach loop, it will execute before the count queries have finished. However, if I include it in the loop, it will execute for each entry. What is the proper way of doing this?
I'd recommend using the popular NodeJS package, async. It's far easier than doing the work/counting, and eventual error handling would be needed by another answer.
In particular, I'd suggest considering each (reference):
getBooks(function(docs){
var counts = {};
async.each(docs, function(doc, callback){
getAllCount(entry, ...);
// call the `callback` with a error if one occured, or
// empty params if everything was OK.
// store the value for each doc in counts
}, function(err) {
// all are complete (or an error occurred)
// you can access counts here
res.render(...);
});
});
or you could use map (reference):
getBooks(function(docs){
async.map(docs, function(doc, transformed){
getAllCount(entry, ...);
// call transformed(null, theCount);
// for each document (or transformed(err); if there was an error);
}, function(err, results) {
// all are complete (or an error occurred)
// you can access results here, which contains the count value
// returned by calling: transformed(null, ###) in the map function
res.render(...);
});
});
If there are too many simultaneous requests, you could use the mapLimit or eachLimit function to limit the amount of simultaneous asynchronous mongoose requests.
forEach probably isn't your best bet here, unless you want all of your calls to getAllCount happening in parallel (maybe you do, I don't know — or for that matter, Node is still single-threaded by default, isn't it?). Instead, just keeping an index and repeating the call for each entry in docs until you're done seems better. E.g.:
getBooks(function(docs){
var counts = {},
index = 0,
entry;
loop();
function loop() {
if (index < docs.length) {
entry = docs[index++];
getAllCount(entry, gotCount);
}
else {
// Done, call `res.render` with the result
}
}
function gotCount(count) {
// ...store the count, it relates to `entry`...
// And loop
loop();
}
});
If you want the calls to happen in parallel (or if you can rely on this working in the single thread), just remember how many are outstanding so you know when you're done:
// Assumes `docs` is not sparse
getBooks(function(docs){
var counts = {},
received = 0,
outstanding;
outstanding = docs.length;
docs.forEach(function(entry){
getAllCount(entry, function(count) {
// ...store the count, note that it *doesn't* relate to `entry` as we
// have overlapping calls...
// Done?
--outstanding;
if (outstanding === 0) {
// Yup, call `res.render` with the result
}
});
});
});
In fact, getAllCount on first item must callback getAllCount on second item, ...
Two way: you can use a framework, like async : https://github.com/caolan/async
Or create yourself the callback chain. It's fun to write the first time.
edit
The goal is to have a mechanism that proceed like we write.
getAllCountFor(1, function(err1, result1) {
getAllCountFor(2, function(err2, result2) {
...
getAllCountFor(N, function(errN, resultN) {
res.sender tout ca tout ca
});
});
});
And that's what you will construct with async, using the sequence format.
I am able to insert and retrieve data from an neDB database in nodejs. But I cannot pass the data outside of the function that retrieves it.
I have read through the neDB documentation and I searched for and tried different combinations of callbacks and returns (see code below) without finding a solution.
I'm new to javascript so I do not know if I am misunderstanding how to use variables in general or if this issue is related to using neDB specifically or both.
Could someone please explain why "x" in my code does not ever contain the docs JSON results from the database? How can I make it work?
var fs = require('fs'),
Datastore = require('nedb')
, db = new Datastore({ filename: 'datastore', autoload: true });
//generate data to add to datafile
var document = { Shift: "Late"
, StartTime: "4:00PM"
, EndTime: "12:00AM"
};
// add the generated data to datafile
db.insert(document, function (err, newDoc) {
});
//test to ensure that this search returns data
db.find({ }, function (err, docs) {
console.log(JSON.stringify(docs)); // logs all of the data in docs
});
//attempt to get a variable "x" that has all
//of the data from the datafile
var x = function(err, callback){
db.find({ }, function (err, docs) {
callback(docs);
});
};
console.log(x); //logs "[Function]"
var x = db.find({ }, function (err, docs) {
return docs;
});
console.log(x); //logs "undefined"
var x = db.find({ }, function (err, docs) {
});
console.log(x); //logs "undefined"*
Callbacks are generally asynchronous in JavaScript meaning you can not use assignment operator, consequently you do not return anything from the callback function.
When you call an async function execution of you programme carries on, passing the 'var x = whatever' statement. The assignment to a variable, the result of whatever callback is received, you need to perform from within the callback itself... what you need is something in the lines of ...
var x = null;
db.find({ }, function (err, docs) {
x = docs;
do_something_when_you_get_your_result();
});
function do_something_when_you_get_your_result() {
console.log(x); // x have docs now
}
EDIT
Here is a nice blog post about asynchronous programming. And there is a lot more resources on this topic that you can pick up.
This is a popular library to help with async flow-control for node.
P.S.
Hope this helps. Please, by all means ask if you need something clarified :)
I ran into the same problem. In the end I used a combination between async-await and a promise with resolve to solve it.
In your example the following would work:
var x = new Promise((resolve,reject) {
db.find({ }, function (err, docs) {
resolve(docs);
});
});
console.log(x);
I had to learn a bit about async functions to get it right. For those who are looking for specific help about getting a return value from nedb, here's a snippet of what worked for me. I was using it in electron.
function findUser(searchParams,callBackFn)
{
db.find({}, function (err, docs))
{
//executes the callback
callBackFn(docs)
};
}
usage
findUser('John Doe',/*this is the callback -> */function(users){
for(i = 0; i < users.length; i++){
//the data will be here now
//eg users.phone will display the user's phone number
}})
I am trying to solve an exam problem, so I cannot post my exam code as it is. So I have simplified such that it addresses the core concept that I do not understand. Basically, I do not know how to slow down node's asynchronous execution so that my mongo code can catch up with it. Here is the code:
MongoClient.connect('mongodb://localhost:27017/somedb', function(err, db) {
if (err) throw err;
var orphans = [];
for (var i; i < 100000; i++) {
var query = { 'images' : i };
db.collection('albums').findOne(query, function(err, doc_album) {
if(err) throw err;
if (doc_album === null) {
orphans.push(i);
}
});
}
console.dir(orphans.length);
return db.close();
});
So I am trying to create an array of those images who do not match my query criteria. I end up with a orphans.length value of 0 since Node does not wait for the callbacks to finish. How can I modify the code such that the callbacks finish executing before I count the number of images in the array that did not meet my query criteria?
Thanks in advance for your time.
Bharat
I assume you want to do 100000 parallel DB calls. To "wait" 10000 calls completion in each call callback we increase finished calls counter and invoke main callback when last one finished. Note that very common mistake here is to use for loop variable as a closure inside callback. This does not work as expected as all 10000 handlers scheduled first and by the time first is executed loop variable is of the same, maximum value.
function getOrphans(cb) {
MongoClient.connect('mongodb://localhost:27017/somedb', function(err, db) {
if (err) cb(err);
var orphans = [];
var numResponses = 0;
var maxIndex = 100000
for (var i = 0; i < maxIndex; i++) {
// problem: by the time you get reply "i" would be 100000.
// closure variable changed to function argument:
(function(index) {
var query = { 'images' : index };
db.collection('albums').findOne(query, function(err, doc_album) {
numResponses++;
if(err) cb(err);
if (doc_album === null) {
orphans.push(index);
}
if (numResponses == maxIndex) {
db.close();
cb(null, orphans);
}
});
})(i); // this is "immediately executed function
}
});
}
getOrphans(function(err, o) {
if (err)
return console.log('error:', err);
console.log(o.length);
});
Im not suggesting this is the best way to handle this specific problem in Mongo, but if you need to wait to the DB to reply before continuing then just use the callback to start next request.
This is not obvious at first, but you can refer to the result processing function inside the function itself:
var i = 0;
var mycback = function(err, doc_album) {
// ... process i-th result ...
if (++i < 100000) {
db.collections("album").findOne({'images': i}, mycback);
} else {
// request is complete, "return" result
result_cback(null, res);
}
};
db.collections('album').findOne({'images': 0}, mycback);
This also means that your function itself will be async (i.e. will want a result_cback parameter to call with the result instead of using return).
Writing a sync function that calls an async one is just not possible.
You cannot "wait" for an event in Javascript... you must set up an handler for the result and then terminate.
Waiting for an event is done in event-based processing by writing a "nested event loop" and this is for example how message boxes are handled in most GUI frameworks. This is a capability that Javascript designers didn't want to give to programmers (not really sure why, though).
Since you know it does not wait for the call to come back. You can do the console.dir inside your callback function, this should work (although I haven't tested it)
db.collection('albums').findOne(query, function(err, doc_album) {
if(err) throw err;
if (doc_album === null) {
orphans.push(i);
}
console.dir(orphans.length);
});
You don't need to slow anything down. If you are simply trying to load 100,000 images from the albums collection, you could consider using the async framework. This will let you assign tasks until the job is complete.
Also, you probably don't want request 100,000 records one-by-one. Instead, you probably want to page them.
I'm trying to execute several async methods of redis with the following code
var redis = require("redis");
var client = redis.createClient();
var async = require("asyncjs");
async.list([
client.hincrby("traffic:" + siteId, 'x', 1),
client.hincrby("traffic:" + siteId, 'y', 1),
client.hincrby("traffic:" + siteId, 'z', 1)
]).call().end(function(err, result)
{
console.log(err); // returns error [TypeError: Object true has no method 'apply']
console.log(result); // undefined
if(err) return false;
return result;
});
All the methods execute successfully
but i get the error [TypeError: Object true has no method 'apply']
the method is executed and returns true, and its probably interpreting that as true, but i don't see why it has to use the method apply on it?
I can get the result of the increment by adding a function(err, result) to client.hincrby as last element.. but how do i get all the results in the result variable in the end function?
I suppose the asyncjs module you use is the one documented at:
https://github.com/fjakobs/async.js
In your code:
list() is a generator. It allows the array to be iterated by asyncjs. The array is an array of values.
call() is a mapper which calls each item. The items has therefore to be callable (i.e. they have to be callbacks).
end() is a termination end point, called when the iteration is over. As a parameter, you only get the last value of the sequence (not the whole sequence).
You got the "[TypeError: Object true has no method 'apply']" error because the list you have built is not a list of callbacks. It is a list of values.
Here is some code which should do what you want:
var redis = require("redis");
var client = redis.createClient();
var async = require("asyncjs");
function main() {
var siteId = 1;
async
.list(['x','y','z'])
.map( function (item,next) {
client.hincrby('traffic:' + siteId, item, 1, function (err,res) {
next(err,res)
})
})
.toArray( function(err,res) {
console.log(err);
console.log(res);
});
}
main()
Please note here we use map() instead of call(), and toArray() instead of end().