I'm trying to use a callback to get the result of a query from a database, but I am still getting undefined returned to me. Here is the code:
function onComplete(x){
return x;
}
function getRecords(callBack){
mongo.MongoClient.connect("mongodb://localhost:27017/nettuts", function(err, db){
if(err){
console.log(err);
return;
}
console.log('Connected to mongo');
db.collection('nettuts').find().toArray(function(err, records){
if(typeof callBack === "function"){
callBack(records);
}
});
});
}
app.get('/', function (req, res) {
var records = getRecords(onComplete);
console.log(records);
res.render("index", {title: "Hello!", people: records});
});
On the third to last line, I am getting an undefined.
As stated, you are following this the wrong way around for asynchronous programming, if you want "re-use" then you are passing in the callback to the function you are re-using and not the other way around:
var mongo = require("mongodb"),
MongoClient = mongo.MongoClient;
function getRecords(db,callback) {
if (typeof callback !== 'function')
throw "getRecords() requires a callback as the second arg";
db.collection("test").find().toArray(function(err,records) {
callback(err,records);
});
}
MongoClient.connect('mongodb://localhost/test',function(err,db) {
getRecords(db,function(err,result) {
if (err) throw err;
console.log(result);
});
});
Also noting here that any of your active code needs to happen "after" the connection is present, too broad to cover here on the best way to do that. But you certainly do not establish a connection with every request, as that would be very bad.
You "re-usable" function then just accepts the connection details and the actual functionality you want to happen when the operation is complete. That is the basic principle of callbacks.
Related
I'm just starting to work with Javascript and Node, and Async and callbacks concepts are not something I have under control right now.
I have to call a function for each element of a documents Array. This function will call to DB and get me an array of the document annotations. I want to get all the annotations and put them on the same array. Something similar to this:
//function in an async waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc, callback2) => {
getAnnotationsFromDocument(doc.Id, callback2);
}, function (err,annotations){
if (err){
callback(err);
}
annotationsArray = annotationsArray.concat(annotations);
callback(null, annotationsArray);
});
},
//Next waterfall function
About the getAnnotationsFromDocument function, this is a simplified structure of it:
function getAnnotationsFromDocument(docId,callback){
initDB();
var async = require('async');
async.waterfall([
function authorize(callback){
//checkAuthorization
(...)
},
function getRfpdocAnnotations(auth, metadata, callback){
//call to DB
(...)
},
function processRfpdocAnnotations(rfpDocAnnotations,metadata,callback){
(...)
callback(null,annotationsList);
}
], function (err, result) {
if(err) {
callback(err);
} else {
callback(null, result);
}
});
}
Unfortunately, I'm unable to code it properly. I'm unable to get the results from the function before exiting the async.each. Could somebody explain me how to structurate the code for this?
Debugging I've found that the function getAnnotationsFromDocument gets the data and execute the last callback(null, result); properly, but when I get to function (err,annotations){, annotations is undefined.
Ok, I think I got it:
First problem was that async.each doesn't return the results on the callback like I was expecting. Unlike waterfall, it just returns the errors. I should have payed more attention reading the documentation.
Secondly, I had to create a callback on the getAnnotationsFromDocument call to process the results.
And finally, I was not executing the call to the callback of async.each, so the execution didn't get to the async.each callback and didn't continue to the next async.waterfall function.
To be quite honest, I'm not sure it's a correct answer, but it does what I was trying to achieve.
// function part of an async.waterfall
function(docs,callback){
let annotationsArray = [];
async.each(docs, (doc,callback2) => {
getAnnotationsFromDocument(doc._id, function(err,result){
if (err){
callback2(err);
}else{
annotationsArray = annotationsArray.concat(result);
}
callback2();
})
}, (err) =>{
if( err ) {
callback(err);
} else {
callback(null,annotationsArray); //to the next waterfall function
}
});
I'm having an issue in Express. The following piece of code finds all the value in the pokemon collection and one by one, checks another collection to find matches. However, the code reaches res.send(documents) before all the items are finished inserting (display.insert(docs)). I know this is because of the way node works asynchronously, but I can't find a way to get past this issue. How can I ensure all of the documents get inserted?
pokeRouter.get('/sightings/:type([a-z]+)', function(req, res) {
display.deleteMany({}, function(err, bool) {
if (err) throw err;
if (bool) {
pokemon.find().each(function(err, item) {
if (err) throw err;
if (item == null) {
display.find().toArray(function(err, documents) {
if (err) throw err;
res.send(documents);
})
} else if ((req.params.type == item.type1) || (req.params.type == item.type2)) {
sightings.find({
pokedex_id: item._id
}).toArray(function(err, docs) {
if (docs == null) {
return null;
} else {
display.insert(docs);
}
});
}
});
}
});
});
Your display.insert(...) function is probably also async. So the function inside the find().each(...) is returning before the insert is finished.
I highly recommend converting your the callbacks to Promises or use the async module to handle your async stuff.
without using Promises or async module, you can refactor your code to have documents variable outside the async code and accumulate all the insertable items, and in your async code you would have some type of check (if everything is inserted) then I would call res.send(documents)
app.get('/ratings', function (req, res){
db.database.find({rating:5}).count(function(err, doc) {
review.FiveStarCount=doc;
});
db.database.find({rating:4}).count(function(err, doc) {
review.FourStarCount=doc;
});
I am new to using the MEAN stack and trying to return the values of the number of 5 and 4 star reviews. I can retrieve the number of 5 stars just fine, but when I do multiple , I seem to be running into trouble and not sure how to set it up.
You kind of need to get the gist of how to handle control flow while doing async operations.
The function execution would not stop, unlike php, when you are running an async operation.
To accomplish what you are trying to do, the second db call needs to be inside the callback of the first. Try this out.
app.get('/ratings', function (req, res, next) {
db.database.find({rating:5}).count(function(err, doc) {
// always handle error first
if (err) {
return next(err);
}
if (doc) {
review.FiveStarCount = doc;
}
db.database.find({rating:4}).count(function(err, doc) {
if (err) {
return next(err);
}
if (doc) {
review.FourStarCount = doc;
}
console.log(review.FiveStarCount, review.FourStarCount);
});
});
});
So it seems like you are running against the async nature of JavaScript. in the case of the functions above, they will not execute in the order you have written (and you wouldn't want them too!) because they are async. There are few simple solutions to get you going while you learn more about the problem.
Solution 1: Nested callbacks. Hard to read. also, it waits for the first to complete before firing the second, possibly slowing everything down.
app.get('/ratings', function(req, res) {
var review = {};
db.database.find({rating:5}).count(function(err, doc) {
review.FiveStarCount = doc;
db.database.find({rating:4}).count(function(err, doc) {
review.FourStarCount = doc;
// DO something with data
res.json(review);
});
});
Solution 2: Latch. This has the advantage of making the requests in parallel and not being as awful to read.
app.get('/ratings', function(req, res) {
var review = {};
db.database.find({rating:5}).count(function(err, doc) {
review.FiveStarCount=doc;
if (review.FiveStarCount && review.FourStarCount) {
// DO something here
// Will only execute if the other has finished
}
});
db.database.find({rating:4}).count(function(err, doc) {
review.FourStarCount=doc;
if (review.FiveStarCount && review.FourStarCount) {
// DO something here
// Will only execute if the other has finished
}
});
});
There are course more solutions, such as maybe a better query or promises. Maybe read a bit more about async JavaScript here to get a better understanding.
I'm trying out the framework node.js on one of my projects.
I'm really seeing some good advantages on what they called "event-driven, non-blocking I/O model" however if my project there are some moments where I don't necessarily want to have some asynchronous calls and to be able to several operation before launching some asynchronous call.
Especially when I want to do some factorization and create some functions.
Typically I have the following case:
I know that in several part of my program I have to check if a media is existing in my database for a given string or id.
So as a guy who tried to stay organize I want to create a function that I will call each time I need to check this.
However, I did not find the way to do that with node.js and pg (the npm PostgreSQL library (https://github.com/brianc/node-postgres/) . Indeed, there is always a callback in the function and the return is null because of the callback. Here is an example below
/*
Function which is supposed to check if a media existing
*/
function is_media_existing (url_or_id){
log.debug("is_media_existing : begin of the function", {"Parameter" : url_or_id});
pg.connect(connectionstring, function (err, client, done) {
if (err) {
log.warning("is_media_existing : Problem with Database connection", {
"Parameter": url_or_id,
"Error": err
});
}
if (isNaN(url_or_id)) {
// Case is parameter is not a number (string)
var query = client.query('SELECT COUNT(*) as count FROM media WHERE url = $1::string ', url_or_id);
query.on('error', function (error) {
log.warning("is_media_existing : Problem with Database query (connection to db passed but not query " +
"", {"Parameter": url_or_id, "Error": error});
});
return query;
} else {
// Case is parameter is a int
log.debug("is_media_existing : Type of Parameter is a string");
// Case is parameter is not a number (string)
var query = client.query('SELECT COUNT(*) as count FROM media WHERE id = $1::id ', url_or_id);
query.on('error', function (error) {
log.warning("is_media_existing : Problem with Database query (connection to db passed but not query " +
"", {"Parameter": url_or_id, "Error": error});
});
return query;
}
});
}
// Executing the function
var test = is_media_existing("http://random_url_existing_in_db");
// test is always null as the return is in a callback and the callback is asynchronous
i have the feeling my question is touching the core concepts of node.js, and perhaps my approach is wrong and I apologize in advance.
I know it's not good to wait for a response before doing something.
But what's the alternative? How can I factorize my code into functions when I need some functionalities in several part of my code?
So if there would be anyone who could explain how to do that with a best practice of programming it would be great.
Thanks
Anselme
As Cody says, you probably dont want to do synchronous function.
The way you should handle the situation in your example is to pass in your own callback like this
function is_media_existing (url_or_id, callback){
and then instead of return query; use your callback like this-
callback(query);
or probably better to follow the node convention for callback functions to have two parameters (err, result) so your callback would look like this
callback(null, query);
Here is a rework of your sample
function is_media_existing (url_or_id, callback){ /* callback(err, result) */
log.debug("is_media_existing : begin of the function", {"Parameter" : url_or_id});
pg.connect(connectionstring, function (err, client, done) {
if (err) {
done(err);
log.warning("is_media_existing : Problem with Database connection", {
"Parameter": url_or_id,
"Error": err
});
return callback(err, null);
/* note that this return is simply used to exit the connect's callback and the return value is typically
* not used it is the call to callback() that returns the error value */
}
var qrystr;
if (isNaN(url_or_id)) {
log.debug("is_media_existing : Type of Parameter is a string");
qrystr = 'SELECT COUNT(*) as count FROM media WHERE url = $1::string;';
} else {
qrystr = 'SELECT COUNT(*) as count FROM media WHERE id = $1::id;';
}
client.query(qrystr, [url_or_id], function(err, result){
done();
if(err){
/* .. */
}
callback(err, result);
});
});
}
// Executing the function
var test = is_media_existing("http://random_url_existing_in_db", function(err, result){
if(err){
}else {
}
});
If you end up with a hard nest of callbacks, promises are really worth looking into.
I don't think you really do want a synchronous call. The problem with synchronous calls in node is that it stops the entire process from doing anything while a synchronous function is running as it will stop the event loop. As an example lets say your sync function takes 2 seconds to complete. Your server will then do nothing for 2 full seconds. That 2 seconds includes everything (accepting new connections, everything else, etc). The reason blocking functions don't exist is because they are (very) bad. Here is an example how your function will react in an async manor.
is_media_existing("http://random_url_existing_in_db", function(exists){
if (exists){
//do stuff
} else {
//do this other stuff
}
});
Then within is_media_existing you will need to call that callback function when your query completes.
//psuedo
function is_media_existing(url, callback){
query('select COUNT(*) as count FROM media WHERE id = $1::id '. [url], function(err, result){
if (err)
callback(false)
else
callback(result.count > 0)
})
}
With the new ES6 plus async stuff and babel its simpler. You can npm i -g babel npm i babel-runtime then compile and run the following with babel test.js --optional runtime --stage 2 | node. Please read the following example carefully to see how to adapt it to your use case:
let testData = [
{ id: 0, childIds: [1,2]},
{ id: 1, childIds:[] }
];
function dbGet(ids) {
return new Promise( r=> {
// this an example; you could do any db
// query here and call r with the results
r(ids.map((id) => { return testData[id];}));
});
}
async function dbExists(ids) {
let found = await dbGet(ids);
return (found && found.length>0);
}
async function test() {
var exists = await dbExists([0]);
console.log(exists);
}
test().then(f=>{}).catch( e=> {console.log('e',e)});
So right now I'm trying to use Nodejs to access files in order to write them to a server and process them.
I've split it into the following steps:
Traverse directories to generate an array of all of the file paths
Put the raw text data from each of file paths in another array
Process the raw data
The first two steps are working fine, using these functions:
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function(file) {
file = path.resolve(dir, file);
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
results.push(file);
if (!--pending) done(null, results);
}
});
});
});
};
function processfilepaths(callback) {
// reading each file
for (var k in filepaths) { if (arrayHasOwnIndex(filepaths, k)) {
fs.readFile(filepaths[k], function (err, data) {
if (err) throw err;
rawdata[k] = data.toString().split(/ *[\t\r\n\v\f]+/g);
for (var j in rawdata[k]) { if (arrayHasOwnIndex(rawdata[k], j)) {
rawdata[k][j] = rawdata[k][j].split(/: *|: +/);
}}
});
}}
if (callback) callback();
}
Obviously, I want to call the function processrawdata() after all of the data has been loaded. However, using callbacks doesn't seem to work.
walk(rootdirectory, function(err, results) {
if (err) throw err;
filepaths = results.slice();
processfilepaths(processrawdata);
});
This never causes an error. Everything seems to run perfectly except that processrawdata() is always finished before processfilepaths(). What am I doing wrong?
You are having a problem with callback invocation and asynchronously calling functions. IMO I'll recommend that you use a library such as after-all to execute a callback once all your functions get executed.
Here's a example, here the function done will be called once all the functions wrapped with next are called.
var afterAll = require('after-all');
// Call `done` once all the functions
// wrapped with next() get called
next = afterAll(done);
// first execute this
setTimeout(next(function() {
console.log('Step two.');
}), 500);
// then this
setTimeout(next(function() {
console.log('Step one.');
}), 100);
function done() {
console.log("Yay we're done!");
}
I think for your problem, you can use async module for Node.js:
async.series([
function(){ ... },
function(){ ... }
]);
To answer you actual question, I need to explain how Node.js works:
Say, when you call an async operation (say mysql db query), Node.js sends "execute this query" to MySQL. Since this query will take some time (may be some milliseconds), Node.js performs the query using the MySQL async library - getting back to the event loop and doing something else there while waiting for MySQL to get back to us. Like handling that HTTP request.
So, In your case both functions are independent and executes almost in parallel.
For more information:
Async.js for use with Node.js
function processfilepaths(callback) {
// reading each file
for (var k in filepaths) { if (arrayHasOwnIndex(filepaths, k)) {
fs.readFile(filepaths[k], function (err, data) {
if (err) throw err;
rawdata[k] = data.toString().split(/ *[\t\r\n\v\f]+/g);
for (var j in rawdata[k]) { if (arrayHasOwnIndex(rawdata[k], j)) {
rawdata[k][j] = rawdata[k][j].split(/: *|: +/);
}}
});
}}
if (callback) callback();
}
Realize that you have:
for
readfile (err, callback) {... }
if ...
Node will call each readfile asynchronously, which only sets up the event and callback, then when it is done calling each readfile, it will do the if, before the callback probably even has a chance to get invoked.
You need to use either Promises, or a promise module like async to serialize it. What you would then do looks like:
async.XXXX(filepaths, processRawData,
function (err, ...) {
// function for when all are done
if (callback) callback();
}
);
Where XXXX is one of the functions from the library like series, parallel, each, etc... The only thing you also need to know is in your process raw data, async gives you a callback to call when done. Unless you really need sequential access (I don't think you do) use parallel so that you can queue up as many i/o events as possible, it should execute faster, maybe only marginally, but it'll better leverage the hardware.