I am able to insert and retrieve data from an neDB database in nodejs. But I cannot pass the data outside of the function that retrieves it.
I have read through the neDB documentation and I searched for and tried different combinations of callbacks and returns (see code below) without finding a solution.
I'm new to javascript so I do not know if I am misunderstanding how to use variables in general or if this issue is related to using neDB specifically or both.
Could someone please explain why "x" in my code does not ever contain the docs JSON results from the database? How can I make it work?
var fs = require('fs'),
Datastore = require('nedb')
, db = new Datastore({ filename: 'datastore', autoload: true });
//generate data to add to datafile
var document = { Shift: "Late"
, StartTime: "4:00PM"
, EndTime: "12:00AM"
};
// add the generated data to datafile
db.insert(document, function (err, newDoc) {
});
//test to ensure that this search returns data
db.find({ }, function (err, docs) {
console.log(JSON.stringify(docs)); // logs all of the data in docs
});
//attempt to get a variable "x" that has all
//of the data from the datafile
var x = function(err, callback){
db.find({ }, function (err, docs) {
callback(docs);
});
};
console.log(x); //logs "[Function]"
var x = db.find({ }, function (err, docs) {
return docs;
});
console.log(x); //logs "undefined"
var x = db.find({ }, function (err, docs) {
});
console.log(x); //logs "undefined"*
Callbacks are generally asynchronous in JavaScript meaning you can not use assignment operator, consequently you do not return anything from the callback function.
When you call an async function execution of you programme carries on, passing the 'var x = whatever' statement. The assignment to a variable, the result of whatever callback is received, you need to perform from within the callback itself... what you need is something in the lines of ...
var x = null;
db.find({ }, function (err, docs) {
x = docs;
do_something_when_you_get_your_result();
});
function do_something_when_you_get_your_result() {
console.log(x); // x have docs now
}
EDIT
Here is a nice blog post about asynchronous programming. And there is a lot more resources on this topic that you can pick up.
This is a popular library to help with async flow-control for node.
P.S.
Hope this helps. Please, by all means ask if you need something clarified :)
I ran into the same problem. In the end I used a combination between async-await and a promise with resolve to solve it.
In your example the following would work:
var x = new Promise((resolve,reject) {
db.find({ }, function (err, docs) {
resolve(docs);
});
});
console.log(x);
I had to learn a bit about async functions to get it right. For those who are looking for specific help about getting a return value from nedb, here's a snippet of what worked for me. I was using it in electron.
function findUser(searchParams,callBackFn)
{
db.find({}, function (err, docs))
{
//executes the callback
callBackFn(docs)
};
}
usage
findUser('John Doe',/*this is the callback -> */function(users){
for(i = 0; i < users.length; i++){
//the data will be here now
//eg users.phone will display the user's phone number
}})
Related
Thank you in advance for anyone that reads this. I really appreciate any and all help.
so this is my first app personal app. I have setup a DB in Atlas on mongodb.com and I can write to it with out an issue. But when my app tries and pull from my db I can get it to print to console. But I am able to assign the data to any variable to use anywhere else in my my app.
here is my code that works to print to the console. But not sure what setup or package I am missing so i can store it as a local variable. I am using a callback function to return the api call and the console print out works. just dont know what to do next
function getTerms() {
let allTerms = []
termAdd.find({}, '_id', (err, term) => {
term.map((term) => {
allTerms.push(term)
// if I understand Push() correctly this should store my output to allTerms.
});
//this works to print out to console in JSON.
console.log(allTerms, 'getTerms')
return allTerms;
});
};
if (res.statusCode === 200) {
//callback function to return the console from the function
getTerms()
// when i do a let foo = getTerms() it will return undefined
//so I am really dont understand how to assign the return from
//the function to a variable.
//How do I assign this console output to variable to use for output
console.log('200 statusCode')
};
//Random Number Generator
let ranNum = Math.floor((Math.random() * 10) + 1);
console.log('random number = ',ranNum);
res.render('fs', {
flashCard : 'Test Card',
items: ranNum
});
});
I have the full code on GitHub if that would help I can link to this project.
Should I be using a delay in anyway. I was given a suggestion. But I did not understand what I was reading.
Before anything, please abstain of pushing your .env file to Github or sharing anything in it publicly.
You separated the functionality of getting terms into its own function, which is good:
// Get function for all items in mongodb
function allItems(all) {
let allTerms = []
termAdd.find({}, 'term', (err, term) => {
term.map((term) => {
allTerms.push(term)
});
//this works to print out to console.
console.log(allTerms, 'function allTerms')
//Question is how to I get this JSON to save to a VAR or be passed to another function
});
};
First of all I would rename it to getAllTerms, since it's what it does. And it seems the all parameter is not necessary.
**
Anyway, usually, what you would do is simply return the allTerms variable as such:
// Get function for all items in mongodb
function getAllTerms() {
let allTerms = []
termAdd.find({}, 'term', (err, term) => {
term.map((term) => {
allTerms.push(term)
});
//this works to print out to console.
console.log(allTerms, 'function allTerms')
//Question is how to I get this JSON to save to a VAR or be passed to another function
});
return allTerms
}
However, this wouldn't work because since you are making a call to a database, it might take some time for the database to get the terms; in this case, allTerms might return an empty array, [].
What you have to do is wait for the database to return the terms, push them into the allTerms array, and finally return it.
// Get function for all items in mongodb
async function getAllTerms() {
let allTerms = []
const fetchedTerms = await termAdd.find({}, 'term')
fetchedTerms.forEach(fetchedTerm => allTerms.push(fetchedTerm))
return allTerms
}
If you don't know what async and await are, no worries, here is a good article explaining the why and when to use them.
If you still have any questions, let me know.
Using Async/Await the code above can be optimized as the code below:
`
async function getTerms() {
//using async/await
try{
const terms = await termAdd.find({}, '_id')
const allTerms = terms.map((term)=>term)
// return allTerms here
return allTerms
}catch(err){
throw err
}
}
`
for more insight on Async/Await here is a good read
I am creating a node API using javascript. I have used redis as my key value store.
I created a redis-client in my app and am able to get values for perticular key.
I want to retrieve all keys along with their values.
So Far I have done this :
app.get('/jobs', function (req, res) {
var jobs = [];
client.keys('*', function (err, keys) {
if (err) return console.log(err);
if(keys){
for(var i=0;i<keys.length;i++){
client.get(keys[i], function (error, value) {
if (err) return console.log(err);
var job = {};
job['jobId']=keys[i];
job['data']=value;
jobs.push(job);
});
}
console.log(jobs);
res.json({data:jobs});
}
});
});
but I always get blank array in response.
is there any way to do this in javascript?
Thanks
First of all, the issue in your question is that, inside the for loop, client.get is invoked with an asynchronous callback where the synchronous for loop will not wait for the asynchronous callback and hence the next line res.json({data:jobs}); is getting called immediately after the for loop before the asynchronous callbacks. At the time of the line res.json({data:jobs}); is getting invoked, the array jobs is still empty [] and getting returned with the response.
To mitigate this, you should use any promise modules like async, bluebird, ES6 Promise etc.
Modified code using async module,
app.get('/jobs', function (req, res) {
var jobs = [];
client.keys('*', function (err, keys) {
if (err) return console.log(err);
if(keys){
async.map(keys, function(key, cb) {
client.get(key, function (error, value) {
if (error) return cb(error);
var job = {};
job['jobId']=key;
job['data']=value;
cb(null, job);
});
}, function (error, results) {
if (error) return console.log(error);
console.log(results);
res.json({data:results});
});
}
});
});
But from the Redis documentation, it is observed that usage of
Keys are intended for debugging and special operations, such as
changing your keyspace layout and not advisable to production
environments.
Hence, I would suggest using another module called redisscan as below which uses SCAN instead of KEYS as suggested in the Redis documentation.
Something like,
var redisScan = require('redisscan');
var redis = require('redis').createClient();
redisScan({
redis: redis,
each_callback: function (type, key, subkey, value, cb) {
console.log(type, key, subkey, value);
cb();
},
done_callback: function (err) {
console.log("-=-=-=-=-=--=-=-=-");
redis.quit();
}
});
Combination of 2 requests:
import * as ioredis from 'ioredis';
const redis = new ioredis({
port: redisPort,
host: redisServer,
password: '',
db: 0
});
const keys = await redis.collection.keys('*');
const values = await redis.collection.mget(keys);
Order will be the same for both arrays.
This will get all keys but with no values:
const redis = require('redis');
const client = redis.createClient();
client.keys('*', (err, keys) => {
// ...
});
Now you need to get the values for those keys in a usual way. For example:
Promise.all(keys.map(key => client.getAsync(key))).then(values => {
// ...
});
or with async module or in any way you like.
You should never do this. First off, it is not recommended to use KEYS * in production. Second, this does not scale (cluster).
You can organise your cached entries into SETs and query for the items within the SET, then retrieve the references keys. This also makes invalidation easier.
Have a look at some data storage best practices.
https://redis.io/topics/data-types-intro
how to get all keys and values in redis in javascript?
You may find something useful in this link
https://github.com/NodeRedis/node_redis/tree/master/examples
I'm trying out the framework node.js on one of my projects.
I'm really seeing some good advantages on what they called "event-driven, non-blocking I/O model" however if my project there are some moments where I don't necessarily want to have some asynchronous calls and to be able to several operation before launching some asynchronous call.
Especially when I want to do some factorization and create some functions.
Typically I have the following case:
I know that in several part of my program I have to check if a media is existing in my database for a given string or id.
So as a guy who tried to stay organize I want to create a function that I will call each time I need to check this.
However, I did not find the way to do that with node.js and pg (the npm PostgreSQL library (https://github.com/brianc/node-postgres/) . Indeed, there is always a callback in the function and the return is null because of the callback. Here is an example below
/*
Function which is supposed to check if a media existing
*/
function is_media_existing (url_or_id){
log.debug("is_media_existing : begin of the function", {"Parameter" : url_or_id});
pg.connect(connectionstring, function (err, client, done) {
if (err) {
log.warning("is_media_existing : Problem with Database connection", {
"Parameter": url_or_id,
"Error": err
});
}
if (isNaN(url_or_id)) {
// Case is parameter is not a number (string)
var query = client.query('SELECT COUNT(*) as count FROM media WHERE url = $1::string ', url_or_id);
query.on('error', function (error) {
log.warning("is_media_existing : Problem with Database query (connection to db passed but not query " +
"", {"Parameter": url_or_id, "Error": error});
});
return query;
} else {
// Case is parameter is a int
log.debug("is_media_existing : Type of Parameter is a string");
// Case is parameter is not a number (string)
var query = client.query('SELECT COUNT(*) as count FROM media WHERE id = $1::id ', url_or_id);
query.on('error', function (error) {
log.warning("is_media_existing : Problem with Database query (connection to db passed but not query " +
"", {"Parameter": url_or_id, "Error": error});
});
return query;
}
});
}
// Executing the function
var test = is_media_existing("http://random_url_existing_in_db");
// test is always null as the return is in a callback and the callback is asynchronous
i have the feeling my question is touching the core concepts of node.js, and perhaps my approach is wrong and I apologize in advance.
I know it's not good to wait for a response before doing something.
But what's the alternative? How can I factorize my code into functions when I need some functionalities in several part of my code?
So if there would be anyone who could explain how to do that with a best practice of programming it would be great.
Thanks
Anselme
As Cody says, you probably dont want to do synchronous function.
The way you should handle the situation in your example is to pass in your own callback like this
function is_media_existing (url_or_id, callback){
and then instead of return query; use your callback like this-
callback(query);
or probably better to follow the node convention for callback functions to have two parameters (err, result) so your callback would look like this
callback(null, query);
Here is a rework of your sample
function is_media_existing (url_or_id, callback){ /* callback(err, result) */
log.debug("is_media_existing : begin of the function", {"Parameter" : url_or_id});
pg.connect(connectionstring, function (err, client, done) {
if (err) {
done(err);
log.warning("is_media_existing : Problem with Database connection", {
"Parameter": url_or_id,
"Error": err
});
return callback(err, null);
/* note that this return is simply used to exit the connect's callback and the return value is typically
* not used it is the call to callback() that returns the error value */
}
var qrystr;
if (isNaN(url_or_id)) {
log.debug("is_media_existing : Type of Parameter is a string");
qrystr = 'SELECT COUNT(*) as count FROM media WHERE url = $1::string;';
} else {
qrystr = 'SELECT COUNT(*) as count FROM media WHERE id = $1::id;';
}
client.query(qrystr, [url_or_id], function(err, result){
done();
if(err){
/* .. */
}
callback(err, result);
});
});
}
// Executing the function
var test = is_media_existing("http://random_url_existing_in_db", function(err, result){
if(err){
}else {
}
});
If you end up with a hard nest of callbacks, promises are really worth looking into.
I don't think you really do want a synchronous call. The problem with synchronous calls in node is that it stops the entire process from doing anything while a synchronous function is running as it will stop the event loop. As an example lets say your sync function takes 2 seconds to complete. Your server will then do nothing for 2 full seconds. That 2 seconds includes everything (accepting new connections, everything else, etc). The reason blocking functions don't exist is because they are (very) bad. Here is an example how your function will react in an async manor.
is_media_existing("http://random_url_existing_in_db", function(exists){
if (exists){
//do stuff
} else {
//do this other stuff
}
});
Then within is_media_existing you will need to call that callback function when your query completes.
//psuedo
function is_media_existing(url, callback){
query('select COUNT(*) as count FROM media WHERE id = $1::id '. [url], function(err, result){
if (err)
callback(false)
else
callback(result.count > 0)
})
}
With the new ES6 plus async stuff and babel its simpler. You can npm i -g babel npm i babel-runtime then compile and run the following with babel test.js --optional runtime --stage 2 | node. Please read the following example carefully to see how to adapt it to your use case:
let testData = [
{ id: 0, childIds: [1,2]},
{ id: 1, childIds:[] }
];
function dbGet(ids) {
return new Promise( r=> {
// this an example; you could do any db
// query here and call r with the results
r(ids.map((id) => { return testData[id];}));
});
}
async function dbExists(ids) {
let found = await dbGet(ids);
return (found && found.length>0);
}
async function test() {
var exists = await dbExists([0]);
console.log(exists);
}
test().then(f=>{}).catch( e=> {console.log('e',e)});
i am taking my first steps with node.js and i came across this issue with passing variable in asynchronous way. I have this piece of code im using to create Facebook user:
req.tmpPassport = {};
var fb = new fbgraph.Facebook(accessToken, 'v2.2');
function initUser() {
fb.me(function (err, me) {
req.tmpPassport.me = me;
console.log(req.tmpPassport.me) // works
});
}
console.log(req.tmpPassport.me) // not working -> undefined var
i tried to figure out why the second log isn't working and i ended up reading about synchronous and asynchronous functions, so in attempt to implement what i read i tried coming up with a solution using callbacks, but no success.
my last attempt was this:
req.tmpPassport = {};
var fb = new fbgraph.Facebook(accessToken, 'v2.2');
function initUser() {
fb.me(function (err, me) {
req.tmpPassport.me = me;
});
fb.my.events(function (err, events) {
//console.log(events);
req.tmpPassport.events = events;
});
fb.my.friends(function (err, result) {
req.tmpPassport.results = result;
});
}
function passUser(){
console.log(req.tmpPassport);
return req.tmpPassport;
}
cp.exec(initUser, passUser);
but its not working...
what i am actually trying to achieve its to render this object with my express router var which looks like this:
router.get('/welcome', securePages, function(req, res, next){
res.render('welcome', {title:'Welcome to aDating', user:req.tmpPassport});
})
but i cant figure out how to pass this object only after created...any help please?
A method of chaining function calls when certain async tasks are done is one way to deal with this.
Looking at the first snippet of code, it would be rewritten as follows:
req.tmpPassport = {};
var fb = new fbgraph.Facebook(accessToken, 'v2.2');
function initUser() {
fb.me(function (err, me) {
console.log(req.tmpPassport.me) // works
req.tmpPassport.me = me;
// triggers execution of the next step
post_populating_passport();
});
}
function post_populating_passport() {
// this function is only executed after the callback from the async call
console.log(req.tmpPassport.me);
}
Node.js is an asynchronous programming language, a fundamental concept in its core. You either need to use function callbacks (not a good practice) or available npms utilities for async flow.
I am using nodejs to query data from Mongodb throught Mongoose.
After get the data, I want do something on that data before responding it to client. But I can not get the return-value. After looking on Google, I have learned Node.js functions is asynchronous javascript function (non I/O blocking). I try this tut (http://www.youtube.com/watch?v=xDW9bK-9pNY) but it is not work. Below is my code. The myObject is valued inside "find()" function and undefined outside "find()" function. So what should I do to get the data? Thanks!
var Person = mongoose.model('Person', PersonSchema);
var Product = mongoose.model('Product', ProductSchema);
var myObject = new Object();
Person.find().exec(function (err, docs) {
for (var i=0;i<docs.length;i++)
{
Product.find({ user: docs[i]._id},function (err, pers) {
myObject[i] = pers;
console.log(myObject[i]); //return the value is ok
});
console.log(myObject[i]); //return undefined value
}
console.log(myObject); //return undefined value
});
console.log(myObject); //return undefined value
app.listen(3000);
console.log('Listening on port 3000');
The reason you're getting undefined values is because the find function is asynchronous, and can finish at any time. In your case, it is finishing after you're using console.log(), so the values are undefined when you're accessing them.
To fix this problem, you can only use the values inside the find function's callback. It would look something like this:
var Person = mongoose.model('Person', PersonSchema);
var Product = mongoose.model('Product', ProductSchema);
var myObject = new Object();
function getData(docs, callback) {
function loop(i) {
Product.find({ user: docs[i]._id}, function (err, pers) {
myObject[i] = pers;
if (i < docs.length) {
loop(i + 1);
} else {
callback();
}
});
};
loop(0);
};
Person.find().exec(function(err, docs) {
getData(docs, function() {
// myObject has been populated at this point
});
});
The data processing has been moved to a loop that waits for the previous iteration to complete. This way, we can determine when the last callback has fired in order to fire the callback in the wrapper function.
Keep in mind that by the time the console.log functions are executed, the query has not yet finished, thus will display "undefined". That's the essence of nodeJS's asynchronicity.
For example,
Person.find().exec(function (err, docs) {
for (var i=0;i<docs.length;i++)
{
Product.find({ user: docs[i]._id},function (err, pers) {
myObject[i] = pers;
console.log(myObject[i]); //return the value is ok
});
console.log(myObject[i]); //return undefined value
}
console.log(myObject); //return undefined value
});
console.log(myObject); // <-- Initially, this value will be undefined. After some miliseconds (Or the duration of the .exec function, myObject will contain the results.
If you want to actually wait until the query is finished so you can use the values, I would recommend moving the app.listen(3000); and console.log('Listening on port 3000'); into the function's final callback.
I'd also recommend you to check out this node module. It will help you build asynchronous / synchronous functions more easily, and allow you to execute a callback when all the asynchronous functions are finished.