Working with Nodejs and MongoDB through Node MongoDB native driver. Need to retrieve some documents, and make modification, then save them right back. This is an example:
db.open(function (err, db) {
db.collection('foo', function (err, collection) {
var cursor = collection.find({});
cursor.each(function (err, doc) {
if (doc != null) {
doc.newkey = 'foo'; // Make some changes
db.save(doc); // Update the document
} else {
db.close(); // Closing the connection
}
});
});
});
With asynchronous nature, if the process of updating the document takes longer, then when cursor reaches the end of documents, database connection is closed. Not all updates are saved to the database.
If the db.close() is omitted, all the documents are correctly updated, but the application hangs, never exits.
I saw a post suggesting using a counter to track number of updates, when fall back to zero, then close the db. But am I doing anything wrong here? What is the best way to handle this kind of situation? Does db.close() have to be used to free up resource? Or does a new db connection needs to open?
Here's a potential solution based on the counting approach (I haven't tested it and there's no error trapping, but it should convey the idea).
The basic strategy is: Acquire the count of how many records need to be updated, save each record asynchronously and a callback on success, which will decrement the count and close the DB if the count reaches 0 (when the last update finishes). By using {safe:true} we can ensure that each update is successful.
The mongo server will use one thread per connection, so it's good to either a) close unused connections, or b) pool/reuse them.
db.open(function (err, db) {
db.collection('foo', function (err, collection) {
var cursor = collection.find({});
cursor.count(function(err,count)){
var savesPending = count;
if(count == 0){
db.close();
return;
}
var saveFinished = function(){
savesPending--;
if(savesPending == 0){
db.close();
}
}
cursor.each(function (err, doc) {
if (doc != null) {
doc.newkey = 'foo'; // Make some changes
db.save(doc, {safe:true}, saveFinished);
}
});
})
});
});
It's best to use a pooled connection and then call db.close() in cleanup function at the end of your application's life:
process.on('SIGINT', cleanup);
process.on('SIGTERM', cleanup);
See http://mongodb.github.io/node-mongodb-native/driver-articles/mongoclient.html
A bit old thread, but anyway.
Here an extended example to the answer given by pkopac, since I had to figure out the rest of the details:
const client = new MongoClient(uri);
(async () => await client.connect())();
// use client to work with db
const find = async (dbName, collectionName) => {
try {
const collection = client.db(dbName).collection(collectionName);
const result = await collection.find().toArray()
return result;
} catch (err) {
console.error(err);
}
}
const cleanup = (event) => { // SIGINT is sent for example when you Ctrl+C a running process from the command line.
client.close(); // Close MongodDB Connection when Process ends
process.exit(); // Exit with default success-code '0'.
}
process.on('SIGINT', cleanup);
process.on('SIGTERM', cleanup);
Here is a link to the difference between SIGINT and SIGTERM.
I had to add the process.exit(), otherwise my node web-server didn't exit cleanly when doing Ctrl + C on the running process in command line.
I found that using counter may apply to simple scenario, but may be hard in complicated situations. Here is a solution that I come up by closing the database connection when database connection is idle:
var dbQueryCounter = 0;
var maxDbIdleTime = 5000; //maximum db idle time
var closeIdleDb = function(connection){
var previousCounter = 0;
var checker = setInterval(function(){
if (previousCounter == dbQueryCounter && dbQueryCounter != 0) {
connection.close();
clearInterval(closeIdleDb);
} else {
previousCounter = dbQueryCounter;
}
}, maxDbIdleTime);
};
MongoClient.connect("mongodb://127.0.0.1:27017/testdb", function(err, connection)(
if (err) throw err;
connection.collection("mycollection").find({'a':{'$gt':1}}).toArray(function(err, docs) {
dbQueryCounter ++;
});
//do any db query, and increase the dbQueryCounter
closeIdleDb(connection);
));
This can be a general solution for any database Connections. maxDbIdleTime can be set as the same value as db query timeout or longer.
This is not very elegant, but I can't think of a better way to do this. I use NodeJs to run a script that queries MongoDb and Mysql, and the script hangs there forever if the database connections are not closed properly.
Here's a solution I came up with. It avoids using toArray and it's pretty short and sweet:
var MongoClient = require('mongodb').MongoClient;
MongoClient.connect("mongodb://localhost:27017/mydb", function(err, db) {
let myCollection = db.collection('myCollection');
let query = {}; // fill in your query here
let i = 0;
myCollection.count(query, (err, count) => {
myCollection.find(query).forEach((doc) => {
// do stuff here
if (++i == count) db.close();
});
});
});
I came up with a solution that involves a counter like this. It does not depend on a count() call nor does it wait for a time out. It will close the db after all the documents in each() are exhausted.
var mydb = {}; // initialize the helper object.
mydb.cnt = {}; // init counter to permit multiple db objects.
mydb.open = function(db) // call open to inc the counter.
{
if( !mydb.cnt[db.tag] ) mydb.cnt[db.tag] = 1;
else mydb.cnt[db.tag]++;
};
mydb.close = function(db) // close the db when the cnt reaches 0.
{
mydb.cnt[db.tag]--;
if ( mydb.cnt[db.tag] <= 0 ) {
delete mydb.cnt[db.tag];
return db.close();
}
return null;
};
So that each time you are going to make a call like db.each() or db.save() you would use these methods to ensure the db is ready while working and closed when done.
Example from OP:
foo = db.collection('foo');
mydb.open(db); // *** Add here to init the counter.**
foo.find({},function(err,cursor)
{
if( err ) throw err;
cursor.each(function (err, doc)
{
if( err ) throw err;
if (doc != null) {
doc.newkey = 'foo';
mydb.open(db); // *** Add here to prevent from closing prematurely **
foo.save(doc, function(err,count) {
if( err ) throw err;
mydb.close(db); // *** Add here to close when done. **
});
} else {
mydb.close(db); // *** Close like this instead. **
}
});
});
Now, this assumes that the second to last callback from each makes it through the mydb.open() before the last callback from each goes to mydb.close().... so, of course, let me know if this is an issue.
So: put a mydb.open(db) before a db call and put a mydb.close(db) at the return point of the callback or after the db call (depending on the call type).
Seems to me that this kind of counter should be maintained within the db object but this is my current workaround. Maybe we could create a new object that takes a db in the constructor and wrap the mongodb functions to handle the close better.
Based on the suggestion from #mpobrien above, I've found the async module to be incredibly helpful in this regard. Here's an example pattern that I've come to adopt:
const assert = require('assert');
const async = require('async');
const MongoClient = require('mongodb').MongoClient;
var mongodb;
async.series(
[
// Establish Covalent Analytics MongoDB connection
(callback) => {
MongoClient.connect('mongodb://localhost:27017/test', (err, db) => {
assert.equal(err, null);
mongodb = db;
callback(null);
});
},
// Insert some documents
(callback) => {
mongodb.collection('sandbox').insertMany(
[{a : 1}, {a : 2}, {a : 3}],
(err) => {
assert.equal(err, null);
callback(null);
}
)
},
// Find some documents
(callback) => {
mongodb.collection('sandbox').find({}).toArray(function(err, docs) {
assert.equal(err, null);
console.dir(docs);
callback(null);
});
}
],
() => {
mongodb.close();
}
);
Modern way of doing this without counters, libraries or any custom code:
let MongoClient = require('mongodb').MongoClient;
let url = 'mongodb://yourMongoDBUrl';
let database = 'dbName';
let collection = 'collectionName';
MongoClient.connect(url, { useNewUrlParser: true }, (mongoError, mongoClient) => {
if (mongoError) throw mongoError;
// query as an async stream
let stream = mongoClient.db(database).collection(collection)
.find({}) // your query goes here
.stream({
transform: (readElement) => {
// here you can transform each element before processing it
return readElement;
}
});
// process each element of stream (async)
stream.on('data', (streamElement) => {
// here you process the data
console.log('single element processed', streamElement);
});
// called only when stream has no pending elements to process
stream.once('end', () => {
mongoClient.close().then(r => console.log('db successfully closed'));
});
});
Tested it on version 3.2.7 of mongodb driver but according to link might be valid since version 2.0
Related
In the following code I am generating a uuid and then making sure if it already doesn't exist but the problem is variable 'pid' is not working that is [pid] written after query is not working somehow and I don't why. now plz don't suggest y I am making sure if uuid already does not exist cuz they are unique all the time, just tell me the fact why js variabel var pid = uuid() is not being accepted by mySql connection.query
FIRST EDIT: The first connection.query is not even getting executed.
SECOND EDIT: After hours of debugging the problem still remains unsolved but now I know the cause which is not javascript variable but the result of the connection.query which is not readily available so the loop continues to run not waiting for the result. After reading articles I somehow know that promises may do my work but idk how to use them so see ya later until I learn them(also I am changing title of the question since error is not with js variable but with the returning of result of mysql connection.query function).
Title changed from:mySQL query is not selecting upon using Javascript variable nodejsTo: above title
//load user model and mysql
const mysql = require('mysql')
const configDB = require('../config/configDB')
const connection = mysql.createConnection(configDB.connection)
//load uuidv4 to generate user ids
const uuid = require('uuid')
module.exports = function(done){
var exist = true
do{
const pid = uuid.v4()
connection.query("SELECT * FROM PATIENT WHERE PATIENT_ID = ?",[pid], function(err, prows){
//query on the above line is not even getting executed and it is because of [pid] but don't know y
if(err)
return done(err)
if(prows.length){
exist = true
} else {
connection.query("SELECT * FROM DOCTOR WHERE DOCTOR_ID = ?",[pid], function(err, drows){
if(err)
return done(err)
if(drows.length){
exist = true
} else {
connection.query("SELECT * FROM ADMIN WHERE ADMIN_ID = ?",[pid], function(err, arows){
if(err)
return done(err)
if(arows.length){ //corrected from drows to arows after someone answered but the problem is something else and it is still unsolved
exist = true
} else {
exist = false
return pid
}
})
}
})
}
})
}while(exist)
}
In this line of your code
if(drows.length){
exist = true
}
Seems it should be
if(arows.length){ //it should be arows.length because arrows.length is never used
exist = true
}
Earlier my loop was keep generating new pids because all it was doing was generating a new pid then checking if that exist or not but while querying database it was not waiting for the result and since the result was not readily available so the loop was continuing to run till infinity(after debugging via vs code).
I used promise to wait for result of loop like this in generateUserId.js file I defined my function as follows
module.exports = generateUserId
async function generateUserId(){
let exist = true
do{
let pid = uuid.v4()
const id = await uuidExistOrNot(pid)
if(id===pid){
exist = false
return id
}
}while(exist)
}
function uuidExistOrNot(pid){
return new Promise((resolve, reject) => {
const selectPQuery = "SELECT * FROM PATIENT WHERE PATIENT_ID = ?"
const selectDQuery = "SELECT * FROM DOCTOR WHERE DOCTOR_ID = ?"
const selectAQuery = "SELECT * FROM HADMIN WHERE ADMIN_ID = ?"
connection.query(selectPQuery, [pid], (err, prows)=>{
if(!prows.length){
connection.query(selectDQuery, [pid], (err, drows)=>{
if(!drows.length){
connection.query(selectAQuery, [pid], (err, arows)=>{
if(!arows.length){
resolve(pid)
} else {
reject(err)
}
})
} else {
reject(err)
}
})
} else {
reject(err)
}
})
})
}
then when to use it I m importing in that file where I wanna use and then use it as:
generateUserId()
.then(function(id){
//do what u wanna do with id here that is being generated by generateUserId()
})
.catch(err=>console.log(err))
Basically, my code here is saying that if a user sends a message !submit ___ then the file leaderboard.json will up their count by one.
This all works perfectly however say for example their count goes from 0 to 1, the next time that same person sends !submit, their count should go from 1 to 2 without me having to restart the script every time. This isn't happening unfortunately... I send !submit and my count goes from 0 to 1, but then I send it again and it stays going from 0 to 1.
Leaderboard.json:
{
"usercount<#386679122614681600>": 0
}
index.js:
client.on('message', msg => {
if (msg.content.startsWith("!submit ")){
var shoe = msg.content.substr("!submit ".length);
var fs = require('fs')
fs.readFile('leaderboard.json', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
var user = msg.member;
var usercount = 'usercount'+user
var username = 'usercount'+user
var LEADERBOARD = require('./leaderboard.json');
var countvalue = LEADERBOARD[username]
var countvalue2 = countvalue+1
var replacetext = ('"'+usercount+'": '+countvalue).toString()
var newtext = ('"'+usercount+'": '+(countvalue2)).toString()
fs.writeFile('leaderboard.json', data.replace(replacetext, newtext),
'utf8', function () {
if (err) return console.log(err);
});
console.log('NEW SUBMISSION: '+replacetext+' >>>> '+newtext)
});
}
Here is what my console looks like after sending !submit twice:
When technically the second line should go from 1 to 2, without me having to close and restart the script.
I know this may seem a bit complicated but any help would be appreciated!
This is what I'd suggest:
const fs = require('fs')
client.on('message', msg => {
if (msg.content.startsWith("!submit ")) {
let shoe = msg.content.substr("!submit ".length);
// read leaderboard file and parse the JSON into a Javascript object
fs.readFile('leaderboard.json', 'utf8', function(err, data) {
if (err) {
console.log("Error reading leaderboard.json", err);
return;
}
let leaderboard;
try {
leaderboard = JSON.parse(data);
} catch(err) {
console.log("Error parsing leaderboard JSON", err);
return;
}
const user = msg.member;
const username = 'usercount' + user;
// make sure there's a count for this username
let cnt = leaderboard[username];
if (!cnt) {
cnt = 0;
}
// increment the cnt
++cnt;
// set the new count
leaderboard[username] = cnt;
// now write the data back to the file
fs.writeFile('leaderboard.json', JSON.stringify(leaderboard), 'utf8', function() {
if (err) {
console.log(err);
return;
}
console.log(`New Submission for ${username}, cnt = ${cnt}`);
});
});
}
});
Summary of changes:
Reads leaderboard.json only once using fs.readFile()
After reading the data, it converts it to JSON using JSON.parse().
Initializes user cnt if not already in the file
Updates cnt in the Javsacript object directly
Writes out the changed object using JSON.stringify() to convert the object back to JSON
Puts new submission console message in fs.writeFile() success handler
Switch to const and let from var
Issues not yet incorporated:
Concurrency issues if multiple message events can be "in-flight" at once and conflict.
More complete error handling besides just stopping processing when there's an error (I'm not sure what your application should be doing in that case as that is application-specific).
Your shoe variable is not being used anywhere, not sure what it's doing there.
I have a database class with some methods to easily do some queries without formatting the values into a sql query string. So here is my delete method where the script is stopping.
// Other Module
let filter = {
url: 'http://www.example.com/some/route',
},
DB = new MySQL();
DB.delete(filter, 'found_urls').then(result => {
console.log('The row/s are deleted.');
}).catch(error => {
console.log('An error occured.');
console.log(error);
});
// MySQL.js
class MySQL {
// some methods....
delete(filter, table, operator = '=') {
// Parse where clauses and create query string.
let whereCondition = MySQL.parseColumnValue(filter, operator),
sql = `DELETE FROM ${table}`;
// whereCondition is an array: ["url = 'http://www.example.com/some/route'"]
if (whereCondition.length > 1) {
sql += ` WHERE ${whereCondition.join(' AND ')}`;
}
console.log(sql);
// I get "DELETE FROM found_urls WHERE url = 'http://www.example.com/some/route'"
return new Promise((resolve, reject) => {
console.log('You can see me.');
this.database.query(sql, (error, result) => {
console.log('You never see me because the script stopped.');
if (error) {
reject(error);
return;
}
resolve(result);
});
});
}
}
Some days ago it worked perfect but now it stops. I don't know what's wrong with this code.
UPDATE
This class would be used in a webcrawler so I think it's not necessary to protect it for SQL injection.
The table found_urls has about 200 rows, nothing is locked.
Bug I found my problem: A failure in another module. Without this module the crawler is running (with the delete operations).
I am writing a small Node js application for automatic vehicle location system.
Here is the code for where I am getting trouble.
markerData contains 4 rows but only in the log I can see the last row.
for (var i = 0, len = markerData.length; i < len; i++) {
var thisMarker = markerData[i];
sql.connect(config, function (err) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
}
});
});
}
Please help me to solve the issue.
As var is not a block level variable in terms of scope, when `sql' module takes time to connect to the database asynchronously, the synchronous loop may change the value of the variable that's why you have the last row printed since the variable holds the reference to the last object at the time of successful connection.
Instead of _.each, I would recommend to use async module with async.each since you have few asynchronous operation to get rid of a synchronous loop.
You can check for samples here,
http://justinklemm.com/node-js-async-tutorial/
Here is your updated code with async.each
-> Install async module with npm install async --save
-> Then add the below reference in the required place,
// Reference
var async = require('async');
-> Modified code:
sql.connect(config, function (err) {
if(err) {
console.log('Connection error: ');
console.log(err);
} else {
async.each(markerData, function(thisMarker, callback) {
var request = new sql.Request();
request.input('myval', sql.Int, thisMarker.id);
request.query('SELECT d.id, d.name, d.lastupdate, p.latitude, p.longitude, p.speed, p.course FROM dbo.devices AS d INNER JOIN dbo.positions AS p ON d.positionid = p.id AND d.id = p.deviceid WHERE (d.id = #myval)', function (err, recordset2) {
if(err) {
console.log(err);
callback();
} else {
if (typeof recordset2 != 'undefined') {
thisMarker.position.lat = recordset2[0].latitude;
thisMarker.position.long = recordset2[0].longitude;
console.log(recordset2[0].id);
} else {
console.log('Recordset empty for id: ' + thisMarker.id);
}
callback();
}
});
}, function(err){
if(err) {
console.log(err);
}
});
}
});
I'm not entirely sure how your library works, but presumably recordset2 is an array of records. recordset2[0] is therefore the first record. If you want the next one you should probably try recordset2[1] and so on and so forth.
Arrays: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array
You'll probably need to loop through all the elements in the array at some point. use a for loop for that:
for (var i = 0; i < recordset2.length; i++ {
console.log(recordset2[i])
}
That will print out everything your query returns.
I have a function in my express app that makes multiple queries within a For Loop and I need to design a callback that responds with JSON when the loop is finished. But, I'm not sure how to do this in Node yet. Here is what I have so far, but it's not yet working...
exports.contacts_create = function(req, res) {
var contacts = req.body;
(function(res, contacts) {
for (var property in contacts) { // for each contact, save to db
if( !isNaN(property) ) {
contact = contacts[property];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err) { console.log(err) };
}); // .save
}; // if !isNAN
}; // for
self.response();
})(); // function
}; // contacts_create
exports.response = function(req, res, success) {
res.json('finished');
};
There are a few problems with your code besides just the callback structure.
var contacts = req.body;
(function(res, contacts) {
...
})(); // function
^ you are redefining contacts and res in the parameter list, but not passing in any arguments, so inside your function res and contacts will be undefined.
Also, not sure where your self variable is coming from, but maybe you defined that elsewhere.
As to the callback structure, you're looking for something like this (assuming contacts is an Array):
exports.contacts_create = function(req, res) {
var contacts = req.body;
var iterator = function (i) {
if (i >= contacts.length) {
res.json('finished'); // or call self.response() or whatever
return;
}
contact = contacts[i];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err)
console.log(err); //if this is really a failure, you should call response here and return
iterator(i + 1); //re-call this function with the next index
});
};
iterator(0); //start the async "for" loop
};
However, you may want to consider performing your database saves in parallel. Something like this:
var savesPending = contacts.length;
var saveCallback = function (i, err) {
if (err)
console.log('Saving contact ' + i + ' failed.');
if (--savesPending === 0)
res.json('finished');
};
for (var i in contacts) {
...
newContact.save(saveCallback.bind(null, i));
}
This way you don't have to wait for each save to complete before starting the next round-trip to the database.
If you're unfamiliar with why I used saveCallback.bind(null, i), it's basically so the callback can know which contact failed in the event of an error. See Function.prototype.bind if you need a reference.