I am using node for my project and using connection pooling, whenever I query show processlist I found more than 200 sleep connections even though i am releasing it after every query like this-
return new Promise((resolve, reject) => {
sql.getConnection(function (err, conn) {
if (err) {
conn.release();
reject(err)
}
conn.query('QUERY', function (err, rows) {
conn.release();
if (err) {
reject(err)
}
else {
resolve(rows[0])
}
})
})
})
Still i found 200 plus sleep connections. Is there any way to kill useless sleeps connection through node? Or is it fine having so many sleep connections?
Thanks in advance!
If you are using connection pooling, you will need to close all the connections in the pool when you're finished using them. Otherwise the connections will stay open until they are closed by the MySQL server.
pool.end(function (err) {
// all connections in the pool have ended
});
This GitHub issue explains a little more about the nuance between releasing a connection (conn.release()) and closing the underlying connection pool (pool.end()).
Related
Github repo. I am trying to use MongoDB Atlas database with my node JS Login & Signup app for storing data. The problem is that the data is not saving to the database or in other words the request isn't going through even if my app is connected to Atlas. Full code available on www.github.com/tahseen09/login
// Connection to mongodb atlas
const uri = "mongodb+srv://tahseen09:<PASSWORD>#cluster0-pirty.mongodb.net/userdb"
MongoClient.connect(uri, function(err, client) {
if(err) {
console.log('Error occurred while connecting to MongoDB Atlas...\n',err);
}
console.log('Connected to Atlas');
const collection = client.db("userdb").collection("credentials");
client.close();
});
//New User Registration
app.post('/register', function(req,res){
var cred= new credential();
cred.uname=req.body.uname;
const hash = bcrypt.hashSync(req.body.password, 10);
cred.password=hash;
collection.save(function(err,newuser){
if(err){
res.status(500).send("Username exists");
}
else{
res.status(200).send("New User Created");
}
})
})
The code that is important is attached as a snippet and the rest of the code is available on www.github.com/tahseen09/login
Note: I am running this app on localhost.
Let me describe your flow so you can understand wrong point there :)
Connect to MongoDB
Create reference to the collection
Close connection
When someone tries to access /register route, you already have closed connection by that time. Thus, any operation attempt to the database will end up with connection error.
From the documentation it's recommended calling MongoClient.connect once and reusing the database variable returned by the callback, i.e. do not close connection manually, driver will just create and use pool of connections, so don't worry about closing connection. Check out example code in the documentation.
Lets step through the code to see what happens:
MongoClient.connect(uri, function(err, client) {
A connection to mongodb is created, then somewhen the connection is established or it fails, then the callback gets called back. Now you create a local variable holding the database reference:
const collection = client.db("userdb").collection("credentials");
And then you close the connection:
client.close();
Then the callback ends:
});
which means that a variables inside (connection) can't be accessed anymore and get therefore recycled.
Now somewhen (that might even happen before the db connection gets established), someone requests the webpage and you try to do:
collection.save(/*...*/);
That won't work for various reasons:
1) The db might not even be opened
2) If it was opened already, it was also closed already.
3) Even if it is open at the moment, you still cannot access connection as it is not in scope.
Now to resolve that we have to:
1) only start the webserver when the db connection is establishee
2) don't close the connection
3) expose the connection so that it can be used elsewhere
For that it makes sense to create a function that establishes the connection and calls back with the db:
function withCredentials(callback) {
const uri = "mongodb+srv://tahseen09:<PASSWORD>#cluster0-pirty.mongodb.net/userdb"
MongoClient.connect(uri, function(err, client) {
if(err) {
console.log('Error occurred while connecting to MongoDB Atlas...\n',err);
} else {
console.log('Connected to Atlas');
const collection = client.db("userdb").collection("credentials");
callback(collection);
}
});
}
So now you can use that:
withCredentials(function(credentials) {
app.post('/register', function(req,res){
const cred = { };
cred.uname = req.body.uname;
cred.password = bcrypt.hashSync(req.body.password, 10);
credentials.insertOne(cred, function(err,newuser){
if(err){
res.status(500).send("Username exists");
} else {
res.status(200).send("New User Created");
}
})
});
});
I am currently using angularJS as my frontend framework while expressJS on node.JS to provide the REST API as my backend framework.
For my insertService function in node.JS, after I insert some value into the database, I want to commit and release the connection. However, I am getting the following error:
NJS-032: connection cannot be released because a database call is in progress
These are my commit and release functions:
function doRelease(connection)
{
console.log("before release");
connection.release(
function(err) {
if (err)
console.error(err.message);
});
console.log("after release");
}
function doCommit(connection)
{
console.log("before commit");
connection.commit(
function(err) {
if (err)
console.error(err.message);
});
console.log("after commit");
doRelease(connection);
}
This is how I am calling them:
app.post('/addService', function(req, res)
{
console.log("addService is called");
oracledb.getConnection(
DBconfig,
function(err, connection)
{
if (err) { console.error(err); return; }
connection.execute(
"Insert into mylist Values (MYLIST_ID_SEQUENCE.nextval ,'"+req.body.name+"','"+req.body.description+"')",
function(err, result)
{
if (err) { console.error(err); doRelease(connection); return; }
console.log("added mylist: "+req.body.name);
doCommit(connection);
}
);
}
);
})
This is the print out:
addService is called
before commit
after commit
before release
after release
NJS-032: connection cannot be released because a database call is in progress
How should I handle this issue? Should I sleep for 1 second before calling release? Should I recursively call release until it is successful?
Thanks
Before I answer your question I have to point out that your code is currently open to SQL injection vulnerabilities. Values from end users (in this case from req.body) should not be concatenated into the SQL, they should be "bound" in with bind variables.
Also, you're API will not scale if you're getting one off connections. You should create a connection pool and get connections from the pool.
Finally, you can use autoCommit (in the execute options object) to save an unnecessary round trip.
Now to your question, you have to wait until the commit finishes before releasing the connection. In doCommit, move the call to doRelease so that it's in the callback to connection.commit:
function doCommit(connection) {
console.log("before commit");
connection.commit(function(err) {
if (err) {
console.error(err.message);
}
console.log("after commit");
doRelease(connection);
});
}
On another note, I have a series on building REST APIs you might want to check out. In part 2, on database basics, I show how you can simplify these types of simple statement executions. There are links to the code in GitHub so you should be able to pull it down to see how it works for you.
I'm creating a web server that stores a user's data in a MongoDB database. The code behind the web requests uses asynchronous functions to insert a document into the database, but because these functions are asynchronous it means that for every request a new connection is made with the server.
exports.create_user = function(username, password, callback) {
mongo.connect(url, function(err, db) {
db.collection('users').insertOne({username: username, password: password}, function(err, result) {
callback(result)
db.close()
})
})
}
I'm under the impression that doing it this way is not the best practise, but I can't think a way to do it using the module model that I'm using above. Any suggestions or advice would be appreciated.
I stumbled upon this on my own research whether to use a new connection for mongodb on each query is the best practice or to use connection pooling. Turns out, that mongodb suggests connection pooling for most use-cases.
Citing from the docs:
A Connection Pool is a cache of database connections maintained by the driver so that connections can be re-used when new connections to the database are required. To reduce the number of connection pools created by your application, we recommend calling MongoClient.connect once and reusing the database variable returned by the callback
I am usually using the following form to establish and reuse a connection while firing queries:
// db.js
import { MongoClient } from 'mongodb';
// this will hold our cached database connection, which will itself hold multiple connections in a pool to be used
let connection,
database;
export {
connect: (next) => {
// already established? => return connection
if (database) return next(undefined, database);
// establish connection
MongoClient.connect('http://localhost:27017/admin', (err, db) => {
if (err) return next(err);
// save connection
connection = db;
// connect to database
database = db.db('myDatabase');
// call callback
next(undefined, database);
});
},
disconnect: (next) => {
if (!connection) return next();
// close connection
connection.close();
next();
}
};
Firing queries:
import db from './db';
db.connect((err, db) => {
if (err) return next(err);
db.collection('myUsers').insertOne({name: 'test'}, (err) => {
if (err) throw err;
db.disconnect((err) => {
if (err) throw err;
console.log('Everything finished, database connection closed');
});
});
});
Note: It is possible to determine the maximum amount of pooled connections manually (afaik the default is 5?). Refer to the docs about how to set the amount of opened connections via the mongodb url.
By doing db.close() you can close the connection, If you don't close your connection, event loop will keep the connection open and your process will not exit. If you are building a web server where your process will not be terminated, it's not necessary for you to close the connection.
For a reference node-mongodb-native
I'm using mysql connection pools in Node JS. After some idle time, the connections expire and the next time I perform a query, a new connection needs to be created. This can cause a delay of several seconds. Unacceptable!
I would like to implement keepalive functionality to periodically poll the database and ensure the consistent health of connections to the backend. I am looking for input from others who have attempted the same, or feedback on my approach.
const mysql = require('mysql');
const pool = createConnectionPool();
setInterval(keepalive, 180000); // 30 mins
function keepalive() {
pool._freeConnections.forEach((connection) => pool.acquireConnection(connection,function () {
connection.query('SELECT 1 + 1 AS solution', function (err) {
if (err) {
console.log(err.code); // 'ER_BAD_DB_ERROR'
}
console.log('Keepalive RDS connection pool using connection id', connection.threadId);
})
}));
}
This keepalive has been somewhat successful:
- once a connection is opened, it stays open
- connections never time out
- if a connection is lost, it is recreated on the next interval
This keepalive is not ideal:
- the mysql connection pool is lazy, only creating and restoring connections as needed. with this keepalive, the pool is no longer lazy. once a connection is opened, the keepalive will keep it open. the pool no longer scales depending on traffic.
- i'm not confident that my method of iterating through the list of free connections and performing a query will be a wise approach. is it possible for the application to check out the same connection from the pool while the same connection is being used by keepalive?
Another possible approach is to ditch the keepalive functionality within the application, and rely on heartbeat traffic to keep a minimum of connections alive.
Has anybody attempted to implement keepalive functionality, use a package or tool that provides this feature, or ended up using a different approach?
did you try this one in case if using pool connection
const pool = mysql.createPool({...});
function keepAlive() {
pool.getConnection(function(err, connection){
if(err) { console.error('mysql keepAlive err', err); return; }
console.log('ping db')
connection.ping(); // this is what you want
connection.release();
});
}
setInterval(keepAlive, 60000); // ping to DB every minute
I don't use pool and this code works
function pingdb() {
var sql_keep = `SELECT 1 + 1 AS solution`;
con.query(sql_keep, function (err, result) {
if (err) throw err;
console.log("Ping DB");
});
}
setInterval(pingdb, 40000);
Getting started with Node.js and Heroku; I am trying to make sense of the following code, in order to build something of my own:
app.get('/db', function (request, response) {
pg.connect(process.env.DATABASE_URL, function(err, client, done) {
client.query('SELECT * FROM test_table', function(err, result) {
done();
if (err)
{ console.error(err); response.send("Error " + err); }
else
{ response.render('pages/db', {results: result.rows} ); }
});
});
});
Where can I find a tutorial or some comments or explanations for that?
Even though I can do some guessing, a good deal of this code is pretty mysterious.
Currently my main concerns are:
What happens if I change the SQL query, replacing it by 'SELECT
count(*) FROM test_table'? How do I then render the result?
What does "done();" do? Is it something I can modify or make use
of?
The parameter "request" is never used. Can it be used for
something at some point?
Before handling heroku, you should first look at tutorials about web application in node.js which will answers your last question.
You can see how works express.js, a web framework.
Then look at node-postgre documentation. You will find your answers about the second question here :
//this initializes a connection pool
//it will keep idle connections open for a 30 seconds
//and set a limit of maximum 10 idle clients
var pool = new pg.Pool(config);
// to run a query we can acquire a client from the pool,
// run a query on the client, and then return the client to the pool
pool.connect(function(err, client, done) {
if(err) {
return console.error('error fetching client from pool', err);
}
client.query('SELECT $1::int AS number', ['1'], function(err, result) {
//call `done()` to release the client back to the pool
done();
if(err) {
return console.error('error running query', err);
}
console.log(result.rows[0].number);
//output: 1
});
});
And finanlly, why don't you just log result output after changing the SQL query and look what you get ?