We have a server running that has 2 connections to the database, they are both created using a pool.
Example:
var connection = mysql.createPool({
host: config.mysql.host,
user: config.mysql.user,
password: config.mysql.pass,
database: config.mysql.database
});
However, there's a lot of queries we do but a pool takes care of handling/closing active connections I was told.
Now connections are just piling up and old connections are not being taken care of. Eventually we will run out of available connections and the server will go hanging.
Query function;
var query = function(query){
return new Promise(function(resolve, reject){
con.query(query, function(err, row){
if(err){
reject(err);
}
resolve(row);
});
});
}
Usage of function;
db.query('SELECT * FROM `users` WHERE `isBanned` = 0').then(function(users){
//active users
}).catch(function(err){
...
});
All connections that are being piled up are in command Sleep
Edit: merged the connections into one database pool
Main.js
var pool = mysql.createPool({
connectionLimit: 250,
host: config.mysql.host,
user: config.mysql.user,
password: config.mysql.pass,
database: config.mysql.database,
charset: 'utf8_general_ci'
});
module.exports.pool = pool;
Database.js (pretty much just for the query function)
var main = require('./server.js');
var query = function(query){
return new Promise(function(resolve, reject){
main.pool.query(query, function(err, row){
if(err){
reject(err);
}
resolve(row);
});
});
}
module.exports = {
con: main.pool,
query: query
};
Related
Hereby I have attached my code,
server.js
var Redshift = require('node-redshift');
const config = {
user: *****,
database: *****,
password: *****,
port: *****,
host: '*****.redshift.amazonaws.com',
};
// The values passed in to the options object will be the difference between a connection pool and raw connection
var redshiftClient = new Redshift(config, {rawConnection:true});
exports.handler = async (event) => {
return new Promise(function(resolve, reject){
redshiftClient.connect(function(err){
if(err){
console.log("errror");
throw err;
}
else{
redshiftClient.parameterizedQuery('SELECT top 10 * FROM user', [event.user_id], {raw: true}, function(err, data){
if(err){
console.log("errror");
}
else{
console.log(data);
resolve(data);
redshiftClient.close();
}
});
}
});
});
};
I have been trying to establish connection with redshift for data retrival, but end up with nothing displayed in the terminal for the given console.log command..
My question is similar to this post but the solution didnt work for me probably because im using a different type of mysql connection (pool). This is my code:
let config= {
host: '***',
user: 'admin',
password: '***,
port: '3306',
database: '***',
multipleStatements: true
};
const con = mysql.createPool(config);
select();
function select(){
return new Promise((resolve, reject) => {
con.getConnection(function (err, connection) {
if (err) throw err;
else
console.log("Connected!");
let sql = "SELECT * FROM bidPrice WHERE idExchangePlatform = 2;";
connection.query(sql, function (err, results, fields) {
connection.release();
connection.destroy();
if (err) throw err;
console.log(results)
resolve(results);
});
});
});
}
I also important to mention that im running this function using the following command
node --max-old-space-size=31744 index.js # Increase to 31 GB
This is because im working with millions of records from the database query
If i run this with regular node command i would be getting Javascript heap out of memory
When i tried integrating the solution i mentioned earlier to my code i just get a "killed" log after a while and then the process stops, should i handle server disconnect in a different way when using mysql.pool?
If you have big table with many rows, you will must check index for column 'idExchangePlatform' and create if doesn't make it
And simple variant your code:
function select(){
return new Promise((rs, rj) => {
let sql = "SELECT * FROM bidPrice WHERE idExchangePlatform = 2;";
pool.query(sql, (err, rows) => {
if(err)
return rj(err);
return rs(rows);
})
});
}
Currently I am playing around with the MySQL library in Node.js however I have a question about the correct/most efficient way to be using this library.
According to w3schools the correct way to make a single query is to use code like this
var mysql = require('mysql');
var con = mysql.createConnection({
host: "localhost",
user: "yourusername",
password: "yourpassword",
database: "mydb"
});
con.connect(function(err) {
if (err) throw err;
con.query("SELECT * FROM customers", function (err, result, fields) {
if (err) throw err;
console.log(result);
});
});
However, say I wanted to make multiple queries which would be executed by an event for example how would I handle this? Should I create an "initialise" function which is executed as soon as the program runs such as this?
var mysql = require('mysql');
var database;
//Initialise database
function setupDatabase() {
database = mysql.createConnection({
host: token.host,
user: token.user,
password: token.password,
database: token.database,
port: token.port
});
}
//Imagine this could be called at any time after execution
function event() {
if(database != null) {
database.connect(function(err) {
if (err) throw err;
database.query("SELECT * FROM customers", function (err, result, fields) {
if (err) throw err;
console.log(result);
});
});
}
}
And also do I have to connect to the database each time I make a query or can I add the "database.connect" call to my setupDatabase function such as this?
var mysql = require('mysql');
var database;
//Initialise database
function setupDatabase() {
database = mysql.createConnection({
host: token.host,
user: token.user,
password: token.password,
database: token.database,
port: token.port
});
if(database != null) {
database.connect(function(err) {
if (err) throw err;
});
}
}
//Imagine this could be called at any time after execution
function event() {
if(database != null) {
database.query("SELECT * FROM customers", function (err, result, fields) {
if (err) throw err;
console.log(result);
});
}
}
My main concern is that calling the con.connect function every single time I make a query would be slow and although these are asynchronous I want to be using the correct/most efficient way possible. Feel free to correct me on any mistakes with the last two code snippets I have only tested the first one so far.
You have to make database connection only once per application livetime (unless you have disconnects). Then you may have as much queries as you want.
Just put database connection routine somewhere in sepparate file and then require it in your applicatin initialisation step.
// mysql.js
const mysql = require('mysql');
module.exports = mysql.createConnection({
host: "localhost",
user: "yourusername",
password: "yourpassword",
database: "mydb"
});
Or require it anywhere you need database connection - it will return connected database object without reruning that code again and again.
// inex.js
const databse = require('./mysql')
database.query("SELECT * FROM customers")
I created a file which include a function that holds a pool and handles the connection to the database like this
let _this = {};
let POOL = null;
function getPool() {
return new Promise((resolve, reject) => {
if(POOL != null) {
resolve(POOL);
} else {
//create connection pool
POOL = connectionPool;
resolve(POOL);
}
});
}
function closePool(){
// close pool here
}
_this.getPool = getPool;
_this.closePool = closePool;
module.exports = _this;
Now you can call getPool() and will recive a pool of connections where you can execute your queries with.
I'm using the mysql module in Node.js. In my model file, currently, I'm specifying the connection constants in each method. However, this is taking up a lot of space and I know it's not ideal.
Here's what that looks like.
doSomething: () => {
var connection = mysql.createConnection({
host : config.database.host,
database : config.database.database,
user : config.database.user,
password : config.database.password
});
connection.query( ... );
connection.destroy();
},
doSomethingElse: () => {
var connection = mysql.createConnection({
host : config.database.host,
database : config.database.database,
user : config.database.user,
password : config.database.password
});
connection.query( ... );
connection.destroy();
},
Could anyone recommend a way to tidy things up a bit and reduce the redundant code here?
Create the connection once and pass it to module exports.
const mysql = require("mysql");
const conn = mysql.createConnection({
host: "localhost",
user: "root",
password: "",
database: "db"
});
module.exports = conn;
Then you can import it in other files and use it.
var dbConnection = require('./dbConnection');
dbConnection.query();
However, instead of using createConnection, I recommend using createPool instead.
Connections can be pooled to ease sharing a single connection, or
managing multiple connections.
const mysql = require("mysql");
const conn = mysql.createPool({
host: "localhost",
user: "root",
password: "",
database: "db"
});
module.exports = conn;
You can use it like this. Make sure to release the connection after fetching data from the table:
var connectionPool = require('./dbConnection');
connectionPool.getConnection((err, connection) => {
connection.query('SELECT * FROM table', (error, result) {
connection.release();
if(error) throw error;
});
});
To close all connections in the pool:
connectionPool.end(function (err) {
// all connections in the pool have ended
});
I am new to node.js and am trying to learn how to connect to mysql database from ejs file. I tried to search for sample code however the code is not working. Can someone please check it out for me. Thank you.
function loaddata() {
var sql = require("mysql");
var con = mysql.createConnection({});
con.connect(function (err) {
if (err) {
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
con.query('update students set name="sus" where email="smn14#mail.aub.edu"', function (err, rows) {
if (err) throw err;
console.log('Data received from Db:\n');
console.log(rows);
});
con.end(function (err) {
// The connection is terminated gracefully
// Ensures all previously enqueued queries are still
// before sending a COM_QUIT packet to the MySQL server.
});
}
The create connect is worst.
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'me',
password : 'secret',
database : 'my_db'
});
connection.connect();
connection.query('SELECT 1 + 1 AS solution', function(err, rows,
fields) {
if (err) throw err;
console.log('The solution is: ', rows[0].solution); });
connection.end();
From this example, you can learn the following:
Every method you invoke on a connection is queued and executed in sequence.
Closing the connection is done using end() which makes sure all remaining queries are executed before sending a quit packet to the
mysql server.
Docs
I now understand the process of server/clients. Makes sense, otherwise you would be able to see the database passwords stored in Client.js. :-)
But, there is one way that works for me. The client call a javascript-function and send a message to the server. The server receives this message and starts a database query. Send the result to all clients via socket.io
At the client in the file.ejs
<script type='text/javascript'>
let socket = io.connect();
function getSql(userId) {
socket.emit('start-new-sql-querie',{
userId: userId
});
}
socket.on('new-sql-result', function (data){ // listen for the new sql result
console.log(data.userStatus); // foo something with the new data
})
</script>
<button onclick="getSql(1)">Test sql query</button>
database connection.js at server side
const connection = {
connectionLimit: 10,
host: "localhost",
user: "Abc",
password: "1234",
database: "d001",
multipleStatements: true
};
module.exports = connection;
yourapp.js at server side
const express = require('express');
const port = process.env.PORT || 1234;
const app = express();
const server = require('http').createServer(app);
const mysql = require('mysql2');
const config = require('./routes/connection'); // SQL-Connection
const pool = mysql.createPool(config);
let io = require('socket.io')(server);
io.sockets.on('connection', function(socket) {
socket.on('start-new-sql-querie', function(data) { // listen from the clients
let user_id = data.userId;
sql_test.getConnection((error, connection) => { // Connect to sql database
console.log("user_id: ", user_id)
connection.query(`SELECT * FROM user WHERE id='${user_id}'`, (err, result) => {
socket.emit('new-sql-result',{ // send sql result-status to all clients
userStatus: result.result[0].status
})
})
connection.release();
})
});
})