I have a problem to design communication with MySQL database in my Nodejs's app.
The biggest problem is that queries are async, so it becomes complicated to design my projects. For example, I have excercises.js
EXCERCISES.JS
var express = require('express');
var database = require('../database/database.js');
var router = express.Router();
console.log(database.db(saveDbData))
/* GET users listing. */
router.get('/', function(req, res, next) {
res.render('exercises',{title: 'Exercises', ex: #DATABASE RESULT});
});
module.exports = router;
In need to write query's result in ex field.
Then I write a module to handle mysql connection
DATABASE.JS
var pool = mysql.createPool({
connectionLimit: 10000,
host: 'localhost',
user: 'root',
password: 'password',
database: 'Example'
});
var results;
var db = function(){
pool.query('SELECT name FROM Exercises', function(error, results, fields){
if (error) throw error;
res = results;
})
return res;
}
module.exports = {
db: db,
}
Obviously, It doesn't work because pool.query is async.
The only alternative that I've found on the web is something like this:
EXERCISES.JS
var pool = mysql.createPool({
connectionLimit: 10000,
host: 'localhost',
user: 'root',
password: 'password',
database: 'Example'
});
pool.query('SELECT name FROM Exercises', function(error, results, fields){
if (error) throw error;
router.get('/', function(req, res, next) {
res.render('exercises',{title: 'Exercises', ex: result[0].name});
});
})
But in this way, mysql parts and routing/render parts are mixed. Is it still a well-designed solution? Are there more elegant solutions?
EDIT:
I have modified the files and I have used Promise like this
EXERCISES.JS
var express = require('express');
var database = require('../database/database.js');
var router = express.Router();
var data = database.db()
.then(
function(data){
console.log("Resolved");
/* GET users listing. */
router.get('/', function(req, res, next) {
res.render('exercises',{title: 'Exercises', ex: data[0].name});
});
})
.catch(
error => console.error(error));
module.exports = router;
DATABASE.JS
ar mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit: 10000,
host: 'localhost',
user: 'root',
password: 'password',
database: 'Example'
});
var res;
var db = function(){
return new Promise(function(resolve, reject){
pool.query('SELECT name FROM Exercises', function(error, results, fields){
if (error) reject(error);
resolve(results)
})
})
}
module.exports = {
db: db,
}
And it works, but I don't think it is the best solution. For example, what if I want to get data for render from more queries?
I'm new in these technologies so I'm not able to figure out the best way to integrate database and html pages's rendering.
Have you ever wondered why all web frameworks in node requires you to return responses using the res object instead of just return? It's because all web frameworks expect that you need to do something asynchronous.
Consider a web framework design similar to Laravel (PHP) or Spring Framework (Java):
// Theoretical synchronous framework API:
app.get('/path', function (request) {
return "<html><body> Hello World </body></html>";
});
Then if you need to do anything async you will face the issue that the data you're fetching hasn't returned by the time you need to return the HTTP request:
// Theoretical synchronous framework API:
app.get('/path', function (request) {
return ??? // OH NO! I need to return now!!
});
It is for this reason that web frameworks in javascript don't act on return values. Instead it passes you a callback to call when you are done:
// Express.js
app.get('/path', function (request, response) {
doSomethingAsync((err, result) => {
response.send(result);
});
});
So for your code you just need to do:
router.get('/', function(req, res) {
pool.query('SELECT name FROM Exercises', function(error, results, fields){
if (error) throw error;
res.render('exercises',{title: 'Exercises', ex: result[0].name});
});
});
Exporting the database
Exporting the database is as simple as exporting pool:
db.js
var pool = mysql.createPool({
connectionLimit: 10000,
host: 'localhost',
user: 'root',
password: 'password',
database: 'Example'
});
module.exports = {
pool: pool
}
exercises.js
let db = require('./db');
// you can now use db.pool in the rest of your code
// ..
Reusing your queries
Instead of coding SELECT statements in your controllers (routes) you can (and should) code them in your db module(s):
db.js
var pool = mysql.createPool({
connectionLimit: 10000,
host: 'localhost',
user: 'root',
password: 'password',
database: 'Example'
});
function getExerciseNames (callback) {
pool.query('SELECT name FROM Exercises',callback);
}
module.exports = {
pool: pool
}
Then in your controller logic you just need to do:
router.get('/', function(req, res) {
db.getExerciseNames(function(error, results, fields){
if (error) throw error;
res.render('exercises',{
title: 'Exercises',
ex: result[0].name
});
});
});
Caching
If your intention is to query the db only once to cache the value of Exercises then don't invert the Express routing flow. Instead implement the caching at your db layer:
db.js:
var exerciseNamesCache = [];
var exerciseNamesFields = [];
function getExerciseNames (callback) {
if (exerciseNamesCache.length > 0 && exerciseNamesFields.length > 0) {
callback(null, exerciseNamesCache, exerciseNamesFields);
}
pool.query('SELECT name FROM Exercises',function(error, results, fields){
if (!error) {
exerciseNamesCache = results;
exerciseNamesFields = fields;
}
callback(error, results, fields);
});
}
Promises
Promises is a design pattern for handling callbacks. It is comparable to Java's Futures (CompletionStage etc.) only a lot more lightweight. If an API you are using returns a promise instead of accepting a callback then you need to call res.render() inside the promise's .then() method:
router.get('/', function(req, res, next) {
doSomethingAsync()
.then(function(result){
res.send(result);
})
.catch(next); // remember to pass on asynchronous errors to next()
});
If the API you're using accepts a callback then weather or not you wrap it in a promise is more a matter of taste. I personally wouldn't do it unless you are also using another API that returns a promise.
async/await
One advantage of promises is that you can use them with await. Express specifically works well with async/await. Just remember you can only use await inside a function marked with async:
router.get('/', async function(req, res, next) {
let result = await doSomethingAsync();
res.send(result);
});
Multiple asynchronous operations
Fetching multiple asynchronous data can be as simple as:
router.get('/', function(req, res, next) {
doSomethingAsync(function(result1){
doSomethingElse(function(result2) {
res.json([result1, result2]);
});
});
});
With promises that would be:
router.get('/', function(req, res, next) {
doSomethingAsync()
.then(function(result1){
return doSomethingElse()
.then(function(result2) {
return [result1, result2];
});
})
.then(function(results){
res.json(results);
})
.catch(next);
});
But both the above code perform the requests sequentially (get result1 then get result2). If you want to fetch both data in parallel you can do this with Promises:
router.get('/', function(req, res, next) {
Promise.all([
doSomethingAsync(), // fetch in parallel
doSomethingElse()
])
.then(function(results){
res.json(results);
});
})
With callbacks it's a little bit more complicated. There is a design pattern you can use and someone has actually implemented it as a library called async.js but often the easiest solution is to wrap them in Promises and use Promise.all(). Still, do check out async.js since it has functionality useful for things like batching requests, perform async operations while a condition is true etc. (the promise based counterpart of that library is async-q)
You can use npm modules for achieving the async task with MySQL.
I recommended to choose sequilize or jm-ez-mysql. If you go with jm-ez-mysql then your code structure like
server.js
require('./config/database.js');
./config/database.js
const sql = require('jm-ez-mysql');
// Init DB Connection
const connection = My.init({
host: process.env.DBHOST,
user: process.env.DBUSER,
password: process.env.DBPASSWORD,
database: process.env.DATABASE,
dateStrings: true,
charset: 'utf8mb4',
timezone: 'utc',
multipleStatements: true,
});
module.exports = {
connection,
};
After that, you can use MySQL asynchronously.
./exercise.js
const sql = require('jm-ez-mysql');
const exerciseUtil = {};
exerciseUtil.searchUserById = async (id) => {
try {
// table name, db fields, condition, values
const result = await sql.first('users', ['id, name, email'], 'id = ?', [id]);
return result; // Which return your result in object {}
} catch (err) {
console.log(err);
throw err;
}
};
module.exports = exerciseUtil;
I hope it helps.
Happy Coding :)
Related
Does anyone know why my request just gets stuck loading when trying to access my database ?
My database name is test. If set the database: books or something like that for example. Then it returns the error database is unknown: books so I assume that my password is correct it just isn't finding the test data base ?
// To import these packages remember to add "type":"module" to package Json
import express from "express";
import mysql from "mysql";
const app = express();
const db = mysql.createConnection({
host: "localhost",
user: "root",
password: "keks000207",
database: "test",
});
// This is an API request with an Express server
app.get("/", (req, res) => {
res.json("Hello this is the backend");
});
app.get("/books", (req, res) => {
const q = "SELECT * FROM books";
db.query(q, (err, data) => {
if (err) return res.json(err);
return data;
});
});
app.listen(8800, () => {
console.log("Connected to backend!");
});
Try db.connect() or similar method available in the file itself.
And Instead of return data inside the callback of db.query, you should use res.send(data), then you will get the response in the GET /books API.
I know that mongodb has their own way of handling native promises with no rejection issues but I guess it doesnt work the same here with mysql. Is there anyway I can use .then in mysql? I was able to do it using more callbacks but I would like to use promises to make the solution cleaner or even use async/await if that makes it simpler too. I need to use a callback to jump to my server file, I have all my methods defined in my db file.
Server file:
app.get('/api/cows', (req, res) => {
db.reqMethods.getAll((err, data) => {
if (err) {
res.send('Error');
} else {
res.send(data);
}
});
});
db file:
const mysql = require('mysql');
const http = require('http');
const express = require('express');
const path = require('path');
const bodyParser = require("body-parser");
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'password',
database: 'cowlist'
});
connection.connect((err) => {
if (err) {
console.log(err);
} else {
console.log('Connected to MySQL!')
}
});
// Your Database Queries Here!!
module.exports.reqMethods = {
// GET All Cow Info
getAll: function (callback) {
const query = connection.query('SELECT * FROM cows;');
query.then(data => callback(null, data));
});
};
// callback solution that I'd like to simplify:
// getAll: function (callback) {
// connection.query('SELECT * FROM cows;', (err, data) => {
// if (err) {
// callback(err, null);
// } else {
// console.log("DATA: \n", data);
// callback(null, data);
// }
// });
// }
Yes, this is possible by using mysql2 npm package.
So in your database.js file, use this method.
const mysql = require('mysql2');
var pool_connection = mysql.createPool({
host: '127.0.0.1',
port: 3306,
user: process.env.DB_USERNAME,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
connectionLimit: 10,
multipleStatements: true
});
pool_connection.getConnection((err) => {
if (err) console.log(JSON.stringify(err));
else {
console.log('Connected!')
}
});
module.exports = pool_connection.promise();
And in your models, you require the connection as follows and make use of async-await in a try-catch block.
const con = require('/path/to/your/database_file');
module.exports = class Messages {
constructor() { }
static async getMessage(arguments_here) {
const query = "some query here with params if required. Use ? for placing params and do not use string literal to embed params.";
try {
const [response] = await con.execute(query, [params]);
return response;
} catch (error) {
console.log(error);
return null;
}
}
}
And in your controller,
const Messages = require('../models/Messages');
const someFn = async (req, res) =>{
try {
const result = await Messages.getMessages('sample_arguments');
//do something with result
}
catch(err){
console.log(err);
}
}
I am trying to implement a use case where users can request to check his password by providing username.
Issue: Node js back-end app returns result before query is complete. I have checked articles explaining async aspect of javascript. However, I am still unsure if the solutions can be applied when there is a rest call involved in between.
High level flow:
Front-end js calls rest api(node.js application) with username to get password
Node app calls DB to get password
Returned password is displayed on front end
Code:
Front-end:
function getPassword() {//Called by clicking button on ui
var username = 'testuser'; //hardcoding for simplicity
$.ajax({
url: urlWhereNodeJsAppIsHosted,
error: function(password) {
console.log('error' + JSON.stringify(password));
}, success: function(data) {
console.log('success' + JSON.stringify(password)); // Displaying on console for now
}
});
}
Back-end(Node js app):
const express = require('express');
const request = require('request');
const mysql = require('mysql');
const app = express();
var connection = mysql.createConnection({
host: 'localhost',
port: '3306',
user: 'dbuser',
password: 'dbuserpassword',
database: 'dbname'
});
connection.connect(function(err) {
if (err) {console.log(err);}
});
app.use((req, res, next) => {
res.header('Access-Control-Allow-Origin', '*');
next();
});
app.get('/verify', (req, res) => {
var username = req.query.username;
var password = 'noresult';
connection.query(
'SELECT * FROM userpassword where user=?, ${[username]}',
function(err, rows, fields) {
password = extractpasswordfromrows(rows);//iterate and get result
}
);
console.log(connection.threadId); //Value is not undefined
res.send(200, { success: password });
});
const PORT = process.env.PORT || 3001;
app.listen(PORT, () => console.log(`listening on ${PORT}`));
Database:
userpassword(user, password) table with existing data.
I am new to javascript as well as thinking asynchronously. Please help with your inputs. Thanks.
The code sends password outside the success callback without waiting for the query to complete. Try
app.get('/verify', (req, res) => {
var username = req.query.username;
var password = 'noresult';
connection.query(
'SELECT * FROM userpassword where user=?, ${[username]}',
function(err, rows, fields) {
password = extractpasswordfromrows(rows);//iterate and get result
res.send(200, { success: password }); // Ok, have password here
}
);
// Not Ok, don't have password here
console.log(connection.threadId); //Value is not undefined
});
I haven't attempted to put in check code to verify that the err parameter in the call back is not truthy, even if logically seen as impossible.
For general treatment of coding against this error, see "How do I return the result from an asynchronous call".
Processing results outside the callback
To avoid nesting code, callback code can resolve or reject a newly created promise depending on processing results, with result and error handling functions attached to the promise using its then/catch methods.
Alternatively in Express you can supply multiple callbacks (middleware functions) to app.get and use next calls to advance to the next middleware function if and when required. As an example based on the post (compiled but not tested):
app.get('/verify',
(req, res, next) => {
var username = req.query.username;
connection.query(
'SELECT * FROM userpassword where user=?, ${[username]}',
function(err, rows, fields) {
if( err) {
res.status(500).send("no result");
}
else {
res.locals.rows = rows;
next();
}
}
);
},
(req,res) => {
const password = extractpasswordfromrows(res.locals.rows); //iterate and get result
res.send(200, { success: password });
}
);
I have 3 files. db.js, app.js, commentController.js.
I am including my connection in every request in my app so that I wont be repeating the code connection again and again. Is this a bad / unsecure practice? Is there a better/proper way to implement this?
db.js
const mysql = require('mysql');
const pool = mysql.createPool({
host : 'host',
user : 'user',
password : 'password',
database : 'dbname'
});
exports.pool = pool;
app.js
const db = require('./db');
app.use((req, res, next) => {
req.pool = db.pool;
next();
});
commentController.js
exports.showComments = (req, res) => {
req.pool.getConnection((err, conn) => {
conn.query(`SELECT * FROM comments`, (err, results, fields) => {
conn.release();
if (err) throw err;
res.render('comments', { results });
});
});
};
If your only reason for doing this is to avoid duplicating code, then I think it's a bad idea. People looking at your code (or you looking at your code in a year) aren't going to naturally expect a db connection to be a property of req. And you aren't saving yourself any trouble really.
Just require() the database pool in the file and use it.
commentController.js
const db = require('./db');
require() will return the same pool to all your modules.
It's also not clear why you are requesting a connection rather than using the pool (I'm making some assumptions about the lib you're using).
Normally you should be able to do:
const db = require('./db');
exports.showComments = (req, res) => {
db.query(`SELECT * FROM comments`, (error, results, fields) => {
if (err) throw err;
res.render('comments', { results });
});
});
This saves the trouble of requesting and returning connections and just lets the pool do it's work.
I'm trying to use connection from a connection.js file and use it in different file webFrontend.js using exports object. Now what I get on running server is:
{
"Result": "undefinedThis is result"
}
That means connection is not defined. Why is it happening? connection is working fine if getConnection is created in same (webFrontend.js) file, but the problem is when I use getConnection in same exports function in connection.js hence the connection not defined error:
Here are 2 necessary files (routes file has no problem) that explains what I'm doing:
connection.js
var mysql = require('mysql');
exports.connExport = function () {
var connectionPool = mysql.createPool({
host: 'localhost',
user: 'root',
password: '',
database: 'rockcity_followme'
});
if(connectionPool) {
connectionPool.getConnection(function (err, connection) {
if (err) {
return err;
} else {
return connection;
}
});
}else{
var abc="return error";
return abc;
}
}
webFrontend.js
var connObj=require('../Routes/connection.js');
var connection=connObj.connExport();
exports.getIndivRecords= function(req, res, next){
res.send({
Result: connection+"This is result"
});
return next();
};
No need for the .js file extension, it's automagically added for you.
The code below uses standard error-first callbacks
webFrontend.js
var connection = require('../Routes/connection');
exports.getIndivRecords = function(req, res, next){
// connection takes a standard error-first callback
connection(function(err, conn){
if (err) {
// Handle the error returned
console.log(err);
}
// The database connection is available here as conn
console.log( "Connection:" + conn);
// presumably you want to do something here
// before sending the response
res.send({
Result: conn + "This is result"
});
});
return next();
};
connection.js
var mySQL = require('mysql');
var connectionPool = mySQL.createPool({
host: 'localhost',
user: 'root',
password: '',
database: 'rockcity_followme'
});
var getConnection = function (cb) {
connectionPool.getConnection(function (err, connection) {
// pass the error to the callback
if (err) {
return cb(err);
}
cb(null, connection);
});
};
module.exports = getConnection;
First of all #Dan Nagle was right no need of .js
Second You are getting the connection undefinded because still the method doesnt returned with result.
Use promise to call your Connection.js method
Your node is single threaded async execution,
He doest wait for the method to return a result
1) Problem with your javascript is that
var connection=connObj.connExport();
in Creation stage connection was defined by javascript as undefined and as
connObj.connExport(); as still not returned with answer
it executed this function in which connection was undefined
exports.getIndivRecords= function(req, res, next){
res.send({
Result: connection+"This is result"
});
Use promise read this first so you can understand something about promise and callback if you are unable to solve than comment i will play with it.But first you try.Thanku
Understanding promises in node.js
Ok Try This I have used promise here
var connObj = require('../Routes/connection');
connObj.connExport().then(
function (connection) {
exports.getIndivRecords = function (req, res, next) {
res.send({
Result: connection + "This is result"
});
return next();
};
}).catch(function (err) {
res.status(400).send(err);
return;
});
var mysql = require('mysql');
exports.connExport = function () {
return new Promise(function (fulfill, reject) {
var connectionPool = mysql.createPool({
host: 'localhost',
user: 'root',
password: '',
database: 'rockcity_followme'
});
if (connectionPool) {
connectionPool.getConnection(function (err, connection) {
if (err) {
return reject(err);
} else {
return fulfill(connection);
}
});
} else {
var abc = "return error";
return reject(abc);
}
});
}