I'm currently using MySQL for a huge project that involves many tables and we are using NodeJS for the backend. We chose MySQL because of the highly relational nature of the data.
I've used MySQL and transactions before and also NodeJS before but never both together.
So we settled on the mysql2/promise package since it supports async/await. However, transactions are ALWAYS locking when dealing with many tables and it is very annoying. I can't seem to figure out for to get it working. As we are ways in the project, changing the library is not an option. We would like to maintain the library but find a way to get it working, hence I'm here hoping someone can shed some light for me on this.
The code is below. It's split into 2 files. One is the database.js and the other is a service file where we execute and run everything called service.js
This is the database.js file that we use to initiate the connection.
// database.js
const mysql = require("mysql2/promise");
const pool = mysql.createPool({
port: process.env.DB_PORT,
host: process.env.DB_HOST,
user: process.env.DB_USERNAME,
password: process.env.DB_PASSWORD,
database: process.env.DB_DATABASE,
connectionLimit: 100,
});
module.exports = pool;
This is the service.js file that we are running the function from.
// service.js
const db = require("../../config/database");
module.exports = {
save: async (data, callBack) => {
try {
await db.query(
"SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED"
);
await db.query("START TRANSACTION");
try {
await db.query(`UPDATE QUERY FOR TABLE A`, [myParams]);
await db.query(`DELETE QUERY FOR SOME DATA ON TABLE B`, [myParam]);
await db.query(`DELETE QUERY FOR SOME DATA ON TABLE C`, [myParam]);
await db.query(`DELETE QUERY FOR SOME DATA ON TABLE D`, [myParam]);
await db.query(`DELETE QUERY FOR SOME DATA ON TABLE E`, [myParam]);
await db.query(`DELETE QUERY FOR SOME DATA ON TABLE F`, [myParam]);
await db.query(`DELETE QUERY FOR SOME DATA ON TABLE G`, [myParam]);
for (var b = 0; b < data.b.length; b++) {
await db.query(`INSERT QUERY FOR TABLE B`, [myParams]);
}
for (var c = 0; c < data.c.length; c++) {
await db.query(`INSERT QUERY FOR TABLE C`, [myParams]);
}
for (var d = 0; d < data.d.length; d++) {
await db.query(`INSERT QUERY FOR TABLE D`, [myParams]);
}
for (var e = 0; e < data.e.length; e++) {
await db.query(`INSERT QUERY FOR TABLE E`, [myParams]);
}
for (var f = 0; f < data.f.length; f++) {
await db.query(`INSERT QUERY FOR TABLE F`, [myParams]);
}
for (var g = 0; g < data.g.length; g++) {
await db.query(`INSERT QUERY FOR TABLE G`, [myParams]);
}
await db.query("COMMIT");
console.log("NEXT");
return callBack(null, someReturnData);
} catch (err) {
console.log("Error");
await db.query("ROLLBACK");
console.log("Rollback successful");
console.log(err);
return callBack(err);
}
} catch (err) {
console.log("Error");
await db.query("ROLLBACK");
console.log("Rollback successful");
console.log(err);
return callBack(err.code);
}
},
};
This is a use case where we needed to clear all the data of a few tables before we insert the new data hence we used it in this way. And since the data is quite huge, we are looping it to insert multiple at the same time. However, this just causes a lock or freezes.
I've tried changing SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED to everything else there is or even just removed it entirely. It always causes more issues.
Anyone who has had issues, please do assist in helping to smoothen this issue out. NodeJS and MySQL has been great but this particular issue with the mysql2/promise package is driving us nuts. The async/await nature of it might be causing us to have issues where we can't insert in since the previous one hasn't completed and that is just raising exponentially till we hit a timeout.
Thanks and Cheers!
With every call of pool.query the query might be executed with another connection of the pool.
So the START TRANSACTION might be executed in an entirely different connection then all of your other commands.
You have to request a connection using getConnection and use that for your entire transaction. And once you finished you have to release it (conn.release();) so that it is put back to the pool and is available again.
So it has to look something like that:
module.exports = {
save: async (data, callBack) => {
let connection
try {
connection = await db.getConnection()
await connection.query(
"SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED"
);
await connection.query("START TRANSACTION");
try {
await connection.query(`UPDATE QUERY FOR TABLE A`, [myParams]);
await connection.query(`DELETE QUERY FOR SOME DATA ON TABLE B`, [myParam]);
await connection.query(`DELETE QUERY FOR SOME DATA ON TABLE C`, [myParam]);
await connection.query(`DELETE QUERY FOR SOME DATA ON TABLE D`, [myParam]);
await connection.query(`DELETE QUERY FOR SOME DATA ON TABLE E`, [myParam]);
await connection.query(`DELETE QUERY FOR SOME DATA ON TABLE F`, [myParam]);
await connection.query(`DELETE QUERY FOR SOME DATA ON TABLE G`, [myParam]);
for (var b = 0; b < data.b.length; b++) {
await connection.query(`INSERT QUERY FOR TABLE B`, [myParams]);
}
for (var c = 0; c < data.c.length; c++) {
await connection.query(`INSERT QUERY FOR TABLE C`, [myParams]);
}
for (var d = 0; d < data.d.length; d++) {
await connection.query(`INSERT QUERY FOR TABLE D`, [myParams]);
}
for (var e = 0; e < data.e.length; e++) {
await connection.query(`INSERT QUERY FOR TABLE E`, [myParams]);
}
for (var f = 0; f < data.f.length; f++) {
await connection.query(`INSERT QUERY FOR TABLE F`, [myParams]);
}
for (var g = 0; g < data.g.length; g++) {
await connection.query(`INSERT QUERY FOR TABLE G`, [myParams]);
}
await connection.query("COMMIT");
console.log("NEXT");
return callBack(null, someReturnData);
} catch (err) {
console.log("Error");
await connection.query("ROLLBACK");
console.log("Rollback successful");
console.log(err);
return callBack(err);
}
} catch (err) {
console.log("Error");
await connection.query("ROLLBACK");
console.log("Rollback successful");
console.log(err);
return callBack(err.code);
}
connection.release()
},
};
Related
I have been struggling for a few days with a problem that seems simple to me. I want to insert a value if it does not exist in a mysql database with nodejs.
I have an array of array, and I need to loop inside a child to check every data.
Example of data
finalRow = [['a > 'b'], ['a' > 'c']];
My code
for(let i=0; i < finalRow.length-1; i++) {
var splitcat = finalRow[i].split(' > ');
(async function(){
for(let j=0; splitcat.length-1; j++) {
var query = "SELECT * from category where name = '" + splitcat[j] + "'";
await con.query(query, async function (err, result, fields) {
if (err) throw err;
if(result && result.length) {
console.log("nothing");
} else {
console.log("insert into db");
await con.query(`INSERT INTO category (name, url, description, parent_id) VALUES (?, ?, ?, ?)`,
[splitcat[j], '', '', null], function (err, result) {
if (err) throw err;
});
}
});
}
})();
I tried several versions of this code, but
the loop does not wait for the existence check and inserts all the data.
Thanks for your help
I'm using MovieDB API to create a website. I'm randomly getting movie id's and generating the genres this way. However, sometimes, an id which doesn't exist gets generated and it throws a 404 error. I want to write something like: if response == 404 then rerun my code or refresh the page. I tried writing this logic many ways but I can't seem to get it to work. Thank you in advance.
async function getAPI(api_url) {
const response = await fetch(api_url)
var data = await response.json()
var g = data.genres
let user_genre = document.getElementById("input").value
for (let i = 0; i < g.length; i++) {
console.log(g[i].name)
}
}
You can check response.ok to check if the fetch succeeded, or response.status for the exact HTTP status.
const response = await fetch(api_url);
if (!response.ok) location.reload();
you can repeat function
async function getAPI(api_url) {
const response = await fetch(api_url)
if(!response.ok) {
return getAPI(api_url)
}
var data = await response.json()
var g = data.genres
let user_genre = document.getElementById("input").value
for (let i = 0; i < g.length; i++) {
console.log(g[i].name)
}
}
I have the async function below. DeviceStatus is a Mongo Collection of devices which has the property SerialNumber defined. Here, I'm collecting an array of of all devices in the DeviceStatus Collection and then iterating through each of them. I'd like to add a new property called NumberOfRetries and set it to zero. The function below works for the first DeviceStatus document. That is, in the loop it updates the document and adds the NumberOfRetries for the first document but does not complete the update for the remaining documents. Does anyone know what changes are required here so that I'm able to update all Mongo documents in the DeviceStatus collection.
static async addNumberOfRetriesCap(){
const devices = await DeviceStatus.find().exec();
for(let i = 0; i < devices.length; i++){
let device = devices[i];
await DeviceStatus.updateOne({"SerialNumber": device.SerialNumber}, {$set: {"NumberOfRetries" : "0"}})
}
}
you can use the $in operator and updateMany functionality.
const addNumberOfRetriesCap = async() => {
const devices = await DeviceStatus.find({}, {
SerialNumber: true
}).exec();
let deviceSerialNumber = []
for (let i = 0; i < devices.length; i++) deviceSerialNumber.push(devices[i].SerialNumber)
await DeviceStatus.updateMany({
"SerialNumber": {
$in: deviceSerialNumber
}
}, {
$set: {
"NumberOfRetries": "0"
}
})
}
If you want to update all values of the collection DeviceStatus you can direclty update the db like in the following example
const addNumberOfRetriesCap = async() => {
await DeviceStatus.updateMany({}, {
$set: {
"NumberOfRetries": "0"
}
})
}
I have multiple sqlite tables and I want to fetch data from these tables together from Ionic 3 app for android and iOS platform and send it to server
Here's the code I wrote for this functionality
function fetchDataFromSqlite(){
let tableNames = ['table1','table2','table3'];
var dataFromSQLite= {};
for (var i = 0; i < tableNames.length; i++)
{
let tableName = tableNames[i];
let queryParams = [1];
let query = `SELECT * FROM ${tableName} WHERE status= ?`;
this.databaseprovider.SelectQry(query, queryParams).then(data => {
dataFromSQLite[tableName] = data;
});
return dataFromSQLite;
}
}
Above function is what I wrote to perform this action. I call this function in my app component
SelectQry() in databaseprovider looks like this:
SelectQry(sql, params) {
return this.database.executeSql(sql, params).then(
data => {
return data;
},
err => {
console.log("Error: ", JSON.stringify(err));
return [];
}
);
}
When I alert() the data returned from fetchDataFromSqlite() , I get {}
Can anyone tell me what I'm doing wrong or why I'm not getting any output when there's data in SQLite tables?
I have an ajax request that adds one or multiple records to a table (the following code is server side):
app.post('/saveLesson',function(req, res) {
let sections = JSON.parse(req.body.sections);
let sql = 'INSERT INTO sections (title, content, duration) VALUES ';
for (let i = 0; i < sections.length; i++) {
if (i == sections.length-1) {
sql += '("' + sections[i].title + '","' + sections[i].content + '","' + sections[i].duration + '");';
} else {
sql += '("' + sections[i].title + '","' + sections[i].content + '","' + sections[i].duration + '"),';
}
}
connection.query(sql,
function (error, result) {
if (error) throw error;
});
});
I want to prevent SQL injection, but I'm not sure how to do it for multiple records.
Generally I know that I would have to build my sql statement as follows:
connection.query("SELECT * FROM bank_accounts WHERE dob = ? AND bank_account = ?",
[
req.body.dob,
req.body.account_number
],
function(error, results) {
}
);
But I'm not sure how to achieve this with multiple records (not knowing how many they are). Is the .query parameter just a regular array?
Also, I need to store somewhere the created IDs and send them back to the client page. How can I achieve this? Thank you.
***************************** UPDATE *****************************
Although someone posted a solution, I thought this might be useful. With the following code you can add multiple records preventing SQL injections.
app.post('/saveLesson',function(req, res) {
let sections = JSON.parse(req.body.sections);
console.log(sections);
let sql = 'INSERT INTO sections (title, duration, content) VALUES ';
// I make a new array to pass the list of values to the query
let sectionsParamList = [];
for (let i = 0; i < sections.length; i++) {
if (i == sections.length-1) {
sql += '(?,?,?);';
} else {
sql += '(?,?,?),';
}
sectionsParamList.push(sections[i].title);
sectionsParamList.push(sections[i].duration);
sectionsParamList.push(sections[i].content);
}
connection.query(sql, sectionsParamList,
function (error, result) {
if (error) throw error;
});
});
The way MySQL works, when you perform a multi-row INSERT operation like the one you propose, you only get back the automatically generated unique id of the last row inserted. It shows up in your result object as result.insertId. Don't try to guess the id values of the other rows, by subtraction for example, because there's no guarantee of that.
The fact that you need that id for every row you insert means you should not use a multi-row insert, but rather a sequence of single row inserts. That neatly solves your SQL injection issue too.
But you will have to figure out how to do a sequence of INSERT operations. You may want to do it with an async / await / promise setup. Something like this, not debugged.
/* do one insert with a Promise so you can await it */
function doInsert (section, connection) {
const values = [section.title, section.content, section.duration];
return new Promise( function ( resolve, reject ) {
const sql = "INSERT INTO sections (title, content, duration) VALUES (?,?,?);"
connection.query (sql, values, function ( error, result ) {
if (error) reject (error)
resolve (result.insertId)
} )
} )
}
/* do all the inserts, awaiting each one */
async function doInserts (sections, connection) {
let ids = []
for (let i = 0; i < sections.length; i++) {
const id = await doInsert (sections[i], connection)
ids.push(id)
}
return ids
}
/* handle your post */
app.post('/saveLesson',function(req, res) {
let sections = JSON.parse(req.body.sections)
/* get the result back from an async function with .then / .catch */
doInserts (sections, connection)
.then (function (resultIds) {
/* respond with the id values in a JSON object */
res.status(200).json(resultIds)
} )
.catch ( function (error) {
/* respond with an error */
res.status(500).json(error)
} )
} )
These async / await and Promise language constructions are really worth your trouble to learn, if you don't already know them.