We Keep Coding

Wait for mysql to finish queries in a for loop before finishing a function in nodeJS

I am trying to run a query to get some data from a table, then use that array of data to get some data from another table to then return it as JSON.
I have been trying for a while but I cannot seem to figure out async and await. Right now it does sort of work but doesn't wait for my second query in the for loop to finish before returning data.
app.get("/get-top-trending", (request, response) => {
const req = request.query
let query = 'SELECT Ticker, Mentions FROM trend_data ORDER BY Date DESC, ' + req.by + ' DESC LIMIT 3';
let returnData = {};
cryptoDB.query(query, (err, tickers) => {
if (err) throw err;
getData(tickers).then(function() {
response.send(returnData)
});
});
async function getData(tickers) {
for (let i = 0; i < tickers.length; i++) {
cryptoDB.query('SELECT HistoricalJSON FROM historical_trend_data WHERE Ticker=? LIMIT 1', [tickers[i]['Ticker']], (err, rows2) => {
if (err) throw err;
returnData[tickers[i]['Ticker']] = rows2[0]['HistoricalJSON'];
});
}
}
});
I assume that something has to be done in the getData async function, however I am not particularly sure how to implement a working solution. I have tried promises but they don't seem to work the way that I expect.
Any guidance would be appreciated.

first solution:
app.get("/get-top-trending", (request, response) => {
const req = request.query
let query = 'SELECT Ticker, Mentions FROM trend_data ORDER BY Date DESC, ' + req.by + ' DESC LIMIT 3';
cryptoDB.query(query, (err, tickers) => {
if (err) throw err;
getData(tickers).then(function (returnData) {
response.send(returnData)
});
});
async function getData(tickers) {
const returnData = {};
const querys = ((ticker) => {
return new Promise((resolve, reject) => {
cryptoDB.query('SELECT HistoricalJSON FROM historical_trend_data WHERE Ticker=? LIMIT 1', [ticker['Ticker']], (err, rows2) => {
if (err) reject(err);
returnData[ticker['Ticker']] = rows2[0]['HistoricalJSON'];
resolve();
});
})
})
for (let i = 0; i < tickers.length; i++) {
await querys(tickers[i]);
}
return returnData
}
});
second solution:
app.get("/get-top-trending", (request, response) => {
const req = request.query
let query = 'SELECT Ticker, Mentions FROM trend_data ORDER BY Date DESC, ' + req.by + ' DESC LIMIT 3';
cryptoDB.query(query, (err, tickers) => {
if (err) throw err;
getData(tickers).then(function(returnData) {
response.send(returnData)
}).catch(error => throw error);
});
async function getData(tickers) {
let returnData = {};
for (let i = 0; i < tickers.length; i++) {
returnData[tickers[i]['Ticker']] = await getTickerQuery([tickers[i]['Ticker']]);
}
return returnData;
}
function getTickerQuery(ticker) {
return new Promise((resolve, reject) => {
cryptoDB.query('SELECT HistoricalJSON FROM historical_trend_data WHERE Ticker=? LIMIT 1', ticker, (err, rows2) => {
if (err) throw reject(err);
resolve(rows2[0]['HistoricalJSON']);
});
})
}
});
I recommend second solution for readability

Nodejs api structure on calling sql helpers inside another helper all called by a controller

I'm studying to create a simple API with mysql. I've understood and implemented the simple structure in which the app call the router, that call the controller, that call the service. But now i'm developing a multiple tag service module and I've realized that I need to call the same sql queries services declared in it. I show you the code for a better understanding:
tag_service.js:
const mysql = require("../../config/database");
module.exports = {
insertTags: async (data, callBack) => {
const connection = await mysql.connection();
let results = '';
const tagsArray = data.tags.map(tag => [data.id_manager,data.cod_table,data.id_record,tag])
try {
//console.log("at insertCallout...");
await connection.query("START TRANSACTION");
results = await connection.query(
`INSERT INTO s_com_tags (id_manager,cod_table,id_record,tag)
VALUES (?,?,?)`,
[tagsArray]
);
await connection.query("COMMIT");
} catch (err) {
await connection.query("ROLLBACK");
//console.log('ROLLBACK at insertCallout', err);
throw err;
} finally {
await connection.release();
return callBack(null, results);
}
},
deleteTags: async (data, callBack) => {
//console.log(data);
let results = '';
const connection = await mysql.connection();
try {
//console.log("at deleteCallouts...");
await connection.query("START TRANSACTION");
results = await connection.query(
`DELETE FROM s_com_tags
WHERE cod_table = ? AND id_record = ? AND tag IN (?)`,
[data.code_table, data.id_record,data.tags]
);
//console.log(res);
await connection.query("COMMIT");
} catch (err) {
await connection.query("ROLLBACK");
//console.log('ROLLBACK at deleteCallouts', err);
throw err;
} finally {
await connection.release();
return callBack(null, Callouts);
}
},
};
controller's structure that will use the service:
module.exports = {
updateLabDesc: async (req, res, next) => {
try {
const body = req.body;
if(!body.internal_code){
updateLabDesc(body.manager, async (err, results) => {
if (err) {
return next(createError.InternalServerError())
}
});
}
updateTags(body, async (err, results) => {
if (err) {
return next(createError.InternalServerError())
}
return res.json({
success: (results ? 1 : 0 ),
message: (results || 0) + " LabDesc inserted successfully"
});
});
} catch (error) {
next(error)
}
},
};
But the update is something like
updateTag function => {
try {
const current_tags = await getTags(req.body);
let newTags = [];
let oldTags = [];
req.body.tags.forEach(tag => {
if(!current_tags.includes(tag))
newTags.push(tag)
});
await insertTags(newTags);
current_tags.tags.forEach(tag => {
if(!req.body.tags.includes(tag))
oldTags.push(tag)
});
await deleteTags(oldTags);
} catch (error) {
next(error)
}
},
Basically, the tag_service has insertTags and deleteTags but I need the updateTags to call these functions as well. The final controller will call insertTags, deleteTags and updateTags. How can I structure these calls?
It is a controller that could call 2 helpers (insertTag and deleteTags) and another helper (updateTags) that call these 2 helpers. Any ideas?

How to troubleshoot response when using Query inside a For statement

I'm currently writing an API with a query that takes input as an array and then deletes it in turn via a for statement.
However, if it is not a normal request (ex) result.affectedRows === 0) the DB will generate an UnhandledPromiseRejectionWarning: Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client.
How can I solve this?
API CODE
exports.delete = (req, res, next) => {
const arr = req.body.arr
for (let i = 0; i < arr.length; i++) {
let sql = `DELETE FROM post WHERE postId = ?`;
Post.query(sql, [arr[i]], (err, result) => {
if (err) {
return next(err);
}
if (result.affectedRows === 0) {
res.send('Delete failed');
return;
}
});
}
res.send('Successfuly Delete');
};

That error is caused by your code sending multiple http responses, i.e. calling res.send more than once. This happens in your code because return is only breaking out of the callback function passed to Post.query, so the last res.send in the exports.delete function still gets called, causing the error.
You can restructure your code to something like this to avoid sending multiple responses:
function runQuery (sql, data) {
return new Promise((resolve, reject) => {
Post.query(sql, [data], (err, result) => {
if (err) {
reject(err);
} else if (result.affectedRows === 0) {
reject('Delete failed');
} else {
resolve();
}
});
})
}
exports.delete = (req, res, next) => {
const arr = req.body.arr
let sql = `DELETE FROM post WHERE postId = ?`;
let deletes = [];
for (let i = 0; i < arr.length; i++) {
deletes.push(runQuery(sql, arr[i]));
}
Promise.all(deletes).then(() => {
res.send('Successfuly Delete');
}).catch((err) => {
if(err === 'Delete failed') {
res.send(err);
} else {
next(err);
}
})
};

getrows method of the recordset object does not call in sequence

Calling Oracle SP which fills the data in the record set.
Fetching rows through the recordset getrows() method.
When called the getrows method and pass a function for fetching the inner methods always run in the end.
The first method calls the inner method, and the inner method received the data, and recordset is being send to inner function.
Example in below code functioncall method return empty data and then responseObj.push run. After this the getrows method process.
function lookups(req, res, next) {
rows = functioncall(context);
responesObj.push({ "Return Data": rows });
}
function simpleProcedureExecute(query, bindvars, opts = {}) {
return new Promise((resolve, reject) => {
oracledb.getConnection(
conn,
function (err, connection) {
if (err) throw err;
connection.execute(
query,
bindvars,
function (err, result) {
if (err) {
console.error(err.message);
reject(err);
}
procJson = [];
function processResultSet() {
console.log("internal method");
console.log(result.outBinds.result);
try {
result.outBinds.result.getRows(1000, function (err, rows) {
if (err) console.log(err);
if (rows.length) {
for (var i = 0; i < rows.length; i++) {
procJson.push({});
for (var j = 0; j < result.outBinds.result.metaData.length; j++) {
procJson[i][result.outBinds.result.metaData[j].name.toLowerCase()] = rows[i][j];
}
}
processResultSet();
return;
}
resultSet.close(function (err) {
if (err) console.error(err.message);
conn.release(function (err) {
if (err) console.error(err.message);
});
});
});
}
catch (err) {
console.log(err);
}
}
processResultSet();
}
);
}
);
resolve(procJson);
});
}

The most obvious issue is the timing of when you're resolving the promise - which is way too soon. You're invoking resolve outside of the call to oracledb.getConnection. You don't have the connection yet, haven't executed the query yet, and haven't collected the rows yet. You have to do all of that first, then invoke resolve and pass along the data.
This is one of the harder things to wrap your head around when you're new to Node.js. See this video, it may help some: https://www.youtube.com/watch?v=iAdeljxq_hs&t=0s&index=2&list=PL_lVOJzXeE__1Kh3ko0F-39-cpNLPXbJL
Also, see this series which covers different async patterns in Node.js. Most Node.js developers start with callbacks and then move to alternative patterns for async work: https://jsao.io/2017/06/how-to-get-use-and-close-a-db-connection-using-various-async-patterns/
Finally, here's an example of how one can iterate a result set with async/await:
const oracledb = require('oracledb');
const config = require('./dbConfig.js');
async function runTest() {
let conn;
let result;
try {
conn = await oracledb.getConnection(config);
result = await conn.execute(
'select * from all_objects where rownum < 100',
[],
{
resultSet: true
}
);
let row;
while (row = await result.resultSet.getRow()) {
console.log(row);
}
} catch (err) {
console.error(err);
} finally {
if (result && result.resultSet) {
try {
await result.resultSet.close();
} catch (err) {
console.error(err);
}
}
if (conn) {
try {
await conn.close();
} catch (err) {
console.error(err);
}
}
}
}
runTest();
Depending on the workload, it may be better to get more rows at once with getRows:
const oracledb = require('oracledb');
const config = require('./dbConfig.js');
async function runTest() {
let conn;
let result;
try {
conn = await oracledb.getConnection(config);
result = await conn.execute(
'select * from all_objects where rownum < 100',
[],
{
resultSet: true
}
);
let rows = await result.resultSet.getRows(50);
while (rows.length) {
for (let x = 0; x < rows.length; x += 1) {
console.log(rows[x]);
}
rows = await result.resultSet.getRows(50);
}
} catch (err) {
console.error(err);
} finally {
if (result && result.resultSet) {
try {
await result.resultSet.close();
} catch (err) {
console.error(err);
}
}
if (conn) {
try {
await conn.close();
} catch (err) {
console.error(err);
}
}
}
}
runTest();

After getting direction from Dan to use getrows directly without calling inner function. Find below my code to resolve it.
async function simpleProcedureExecute(query, bindvars, opts = {}) {
let rowss;
let conn;
let procJson = [];
try {
conn = await oracledb.getConnection();
result = await conn.execute(query, bindvars);
rowss = await result.outBinds.result.getRows(1000);
if (rowss.length) {
for (var i = 0; i < rowss.length; i++) {
procJson.push({});
for (var j = 0; j < result.outBinds.result.metaData.length; j++) {
procJson[i][result.outBinds.result.metaData[j].name.toUpperCase()] = rowss[i][j];
}
}
}
return procJson;
} catch (err) {
console.log(err);
} finally {
if (conn) { // conn assignment worked, need to close
try {
await conn.close();
} catch (err) {
console.log(err);
}
}
}
}

howto Node module.exports

I want to separate the code for executing mysql query in Node, so I am trying to use the Revealing Module pattern here
/* pool -create connection pool mysql*/
var sqlQuery = function (sqlString) {
var _result = pool.getConnection(function (err, connection) {
/* error handling*/
connection.query(sqlString, function (err, rows) {
console.log(this.sql);
if (!err) {
return _result = rows; <============
}
connection.release();
});
return;
});
console.log(_result);
return { recordSet : _result }
};
module.exports = sqlQuery;
How can I return rows to my app.js. the code below for calling sqlQuery is not working
var SqlQuery = require(./path/to/sqlQueryFile);
var rows = SqlQuery('pass sql here').recordSet;
console.log(row);
res.json(rows);

Your code is asynchronous, but you're calling it synchronously.
If you wanted to do it like this, you'll also need to pass a callback to SqlQuery.
/* pool -create connection pool mysql*/
var sqlQuery = function (sqlString, callback) {
var _result = pool.getConnection(function (err, connection) {
/* error handling*/
connection.query(sqlString, function (err, rows) {
console.log(this.sql);
if (!err) {
callback(rows);
}
connection.release();
});
});
};
module.exports = sqlQuery;
And then call it with:
var SqlQuery = require(./path/to/sqlQueryFile);
var rows = SqlQuery('pass sql here', function(recordSet){
console.log(recordSet);
res.json(recordSet);
});
Edit: If you're using newer versions of JavaScript, you have a few more options.
If you have access to Promises, you can do this:
function sqlQuery (sqlString) {
return new Promise((resolve, reject) => {
pool.getConnection(function (err, connection) {
if (err) { return reject(err); } // error handling
connection.query(sqlString, function (err, rows) {
if (err) { return reject(err); }
resolve(rows);
connection.release();
});
});
});
}
module.exports = sqlQuery;
And then you'd use it like:
var SqlQuery = require(./path/to/sqlQueryFile);
SqlQuery('pass sql here')
.then(function(recordSet) {
console.log(recordSet);
res.json(recordSet);
})
.catch(function(err) {
// do your error handling
res.status(500).json({ err: 'Sorry there was an error' });
});
If you're using even newer JavaScript, you can use the async/await syntax (currently available via Babel, and I think in FireFox. Chrome in V55).
var SqlQuery = require(./path/to/sqlQueryFile);
async handleQuery(query) {
try {
var rows = await SqlQuery(query);
res.json(rows);
} catch (e) {
console.log('Error!', e);
}
}
To chain multiple queries together:
async handleQuery(query) {
try {
return await SqlQuery(query);
} catch (e) {
console.log('Error!', e);
}
}
var rows = await handleQuery('select * from tablename');
var rowsToReturn = await handleQuery('select id from another_table where name = "' + rows[0].name + '"');