We Keep Coding

Wait for mysql to finish queries in a for loop before finishing a function in nodeJS

I am trying to run a query to get some data from a table, then use that array of data to get some data from another table to then return it as JSON.
I have been trying for a while but I cannot seem to figure out async and await. Right now it does sort of work but doesn't wait for my second query in the for loop to finish before returning data.
app.get("/get-top-trending", (request, response) => {
const req = request.query
let query = 'SELECT Ticker, Mentions FROM trend_data ORDER BY Date DESC, ' + req.by + ' DESC LIMIT 3';
let returnData = {};
cryptoDB.query(query, (err, tickers) => {
if (err) throw err;
getData(tickers).then(function() {
response.send(returnData)
});
});
async function getData(tickers) {
for (let i = 0; i < tickers.length; i++) {
cryptoDB.query('SELECT HistoricalJSON FROM historical_trend_data WHERE Ticker=? LIMIT 1', [tickers[i]['Ticker']], (err, rows2) => {
if (err) throw err;
returnData[tickers[i]['Ticker']] = rows2[0]['HistoricalJSON'];
});
}
}
});
I assume that something has to be done in the getData async function, however I am not particularly sure how to implement a working solution. I have tried promises but they don't seem to work the way that I expect.
Any guidance would be appreciated.

first solution:
app.get("/get-top-trending", (request, response) => {
const req = request.query
let query = 'SELECT Ticker, Mentions FROM trend_data ORDER BY Date DESC, ' + req.by + ' DESC LIMIT 3';
cryptoDB.query(query, (err, tickers) => {
if (err) throw err;
getData(tickers).then(function (returnData) {
response.send(returnData)
});
});
async function getData(tickers) {
const returnData = {};
const querys = ((ticker) => {
return new Promise((resolve, reject) => {
cryptoDB.query('SELECT HistoricalJSON FROM historical_trend_data WHERE Ticker=? LIMIT 1', [ticker['Ticker']], (err, rows2) => {
if (err) reject(err);
returnData[ticker['Ticker']] = rows2[0]['HistoricalJSON'];
resolve();
});
})
})
for (let i = 0; i < tickers.length; i++) {
await querys(tickers[i]);
}
return returnData
}
});
second solution:
app.get("/get-top-trending", (request, response) => {
const req = request.query
let query = 'SELECT Ticker, Mentions FROM trend_data ORDER BY Date DESC, ' + req.by + ' DESC LIMIT 3';
cryptoDB.query(query, (err, tickers) => {
if (err) throw err;
getData(tickers).then(function(returnData) {
response.send(returnData)
}).catch(error => throw error);
});
async function getData(tickers) {
let returnData = {};
for (let i = 0; i < tickers.length; i++) {
returnData[tickers[i]['Ticker']] = await getTickerQuery([tickers[i]['Ticker']]);
}
return returnData;
}
function getTickerQuery(ticker) {
return new Promise((resolve, reject) => {
cryptoDB.query('SELECT HistoricalJSON FROM historical_trend_data WHERE Ticker=? LIMIT 1', ticker, (err, rows2) => {
if (err) throw reject(err);
resolve(rows2[0]['HistoricalJSON']);
});
})
}
});
I recommend second solution for readability

Nodejs api structure on calling sql helpers inside another helper all called by a controller

I'm studying to create a simple API with mysql. I've understood and implemented the simple structure in which the app call the router, that call the controller, that call the service. But now i'm developing a multiple tag service module and I've realized that I need to call the same sql queries services declared in it. I show you the code for a better understanding:
tag_service.js:
const mysql = require("../../config/database");
module.exports = {
insertTags: async (data, callBack) => {
const connection = await mysql.connection();
let results = '';
const tagsArray = data.tags.map(tag => [data.id_manager,data.cod_table,data.id_record,tag])
try {
//console.log("at insertCallout...");
await connection.query("START TRANSACTION");
results = await connection.query(
`INSERT INTO s_com_tags (id_manager,cod_table,id_record,tag)
VALUES (?,?,?)`,
[tagsArray]
);
await connection.query("COMMIT");
} catch (err) {
await connection.query("ROLLBACK");
//console.log('ROLLBACK at insertCallout', err);
throw err;
} finally {
await connection.release();
return callBack(null, results);
}
},
deleteTags: async (data, callBack) => {
//console.log(data);
let results = '';
const connection = await mysql.connection();
try {
//console.log("at deleteCallouts...");
await connection.query("START TRANSACTION");
results = await connection.query(
`DELETE FROM s_com_tags
WHERE cod_table = ? AND id_record = ? AND tag IN (?)`,
[data.code_table, data.id_record,data.tags]
);
//console.log(res);
await connection.query("COMMIT");
} catch (err) {
await connection.query("ROLLBACK");
//console.log('ROLLBACK at deleteCallouts', err);
throw err;
} finally {
await connection.release();
return callBack(null, Callouts);
}
},
};
controller's structure that will use the service:
module.exports = {
updateLabDesc: async (req, res, next) => {
try {
const body = req.body;
if(!body.internal_code){
updateLabDesc(body.manager, async (err, results) => {
if (err) {
return next(createError.InternalServerError())
}
});
}
updateTags(body, async (err, results) => {
if (err) {
return next(createError.InternalServerError())
}
return res.json({
success: (results ? 1 : 0 ),
message: (results || 0) + " LabDesc inserted successfully"
});
});
} catch (error) {
next(error)
}
},
};
But the update is something like
updateTag function => {
try {
const current_tags = await getTags(req.body);
let newTags = [];
let oldTags = [];
req.body.tags.forEach(tag => {
if(!current_tags.includes(tag))
newTags.push(tag)
});
await insertTags(newTags);
current_tags.tags.forEach(tag => {
if(!req.body.tags.includes(tag))
oldTags.push(tag)
});
await deleteTags(oldTags);
} catch (error) {
next(error)
}
},
Basically, the tag_service has insertTags and deleteTags but I need the updateTags to call these functions as well. The final controller will call insertTags, deleteTags and updateTags. How can I structure these calls?
It is a controller that could call 2 helpers (insertTag and deleteTags) and another helper (updateTags) that call these 2 helpers. Any ideas?

Javascript/ Node.js not waiting for sql query to come back with a response before it continues the code

I have a .js file that calls an external .js file that runs the following code:
const sql = require('../../node_modules/mysql');
module.exports =
{
connect_to_db: function (sql_query)
{
let con = sql.createConnection({
host: "localhost",
user: config.server_username,
password: config.server_password,
database: config.database_name
});
con.connect((err)=> {
if (err){
console.log("Problem connecting to the DB!");
return;
}
console.log("Connected to the DB!");
});
con.query(sql_query, (err, result) => {
if (err) throw err;
console.log('Data received from the DB');
console.log(result);
return result;
});
con.end((err) => {});
}
};
Which is run like:
const connect_to_DB = require('DB_Connection');
let sql_query = "SELECT * FROM table";
database_results.push(connect_to_DB.connect_to_db(sql_query));
console.log(database_results);
however this results in the code finishing before the sql query comes back with a result (data removed):
[ undefined ]
Connected to the DB!
Data received from the DB
[ RowDataPacket {
mail_id: ,
from: ,
to: ',
subject: ,
message: ,
date:,
read_date: } ]
Process finished with exit code 0
It looks like the push of the result is coming back as undefined as there is nothing to push at the point it does this. However I want it to wait until the response from the query comes back before it continues.
I was thinking of a promise perhaps but not sure if that would work something like:
const sql = require('../../node_modules/mysql');
module.exports =
{
connect_to_db: function (sql_query)
{
return new Promise((resolve, reject) => {
(async () => {
let con = sql.createConnection({
host: "localhost",
user: config.server_username,
password: config.server_password,
database: config.database_name
});
con.connect((err)=> {
if (err){
console.log("Problem connecting to the DB!");
return;
}
console.log("Connected to the DB!");
});
con.query(sql_query, (err, result) => {
if (err) throw err;
console.log('Data received from the DB');
console.log(result);
resolve();
return result;
});
con.end((err) => {});
})();
});
}
};
but when I run this I get this back:
[ Promise { <pending> } ]
I just need some help in order for the result to come back then the code to continue.

According to my view, The best possible way of solving this is by using callbacks in Node js.
Node js executes codes sychronously, let me explain by explaining what happened at your code
database_results.push(connect_to_DB.connect_to_db(sql_query));
console.log(database_results);
Here in your code console.log(database_results) is returned before executing the function connect_to_DB.connect_to_db(sql_query))
Your DBConnection.js can be modified as:
const sql = require('mysql');
exports.connect_to_db = function (sql_query, callback) {
let con = sql.createConnection({
host: "localhost",
user: config.server_username,
password: config.server_password,
database: config.database_name
});
con.connect((err) => {
if (err) {
console.log("Problem connecting to the DB!");
return;
}
console.log("Connected to the DB!");
});
con.query(sql_query, (err, result) => {
if (err) callback(err);
console.log('Data received from the DB');
console.log(result);
callback(result);
});
con.end((err) => { });
};
and the external js that calls the function connect_to_db function can be modified as:
'use strict';
const connect_to_DB = require('DB_Connection');
let sql_query = "SELECT * FROM table";
connect_to_DB.connect_to_db(sql_query, function (data, err) {
if (err) {
console.log(err);
}
else {
database_results.push(data);
}
});
console.log(database_results);
to know more about callbacks visit

You don't need to use promises and async/await in the same piece of code. Try something like this:
module.exports =
{
connect_to_db: async function (sql_query)
{
let con = sql.createConnection({
host: "localhost",
user: config.server_username,
password: config.server_password,
database: config.database_name
});
con.connect((err)=> {
if (err){
console.log("Problem connecting to the DB!");
return;
}
console.log("Connected to the DB!");
});
return await con.query(sql_query);
}
};
and then
const connect_to_DB = require('DB_Connection');
let sql_query = "SELECT * FROM table";
database_results.push(await connect_to_DB.connect_to_db(sql_query));
console.log(database_results);
Note that sice await keyword id only allowed inside async functions, this line database_results.push(await connect_to_DB.connect_to_db(sql_query)); should be inside an async function to work

Node mysql based module not working

I have a node based app that will look into a database for data. Because the database is fairly large with several tables, I am writing a module to help modularize the task. The problem is that I cannot get the main code to return all the data from the database lookup because I believe the program exits before it is executed. How do I get my node module working? My intention is to have the DB helper functions reside in the SomethingHelper.js module. I have the main code silly.js that looks like this:
// silly.js
var sh = require('./SomethingHelpers.js');
helper = new sh();
helper.then(function(res) {
var promise = helper.getAllForUsername('sonny');
promise.then(function(res) {
console.log('worked', res);
});
promise.catch(function(err) {
console.log('err: ', err);
});
});
helper.catch(function(err) {
console('Could not create object: ', err);
});
SomethingHelpers.js looks like this:
var mysql = require("mysql");
function SomethingHelpers() {
return new Promise(function(resolve, reject) {
this.connectionPool = mysql.createPool({
connectionLimit: 100,
host: 'server.somewhere.com',
user: "username",
password: "somepass",
database: "sillyDB",
debug: false
});
});
}
SomethingHelpers.prototype.getAllSomethingForUsername = function(username) {
var result = [];
return new Promise(function(resolve, reject) {
this.connectionPool.getConnection(function(err, connection) {
if (err) {
console.log('Error connecting to the silly database.');
return;
} else {
console.log('Connection established to the silly database. Super-Duper!');
return connection.query('SELECT something FROM somethingTable where username=\"' + username + '\"',
function(err, rows, field) {
connection.release();
if (!err) {
//console.log (rows.something);
rows.forEach(function(item) {
var allSomething = JSON.parse(item.something);
console.log(allSomething.length);
result.push(allSomething);
for (var i = 0; i < allSomething.length; i++) {
console.log(allSomething[i].handle);
}
console.log('\n\n');
});
console.log('Done');
return result;
} else {
console.log('Eeeeeeeek!');
//console.log (result);
return result;
}
});
}
});
});
} // End of getAllSomething ()
module.exports = SomethingHelpers;

I figured out the answer to my own question. Here's how I solved it. First, SomethingHelpers.js:
//SomethingHelpers.js
var mysql = require('promise-mysql');
function SomethingHelpers () {
this.pool = mysql.createPool({
connectionLimit: 100,
host: 'server.somewhere.com',
user: "username",
password: "somepass",
database: "sillyDB",
debug: false
});
}
SomethingHelpers.prototype.getAllSomethingsForThisUsername = function (username) {
let pool = this.pool;
return new Promise(function (resolve, reject) {
pool.getConnection ().then(function(connection) {
connection.query('SELECT something FROM somethingsTable where username=\"'+
username+'\"').
then (function (rows) {
resolve (getAllSomethings (rows));
}).catch (function (error) {
console.log ('Error: ', error);
});
});
});
}
function getAllSomethings (rows)
{
var result = [];
rows.forEach (function (item) {
var allSomethings = JSON.parse(item.something);
result.push (allSomethings);
});
return result;
}
module.exports = SomethingHelpers;
With the glory of Promise, the bounty from the helper module can be enjoyed thusly:
//silly.js
var hh = require ('./SomethingHelpers');
helper = new hh ();
thePromiseOfSomething = helper.getAllSomethingsForThisUsername ('sonny');
thePromiseOfSomething.then(function (rows) {
console.log (rows);
});
Thus releasing me from the tyranny of asynchronous thinking (J/K).

howto Node module.exports

I want to separate the code for executing mysql query in Node, so I am trying to use the Revealing Module pattern here
/* pool -create connection pool mysql*/
var sqlQuery = function (sqlString) {
var _result = pool.getConnection(function (err, connection) {
/* error handling*/
connection.query(sqlString, function (err, rows) {
console.log(this.sql);
if (!err) {
return _result = rows; <============
}
connection.release();
});
return;
});
console.log(_result);
return { recordSet : _result }
};
module.exports = sqlQuery;
How can I return rows to my app.js. the code below for calling sqlQuery is not working
var SqlQuery = require(./path/to/sqlQueryFile);
var rows = SqlQuery('pass sql here').recordSet;
console.log(row);
res.json(rows);

Your code is asynchronous, but you're calling it synchronously.
If you wanted to do it like this, you'll also need to pass a callback to SqlQuery.
/* pool -create connection pool mysql*/
var sqlQuery = function (sqlString, callback) {
var _result = pool.getConnection(function (err, connection) {
/* error handling*/
connection.query(sqlString, function (err, rows) {
console.log(this.sql);
if (!err) {
callback(rows);
}
connection.release();
});
});
};
module.exports = sqlQuery;
And then call it with:
var SqlQuery = require(./path/to/sqlQueryFile);
var rows = SqlQuery('pass sql here', function(recordSet){
console.log(recordSet);
res.json(recordSet);
});
Edit: If you're using newer versions of JavaScript, you have a few more options.
If you have access to Promises, you can do this:
function sqlQuery (sqlString) {
return new Promise((resolve, reject) => {
pool.getConnection(function (err, connection) {
if (err) { return reject(err); } // error handling
connection.query(sqlString, function (err, rows) {
if (err) { return reject(err); }
resolve(rows);
connection.release();
});
});
});
}
module.exports = sqlQuery;
And then you'd use it like:
var SqlQuery = require(./path/to/sqlQueryFile);
SqlQuery('pass sql here')
.then(function(recordSet) {
console.log(recordSet);
res.json(recordSet);
})
.catch(function(err) {
// do your error handling
res.status(500).json({ err: 'Sorry there was an error' });
});
If you're using even newer JavaScript, you can use the async/await syntax (currently available via Babel, and I think in FireFox. Chrome in V55).
var SqlQuery = require(./path/to/sqlQueryFile);
async handleQuery(query) {
try {
var rows = await SqlQuery(query);
res.json(rows);
} catch (e) {
console.log('Error!', e);
}
}
To chain multiple queries together:
async handleQuery(query) {
try {
return await SqlQuery(query);
} catch (e) {
console.log('Error!', e);
}
}
var rows = await handleQuery('select * from tablename');
var rowsToReturn = await handleQuery('select id from another_table where name = "' + rows[0].name + '"');