I'm wondering if I implemented Promises correctly with MySQL statements. The problem is that it will timeout waiting for the query results. I find this strange because I implemented the code the same way using callback functions and it never timed out. I think I'm using Promises incorrectly but can't figure out how. I confirmed that the code using Promises does work on retrieving small datasets (hundreds), but times out when retrieving thousands of records.
Using Promises
function innermostRetrieveDependencyByResource(dep_name_version){
var sql = "SELECT * FROM dependent WHERE name_version = '" + dep_name_version + "' LIMIT 1";
return database.query(sql).then(result => {
return result;
})
}
function innerRetrieveDependencyByResource(scan_id){
var depList = [];
var sql = "SELECT DISTINCT dep_name_version FROM branch WHERE scanid = '" + scan_id + "'";
return database.query(sql).then(result => {
promiseList = [];
for (r in result){ // iterate over branch scans
dep_name_version = result[r];
promiseList.push(innermostRetrieveDependencyByResource(dep_name_version))
}
return Promise.all(promiseList).then(result => {
return result;
})
})
}
function retrieveDependencyByResource(resource, cb){
promiseList = []
var sql = "SELECT (scanid) FROM scan WHERE resource = '" + resource + "'";
database.query(sql).then(result => { // note: query returns a Promise
var scanPending = result.length;
for (s in result){
scan_id = result[s].scanid;
promiseList.push(innerRetrieveDependencyByResource(scan_id))
}
Promise.all(promiseList).then(result => {
cb(result);
})
})
}
Using Callbacks:
function retrieveDependency(projname, branch, cb){
depList = []
var sql = "SELECT (scanid) FROM scan WHERE project = '" + projname + "' AND branch = '" + branch + "'";
connection.query(sql, function (err, result) { // note: query returns a Promise
if (err) throw err;
scan_id = result[0].scanid;
var sql = "SELECT DISTINCT dep_name_version FROM branch WHERE scanid = '" + scan_id + "'";
connection.query(sql, function (err, result) {
if (err) throw err;
pending = result.length;
for (r in result){
dep_name_version = result[r].dep_name_version;
var sql = "SELECT * FROM dependent WHERE name_version = '" + dep_name_version + "'";
connection.query(sql, function(err, result){
if (err) throw err;
depList.push(result);
if ( 0 === --pending ){
cb(depList);
}
})
}
});
});
}
For proper query promises, use mysql2/promise or mysql21.
Also, I'd recommend loading results with a single query:
// with mysql21
let query = `SELECT scanid, dep_name_version, dependent*
FROM scan
JOIN branch USING (scanid)
JOIN dependent ON (name_version = dep_name_version)
WHERE resource = ?`;
const retrieveDependencyByResource = async (resource) =>
connection.query(query, [resource]);
// await results
let dependencies = await retrieveDependencyByResource(resource));
// or use then
retrieveDependencyByResource(resource)
.then(dependencies => {
...
});
Related
I have this function:
var arrayOfNumbers = []
var getTexts = new cronJob('45 * * * * *', function() {
var viewConformationEmails = "select * from clients";
ibmdb.open(ibmdbconn, function(err, conn) {
if (err) return console.log(err);
conn.query(viewConformationEmails, function(err, rows) {
if (err) {
console.log(err);
} else if (!err) {
console.log("Success")
}
for (var i = 0; i < rows.length; i++) {
// arrayOfNumbers.push(rows[i].NAME)
arrayOfNumbers.push(rows[i]["PHONE_NUMBER"])
var stringg = rows[i]["MINUTE"] + " " + rows[i]["HOUR"] + " * " + "* " + "*"
var textJob = new cronJob(stringg, function() {
client.messages.create({
to: arrayOfNumbers[i],
from: '12055578708',
body: 'Your training session just finished.'
}, function(err, data) {});
}, null, true);
}
conn.close(function() {
// console.log("closed the function /login");
});
});
});
}, null, true)
what it does is loop through my DB table clients, and then what I want it to do is in the loop grab the phone number, as well as the minute and hour in which the user needs a text message sent to them. However, what I need it to do is ONLY SEND THE MESSAGE TO THE PHONE_NUMBER that has the time slot at x time.
So if the db returns 5 people, but only 1 of those people have a time slot at 9am, then how do i only save that one person, or check for it.
This is a relativley general question to fixing returned loop data so I am just trying to figure out how to fix this. Any ideas?
I have the code below, from a REST API, that inserts data in Mysql. I use Node and Express (this is, in fact, my 1st Node project, so please bear in mind I don't understand much about Node).
What I need is that response to client (browser, web application, Postman for testing or whatever access to the API) is returned only when the forEach loop and data insertion into DB terminates, so I get a JSON object with the list error messages, if any.
I've been hitting my head on the wall for half a day, and this is what I got so far.
var wait=require('wait.for');
var async = require('async');
var Promise = require('promise');
var Q = require('q');
var errmsg = [];
router.route('/subscriber').post((req, res, callback) => {
const data = req.body;
var subscriberCollection = data;
this.errmsg = [];
let asyncCall =
(async () => {
let rr = await new Promise (resolve => subscriberCollection.forEach(function (value, key){
var phoneNumber = value.phoneNumber;
var msg = "";
if (phoneNumber == ""){
msg = "ERROR","missing phoneNumber for subscriber index #" + key + ";Skipping";
console.log(msg);
errmsg[key] = msg
return;
}
var sql = "call insertSubscriber(?)";
console.log("INFO",`Inserting subscriber ${phoneNumber} index ${key}`);
connection.query(sql,[ phoneNumber ] ,function (err, data) {
if (err){
var msg = err.errno + " - " + err.sqlMessage;
console.log("ERROR" , msg);
errmsg[key] = msg;
}
});
}) //end forEach
); //end Promise
})();
asyncCall.then(console.log("ENDING!!") ); // THIS IS NOT WORKING
});
On the console, I get this:
INFO Inserting 916311145 for index 0
INFO Inserting 916311146 for index 1
ENDING!!
ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'
ERROR 1062 - Duplicate entry '916311146' for key 'phoneNumber_UNIQUE'
but what I need it to be is:
INFO Inserting 916311145 for index 0
INFO Inserting 916311146 for index 1
ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'
ERROR 1062 - Duplicate entry '916311146' for key 'phoneNumber_UNIQUE'
ENDING!!
Also, when all subscriber objects are saved on DB, I need to return a response to client, something like:
[{"key 0" : "ok"},{"key 1" : "ok"}, {"key 3": "ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'"}...]
and again, the response should only appear when all processing has finished.
How can I get this work?
Hmm try this:
var wait = require('wait.for');
var async = require('async');
var Promise = require('promise');
var Q = require('q');
router.route('/subscriber').post(async (req, res, callback) => {
const data = req.body;
var subscriberCollection = data;
const response = await Promise.all(
subscriberCollection.map(function (value, key) {
var phoneNumber = value.phoneNumber;
var msg = '';
const obj = {};
if (phoneNumber == '') {
msg = 'ERROR missing phoneNumber for subscriber index #' + key + ';Skipping';
console.log(msg);
obj[key] = msg;
Promise.resolve(obj);
return;
}
var sql = 'call insertSubscriber(?)';
console.log('INFO', `Inserting subscriber ${phoneNumber} index ${key}`);
return new Promise((resolve) => {
connection.query(sql, [phoneNumber], function (err, data) {
if (err) {
var msg = 'ERROR' + err.errno + ' - ' + err.sqlMessage;
console.log(msg);
obj[key] = msg;
resolve(obj);
return;
}
obj[key] = 'ok';
resolve(obj);
});
});
}) //end forEach
); //end Promise
console.log('ENDING!!');
res.send(response);
});
I am trying to make an ajax call to a node page, which performs some operations to the database, based on the parameters passed in the ajax calls.
The issue I am facing is that the control is passed back to the html page(I am using an alert to test the request completion) which initiated the ajax call even before all the DB transactions are completed on the nodejs end.
I have added a smaller version of the code for better understanding.
Origin: HTML page
<div id="paramList">
<form action='import/other' method='POST' style='display:none;' id="h_form">
<label><input type="checkbox" value="opt1" />Option 1</label>
<label><input type="checkbox" value="opt2" />Option 2</label>
<label><input type="checkbox" value="opt3" />Option 3</label>
<label><input type="checkbox" value="opt4" />Option 4</label>
<label><input type="checkbox" value="opt5" />Option 5</label>
</form>
</div>
<button onclick="startValidation()">Submit</button>
Handler: JS
function startValidation() {
var paramlist = '';
$("#paramList input[type='checkbox']").each(function() {
if ($(this).is(":checked")) {
paramlist += "'" + $(this).val() + "',";
}
});
paramlist = paramlist.substr(0, paramlist.length - 1);
var req = $.ajax({
url: '/import/validate',
type: 'POST',
data: { paramList: paramlist, fileName: finalName }
})
req.done(function(data, status) {
alert('Data validated successfully. Select Ok to go to validate sheet: ' + status);
var form = $("#h_form");
form.submit();
//redirect to the other page and load data from the latest insert
});
req.fail(function(xOptions, textStatus) {
alert('Error occured!: ' + textStatus);
});
}
Server side script
NodeJS: Router
router.post('/validate',function(req,res){
var paramlist = req.body.paramList;
var fileName = req.body.fileName;
var cli = modImport.parseFile(fileName,paramlist);
res.send(cli);
});
//Import module: modImport.js
module.exports.executeQuery = function(strSQL, operationType, tableName, cb, param) {
logger.log('debug','running query: '+strSQL);
var request = new sql.Request(connection);
request.query(strSQL,function(err, recordset) {
if(err){
logger.error('ERROR in '+operationType+' ON '+tableName+': '+err);
}
logger.info(operationType+' ON '+tableName+' successful!');
if(cb){
cb(param);
}
});
},
module.exports.parseFile: function(filePath, validateParam){
sql.connect(config).then(function() {
var arr = [];
arr.push(data);arr.push(validateParam);
arr.push(tName);
var delQ = "DELETE FROM [Tbl_TempData]";
util.executeQuery(delQ,'DELETION','[Tbl_TempData]', module.exports.bulkImportIntoTempData, arr);
console.log("deletion completed");
}).catch(function(err) {
logger.error('other error: '+err);
});
},
module.exports.bulkImportIntoTempData: function(arr){
var data = arr[0];
var validateParam = arr[1];
var tName = arr[2];
var bInQ = "INSERT INTO [Tbl_TempData] (field1,field2,field3) VALUES ";
data.forEach(function(rec, index){
var keys = Object.keys(rec);
var kLen = keys.length - 1;
keys.forEach(function(datum,cursor){
bInQ += "('" + datum + "','" + rec[datum] + "','" + tName + ")";
if(cursor < kLen){
bInQ += ",";
}
});
});
module.exports.executeQuery(bInQ,'BULK INSERTION','[Omni_TempSurveyData]',module.exports.processForTempCalc,validateParam);
},
module.exports.processForTempCalc: function(validateParam){
var strSQL = "DELETE FROM [Tbl_TempCalc]";
util.executeQuery(strSQL,'DELETION','[Tbl_TempCalc]',module.exports.insertIntoTempCalc,validateParam);
},
module.exports.insertIntoTempCalc: function(validateParam){
var strSQL = "INSERT INTO .....";
//some transformations here from other tables
util.executeQuery(strSQL,'INSERTION','[Tbl_TempCalcData]');
},
module.exports.insertIntoBlankCalc: function(validateParam){
var strSQL = "INSERT INTO .....";
util.executeQuery(strSQL,'INSERTION','[Tbl_BlankCalcData]');
//TODO: return the ajax call
}
After the ajax completion, the user is to be redirected to a page which is loading data from the last inserted table. But since the alert is popped up before final insertion, the redirected page is shown blank to the user.
Please suggest how to overcome this scenario.
if other functions work fine this will resolve the problem:
NodeJS: Router
router.post('/validate',function(req,res){
var paramlist = req.body.paramList;
var fileName = req.body.fileName;
var cli = modImport.parseFile(fileName,paramlist,function(){
res.send(cli);
});
});
//Import module: modImport.js
module.exports.executeQuery = function(strSQL, operationType, tableName, cb, param) {
logger.log('debug','running query: '+strSQL);
var request = new sql.Request(connection);
request.query(strSQL,function(err, recordset) {
if(err){
logger.error('ERROR in '+operationType+' ON '+tableName+': '+err);
}
logger.info(operationType+' ON '+tableName+' successful!');
if(cb){
cb(param);
}
});
},
module.exports.parseFile: function(filePath, validateParam,callback){
sql.connect(config).then(function() {
var arr = [];
arr.push(data);arr.push(validateParam);
arr.push(tName);
var delQ = "DELETE FROM [Tbl_TempData]";
util.executeQuery(delQ,'DELETION','[Tbl_TempData]', module.exports.bulkImportIntoTempData, arr);
console.log("deletion completed");
callback()
}).catch(function(err) {
logger.error('other error: '+err);
callback()
});
},
module.exports.bulkImportIntoTempData: function(arr){
var data = arr[0];
var validateParam = arr[1];
var tName = arr[2];
var bInQ = "INSERT INTO [Tbl_TempData] (field1,field2,field3) VALUES ";
data.forEach(function(rec, index){
var keys = Object.keys(rec);
var kLen = keys.length - 1;
keys.forEach(function(datum,cursor){
bInQ += "('" + datum + "','" + rec[datum] + "','" + tName + ")";
if(cursor < kLen){
bInQ += ",";
}
});
});
module.exports.executeQuery(bInQ,'BULK INSERTION','[Omni_TempSurveyData]',module.exports.processForTempCalc,validateParam);
},
module.exports.processForTempCalc: function(validateParam){
var strSQL = "DELETE FROM [Tbl_TempCalc]";
util.executeQuery(strSQL,'DELETION','[Tbl_TempCalc]',module.exports.insertIntoTempCalc,validateParam);
},
module.exports.insertIntoTempCalc: function(validateParam){
var strSQL = "INSERT INTO .....";
//some transformations here from other tables
util.executeQuery(strSQL,'INSERTION','[Tbl_TempCalcData]');
},
module.exports.insertIntoBlankCalc: function(validateParam){
var strSQL = "INSERT INTO .....";
util.executeQuery(strSQL,'INSERTION','[Tbl_BlankCalcData]');
//TODO: return the ajax call
}
if i am right the "modImport.parseFile" function does not return anything and just calls "util.executeQuery" in a promise.
so in this line "var cli = modImport.parseFile(fileName,paramlist);" cli is not valued and raises an error;
i think you should send "res.send(cli)" as callback for "modImport.parseFile" as parameter and then send it for "util.executeQuery" or you should change your functions
update:
i changed your code this way. so i can not run it but i guess it will works. hope it helps.
router.post('/validate',function(req,res){
var paramlist = req.body.paramList;
var fileName = req.body.fileName;
var GLOBAL.response_function = function(err,data)
{
//here you can do anything with your data or err;
// like that
if (err)
{
return res.status(404).send(err.toString());
}
else
{
// if your data is json
return res.status(200).send(JSON.stringify(data));
}
};
modImport.parseFile(fileName,paramlist) ;
});
module.exports.executeQuery = function(strSQL, operationType, tableName, cb, param) {
logger.log('debug','running query: '+strSQL);
var request = new sql.Request(connection);
request.query(strSQL,function(err, recordset) {
if(err){
logger.error('ERROR in '+operationType+' ON '+tableName+': '+err);
}
logger.info(operationType+' ON '+tableName+' successful!');
if(cb){
if (param.is_return_callback)
{
cb(err, recordset)
}
else
{
cb(param);
}
}
});
},
the point is you send this ",GLOBAL.response_function, {is_return_callback:true}" by last function which should execute query
module.exports.insertIntoTempCalc: function(validateParam){
var strSQL = "INSERT INTO .....";
//some transformations here from other tables
util.executeQuery(strSQL,'INSERTION','[Tbl_TempCalcData]',GLOBAL.response_function, {is_return_callback:true});
},
I'm currently trying to retrieve data from a sqlite query in node.js, the sql function is on a diferent file so i'm exporting it as a module and then call the function from the index.js. But when i try to retrieve the data the function returns a null value.
Here is my code
Index.js
var express = require("express");
var body_parser = require("body-parser");
var app = express();
var db = require('./dbhandler.js');
app.set("view engine", "jade");
app.get("/data",function(req,res){
let data = db.select();
res.send(data);
});
app.get("/",function(req,res){
res.render("index");
});
app.listen(8888);
dbhandler.js
var sqlite3 = require("sqlite3");
const file = "hr";
exports.select = function (){
var lista = [];
var db = new sqlite3.Database(file);
db.all("SELECT * FROM usuarios", function(err,rows){
let contador = 0;
rows.forEach(function (row) {
lista[contador] = row.nombre + ";" + row.cedula + ";" + row.edad + ";" + row.pais;
});
});
db.close();
return lista;
}
Node is asynchronous!!!. lista is returned from the module before the db.all function completes.
You either need to pass a callback into the select function or return a promise. The callback approach would look something like this:
exports.select = function (cb){
var lista = [];
var db = new sqlite3.Database(file);
db.all("SELECT * FROM usuarios", function(err,rows){
if(err) return cb(err);
let contador = 0;
rows.forEach(function (row) {
lista[contador] = row.nombre + ";" + row.cedula + ";" + row.edad + ";"
+ row.pais; });
db.close();
return cb(null, lists);
});
}
I have a node.js script which takes in SMDR data from a phone system, then inserts it into a database. The code that I have right now inserts everything except for OperatorID (which it inserts [object Object] into that column). I cannot figure out as to why this is happening. Any ideas?
var net = require('net'),
mysql = require('mysql'),
PORT = 1752,
HOST = '192.168.10.2',
pool = mysql.createPool({
host: 'localhost',
port: 3306,
user: 'root',
password: 'root',
database: 'mydb'
});
function connectionListener(conn) {
console.log('Listening for incoming calls...');
}
function logCall(phonenumber, operator) {
pool.getConnection(function(err, connection) {
if (!err) { // If no error exists
//var operID = '';
var opquery = connection.query('SELECT OperatorID FROM tblOperators WHERE Phone_Login = ' + operator, function(err, rows) {
if (err) {
console.error(err);
}
//operID = rows[0];
//console.log(operID);
console.log(rows[0]); //logs on console okay
var query = connection.query('INSERT INTO incoming_calls(phone_number, OperatorID) VALUES("' + phonenumber + '", "' + rows[0] + '") ON DUPLICATE KEY UPDATE OperatorID=OperatorID, date_created=NOW()', function(err, rows) {//fails to insert rows[0]
if (err) {
console.error(err);
}
connection.release();
});
});
} else {
console.error(err);
}
});
}
function processdata(data) {
var phonedata = data.toString().match(/([0-9]?)([0-9]{10})/),
operdata = data.toString().match(/([*])([0-9]{4})/);
if (phonedata !== null && operdata !== null) {
var phonenumber = phonedata[2],
oper = operdata[2];
oper = oper.replace('*', '');
//phonenumber = phonenumber.slice(0,3)+"-"+phonenumber.slice(3,6)+"-"+phonenumber.slice(6);
logCall(phonenumber, oper);
}
};
logCall('999-999-9999', '1203'); //where 1203 is valid
var conn = net.createConnection(PORT, HOST, connectionListener);
conn.on('data', processdata);
conn.setEncoding('utf8');
You need to specify
rows[0].OperatorID
in your INSERT query as you are trying to insert the whole row right now, which is an object.
var query = connection.query('INSERT INTO incoming_calls(phone_number, OperatorID) VALUES("' + phonenumber + '", "' + rows[0].OperatorID + '") ON DUPLICATE KEY UPDATE OperatorID=OperatorID, date_created=NOW()', function(err, rows) {//...