I have a node based app that will look into a database for data. Because the database is fairly large with several tables, I am writing a module to help modularize the task. The problem is that I cannot get the main code to return all the data from the database lookup because I believe the program exits before it is executed. How do I get my node module working? My intention is to have the DB helper functions reside in the SomethingHelper.js module. I have the main code silly.js that looks like this:
// silly.js
var sh = require('./SomethingHelpers.js');
helper = new sh();
helper.then(function(res) {
var promise = helper.getAllForUsername('sonny');
promise.then(function(res) {
console.log('worked', res);
});
promise.catch(function(err) {
console.log('err: ', err);
});
});
helper.catch(function(err) {
console('Could not create object: ', err);
});
SomethingHelpers.js looks like this:
var mysql = require("mysql");
function SomethingHelpers() {
return new Promise(function(resolve, reject) {
this.connectionPool = mysql.createPool({
connectionLimit: 100,
host: 'server.somewhere.com',
user: "username",
password: "somepass",
database: "sillyDB",
debug: false
});
});
}
SomethingHelpers.prototype.getAllSomethingForUsername = function(username) {
var result = [];
return new Promise(function(resolve, reject) {
this.connectionPool.getConnection(function(err, connection) {
if (err) {
console.log('Error connecting to the silly database.');
return;
} else {
console.log('Connection established to the silly database. Super-Duper!');
return connection.query('SELECT something FROM somethingTable where username=\"' + username + '\"',
function(err, rows, field) {
connection.release();
if (!err) {
//console.log (rows.something);
rows.forEach(function(item) {
var allSomething = JSON.parse(item.something);
console.log(allSomething.length);
result.push(allSomething);
for (var i = 0; i < allSomething.length; i++) {
console.log(allSomething[i].handle);
}
console.log('\n\n');
});
console.log('Done');
return result;
} else {
console.log('Eeeeeeeek!');
//console.log (result);
return result;
}
});
}
});
});
} // End of getAllSomething ()
module.exports = SomethingHelpers;
I figured out the answer to my own question. Here's how I solved it. First, SomethingHelpers.js:
//SomethingHelpers.js
var mysql = require('promise-mysql');
function SomethingHelpers () {
this.pool = mysql.createPool({
connectionLimit: 100,
host: 'server.somewhere.com',
user: "username",
password: "somepass",
database: "sillyDB",
debug: false
});
}
SomethingHelpers.prototype.getAllSomethingsForThisUsername = function (username) {
let pool = this.pool;
return new Promise(function (resolve, reject) {
pool.getConnection ().then(function(connection) {
connection.query('SELECT something FROM somethingsTable where username=\"'+
username+'\"').
then (function (rows) {
resolve (getAllSomethings (rows));
}).catch (function (error) {
console.log ('Error: ', error);
});
});
});
}
function getAllSomethings (rows)
{
var result = [];
rows.forEach (function (item) {
var allSomethings = JSON.parse(item.something);
result.push (allSomethings);
});
return result;
}
module.exports = SomethingHelpers;
With the glory of Promise, the bounty from the helper module can be enjoyed thusly:
//silly.js
var hh = require ('./SomethingHelpers');
helper = new hh ();
thePromiseOfSomething = helper.getAllSomethingsForThisUsername ('sonny');
thePromiseOfSomething.then(function (rows) {
console.log (rows);
});
Thus releasing me from the tyranny of asynchronous thinking (J/K).
Related
I've been slowly learning node.js so that I can integrate things better between a lot of our current ops (mysql) and Xero accounting.
I have several tables in mysql, one each for ["invoices","accounts","items","organisations","receipts","taxRates","users","trackingCategories"]
and each of those tables has a JSON column with the same name as the table it's in. This is NOT a json question.
Sending a query like ""select "+ wewant1[i] + " from "+wewant1[i]" is basically simple "select invoices from invoices" and easy to iterate through.
I can get the mysql results to list, but need to get each separate "list" as an array of results.
I would ultimately like to be able to reference the results from "select invoices from invoices" as an "invoices" array in node (invoices[0], invoices[1], etc).
I've tried avoiding "callback hell" and still cannot get this to work...
Any tips are very welcome!
Here is the current code:
var mysql = require('mysql');
var con = mysql.createConnection({
host: "10.0.1.103",
user: "badayaba",
password: "yadabuba",
database: "xeroetc"
});
(async function() {
let wewant1 = ["invoices", "accounts", "items", "organisations", "receipts", "taxRates", "users", "trackingCategories"];
function getmydata(sql, result, callback) {
var query = con.query(sql);
query.on('result', function(row) {
callback(null, row);
});
};
for (let i = 0; i < wewant1.length; i++) {
var sql = "select " + wewant1[i] + " from " + wewant1[i];
getmydata(sql, wewant1[i], function querydata(err, result) {
console.log(err || result);
return result;
});
};
con.end();
})();
20180910 22:00 GMT-6
Thanks Steven!
I think I got it with this:
const XeroClient = require('xero-node').AccountingAPIClient;
const XeroErrors = require('xero-node').XeroError;
var mysql = require('mysql');
const config = {
appType: "private",
consumerKey: "_____",
consumerSecret: "_____",
privateKeyPath: "../../../ssl/_____.pem"
};
var con = mysql.createConnection({
host: "_____",
user: "_____",
password: "_____",
database: "xeroetc"
});
(async function(){
let wewant1 = ["invoices","accounts","items","organisations","receipts","taxRates","users","trackingCategories"];
let xero = new XeroClient(config);
function getmydata(it, callback) {
var sql = "select "+it+" from "+it;
con.query(sql, function (err, result, fields) {
if (err) throw err;
callback(null,result);
});
};
const promises = wewant1.map(it => {
return new Promise((resolve, reject) => {
getmydata(it, function querydata(err, result) {
if (err) {
reject(err);
} else {
resolve(result);
}
});
})
});
Promise.all(promises)
.then(results => {
//results is a array of the resolved promises
invoices=results[0];
accounts=results[1];
items=results[2];
organisations=results[3];
receipts=results[4];
taxRates=results[5];
users=results[6];
trackingCategories=results[7];
console.log(invoices);
})
.catch(err => {})
.then(() => {
con.end();
})
})();
Map your array to promises, and then use Promise.all. This also solves your unstated problem of closing your connection too early.
var mysql = require('mysql');
var con = mysql.createConnection({
host: "10.0.1.103",
user: "badayaba",
password: "yadabuba",
database: "xeroetc"
});
let wewant1 = ["invoices", "accounts", "items", "organisations", "receipts", "taxRates", "users", "trackingCategories"];
function getmydata(sql, result, callback) {
var query = con.query(sql);
query.on('result', function(row) {
callback(null, row);
});
};
const promises = weweant1.map(it => {
return new Promise((resolve, reject) => {
getmydata(sql, it, function querydata(err, result) {
if (err) {
reject(err);
} else {
resolve(result);
}
});
})
});
Promise.all(promises)
.then(results => {
//results is a array of the resolved promises
})
.catch(err => {})
.then(() => {
// then after catch is a finally block
con.end();
})
Of course you can also use async/await and get rid of the then nesting. You can additionally promisify getmydata. All use the same principle though: Wait on an array of promises.
Using async/await
If getmydata returned a promise or was declared async, the following snippet would do, assuming it was in an async function. So much cleaner...
const promises = weweant1.map(it => getmydata(sql, it))
try {
const results = await Promise.all(promises)
} catch (e) {
// handle error
} finally {
con.end();
}
I have read other related posts and still am not understanding correctly how to use promises.
router.get('/', ensureAuthenticated, function(req, res) {
let promiseToGetResponses = new Promise(function(resolve, reject) {
var indexData = new getIndexData();
resolve(indexData);
console.log('received ' + indexData.length);
});
promiseToGetResponses.then(function(data) {
console.log('then data length ' + data.length);
res.render('index', {rsvpsIn: data});
}).catch(function() { });
});
Console shows this:
received undefined
then data length undefined
returned 1 *** this is from a console.log inside the getIndexData().
The function is getting the data, but my promise usage is not waiting for it.
Thanks.
P.S. I didn't know the getIndexData function was needed. Here it is:
function getIndexData(){
RSVP.find({response: 'in'}, function (err, data) {
if (err) throw err;
// This will be a list of all responses to show in the view
var rsvpsIn = [];
if (data.length > 0) {
// Use because the foreach loop below has async calls.
var responseCounter = data.length;
data.forEach(function(response) {
var foundUser = User.getUserById(response.userId, function(err, user) {
var newRSVP = {userName: user.username, notes: response.notes};
rsvpsIn.push(newRSVP);
// decrement and if we are done, return list
responseCounter -= 1;
if (responseCounter == 0) {
console.log('returned ' + rsvpsIn.length);
return rsvpsIn;
}
});
});
} else {
return rsvpsIn;
}
});
}
Not familiar with RSVP but it appears to be callback based API, so you should wrap it in a promise and just use the promise directly:
function getIndexData(){
return new Promise((resolve, reject) => {
RSVP.find({response: 'in'}, function (err, data) {
if (err) reject(err);
// This will be a list of all responses to show in the view
var rsvpsIn = [];
if (data.length > 0) {
// Use because the foreach loop below has async calls.
var responseCounter = data.length;
data.forEach(function(response) {
var foundUser = User.getUserById(response.userId, function(err, user) {
var newRSVP = {userName: user.username, notes: response.notes};
rsvpsIn.push(newRSVP);
// decrement and if we are done, return list
responseCounter -= 1;
if (responseCounter == 0) {
console.log('returned ' + rsvpsIn.length);
resolve(rsvpsIn);
}
});
});
} else {
resolve(rsvpsIn);
}
});
});
}
then just use it where you use the promise:
router.get('/', ensureAuthenticated, function(req, res) {
getIndexData().then(function(data) {
console.log('then data length ' + data.length);
res.render('index', {rsvpsIn: data});
}).catch(function() { });
});
I want to separate the code for executing mysql query in Node, so I am trying to use the Revealing Module pattern here
/* pool -create connection pool mysql*/
var sqlQuery = function (sqlString) {
var _result = pool.getConnection(function (err, connection) {
/* error handling*/
connection.query(sqlString, function (err, rows) {
console.log(this.sql);
if (!err) {
return _result = rows; <============
}
connection.release();
});
return;
});
console.log(_result);
return { recordSet : _result }
};
module.exports = sqlQuery;
How can I return rows to my app.js. the code below for calling sqlQuery is not working
var SqlQuery = require(./path/to/sqlQueryFile);
var rows = SqlQuery('pass sql here').recordSet;
console.log(row);
res.json(rows);
Your code is asynchronous, but you're calling it synchronously.
If you wanted to do it like this, you'll also need to pass a callback to SqlQuery.
/* pool -create connection pool mysql*/
var sqlQuery = function (sqlString, callback) {
var _result = pool.getConnection(function (err, connection) {
/* error handling*/
connection.query(sqlString, function (err, rows) {
console.log(this.sql);
if (!err) {
callback(rows);
}
connection.release();
});
});
};
module.exports = sqlQuery;
And then call it with:
var SqlQuery = require(./path/to/sqlQueryFile);
var rows = SqlQuery('pass sql here', function(recordSet){
console.log(recordSet);
res.json(recordSet);
});
Edit: If you're using newer versions of JavaScript, you have a few more options.
If you have access to Promises, you can do this:
function sqlQuery (sqlString) {
return new Promise((resolve, reject) => {
pool.getConnection(function (err, connection) {
if (err) { return reject(err); } // error handling
connection.query(sqlString, function (err, rows) {
if (err) { return reject(err); }
resolve(rows);
connection.release();
});
});
});
}
module.exports = sqlQuery;
And then you'd use it like:
var SqlQuery = require(./path/to/sqlQueryFile);
SqlQuery('pass sql here')
.then(function(recordSet) {
console.log(recordSet);
res.json(recordSet);
})
.catch(function(err) {
// do your error handling
res.status(500).json({ err: 'Sorry there was an error' });
});
If you're using even newer JavaScript, you can use the async/await syntax (currently available via Babel, and I think in FireFox. Chrome in V55).
var SqlQuery = require(./path/to/sqlQueryFile);
async handleQuery(query) {
try {
var rows = await SqlQuery(query);
res.json(rows);
} catch (e) {
console.log('Error!', e);
}
}
To chain multiple queries together:
async handleQuery(query) {
try {
return await SqlQuery(query);
} catch (e) {
console.log('Error!', e);
}
}
var rows = await handleQuery('select * from tablename');
var rowsToReturn = await handleQuery('select id from another_table where name = "' + rows[0].name + '"');
I am currently working on a project that requires that I manipulate SQL tables pertinent to a time reporting application. My simple code to connect right now is as follows:
var Connection = require('tedious').Connection;
var config = {
userName: 'my_user_name',
password: 'my_password',
server: 'server_to_access',
database: 'database_in_SQL_Server_Management_Studio'
};
var connection = new Connection(config);
connection.on('connect', function(err) {
console.log("Connected");
});
It shows that a connection is made, but then when I run the following:
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var config = {
userName: 'my_user_name',
password: 'my_password',
server: 'server_to_access',
database: 'database_in_SQL_Server_Management_Studio'
};
var connection = new Connection(config);
connection.on('connect', function(err) {
console.log("Connected");
executeStatement();
});
function executeStatement() {
request = new Request("SELECT * FROM Employees", function(err, rowCount) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
});
request.on('row', function(columns) {
columns.forEach(function(column) {
console.log(column.value);
});
});
connection.execSql(request);
}
This is my output in cmd:
C:\Users\name\attempt>node test.js
Connected
{ [RequestError: Requests can only be made in the LoggedIn state, not the Connec
ting state]
message: 'Requests can only be made in the LoggedIn state, not the Connecting
state',
code: 'EINVALIDSTATE' }
C:\Users\name\attempt>
The application managing the SQL tables is MS SQL Server Management Studio 2008 R2.
Any direction as to what I'm doing wrong would be very appreciated.
Thanks!
You'll need to manage a queue of requests and process based on the availability of the connection (tedious-wrapper might help and I haven't used it in awhile https://github.com/mhingston/tedious-wrapper). This pseudocode is adapted (and not tested) from my own working solution:
connection.on('connect', (err)=>{ RequestWrapper.next(err); });
const queue = [];
class RequestWrapper(){
constructor(sql){ this.sql = sql; }
static next(err){
const next = queue.shift();
if(next) next(err);
}
queue(args){
return new Promise((ok, fail)=>{
queue.push((err)=>{
if(err) fail(err); else ok(this.request(args));
})
})
}
request(args){
if(!connection) return Promise.reject(new Error(`connection required`));
if(!connection.loggedIn || connection.state.name !== 'LoggedIn') return this.queue(args);
return new Promise((ok, fail)=>{
try{
const request = new tedious.Request(this.sql, (err, count, rows)=>{
RequestWrapper.next();
if(err) return fail(err);
ok(rows);
});
...add params here...
connection.execSql(request);
}catch(err){
RequestWrapper.next();
fail(err);
}
});
}
};
I am writing an service, where I retrieve a list of items from a another service, then iterate over result performing keystone.list operation(s).
I am loosing the return status in the find/exec operation. I have tried promises, async, etc.
If someone could point out the correct way to implement this, I would appreciate it.
general implementation:
exports = module.exports = function (req, res) {
var rtn = {
added: 0,
count: 0
}
service(params)
.then(function(svcResult) {
svcResult.forEach(function(item) {
rtn.count++; // <-- correctly seen in apiresponse
Artifact.model.find()
.where({ artifactId: item.id})
.exec(function(err, result) {
if (result.length == 0) {
result = new Artifact.model({
... populate from item ....
});
result.save();
rtn.added++; // <-- not seen in api response
});
});
res.apiResponse(rtn);
});
}
for starters, exec is an async call, which you are ignoring in your res.apiResponse and thus count is incremented and not added, to make life easy, I am moving the exec call outside and wrapping it with promise:
function pExec(id){
return new Promise(function(resolve, reject){
Artifact.model.find()
.where({ artifactId: id})
.exec(function(err, result){
console.log('result: ', result); // there is a possibility that this is not empty array, which seems to be the only case when you increment added value
err? reject(err): resolve(result);
});
});
}
exports = module.exports = function(req, res){ // I think it is 'exports' not 'exposts'
service(params)
.then(function(svcResult) {
var promises = svcResult.map(function(item){
rtn.count++;
return pExec(item.id).then(function(result){
if (result.length == 0) {
result = new Artifact.model({
//... populate from item ....
});
result.save(); // again this might be an async call whose response you might need before incrementing added...
rtn.added++; // <-- not seen in api response
};
});
});
Promise.all(promises).then(function(){
res.apiResponse(rtn);
});
});
}
Thanks... Here is what I have come up with so far....
function getArtifact(id) {
return new Promise(function (resolve, reject) {
Artifact.model.findOne()
.where({artifactId: id})
.exec(function (err, artifact) {
err ? resolve(null) : resolve(artifact);
});
});
}
function createArtifact(item) {
return new Promise(function (resolve, reject) {
var artifact = new Artifact.model({
// ... populate from item ....
});
artifact.save(function (err, artifact) {
err ? resolve(null) : resolve(artifact);
});
});
}
exports = module.exports = function (req, res) {
var rtn = {
success: false,
count: 0,
section: '',
globalLibrary: {
Added: 0,
Matched: 0
},
messages: [],
};
if (!req.user || !req.user._id) {
rtn.messages.push("Requires Authentication");
return res.apiResponse(rtn);
}
if (!req.params.section) {
rtn.messages.push("Invalid parameters");
return res.apiResponse(rtn);
}
var userId = req.user._id;
var section = req.params.section;
rtn.section = section;
service(section)
.then(function (svcResult) {
if (svcResult.length == 0 || svcResult.items.length == 0) {
rtn.messages.push("Retrieved empty collection");
return;
}
rtn.messages.push("Retrieved collection");
var artifacts = svcResult.items(function (item) {
rtn.count++;
return getArtifact(item.objectid)
.then(function (artifact) {
if (!artifact || artifact.length == 0) {
rtn.messages.push("Global Library Adding: " + item.name['$t']);
rtn.globalLibrary.Added++;
artifact = createArtifact(item);
} else {
rtn.globalLibrary.Matched++;
}
return artifact;
})
});
Promise.all(artifacts)
.then(function () {
rtn.success = true;
res.apiResponse(rtn);
});
});
}