We Keep Coding

Get and check a value from JSON read in NodeJS

I'm trying to check if a user exists (registered on a json file).
Unfortunately I don't find a valid solution in all Stack Overflow that gives me a simple "true" in a callback.
The version closest to a solution
Experiment V1 :
let userExist;
function check(){
console.log("CHECK!");
return userExist = true;
}
// check(); if this, return true... obvious.
//// check if user exist
server.readFileSync(filepath, 'utf8', (err, data) =>
{
let json = JSON.parse(data),
userlist = json.allusers;
for (let key in userlist)
{
if ( userlist[key].email == req.body.user_email )
{
console.log("FINDED EQUAL");
check(); // return undefined ???
}
}
});
console.log("userExist value : "+userExist);
differently formulated the debugs also appear, but "true" never returns.
note: yes, JSON is read correctly. If everything works inside the readfile, you immediately notice the same emails.
output: "undefined"
Log: total bypassed
Experiment V2 :
In this case (with asynchronous reading) it returns all the debugging (but the "true" remains undefined)
The problem with the asynchronous is that I have to wait for it to check to continue with the other functions.
//// check if user exist
server.readFile(filepath, 'utf8', (err, data) =>
{
let json = JSON.parse(data),
userlist = json.allusers;
for (let key in userlist)
{
if (/* json.allusers.hasOwnProperty(key) &&*/ userlist[key].email == req.body.user_email )
{
console.log("FINDED EQUAL");
check();
}
}
});
var userExist;
function check(userExist){
console.log("CHECK!");
return userExist=true;
}
console.log("userExist value : "+userExist+"");
server listening on: 8080
userExist value : undefined
CHECK!
FINDED EQUAL
Experiment V3 :
after the various suggestions I come to a compromise by using the syntax for the async functions.
This allowed to reach an ordered code, but despite this it is not possible to wait for the results and export them out of the same function (this is because node itself is asynchronous! Therefore it has already gone on!)
using a "message" variable to check if it could return an object I did so:
//simple output tester
var message;
// create a promise
let loopusers = new Promise( (resolve)=>{
server.readFile( filepath, 'utf8',
(err, data) => {
let json = JSON.parse(data),
userlist = json.allusers,
findedequal;
console.log("CHECK USERS IN DB...for "+userlist.length+" elements");
// loop all items
for (let key in userlist)
{
console.log("Analyzed key ::: "+key);
if ( userlist[key].email == req.body.user_email )
{
console.log("CHECK : user isn't free");
findedequal=true;
resolve(true); // return the result of promise
}
else if(key >= userlist.length-1 && !findedequal )
{
console.log("CHECK : User is free ;)");
resolve(false); // return the result of promise
}
}
// call the action
createuser();
});
});
// when promise finished --> start action
async function createuser(message)
{
let userExist = await loopusers;
console.log("userExist: "+userExist);
if(userExist)
{
message = { Server: "This user already exists, Try new e-mail..."};
}
else
{
message = { Server: "Registration user -> completed..."};
}
// return values
return message;
};
It is also possible to use the classic syntax via "then". For exemple:
//simple output tester
var message;
// create a promise
let loopusers = new Promise( (resolve)=>{
...
});
loopusers.then( (response)=>{
...
})
Then I realized that it was easy to simplify even more by calling the functions directly from the initial one:
var message;
// create a promise --> check json items
server.readFile( filepath, 'utf8',
(err, data) => {
let json = JSON.parse(data),
userlist = json.allusers,
findedequal;
console.log("CHECK USERS IN DB...for "+userlist.length+" elements");
for (let key in userlist)
{
console.log("Analyzed key ::: "+key);
if ( userlist[key].email == req.body.user_email )
{
console.log("CHECK : user isn't free");
findedequal=true;
createuser(true); // call direct function whit params true
}
else if(key >= userlist.length-1 && !findedequal )
{
console.log("CHECK : User is free ;)");
createuser(false); // call direct function whit params false
}
}
});
// start action
function createuser(userExist)
{
if(userExist)
{
message = { Server: "This user already exists, Try new e-mail..."};
}
else
{
message = { Server: "Registration user -> completed!"};
}
// return values
return message;
};
debugging is taken and written
the message is lost outside the aSync function
Experiment V4 Final! :
Finally, after many attempts the solution! (Yes... But know it's not Async)
If we allocate in a variable the reading becomes synchronous the whole model and we return to the simple one
let message,
file = server.readFileSync(filepath, 'utf8'), // read sync
json = JSON.parse(file), // now parse file
userlist = json.allusers, // get your target container object
userExist,
findedequal;
console.log("CHECK USERS IN DB...for "+userlist.length+" elements");
for (let key in userlist)
{
console.log("Analyzed key ::: "+key);
if ( userlist[key].email == req.body.user_email )
{
console.log("CHECK : finded equal value on key ["+key+"] - user isn't free");
findedequal=true;
userExist = true;
}
else if(key >= userlist.length-1 && !findedequal )
{
console.log("CHECK : User is free ;)");
userExist = false;
}
}
if(userExist)
{
console.log("└ EXIT TO CHECK --> Can't create user, function stop.");
message = { Server: "This user already exists, Try new e-mail..."};
}
else
{
console.log("└ Exit to check --> New user registration ...");
message = { Server: "Registration user -> completed!"};
}
}
return message;
Now:
It's all sync and all log is perfect
all var is checked
all return... return
** Final conclusions: **
Is it possible to retrieve an ASync variable in node?
As far as I understand so far ... no.
Node is async by its nature, therefore recovering information that is not saved and then recovered from a DB is left behind among the things to do, becoming unrecoverable if you use it as in this post.
However ... if the purpose is to make reading a file synchronous, the answer was simpler than expected.
A special thanks to: Barmar; Austin Leehealey; C.Gochev;

The problem is that you are calling console.log("userExist value : "+userExist+"");
too early. At the moment that you call that line, userExist is not defined yet. This is because the server.readFile() function requires a callback and that callback function is executed once it has read the file. However, reading files often take time and so the program keeps going. It executes console.log("userExist value : "+userExist+""); and then goes back to the callback function and defines userExist as true.
If you want more information on what callbacks are look at the link below. Callbacks are a defining feature of Nodejs and understanding them is essential to any Node website.
https://medium.com/better-programming/callbacks-in-node-js-how-why-when-ac293f0403ca
Try something like this.
let userExist;
function check(){
console.log("CHECK!");
return userExist = true;
}
// check(); if this, return true... obvious.
//// check if user exist
server.readFileSync(filepath, 'utf8', (err, data) =>
{
let json = JSON.parse(data),
userlist = json.allusers;
for (let key in userlist)
{
if ( userlist[key].email == req.body.user_email )
{
console.log("FINDED EQUAL");
check(); // return undefined ???
console.log("userExist value : "+userExist);
}
}
});

Creating my own sql wrapper for nodejs/express

I'm trying to create my own wrapper for mysql for my nodejs application. I have two questions here one of which theres a work around and one where I'm unsure what to do as my javascript skills are still in the learning phase.
First thing: As of right now when you navigate to /api/finance it directs you to the finance controller and the index method. This is currently just for testing purposes trying to figure out how to this kind of stuff.
FinanceController:
const sql = require('../../sql.js')
module.exports = {
index: (req, res, next) => {
sql.get('test').then((result) => {
res.status(200).json(result);
})
}
}
sql.js
const mysql = require('mysql');
const { DB } = require('./config')
var connection = mysql.createConnection(DB)
module.exports = {
get: function(table, columns = '*') {
return new Promise((resolve, reject) => {
connection.query('SELECT ?? FROM ?? ', [columns, table], function (error, results, fields) {
if (error) throw error;
resolve(results);
});
})
},
all: function(table) {
return new Promise((resolve, reject) => {
connection.query('SELECT * FROM ?? ', table, function (error, results, fields) {
if (error) throw error;
resolve(results);
});
})
},
where: function(){
console.log('where called')
}
}
As you can see, I have a get() and all(). get() allows you to pass the table name and an array of columns for example: ['id', 'name'] would get you the id column and name column. columns = '*' was an attempt on being able to use one function to either get all columns of the table or specify specific columns however it returns an error: Unknown column in 'field list' so all() was my "workaround" however i'd like it to be one function.
Next I can't figure out how to stack/pipe methods? if thats the word.
The goal here would be so I could call the function like this:
index: (req, res, next) => {
sql.all('test').where().then((result) => {
res.status(200).json(result);
})
}
}
obviously within the .where() I would have it like: .where('id', '=', 'userID') or something along those lines.
however I'm unsure on how to go about doing that and would like some guidance if its possible. I receive the error: sql.all(...).where is not a function

Instead of immediately launching the SQL, you should simply register the provided information in an instance object (having the same methods) and return that object, and let each method enrich the SQL until the end of the chain is reached and you call a method that will launch the SQL.
The object that is passed from one method to the next (as this) maintains state, and collects the different elements of the SQL statement.
Here is one way to do it.
NB: In this demo I used a mock-object for connection. This mock object will not actually connect to a database. Its query method will just produce the final SQL (with all ? resolved) instead of a real data set.
// Mock connection object for this snippet only. Just produces the final SQL, not the result
var connection = {
query(sql, params, callback) {
let i = 0;
sql = sql.replace(/\?\??/g, (m) => {
if (m.length > 1) return [].concat(params[i++]).map(p => "`" + p + "`").join(", ");
if (typeof params[i] === "string") return "'" + params[i++].replace(/'/g, "''") + "'";
return params[i++];
});
setTimeout(callback(null, sql));
}
}
// Function to create an instance when a first method call is made on the `sql` object directly
// Chained method calls will keep using that instance
function getInstance(inst) {
if (inst.columns) return inst; // Keep using same instance in the chain
inst = Object.create(inst); // No instance yet: create one
inst.table = null;
inst.params = [];
inst.columns = [];
inst.conditions = [];
inst.order = [];
return inst;
}
// This sql object serves a starting point as well
// as proto object for the instance object that
// passes through the chain:
var sql = {
from(table) {
let inst = getInstance(this);
inst.table = table;
return inst;
},
select(...columns) {
let inst = getInstance(this);
inst.columns = inst.columns.concat(columns);
return inst;
},
where(column, cmp, value) {
if (!["<",">","<=",">=","=","!="].includes(cmp)) throw "invalid operator";
let inst = getInstance(this);
inst.params.push(column, value);
inst.conditions.push(cmp);
return inst;
},
orderBy(...columns) {
let inst = getInstance(this);
inst.order = inst.order.concat(columns);
return inst;
},
promise() {
if (!this.table) throw "no table specified";
// build SQL and parameter list
let sql = "SELECT *";
let params = [];
if (this.columns.length && this.columns != "*") {
sql = "SELECT ??";
params.push(this.columns);
}
sql += " FROM ??";
params.push(this.table);
if (this.conditions.length) {
sql += " WHERE " + this.conditions.map(cmp => `?? ${cmp} ?`).join(" AND ");
params.push(...this.params);
}
if (this.order.length) {
sql += " ORDER BY ??";
params.push(this.order);
}
return new Promise(resolve => {
connection.query(sql, params, function (error, results) {
if (error) throw error;
resolve(results);
});
});
}
};
// demo
sql.from("customer")
.select("id", "name")
.where("name", ">", "john")
.where("name", "<", "thomas")
.orderBy("name", "id")
.promise()
.then(console.log);
Note that in this implementation it does not matter in which order you chain the from, select, where and order method calls. You could even do the following if you wanted to:
sql .orderBy("name", "id")
.where("name", ">", "john")
.from("customer")
.where("name", "<", "thomas")
.select("id", "name")
.promise()
.then(console.log);

Conditional async callbacks

I'm writing an Electron program which takes a CSV file as input, and does file operations depending on the CSV content and file existence (it's to manage MAME arcade roms).
In order to have a progress bar on the UI side, I have switched the code from fully synchronous (because it was much easier) to asynchronous.
I just cannot find out how to reliably display a message to the user when all the lines in the CSV file are processed, and all the zip files are copied or removed.
Here is a (simplified) sample method:
fs.readFile(file, { 'encoding': 'utf8' }, (err, fileContents) => {
let fileCsv = csvparse(fileContents);
let lines = fileCsv.length;
fileCsv.forEach((line) => {
lines--;
let zip = line.name + '.zip';
let sourceRom = path.join(romset, zip);
let destRom = path.join(selection, zip);
this.emit('progress.add', fileCsv.length, fileCsv.length - lines, zip);
if (fs.existsSync(sourceRom) && !fs.existsSync(destRom)) {
fs.copy(sourceRom, destRom, (err) => {
let sourceChd = path.join(romset, game);
if (fs.existsSync(sourceChd)) {
fs.copy(sourceChd, path.join(selection, game), (err) => {
if (lines <= 0) { callback(); } // chd is copied
});
} else {
if (lines <= 0) { callback(); } // no chd, rom is copied
}
});
} else {
if (lines <= 0) { callback(); } // no source found or already exists
}
});
});
The problem is that the CSV file is processed really fast, but the file are not copied as fast. So it decrements the lines counter to 0, then after each file copy, it finds that it's zero and triggers the callback.
How do I reliably trigger the callback at the end of all these nested callbacks and conditions?
Thanks

I tried to change the code without massively overwriting your style - assuming there is a reason to avoid things like bluebird, async/await & native Promises, and the async lib.
You need to decrement lines after a line is processed. I pulled the processing logic out into a function to make this clearer:
function processLine({
sourceRom, destRom, romset, game, callback
}) {
if (fs.existsSync(sourceRom) && !fs.existsSync(destRom)) {
fs.copy(sourceRom, destRom, (err) => {
// *really* should handle this error
let sourceChd = path.join(romset, game);
if (fs.existsSync(sourceChd)) {
fs.copy(sourceChd, path.join(selection, game), (err) => {
// *really* should handle this error
callback();
});
} else {
callback();
}
});
} else {
callback() // no source found or already exists
}
}
fs.readFile(file, { 'encoding': 'utf8' }, (err, fileContents) => {
let fileCsv = csvparse(fileContents);
let lines = fileCsv.length;
fileCsv.forEach((line) => {
let zip = line.name + '.zip';
let sourceRom = path.join(romset, zip);
let destRom = path.join(selection, zip);
this.emit('progress.add', fileCsv.length, fileCsv.length - lines, zip);
processLine({ sourceRom, destRom, game, romset, callback: () => {
lines--;
if (lines <= 0) {
callback();
}
}})
});
});

Node.js Promise loop resolves not every object

I'm trying to parse a XML file, build a object of that infos and push the results to an array. This runs on a node js server so I use promises to do this synchron. In the parseImportantInfos method all parsed objects are printed correct to the console, but after I resolve this objects to the getLocations methode and console.log again only the first situationrecord of one situation will be resolved.
The XML looks like:
...
<situation>
<situationrecord version="123">...</situationrecord>
<situationrecord version="123">...</situationrecord>
</situation>
<situation>
<situationrecord version="456">...</situationrecord>
<situationrecord version="456">...</situationrecord>
</situation>
...
<!-- sometimes there is only one situationrecord -->
<situation>
<situationrecord version="789">...</situationrecord>
</situation>
This is the part of my code where the parsed objects get "lost".
let getLocations = function(xmlArr) {
return new Promise(function(resolve, reject) {
for (var x in xmlArr) {
var $ = cheerio.load(xmlArr[x]);
var promiseList = [];
$('situation').each(function() {
var newPromise = parseImportantInfos($, this)
.then(function(obj) {
console.log(obj); // <---- Here are parsed situations missing, only the first situationrecord of a situation is received
if (obj != "") {
constructionsites.push(obj);
}
});
promiseList.push(newPromise);
});
Promise.all(promiseList)
.then(function() {
resolve(constructionsites);
});
}
});
};
let parseImportantInfos = function($, record) {
return new Promise(function(resolve, reject) {
$(record).find('situationRecord').each(function() {
var startLocationCode = $(this).find('alertCMethod2SecondaryPointLocation').find('specificLocation').text();
var endLocationCode = $(this).find('alertCMethod2PrimaryPointLocation').find('specificLocation').text();
var managementType = $(this).find('roadOrCarriagewayOrLaneManagementType').text();
if (startLocationCode != '' && endLocationCode != '' && managementType != 'roadClosed') {
createSituationRecord($, this, startLocationCode, endLocationCode)
.then(function(obj) {
console.log(obj); // <----------- This log prints all parsed infos correct
resolve(obj);
})
.catch((err) => {
reject("There was an error while parsing the xml");
console.log('err', err.stack);
});
} else {
resolve("");
}
});
});
};
I already debugged the project but can't figure out why only one part of the object where resolved

Unreliable behaviour in Node.js

I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22

}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.

Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};

constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.