We Keep Coding

Get and check a value from JSON read in NodeJS

I'm trying to check if a user exists (registered on a json file).
Unfortunately I don't find a valid solution in all Stack Overflow that gives me a simple "true" in a callback.
The version closest to a solution
Experiment V1 :
let userExist;
function check(){
console.log("CHECK!");
return userExist = true;
}
// check(); if this, return true... obvious.
//// check if user exist
server.readFileSync(filepath, 'utf8', (err, data) =>
{
let json = JSON.parse(data),
userlist = json.allusers;
for (let key in userlist)
{
if ( userlist[key].email == req.body.user_email )
{
console.log("FINDED EQUAL");
check(); // return undefined ???
}
}
});
console.log("userExist value : "+userExist);
differently formulated the debugs also appear, but "true" never returns.
note: yes, JSON is read correctly. If everything works inside the readfile, you immediately notice the same emails.
output: "undefined"
Log: total bypassed
Experiment V2 :
In this case (with asynchronous reading) it returns all the debugging (but the "true" remains undefined)
The problem with the asynchronous is that I have to wait for it to check to continue with the other functions.
//// check if user exist
server.readFile(filepath, 'utf8', (err, data) =>
{
let json = JSON.parse(data),
userlist = json.allusers;
for (let key in userlist)
{
if (/* json.allusers.hasOwnProperty(key) &&*/ userlist[key].email == req.body.user_email )
{
console.log("FINDED EQUAL");
check();
}
}
});
var userExist;
function check(userExist){
console.log("CHECK!");
return userExist=true;
}
console.log("userExist value : "+userExist+"");
server listening on: 8080
userExist value : undefined
CHECK!
FINDED EQUAL
Experiment V3 :
after the various suggestions I come to a compromise by using the syntax for the async functions.
This allowed to reach an ordered code, but despite this it is not possible to wait for the results and export them out of the same function (this is because node itself is asynchronous! Therefore it has already gone on!)
using a "message" variable to check if it could return an object I did so:
//simple output tester
var message;
// create a promise
let loopusers = new Promise( (resolve)=>{
server.readFile( filepath, 'utf8',
(err, data) => {
let json = JSON.parse(data),
userlist = json.allusers,
findedequal;
console.log("CHECK USERS IN DB...for "+userlist.length+" elements");
// loop all items
for (let key in userlist)
{
console.log("Analyzed key ::: "+key);
if ( userlist[key].email == req.body.user_email )
{
console.log("CHECK : user isn't free");
findedequal=true;
resolve(true); // return the result of promise
}
else if(key >= userlist.length-1 && !findedequal )
{
console.log("CHECK : User is free ;)");
resolve(false); // return the result of promise
}
}
// call the action
createuser();
});
});
// when promise finished --> start action
async function createuser(message)
{
let userExist = await loopusers;
console.log("userExist: "+userExist);
if(userExist)
{
message = { Server: "This user already exists, Try new e-mail..."};
}
else
{
message = { Server: "Registration user -> completed..."};
}
// return values
return message;
};
It is also possible to use the classic syntax via "then". For exemple:
//simple output tester
var message;
// create a promise
let loopusers = new Promise( (resolve)=>{
...
});
loopusers.then( (response)=>{
...
})
Then I realized that it was easy to simplify even more by calling the functions directly from the initial one:
var message;
// create a promise --> check json items
server.readFile( filepath, 'utf8',
(err, data) => {
let json = JSON.parse(data),
userlist = json.allusers,
findedequal;
console.log("CHECK USERS IN DB...for "+userlist.length+" elements");
for (let key in userlist)
{
console.log("Analyzed key ::: "+key);
if ( userlist[key].email == req.body.user_email )
{
console.log("CHECK : user isn't free");
findedequal=true;
createuser(true); // call direct function whit params true
}
else if(key >= userlist.length-1 && !findedequal )
{
console.log("CHECK : User is free ;)");
createuser(false); // call direct function whit params false
}
}
});
// start action
function createuser(userExist)
{
if(userExist)
{
message = { Server: "This user already exists, Try new e-mail..."};
}
else
{
message = { Server: "Registration user -> completed!"};
}
// return values
return message;
};
debugging is taken and written
the message is lost outside the aSync function
Experiment V4 Final! :
Finally, after many attempts the solution! (Yes... But know it's not Async)
If we allocate in a variable the reading becomes synchronous the whole model and we return to the simple one
let message,
file = server.readFileSync(filepath, 'utf8'), // read sync
json = JSON.parse(file), // now parse file
userlist = json.allusers, // get your target container object
userExist,
findedequal;
console.log("CHECK USERS IN DB...for "+userlist.length+" elements");
for (let key in userlist)
{
console.log("Analyzed key ::: "+key);
if ( userlist[key].email == req.body.user_email )
{
console.log("CHECK : finded equal value on key ["+key+"] - user isn't free");
findedequal=true;
userExist = true;
}
else if(key >= userlist.length-1 && !findedequal )
{
console.log("CHECK : User is free ;)");
userExist = false;
}
}
if(userExist)
{
console.log("└ EXIT TO CHECK --> Can't create user, function stop.");
message = { Server: "This user already exists, Try new e-mail..."};
}
else
{
console.log("└ Exit to check --> New user registration ...");
message = { Server: "Registration user -> completed!"};
}
}
return message;
Now:
It's all sync and all log is perfect
all var is checked
all return... return
** Final conclusions: **
Is it possible to retrieve an ASync variable in node?
As far as I understand so far ... no.
Node is async by its nature, therefore recovering information that is not saved and then recovered from a DB is left behind among the things to do, becoming unrecoverable if you use it as in this post.
However ... if the purpose is to make reading a file synchronous, the answer was simpler than expected.
A special thanks to: Barmar; Austin Leehealey; C.Gochev;

The problem is that you are calling console.log("userExist value : "+userExist+"");
too early. At the moment that you call that line, userExist is not defined yet. This is because the server.readFile() function requires a callback and that callback function is executed once it has read the file. However, reading files often take time and so the program keeps going. It executes console.log("userExist value : "+userExist+""); and then goes back to the callback function and defines userExist as true.
If you want more information on what callbacks are look at the link below. Callbacks are a defining feature of Nodejs and understanding them is essential to any Node website.
https://medium.com/better-programming/callbacks-in-node-js-how-why-when-ac293f0403ca
Try something like this.
let userExist;
function check(){
console.log("CHECK!");
return userExist = true;
}
// check(); if this, return true... obvious.
//// check if user exist
server.readFileSync(filepath, 'utf8', (err, data) =>
{
let json = JSON.parse(data),
userlist = json.allusers;
for (let key in userlist)
{
if ( userlist[key].email == req.body.user_email )
{
console.log("FINDED EQUAL");
check(); // return undefined ???
console.log("userExist value : "+userExist);
}
}
});

JavaScript function response and chained promises

I have a Parse CloudCode beforeSave function, which roughly does the following:
Runs a query to check if there's a duplicate user to the one being saved;
If there is NO duplicate, I call a response.success(), which means the code can go ahead and allow saving the new user;
If, however, there is a duplicate, I want to take the existing user, fetch a group object and add the existing user to the group.
For this purpose, I am using chained promises to make the code cleaner. The code is below:
Parse.Cloud.beforeSave("Contact", function(request, response) {
var facebookID = request.object.get("facebookID");
var email = request.object.get("email");
var queryFb;
if (facebookID) {
queryFb = new Parse.Query(Parse.User);
queryFb.equalTo("facebookID", facebookID);
}
var queryEmail;
if (email) {
queryEmail = new Parse.Query(Parse.User);
queryEmail.equalTo("email", email);
}
var query;
if (facebookID && email) {
query = new Parse.Query.or(queryFb, queryEmail);
} else if (facebookID) {
query = queryFb;
} else {
query = queryEmail;
}
var user;
query.first().then(function(user) {
if (!user) {
response.success();
} else {
var groupQuery = new Parse.Query("Group");
groupQuery.equalTo("title", "ssAll");
groupQuery.equalTo("owner", request.user);
return groupQuery.first();
}
}).then(function(group) {
group.addUnique("contacts", user);
return group.save();
}).then(function(success) {
response.error("NOT ERROR - new object was NOT created");
}, function(error) {
response.error(error);
});
});
In my test case, the query returns !user, so the response.success() message is called - all good. However, this response seems to then travel down the promise chain, which is intended for the case when the query returns a user object. And so, my function terminates with an error on line group.addUnique("contacts", user); because, obviously, the group object is undefined.
How do I work around this issue?

The code needed a few improvements. The key improvement was to provide consistent starting conditions to the second promise's resolution (the second then block). The OP code called response.success() in the case where there was no existing user. This is fine, except the execution still falls through to the next resolution, in one case with an undefined group parameter.
The new code fixes that by returning either the existingUser (after the group has been updated) or null. Null tells the next promise resolution to call success() and allow the save to proceed, otherwise, block the save.
Also note, it is a mistake for the first block's user parameter to conflict with the var user in the enclosing scope. I tried to use variable naming below to highlight the two different types of users the code considers...
Parse.Cloud.beforeSave("Contact", function(request, response) {
var facebookID = request.object.get("facebookID");
var email = request.object.get("email");
// moved into a function so we can test and deal with it tersely
findUserByEmailOrFB(email, facebookID).then(function(existingUser) {
return (existingUser)? addContactToGroupOwnedBy(request.user, existingUser) : null;
}).then(function(existingUser) {
if (existingUser) {
response.error("NOT ERROR - new object was NOT created");
} else {
response.success();
}
}, function(error) {
response.error(error);
});
});
// find the group owned by ownerUser, add contactUser to its contacts return a promise fulfilled as contactUser
function addContactToGroupOwnedBy(ownerUser, contactUser) {
var groupQuery = new Parse.Query("Group");
groupQuery.equalTo("title", "ssAll");
groupQuery.equalTo("owner", ownerUser);
return groupQuery.first().then(function(group) {
group.addUnique("contacts", contactUser);
return group.save().then(function() { return contactUser; });
});
}
function findUserByEmailOrFB(email, facebookID) {
var queryFb;
if (facebookID) {
queryFb = new Parse.Query(Parse.User);
queryFb.equalTo("facebookID", facebookID);
}
var queryEmail;
if (email) {
queryEmail = new Parse.Query(Parse.User);
queryEmail.equalTo("email", email);
}
var query;
if (facebookID && email) {
query = new Parse.Query.or(queryFb, queryEmail);
} else if (facebookID) {
query = queryFb;
} else {
query = queryEmail;
}
return query.first();
}

The problem is that you are always resolving the first promise regardless of whether checking for the user was successful (no such user yet) or not. However, actually you don't ever have to resolve the promise. I suggest you separate the error case like this:
query.first().then(function(user) {
if (!user) {
response.success();
} else {
addUserToGroup(request.user).then(function() {
response.error("NOT ERROR - new object was NOT created");
}, function(error) {
response.error(error);
});
}
});
function addUserToGroup(user) {
var groupQuery = new Parse.Query("Group");
groupQuery.equalTo("title", "ssAll");
groupQuery.equalTo("owner", user);
return groupQuery.first().then(function(group) {
group.addUnique("contacts", user);
return group.save();
});
}
As you can see, the first promise doesn't ever have to be resolved because the result is not used anyway.

Unreliable behaviour in Node.js

I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22

}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.

Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};

constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.

Meteor.call issue in Meteor?

I need to know about the usage of Meteor.call. I did a simple example as shown below. The problem is that it never goes to insertDetails(). Can you please check the below code and suggest me what to do so that I don't get the Match Failed error.
Client.JS
Meteor.methods
({
//this method doesn't cal when using meteor.cal
insertDetails : function(adData, callback)
{
console.log(">>>>>>>>>>>>>>>>>>> ******* insertDetails ");
checkFields(adData);
var fields =
{
userID: adData.UserID,
fname: adData.fname,
lname: adData.lname,
dob: adData.dob
};
return Client.insert(fields, callback);
}
});
// SERVER-SIDE HELPERS ************************************
var nonEmpty = Match.Where(function(x) {return !!x;});
var checkFields = function(adData)
{
console.log(">>>>>>>>>>>>>>>>>>> checkFields ");
check(adData.userID, nonEmpty);
check(adData.fname, nonEmpty);
};
Insert.js
if (Meteor.isClient)
{
Template.hello.events({
'submit #addnewuserdetails': function (e,t)
{
if (typeof console !== 'undefined')
console.log(">>>>>>>>>>>>>>>>>>> Add button in details ");
e.preventDefault();
saveClientDetails();
}
});
}
var saveClientDetails = function()
{
console.log(">>>>>>>>>>>>>>>>>>> saveClientDetails ");
var fields = {
//ownerId: Meteor.userId(),
UserID : $('#userid').value
,fname : $('#fname').value
,lname :$('#lname').value
,dob : $('#dob').value
};
console.log(">>>>>>>>>>>>>>>>>>> fields.UserID "+fields.UserID);
//here cal to above insertDetails()
Meteor.call("insertDetails", fields, function(err, result)
{
if (!err)
{
console.log(">>>>>>>>>>>>>>>>>>> saveClientDetails Success");
}
else
{
console.log(">>>>>>>>>>>>>>>>>>> saveClientDetails ERROR "+err.reason);
}
});
};

The Match Failed error points to invalid data being rejected by your check function. My guess is that the problem is with user id: when you call the method you use UserID parameter, but then you check lowercase userID. Try fixing that and see whether it works. Also, try commenting out check call and see whether the rest of code is running. Also, how do you verify that the method was not called? Notice that log should be visible in the server console.

Node Fibers Trouble with Meteor

I'm trying to write a simple authentication backend for Meteor that authenticates against an LDAP server. I need the function registered as login handler (the input to Accounts.registerLoginHandler) to return the id the of the user just logged in.
The problem I think lies in the Fiber I've created, getUserId, and that it's not returning id like I want it to. I know it has to be in a Fiber or else meteor gets angry and throws errors. Even though the log right before the yield shows me the id isn't undefined, getUserId.run() always returns undefined.
Any help would be greatly appreciated, thanks!
Accounts.registerLoginHandler(function(loginRequest) {
console.log("In login handler");
return auth.authenticate(loginRequest.username, loginRequest.password, function(err, ldap_user) {
if (err){
// ldap authentications was failed
console.log("Login failed");
return undefined;
}
else {
// authentication was successful
console.log("Login success");
// extracting team name from ldap record
var equals = ldap_user.memberOf.indexOf("=");
var comma = ldap_user.memberOf.indexOf(",");
var team_name = ldap_user.memberOf.slice(equals+1,comma);
// add user if they don't already exist
var getUserId = Fiber( function() { // Meteor code must be ran within a fiber
var id = null;
var user = Meteor.users.findOne({username: loginRequest.username});
if(!user) {
// insert user and kick back id
id = Meteor.users.insert({username: loginRequest.username,
profile : {team : team_name}
});
console.log('no user found, creating' + id);
} else {
id = user._id;
console.log('user found, returning id' + id);
}
console.log('id: '+id);
Fiber.yield(id); // return id
});
// send logged in users if by executing the fiber
return {id: getUserId.run()};
}
});
});

I think the problem is related to instead needing to use Meteor.bindEnvironment to control the scope of the (environment) variables and fibers in use.
A good three-step tutorial on the subject is found here:
https://www.eventedmind.com/feed/nodejs-introducing-fibers
https://www.eventedmind.com/feed/meteor-dynamic-scoping-with-environment-variables
https://www.eventedmind.com/feed/meteor-what-is-meteor-bindenvironment
My take on your code would be something like this (which in a similar problem worked for me):
Accounts.registerLoginHandler(function(loginRequest) {
console.log("In login handler");
var boundAuthenticateFunction = Meteor.bindEnvironment(function(err, ldap_user) {
if (err){
// ldap authentications was failed
console.log("Login failed");
return undefined;
}
else {
// authentication was successful
console.log("Login success");
// extracting team name from ldap record
var equals = ldap_user.memberOf.indexOf("=");
var comma = ldap_user.memberOf.indexOf(",");
var team_name = ldap_user.memberOf.slice(equals+1,comma);
// add user if they don't already exist
var id = null;
var user = Meteor.users.findOne({username: loginRequest.username});
if(!user) {
// insert user and kick back id
id = Meteor.users.insert({username: loginRequest.username,
profile : {team : team_name}
});
console.log('no user found, creating' + id);
} else {
id = user._id;
console.log('user found, returning id' + id);
}
console.log('id: '+id);
return {id: id};
}
}, function(e){throw e;});
return auth.authenticate(loginRequest.username, loginRequest.password, boundAuthenticateFunction);
});
Mind, the code sample above is untested...