Unreliable behaviour in Node.js - javascript

I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22

}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.

Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};

constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.

Related

looping the callback function in node js

This is a piece of code which writes data to a ble device and reads data from it. data is written to the device in the form of a buffer. the value in 'mydata' (AAAD0000) is the command to be written in order to read the data.
function named chara3() consists of write and read function which is a callback function in which the command is passed read back.
My requirement is the 'mydata' value which i said earlier, the last two zeros is the memory address. i need to read the data in different memory addresses starting from zero to 59. That is AAAD0000 to AAAD0059. so of course i need to run a loop. If I'm reading the zeroth location, the code is quite fine and i got the output as well but when i tried to make it inside a loop, the code is all a mess. the read part is not executing.
can any one suggest a better way to read data from zeroth memory location to 59th memory location (AAAD0000 to AAAD0059)???
first command writes to it
then reads data
memory location incremented by 1
this should repeat up to 59
var mydata = 'AAAD0000';
function chara3() {
var buff2 = new Buffer(mydata, 'hex');
SensorCharacteristic.write(buff2, false, function(error) { //[0x002d]
console.log('Writing command SUCCESSFUL',mydata);
if (!error) {
SensorCharacteristic.read((error, data) => {
console.log("i just entered");
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
enter();
}
else {
console.log('contains null value');
} //else
});
}
function enter()
{
mydata = (parseInt(mydata, 16) + 00000001).toString(16);
}
}); //.write
} //chara3
there's no error. But some part of the code is not executing.
You can use the recursion by wrapping your methods into a promise
async function chara3(mydata = 'AAAD0000') {
if (mydata === 'AAAD0059') {
return;
}
var buff2 = new Buffer(mydata, 'hex');
return new Promise((resolve) => {
SensorCharacteristic.write(buff2, false, function (error) { //[0x002d]
console.log('Writing command SUCCESSFUL', mydata);
if (!error) {
SensorCharacteristic.read(async (error, data) => {
console.log("i just entered");
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
let next = await chara3(enter())
return resolve(next);
}
else {
console.log('contains null value');
return resolve();
} //else
});
}
}); //.write
});
} //chara3
function enter() {
return (parseInt(mydata, 16) + 00000001).toString(16);
}
Also if you can convert your methods SensorCharacteristic.write and SensorCharacteristic.read into promises you can simply map
function chara3(mydata) {
var buff2 = new Buffer(mydata, 'hex');
await SensorCharacteristic.write(buff2, false);
console.log('Writing command SUCCESSFUL', mydata);
let data = await SensorCharacteristic.read();
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
enter();
} else {
console.log('contains null value');
}
};
let promiseArray = Array(60).fill().map((_, i) => (parseInt('AAAD0000', 16) + i).toString(16)).map(chara3);
Promise.all(promiseArray).then(() => console.log('done'));

Javascript: How to check if async operation is still pending / In progress?

I would like to know if it is somehow possible to check if an asynchronous operation in Javascript is still pending..
Because I am doing a database request on calling a specific URL... While the db call is still in progress, I want to stop any other incoming db-calls (which means, stop any further calls to that URL in case the db-request is still pending).
Is that somehow possible?
Because the database call takes up to minutes, and I don't want to launch another database-call while the first is still in progress.. The problem is, I somehow cannot figure out how to check if the call has started and is still in progress, because the response comes only after the .then() clause when the process has already finished.
this is my db-call function:
const getWriteIndex = async () => {
return Promise.all(someFunction1, someFunction2...).then(...) {
writeMessageObject = checkDocuments(...);
return Promise.resolve(writeMessageObject);
})).catch((err) => {
return Promise.reject(err);
});
}
This is my URL/Route Call function with express:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
//HOW CAN I CHECK HERE IF THE DB-CALL IS ALREADY IN PROGRESS?
readmsg_obj.message = '';
getReadIndex().then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});
hmm I found a workaround here:
https://ourcodeworld.com/articles/read/317/how-to-check-if-a-javascript-promise-has-been-fulfilled-rejected-or-resolved
so I wrote that function to check for promise stati, but I am still wondering if it's not somehow possible to query for static promise properties to get their actual state ;) (but weirdly, I didn't find any on the web).
const checkPendingRequest= (promise) => {
if (promise.isResolved) return promise;
// Set initial state
var isPending = true;
var isRejected = false;
var isFulfilled = false;
// Observe the promise, saving the fulfillment in a closure scope.
var result = promise.then(
function(v) {
isFulfilled = true;
isPending = false;
return v;
},
function(e) {
isRejected = true;
isPending = false;
throw e;
}
);
result.isFulfilled = function() { return isFulfilled; };
result.isPending = function() { return isPending; };
result.isRejected = function() { return isRejected; };
return result;
}
So I fixed my function for the request:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
readmsg_obj.message = '';
if ((dbQueryPromiseRead != null) && dbQueryPromiseRead.isPending()) {
logger.info(`Database request for Index-read is still pending!`);
return;
}
dbQueryPromiseRead = checkPendingRequest(getReadIndex());
dbQueryPromiseRead.then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});
You need to try add in node.js like global.dbCallState flag if operation is still running.
This global var one for all modules.
Do not change this object like global = new Object();, but you can use child field's.
https://nodejs.org/api/globals.html
You can change it in another module like global.dbCallState = false.
It not best solution, but it can help.
But i don't know, why you want only one connection. Its not good solution to block I/O in node.js

JavaScript function response and chained promises

I have a Parse CloudCode beforeSave function, which roughly does the following:
Runs a query to check if there's a duplicate user to the one being saved;
If there is NO duplicate, I call a response.success(), which means the code can go ahead and allow saving the new user;
If, however, there is a duplicate, I want to take the existing user, fetch a group object and add the existing user to the group.
For this purpose, I am using chained promises to make the code cleaner. The code is below:
Parse.Cloud.beforeSave("Contact", function(request, response) {
var facebookID = request.object.get("facebookID");
var email = request.object.get("email");
var queryFb;
if (facebookID) {
queryFb = new Parse.Query(Parse.User);
queryFb.equalTo("facebookID", facebookID);
}
var queryEmail;
if (email) {
queryEmail = new Parse.Query(Parse.User);
queryEmail.equalTo("email", email);
}
var query;
if (facebookID && email) {
query = new Parse.Query.or(queryFb, queryEmail);
} else if (facebookID) {
query = queryFb;
} else {
query = queryEmail;
}
var user;
query.first().then(function(user) {
if (!user) {
response.success();
} else {
var groupQuery = new Parse.Query("Group");
groupQuery.equalTo("title", "ssAll");
groupQuery.equalTo("owner", request.user);
return groupQuery.first();
}
}).then(function(group) {
group.addUnique("contacts", user);
return group.save();
}).then(function(success) {
response.error("NOT ERROR - new object was NOT created");
}, function(error) {
response.error(error);
});
});
In my test case, the query returns !user, so the response.success() message is called - all good. However, this response seems to then travel down the promise chain, which is intended for the case when the query returns a user object. And so, my function terminates with an error on line group.addUnique("contacts", user); because, obviously, the group object is undefined.
How do I work around this issue?
The code needed a few improvements. The key improvement was to provide consistent starting conditions to the second promise's resolution (the second then block). The OP code called response.success() in the case where there was no existing user. This is fine, except the execution still falls through to the next resolution, in one case with an undefined group parameter.
The new code fixes that by returning either the existingUser (after the group has been updated) or null. Null tells the next promise resolution to call success() and allow the save to proceed, otherwise, block the save.
Also note, it is a mistake for the first block's user parameter to conflict with the var user in the enclosing scope. I tried to use variable naming below to highlight the two different types of users the code considers...
Parse.Cloud.beforeSave("Contact", function(request, response) {
var facebookID = request.object.get("facebookID");
var email = request.object.get("email");
// moved into a function so we can test and deal with it tersely
findUserByEmailOrFB(email, facebookID).then(function(existingUser) {
return (existingUser)? addContactToGroupOwnedBy(request.user, existingUser) : null;
}).then(function(existingUser) {
if (existingUser) {
response.error("NOT ERROR - new object was NOT created");
} else {
response.success();
}
}, function(error) {
response.error(error);
});
});
// find the group owned by ownerUser, add contactUser to its contacts return a promise fulfilled as contactUser
function addContactToGroupOwnedBy(ownerUser, contactUser) {
var groupQuery = new Parse.Query("Group");
groupQuery.equalTo("title", "ssAll");
groupQuery.equalTo("owner", ownerUser);
return groupQuery.first().then(function(group) {
group.addUnique("contacts", contactUser);
return group.save().then(function() { return contactUser; });
});
}
function findUserByEmailOrFB(email, facebookID) {
var queryFb;
if (facebookID) {
queryFb = new Parse.Query(Parse.User);
queryFb.equalTo("facebookID", facebookID);
}
var queryEmail;
if (email) {
queryEmail = new Parse.Query(Parse.User);
queryEmail.equalTo("email", email);
}
var query;
if (facebookID && email) {
query = new Parse.Query.or(queryFb, queryEmail);
} else if (facebookID) {
query = queryFb;
} else {
query = queryEmail;
}
return query.first();
}
The problem is that you are always resolving the first promise regardless of whether checking for the user was successful (no such user yet) or not. However, actually you don't ever have to resolve the promise. I suggest you separate the error case like this:
query.first().then(function(user) {
if (!user) {
response.success();
} else {
addUserToGroup(request.user).then(function() {
response.error("NOT ERROR - new object was NOT created");
}, function(error) {
response.error(error);
});
}
});
function addUserToGroup(user) {
var groupQuery = new Parse.Query("Group");
groupQuery.equalTo("title", "ssAll");
groupQuery.equalTo("owner", user);
return groupQuery.first().then(function(group) {
group.addUnique("contacts", user);
return group.save();
});
}
As you can see, the first promise doesn't ever have to be resolved because the result is not used anyway.

Async Recursion with JavaScript and Node.js

This is probably a noob JavaScript question, but I'm looking to know if my solution to a problem I am having is 'correct'
I have created the following sample application that recreates my error:
Firstly in index.js
var processor = require('./fileProcessor/processor.js');
var container = {
source: "source.txt",
destination: "destination.txt"
};
new processor().process(container);
I create my container object which has the name of the source file and the name of the destination file. This is passed into the process function of the processor:
var fileProcessor = require('./fileProcessor.js');
module.exports = function Processor() {
this.process = function(container) {
var file = new fileProcessor();
if(container.finished === undefined) {
if(container.body === undefined) {
file.read(container, this.process);
} else {
file.write(container, this.process);
}
}
};
};
As you can see this calls the read and write functions passing in the container and the process function as the callback, the fileProcessor looks like this:
var fs = require('fs');
module.exports = function() {
this.read = function(container, callback) {
fs.readFile(container.source, function (err, data) {
if(err) throw err;
container.body = data;
callback(container);
});
};
this.write = function(container, callback) {
fs.writeFile(container.destination, container.body, function(err) {
if(err) {
return console.log(err);
}
container.finished = true;
callback(container);
});
};
};
In simple terms the processor calls file.read, which reads the file and calls back into the process function, which then calls the write function. However at the end of the write function an error is thrown:
callback(container);
^
TypeError: object is not a function
Obviously when passing in this.process to file.write(container, this.process); the this isn't the this I intend it to be!
If I update my processor by adding a processFunction variable:
var fileProcessor = require('./fileProcessor.js');
module.exports = function Processor() {
var processFunction = function(container) {
var file = new fileProcessor();
if(container.finished === undefined) {
if(container.body === undefined) {
file.read(container, processFunction);
} else {
file.write(container, processFunction);
}
}
};
this.process = function(container) {
processFunction(container);
};
};
Everything works fine. Is this a good way to do this or is there a better solution?
I think this is a fine way to do it. There is one possible modification that you might make. Since you are creating a new name in your scope just for the purpose of recursing, you could just name your function and refer to it by its name inside of the function.
module.exports = function Processor() {
this.process = function processFunction(container) {
var file = new fileProcessor();
if(container.finished === undefined) {
if(container.body === undefined) {
file.read(container, processFunction);
} else {
file.write(container, processFunction);
}
}
};
};
Then you can avoid creating a name (processFunction) that will be visible outside the function.
Take a look here for reference:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/function#Named_function_expression

breeze/durandal: configureBreezeManager().saveChanges() doesn't trigger function subscribed hasChangesChanged

I'm been creating a web site following Hottowl's example. However, the value of hasChanges is not changed after saveChanges is called - even the data is actually saved in database table. I use hottowel durandal 2.0.1. I found the durandal version is different from the CCJS-Scratch example. Did I miss anything?
datacontext.js
var manager = configureBreezeManager();
var hasChanges = ko.observable(false);
manager.hasChangesChanged.subscribe(function (eventArgs) {
hasChanges(eventArgs.hasChanges); // The function isn't called when saveChanges is called
});
var cancelChanges = function () {
manager.rejectChanges();
log('Canceled changes', null, true);
};
var saveChanges = function () {
return manager.saveChanges()
.then(saveSucceeded)
.fail(saveFailed);
function saveSucceeded(saveResult) {
log('Saved data successfully', saveResult, true);
}
function saveFailed(error) {
var msg = 'Save failed: ' + getErrorMessages(error);
logError(msg, error);
error.message = msg;
throw error;
}
};
The following code exist in the view model.
var hasChanges = ko.computed(function () {
return datacontext.hasChanges();
});
var cancel = function () {
datacontext.cancelChanges();
};
var canSave = ko.computed(function () {
return hasChanges() && !isSaving();
});
var save = function () {
isSaving(true);
return datacontext.saveChanges().fin(complete);
function complete() {
isSaving(false);
}
};
Maybe you have a timing issue? You have a lot of nested observables and computed observables that I have trouble aligning at this late hour.
What I do know is that the Breeze DocCode:saveTodoTests.js show that hasChangesChanged is called with the correct eventArgs.hasChanges value when both saving changes and reverting (rejectChanges). Here is a passing test from that suite that confirms it.
test("hasChangesChanged event raised after saveChanges", 4, function () {
var em = newTodosEm();
var hasChangesChangedRaised = [];
em.hasChangesChanged.subscribe(
function(eventArgs) {
hasChangesChangedRaised.push(eventArgs.hasChanges);
}
);
// add a Todo (and forget about it)
em.createEntity('TodoItem',{ Description: "Learn to save in breeze" });
stop();
em.saveChanges()
.then ( function() {
equal(hasChangesChangedRaised.length, 2,
"hasChangesChanged should have been raised twice");
ok(hasChangesChangedRaised[0]===true,
"first hasChangesChanged is true after create");
ok(hasChangesChangedRaised[1]===false,
"second hasChangesChanged is false after save");
ok(!em.hasChanges(),
"manager should not have pending changes after save");
})
.fail(handleSaveFailed)
.fin(start);
});
Note that the hasChangesChanged event is raised twice in this test.
Please try debugging again and make sure that the event you're looking at is the one raised after the save, not during some other manipulation.
Also watch out for the nesting of KO events. Race conditions are possible.

Categories