I have a unit test fails to correctly read the JSON in the second request
this is my Config factory
(function() {
'use strict';
angular.module('commercial.factories').factory('config', config);
function config($http) {
var service = {
GetConfig: getConfig
};
return service;
function getConfig(identifier) {
var _config = {};
// Check for url match in mapping json file
var urlMap = $http.get('./app/core/urlMap.json').then(function(response) {
for (var i = 0; i < response.data.length; i++) {
if (identifier.toString().toLowerCase().indexOf(response.data[i].url.toLowerCase()) > -1 || response.data[i].clientId === identifier) {
return response.data[i].contentFolder;
}
}
});
// Retrieve the config for the related client found in the url map (above)
return urlMap.then(function(response) {
var contentFolder = response;
return $http.get('./content/' + response + '/config.json')
.then(function(response) {
if (Object.keys(_config).length === 0) {
_config = response.data;
_config.contentFolder = contentFolder;
}
return _config;
});
});
}
}
})();
and my test...
describe('Config Factory', function() {
var configFactory;
beforeEach(inject(function(_config_) {
configFactory = _config_;
}));
describe('GetConfig()', function() {
it('should get the urlMap from the urlMap.json', function() {
var identifier = '_default';
var mockData = [{ url: identifier, contentFolder: '_default' }];
$httpBackend.expectGET('./content/' + identifier + '/config.json');
$httpBackend.expectGET('./app/core/urlMap.json').respond(mockData);
var promise = configFactory.GetConfig(identifier);
$httpBackend.flush(0);
promise.then(function(result) {
expect(result).toEqual(mockData);
})
});
});
});
and the config.json it trys to read...
{
"clientId":34
}
when i run my test i get an error back from Karama saying ...
uncaught SyntaxError: Unexpected token :
on line 2 of my JSON.
Im suspicious that it may have something to do with having two expectGET's in the same test but i cant be sure?
You might need to call json.stringify(mockData) for the mockData, when responding from the get request. The json parser probably has problems with the single quotes in your mockData array.
I also spotted a missing . in your expectation:
expect(result) toEqual(mockData);
should be:
expect(result).toEqual(mockData);
OK, so this was a bit of a silly mistake on my part.
I noticed that i failed to add a respond to the config.json call. See code below.
it('should get the urlMap from the urlMap.json', function() {
var identifier = '_default';
var mockData = [{ url: identifier, contentFolder: '_default' }];
var mockConfig = { clientId: 34 };
$httpBackend.expectGET('./app/core/urlMap.json').respond(mockData);
$httpBackend.expectGET('./content/' + identifier + '/config.json').respond(mockConfig);
var promise = configFactory.GetConfig(identifier);
$httpBackend.flush();
promise.then(function(result) {
expect(result).toEqual(mockData);
})
});
Related
Original code from the Point of Sale module
In the point_of_sale module there is a list of objects as the following
module.PosModel = Backbone.Model.extend({
models: {
// [...]
{
model: 'pos.session',
fields: ['id', 'journal_ids','name','user_id','config_id','start_at','stop_at','sequence_number','login_number'],
domain: function(self){ return [['state','=','opened'],['user_id','=',self.session.uid]]; },
loaded: function(self,pos_sessions){
self.pos_session = pos_sessions[0];
var orders = self.db.get_orders();
for (var i = 0; i < orders.length; i++) {
self.pos_session.sequence_number = Math.max(self.pos_session.sequence_number, orders[i].data.sequence_number+1);
}
},
},
{
model: 'product.product',
fields: ['display_name', 'list_price','price','pos_categ_id', 'taxes_id', 'ean13', 'default_code',
'to_weight', 'uom_id', 'uos_id', 'uos_coeff', 'mes_type', 'description_sale', 'description',
'product_tmpl_id'],
domain: [['sale_ok','=',true],['available_in_pos','=',true]],
context: function(self){ return { pricelist: self.pricelist.id, display_default_code: false }; },
loaded: function(self, products){
self.db.add_products(products);
},
// [...]
}
And then the information of the data is loaded like this
load_server_data: function(){
var self = this;
var loaded = new $.Deferred();
var progress = 0;
var progress_step = 1.0 / self.models.length;
var tmp = {}; // this is used to share a temporary state between models loaders
function load_model(index){
if(index >= self.models.length){
loaded.resolve();
}else{
var model = self.models[index];
self.pos_widget.loading_message(_t('Loading')+' '+(model.label || model.model || ''), progress);
var fields = typeof model.fields === 'function' ? model.fields(self,tmp) : model.fields;
var domain = typeof model.domain === 'function' ? model.domain(self,tmp) : model.domain;
var context = typeof model.context === 'function' ? model.context(self,tmp) : model.context;
var ids = typeof model.ids === 'function' ? model.ids(self,tmp) : model.ids;
progress += progress_step;
if( model.model ){
if (model.ids) {
var records = new instance.web.Model(model.model).call('read',[ids,fields],context);
} else {
var records = new instance.web.Model(model.model).query(fields).filter(domain).context(context).all()
}
// [...]
What I have tried. First try
So, I would like to change the domain field of the product.product model. I am trying with this
if (typeof jQuery === 'undefined') { throw new Error('Product multi POS needs jQuery'); }
+function ($) {
'use strict';
openerp.pos_product_multi_shop = function(instance, module) {
var PosModelParent = instance.point_of_sale.PosModel;
instance.point_of_sale.PosModel = instance.point_of_sale.PosModel.extend({
load_server_data: function(){
console.log('-- LOAD SERVER DATA');
var self = this;
self.models.forEach(function(elem) {
if (elem.model == 'product.product') {
// return [['id', 'in', [2]]]; // if I return this domain it works well
domain_loaded = function() {
return new instance.web.Model('product.product').call(
'get_available_in_pos_ids',
[self.pos_session.config_id[0]],
)
}
elem.domain = $.when(domain_loaded);
}
})
var loaded = PosModelParent.prototype.load_server_data.apply(this, arguments);
return loaded;
},
});
}
}(jQuery);
If I return a domain directly it works. But if I replace it with a function that calls a python function with call, the domain is not loaded well: [['sale_ok','=',true],['available_in_pos','=',true]]. I've tried with $.when and without it and it does not work.
In addition elem.domain must be a function because self.pos_session only exists when all the previous model information is executed.
Second try
I have tried this following code as well:
if (elem.model == 'product.product') {
// return [['id', 'in', [2]]]; // if I return the domain like this it works
console.log('>> OLD DOMAIN')
console.log(elem.domain);
elem.domain = function() {
console.log('>>> PRODUCT SESSION');
console.log(self.pos_session);
var product_product_obj = new instance.web.Model('product.product');
return product_product_obj.call(
'get_available_in_pos_ids',
[self.pos_session.config_id[0]],
)
}
console.log('>> NEW DOMAIN')
console.log(elem.domain);
}
So first '>> OLD DOMAIN' is printed, then '>> NEW DOMAIN' and, at last '>>> PRODUCT SESSION' is printed. So the function is executed. But the the domains is not being returned well.
Third try. With "then"
And I cannot use then because I need to do the variable assignation. But on the other hand the assignation is well done becase when I print the new domain the function appears in the log.
Even if I use then I am getting the result well from python
var domain_return = product_product_obj.call(
'get_available_in_pos_ids',
[self.pos_session.config_id[0]],
).then(function(result) {
console.log('>> RESULT: ');
console.log(result)
});
I also tried with other promise, but I get a pending result that is ignored and all the products are shown
elem.domain = function() {
return new Promise(function next(resolve, reject) {
console.log('>>> PRODUCT SESSION');
console.log(self.pos_session);
var product_product_obj = new instance.web.Model('product.product');
var domain_return = product_product_obj.call(
'get_available_in_pos_ids',
[self.pos_session.config_id[0]],
).then(function(result) {
console.log('>> RETURN: ');
console.log(result);
resolve(result);
});
console.log('>> DOMAIN RETURN: ');
console.log(domain_return);
});
}
The rest of the domains of the object are calculated without calling python functions. So I can't copy an example from other place
So, is there a way to avoid the pending result? I cannot use async/await yet.
Maybe to make it syncronous will help but I know this should be avoided
Finally I found a workaround overriding the loaded function where all the products are already loaded
var PosModelParent = instance.point_of_sale.PosModel;
instance.point_of_sale.PosModel = instance.point_of_sale.PosModel.extend({
load_server_data: function(){
let self = this;
self.models.forEach(function(elem) {
if (elem.model == 'product.product') {
elem.fields = ['display_name', 'list_price','price','pos_categ_id', 'taxes_id', 'ean13', 'default_code',
'to_weight', 'uom_id', 'uos_id', 'uos_coeff', 'mes_type', 'description_sale', 'description',
'product_tmpl_id', 'available_in_pos_ids'];
elem.loaded = function(self, products){
console.log('>> PRODUCTS: ');
console.log(products);
var shop_id = self.pos_session.config_id[0];
var new_products = [];
products.forEach(function(prod) {
if (prod.available_in_pos_ids.includes(shop_id)) {
new_products.push(prod);
}
})
self.db.add_products(new_products);
}
}
})
var loaded = PosModelParent.prototype.load_server_data.apply(this, arguments);
return loaded;
},
});
I've been trying to code up a search engine using angular js, but I can't copy one array to another. When I initiate the the code (in the service.FoundItems in the q.all then function) new array(foundArray) shows up as an empty array. I searched up how to copy one array to another and tried that method as you can see, but it isn't working. Please help, here is the code, and thank you.
P.S. if you need the html please tell me.
(function () {
'use strict';
angular.module('narrowDownMenuApp', [])
.controller('narrowItDownController', narrowItDownController)
.service('MenuSearchService', MenuSearchService)
.directive('searchResult', searchResultDirective);
function searchResultDirective() {
var ddo = {
templateUrl: 'searchResult.html',
scope: {
items: '<'
},
};
return ddo
}
narrowItDownController.$inject = ['MenuSearchService'];
function narrowItDownController(MenuSearchService) {
var menu = this;
menu.input = "";
menu.displayResult = [];
menu.searchX = function(name) {
menu.displayResult = MenuSearchService.FoundItems(menu.input, name);
console.log(menu.displayResult);
};
}
MenuSearchService.$inject = ['$http', '$q'];
function MenuSearchService($http, $q) {
var service = this;
service.getMatchedMenuItems = function(name, searchTerm) {
var deferred = $q.defer();
var foundItems = [];
var result = $http({
method: "GET",
url: ('https://davids-restaurant.herokuapp.com/menu_items.json'),
params: {
category: name
}
}).then(function (result) {
var items = result.data;
for (var i = 0; i < items.menu_items.length; i++) {
if (searchTerm === ""){
deferred.reject("Please enter search term");
i = items.menu_items.length;
}
else if (items.menu_items[i].name.toLowerCase().indexOf(searchTerm.toLowerCase()) ==! -1){
foundItems.push(items.menu_items[i].name)
deferred.resolve(foundItems);
}else {
console.log("doesn't match search");
}
}
});
return deferred.promise;
};
service.FoundItems = function (searchTerm, name) {
var searchResult = service.getMatchedMenuItems(name, searchTerm);
var foundArray = [];
$q.all([searchResult])
.then(function (foundItems) {
foundArray = foundItems[0].slice(0);
foundArray.reverse();
})
.catch(function (errorResponse) {
foundArray.push(errorResponse);
});
console.log(foundArray);
return foundArray;
};
};
})();
If the goal of the service.FoundItems function is to return a reference to an array that is later populated with results from the server, use angular.copy to copy the new array from the server to the existing array:
service.FoundItems = function (searchTerm, name) {
var foundArray = [];
var searchPromise = service.getMatchedMenuItems(name, searchTerm);
foundArray.$promise = searchPromise
.then(function (foundItems) {
angular.copy(foundItems, foundArray);
foundArray.reverse();
return foundArray;
})
.catch(function (errorResponse) {
return $q.reject(errorResponse);
})
.finally(function() {
console.log(foundArray);
});
return foundArray;
};
I recommend that the promise be attached to the array reference as a property named $promise so that it can be used to chain functions that depend on results from the server.
Frankly I don't recommend designing services that return array references that are later populated with results. If you insist on designing it that way, this is how it is done.
I tried the $promise thing that you recommended. I was wondering how you would get the value from it ie the array.
In the controller, use the .then method of the $promise to see the final value of the array:
narrowItDownController.$inject = ['MenuSearchService'];
function narrowItDownController(MenuSearchService) {
var menu = this;
menu.input = "";
menu.displayResult = [];
menu.searchX = function(name) {
menu.displayResult = MenuSearchService.FoundItems(menu.input, name);
̶c̶o̶n̶s̶o̶l̶e̶.̶l̶o̶g̶(̶m̶e̶n̶u̶.̶d̶i̶s̶p̶l̶a̶y̶R̶e̶s̶u̶l̶t̶)̶;̶
menu.displayResult.$promise
.then(function(foundArray) {
console.log(foundArray);
console.log(menu.displayResult);
}).catch(function(errorResponse) {
console.log("ERROR");
console.log(errorResponse);
});
};
}
To see the final result, the console.log needs to be moved inside the .then block of the promise.
Titus is right. The function always immediately returns the initial value of foundArray which is an empty array. The promise is executed asynchronously so by the time you are trying to change foundArray it is too late. You need to return the promise itself and then using .then() to retrieve the value just like you are currently doing inside the method.
From just quickly looking at your code I think you made have a simple error in there. Are you sure you want
foundArray = foundItems[0].slice(0);
instead of
foundArray = foundItems.slice(0);
I am using sw-precache module with sw-toolbox. When i navigate to the web app url, the api response is cached in the toolbox cache storage and then if i turn off the internet and refresh the url. Service worker loads the api with a 200OK response from Service Worker Cache but if i navigate to some other url and then come back to the main web app. All the dynamic cached api response of the toolbox cache storage gets deleted. I am using gulp to create the service worker file. The gulp code to generate the file is mentioned below:
gulp.task('generate-service-worker', function(callback) {
var path = require('path');
var swPrecache = require('sw-precache');
var rootDir = './src';
swPrecache.write(path.join(rootDir, 'sw.js'), {
staticFileGlobs: [rootDir + '/**/*.{js,html,css,png,jpg,gif}'],
stripPrefix: rootDir,
importScripts: [
'sw-toolbox.js',
'toolbox-routes.js'
]
}, callback);
});
The sw-toolbox.js has the code generated from bower components of sw-toolbox library. The toolbox-routes.js has four calls as mentioned below:
toolbox.router.get(/^https:\/\/.*\/city/, toolbox.cacheFirst, {});
toolbox.router.get(/^https:\/\/.*\/splash/, toolbox.cacheFirst, {});
toolbox.router.get(/^https:\/\/.*amazonaws\.com\/.*/, toolbox.cacheFirst, {});
toolbox.router.get(/^https:\/\/.*\/api\/booking/, toolbox.networkFirst, {});
The city api looks something like this api for which i get a 404 error.
https://www.abcd.com/city?includes=venue,car
I also get a warning message in the console which says "The FetchEvent for "https://www.abcd.com/city?includes=venue,car" resulted in a network error response: the promise was rejected."
The service worker file is this
'use strict';
importScripts("sw-toolbox.js", "toolbox-routes.js");
/* eslint-disable quotes, comma-spacing */
var PrecacheConfig = [
["/app/directives/datepicker/datepicker.html", "fa21126be162d099882132226c9681d2"],
["/app/directives/datepicker/datepicker.js", "5ec9f32a36e10feeb03c40d984e74058"],
["/app/directives/map/map.js", "cfabf0230663391d8c78f46a2b198f89"],
["/index.html", "98b8890a31f58d2362953593bbda427f"],
["/index.notification.js", "0cf8aa857d28c309f7b37b97141c1853"],
["/sw-toolbox.js", "66531e5962e4dccb0526a2b4cd6364a4"],
["/toolbox-routes.js", "e7e168157a8afc939ecfb44da773270d"]
];
/* eslint-enable quotes, comma-spacing */
var CacheNamePrefix = 'sw-precache-v1--' + (self.registration ? self.registration.scope : '') + '-';
var IgnoreUrlParametersMatching = [/^utm_/];
var addDirectoryIndex = function(originalUrl, index) {
var url = new URL(originalUrl);
if (url.pathname.slice(-1) === '/') {
url.pathname += index;
}
return url.toString();
};
var getCacheBustedUrl = function(url, param) {
param = param || Date.now();
var urlWithCacheBusting = new URL(url);
urlWithCacheBusting.search += (urlWithCacheBusting.search ? '&' : '') +
'sw-precache=' + param;
return urlWithCacheBusting.toString();
};
var isPathWhitelisted = function(whitelist, absoluteUrlString) {
// If the whitelist is empty, then consider all URLs to be whitelisted.
if (whitelist.length === 0) {
return true;
}
// Otherwise compare each path regex to the path of the URL passed in.
var path = (new URL(absoluteUrlString)).pathname;
return whitelist.some(function(whitelistedPathRegex) {
return path.match(whitelistedPathRegex);
});
};
var populateCurrentCacheNames = function(precacheConfig,
cacheNamePrefix, baseUrl) {
var absoluteUrlToCacheName = {};
var currentCacheNamesToAbsoluteUrl = {};
precacheConfig.forEach(function(cacheOption) {
var absoluteUrl = new URL(cacheOption[0], baseUrl).toString();
var cacheName = cacheNamePrefix + absoluteUrl + '-' + cacheOption[1];
currentCacheNamesToAbsoluteUrl[cacheName] = absoluteUrl;
absoluteUrlToCacheName[absoluteUrl] = cacheName;
});
return {
absoluteUrlToCacheName: absoluteUrlToCacheName,
currentCacheNamesToAbsoluteUrl: currentCacheNamesToAbsoluteUrl
};
};
var stripIgnoredUrlParameters = function(originalUrl,
ignoreUrlParametersMatching) {
var url = new URL(originalUrl);
url.search = url.search.slice(1) // Exclude initial '?'
.split('&') // Split into an array of 'key=value' strings
.map(function(kv) {
return kv.split('='); // Split each 'key=value' string into a [key, value] array
})
.filter(function(kv) {
return ignoreUrlParametersMatching.every(function(ignoredRegex) {
return !ignoredRegex.test(kv[0]); // Return true iff the key doesn't match any of the regexes.
});
})
.map(function(kv) {
return kv.join('='); // Join each [key, value] array into a 'key=value' string
})
.join('&'); // Join the array of 'key=value' strings into a string with '&' in between each
return url.toString();
};
var mappings = populateCurrentCacheNames(PrecacheConfig, CacheNamePrefix, self.location);
var AbsoluteUrlToCacheName = mappings.absoluteUrlToCacheName;
var CurrentCacheNamesToAbsoluteUrl = mappings.currentCacheNamesToAbsoluteUrl;
function deleteAllCaches() {
return caches.keys().then(function(cacheNames) {
return Promise.all(
cacheNames.map(function(cacheName) {
return caches.delete(cacheName);
})
);
});
}
self.addEventListener('install', function(event) {
event.waitUntil(
// Take a look at each of the cache names we expect for this version.
Promise.all(Object.keys(CurrentCacheNamesToAbsoluteUrl).map(function(cacheName) {
return caches.open(cacheName).then(function(cache) {
// Get a list of all the entries in the specific named cache.
// For caches that are already populated for a given version of a
// resource, there should be 1 entry.
return cache.keys().then(function(keys) {
// If there are 0 entries, either because this is a brand new version
// of a resource or because the install step was interrupted the
// last time it ran, then we need to populate the cache.
if (keys.length === 0) {
// Use the last bit of the cache name, which contains the hash,
// as the cache-busting parameter.
// See https://github.com/GoogleChrome/sw-precache/issues/100
var cacheBustParam = cacheName.split('-').pop();
var urlWithCacheBusting = getCacheBustedUrl(
CurrentCacheNamesToAbsoluteUrl[cacheName], cacheBustParam);
var request = new Request(urlWithCacheBusting, {
credentials: 'same-origin'
});
return fetch(request).then(function(response) {
if (response.ok) {
return cache.put(CurrentCacheNamesToAbsoluteUrl[cacheName],
response);
}
console.error('Request for %s returned a response status %d, ' +
'so not attempting to cache it.',
urlWithCacheBusting, response.status);
// Get rid of the empty cache if we can't add a successful response to it.
return caches.delete(cacheName);
});
}
});
});
})).then(function() {
return caches.keys().then(function(allCacheNames) {
return Promise.all(allCacheNames.filter(function(cacheName) {
return cacheName.indexOf(CacheNamePrefix) === 0 &&
!(cacheName in CurrentCacheNamesToAbsoluteUrl);
}).map(function(cacheName) {
return caches.delete(cacheName);
}));
});
}).then(function() {
if (typeof self.skipWaiting === 'function') {
// Force the SW to transition from installing -> active state
self.skipWaiting();
}
})
);
});
if (self.clients && (typeof self.clients.claim === 'function')) {
self.addEventListener('activate', function(event) {
event.waitUntil(self.clients.claim());
});
}
self.addEventListener('message', function(event) {
if (event.data.command === 'delete_all') {
console.log('About to delete all caches...');
deleteAllCaches().then(function() {
console.log('Caches deleted.');
event.ports[0].postMessage({
error: null
});
}).catch(function(error) {
console.log('Caches not deleted:', error);
event.ports[0].postMessage({
error: error
});
});
}
});
self.addEventListener('fetch', function(event) {
if (event.request.method === 'GET') {
var urlWithoutIgnoredParameters = stripIgnoredUrlParameters(event.request.url,
IgnoreUrlParametersMatching);
var cacheName = AbsoluteUrlToCacheName[urlWithoutIgnoredParameters];
var directoryIndex = 'index.html';
if (!cacheName && directoryIndex) {
urlWithoutIgnoredParameters = addDirectoryIndex(urlWithoutIgnoredParameters, directoryIndex);
cacheName = AbsoluteUrlToCacheName[urlWithoutIgnoredParameters];
}
var navigateFallback = '';
// Ideally, this would check for event.request.mode === 'navigate', but that is not widely
// supported yet:
// https://code.google.com/p/chromium/issues/detail?id=540967
// https://bugzilla.mozilla.org/show_bug.cgi?id=1209081
if (!cacheName && navigateFallback && event.request.headers.has('accept') &&
event.request.headers.get('accept').includes('text/html') &&
/* eslint-disable quotes, comma-spacing */
isPathWhitelisted([], event.request.url)) {
/* eslint-enable quotes, comma-spacing */
var navigateFallbackUrl = new URL(navigateFallback, self.location);
cacheName = AbsoluteUrlToCacheName[navigateFallbackUrl.toString()];
}
if (cacheName) {
event.respondWith(
// Rely on the fact that each cache we manage should only have one entry, and return that.
caches.open(cacheName).then(function(cache) {
return cache.keys().then(function(keys) {
return cache.match(keys[0]).then(function(response) {
if (response) {
return response;
}
// If for some reason the response was deleted from the cache,
// raise and exception and fall back to the fetch() triggered in the catch().
throw Error('The cache ' + cacheName + ' is empty.');
});
});
}).catch(function(e) {
console.warn('Couldn\'t serve response for "%s" from cache: %O', event.request.url, e);
return fetch(event.request);
})
);
}
}
});
Thanks a tonnn in Advance!!!.
I'm using the selenium javascript webdriver, and I'd like to make a general 'getVisibleElement' function. I saw this version in the webdriver source code:
var link = driver.findElement(firstVisibleLink);
function firstVisibleLink(driver) {
var links = driver.findElements(By.tagName('a'));
return webdriver.promise.filter(links, function(link) {
return links.isDisplayed();
}).then(function(visibleLinks) {
return visibleLinks[0];
});
}
Is there a way that I can generalize this such that I can pass in any css selector as parameter and get just the first visible version of it? I've been having issues with 'invalid locator' and 'undefined is not a function' errors the ways i have been trying.
Page:
var webdriverUtils = require('../../utils/extendWebdriver_utils');
function contactCardPage(){
var elements = Object.create({}, {
viewToggle: { get: function(){
return webdriverUtils.getFirstVisibleElement('[name="viewToggle"]');
}
}
this.toggleView = function(){
return elements.viewToggle.click();
}
});
module.exports = new contactCardPage();
WebdriverUtils:
function webdriverUtils(){
this.getFirstVisibleElement = function(selectorString){
var elements = session.driver.findElements(webdriver.By.css(selectorString));
return webdriver.promise.filter(elements, function(el) {
return el.isDisplayed();
}).then(function(visibleElement) {
return visibleElement[0];
});
}
});
module.exports = new webdriverUtils();
Spec:
var expect = require('chai').expect;
var contactCardPage = require('../../pageObjects/contactCardPage.js');
describe('Contacts', function () {
it('Can be expanded', function(done){
contactCardPage.toggleView().then(function(){
//Expect here for making sure expanded elements are now displayed
done();
});
});
});
Currently this gives me an 'Invalid Locator' error.
Is there any way to pass in a selector and get just the visible elements this way?
I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22
}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.
Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.