I have set a timeout for sending back an error. The problem is that if I need to clear the timeout with clearTimeOut() it does indeed kill it, I can see that as the value of errTimeout's _kill shows true in the debugger. But for some reason node still keeps running the script until the timeOutPeriod is over. I guess it won't really create issues in production because the calling function will receive the returned value. But it still kinda pisses me off that it keeps waiting instead of killing the script.
return new Promise((resolve,reject) => {
function checkResponse () {
//creates a timeout that returns an error if data not receieved after the specified time.
let errTimeout = setTimeout(reject, config.timeOutPeriod);
//the +1 is there because we originally reduced one, we need to use the physical number now.
if(layerKeycodes.length !== (rows+1) * (columns+1)){
//if the array is not complete, check again in 100 ms
setTimeout(checkResponse, 100);
} else {
//clear the error timeout
clearTimeout(errTimeout);
//send the layerKeycodes to the calling function
resolve(layerKeycodes);
}
}
It looks like this code is something you're trying to fit into getLayerKeycodes() from this other question to somehow know when all the data has been received from your keyboard hardware.
I'll illustrate how you can plug into that without using timers. Here's what you started with in that other question:
Your original function
const getLayerKeycodes = (keyboard, layer, rows, columns) => {
//array that stores all the keycodes according to their order
let layerKeycodes = [];
//rows and columns start the count at 0 in low level, so we need to decrease one from the actual number.
columns --;
rows --;
//loop that asks about all the keycodes in a layer
const dataReceived = (err, data) => {
if(err) {
return err;
}
// push the current keycode to the array
// The keycode is always returned as the fifth object.
layerKeycodes.push(data[5]);
console.log(layerKeycodes);
};
for (let r = 0 , c = 0;c <= columns; r ++){
//callback to fire once data is receieved back from the keyboard.
if(r > rows){
c++;
//r will turn to 0 once the continue fires and the loop executes again
r = -1;
continue;
}
//Start listening and call dataReceived when data is received
keyboard[0].read(dataReceived);
//Ask keyboard for information about keycode
// [always 0 (first byte is ignored),always 0x04 (get_keycode),layer requested,row being checked,column being checked]
keyboard[0].write([0x01,0x04,layer,r,c]);
}
console.log(layerKeycodes);
}
Manually created promise to resolve upon completion of all rows/columns
And, you can incorporate the completion detection code inside of the dataReceived() function without any timers and without reworking much of the rest of your logic like this:
const getLayerKeycodes = (keyboard, layer, rows, columns) => {
return new Promise((resolve, reject) => {
//array that stores all the keycodes according to their order
const layerKeycodes = [];
const totalCells = rows * columns;
let abort = false;
//rows and columns start the count at 0 in low level, so we need to decrease one from the actual number.
columns--;
rows--;
// function that gets with keyboard data
function dataReceived(err, data) => {
if (err) {
abort = true; // set flag to stop sending more requests
reject(err);
return;
}
// push the current keycode to the array
// The keycode is always returned as the fifth object.
layerKeycodes.push(data[5]);
// now see if we're done with all of them
if (layerKeycodes.length >= totalCells) {
resolve(layerKeycodes);
}
}
// loop that asks about all the keycodes in a layer
for (let r = 0, c = 0; c <= columns; r++) {
// stop sending more requests if we've already gotten an error
if (abort) {
break;
}
//callback to fire once data is receieved back from the keyboard.
if (r > rows) {
c++;
//r will turn to 0 once the continue fires and the loop executes again
r = -1;
continue;
}
//Start listening and call dataReceived when data is received
keyboard[0].read(dataReceived);
//Ask keyboard for information about keycode
// [always 0 (first byte is ignored),always 0x04 (get_keycode),layer requested,row being checked,column being checked]
keyboard[0].write([0x01, 0x04, layer, r, c]);
}
}
}
}
A simplified version by promisifying the read function
And, here's a bit simpler version that promisifies the read function so we can use await on it and then just use an async function and a dual nested for loop for simpler loop mechanics.
const util = require('util');
async function getLayerKeycodes(keyboard, layer, rows, columns) => {
// promisify the keyboard.read()
const readKeyboard = util.promisify(keyboard[0].read).bind(keyboard[0]);
//array that stores all the keycodes according to their order
const layerKeycodes = [];
// loop that asks about all the keycodes in a layer
for (let rowCntr = 0; rowCntr < rows; rowCntr++) {
for (let colCntr = 0; colCntr < columns; colCntr++) {
// Start listening and collect the promise
const readPromise = readKeyboard();
// Ask keyboard for information about keycode
// [always 0 (first byte is ignored),always 0x04 (get_keycode),layer requested,row being checked,column being checked]
keyboard[0].write([0x01, 0x04, layer, rowCntr, colCntr]);
// wait for data to come in
const data = await readPromise;
// push the current keycode to the array
// The keycode is always returned as the fifth object.
layerKeycodes.push(data[5]);
}
}
return layerCodes;
}
This also makes sure that we send a write, then wait for the data from that write to come back before we sent the next write which seems like a potentially safer way to handle the hardware. Your original code fires all the writes at once which might work, but it seems like the reads could come back in any order. This guarantees sequential order in the layerCodes array which seems safer (I'm not sure if that matters with this data or not).
Error handling in this last version is somewhat automatically handled by the async function and the promises. If the read returns an error, then the readPromise will automatically reject which will abort our loop and in turn reject the promise that the async function returned. So, we don't have to do the abort checking that the previous function with the manually created promise had.
Now, of course, I don't have the ability to run any of these to test them so it's possible there are some typos somewhere, but hopefully you can work through any of those and see the concept for how these work.
Related
I have a web socket that receives data from a web socket server every 100 to 200ms, ( I have tried both with a shared web worker as well as all in the main.js file),
When new JSON data arrives my main.js runs filter_json_run_all(json_data) which updates Tabulator.js & Dygraph.js Tables & Graphs with some custom color coding based on if values are increasing or decreasing
1) web socket json data ( every 100ms or less) -> 2) run function filter_json_run_all(json_data) (takes 150 to 200ms) -> 3) repeat 1 & 2 forever
Quickly the timestamp of the incoming json data gets delayed versus the actual time (json_time 15:30:12 vs actual time: 15:31:30) since the filter_json_run_all is causing a backlog in operations.
So it causes users on different PC's to have websocket sync issues, based on when they opened or refreshed the website.
This is only caused by the long filter_json_run_all() function, otherwise if all I did was console.log(json_data) they would be perfectly in sync.
Please I would be very very grateful if anyone has any ideas how I can prevent this sort of blocking / backlog of incoming JSON websocket data caused by a slow running javascript
function :)
I tried using a shared web worker which works but it doesn't get around the delay in main.js blocked by filter_json_run_all(), I dont thing I can put filter_json_run_all() since all the graph & table objects are defined in main & also I have callbacks for when I click on a table to update a value manually (Bi directional web socket)
If you have any ideas or tips at all I will be very grateful :)
worker.js:
const connectedPorts = [];
// Create socket instance.
var socket = new WebSocket(
'ws://'
+ 'ip:port'
+ '/ws/'
);
// Send initial package on open.
socket.addEventListener('open', () => {
const package = JSON.stringify({
"time": 123456,
"channel": "futures.tickers",
"event": "subscribe",
"payload": ["BTC_USD", "ETH_USD"]
});
socket.send(package);
});
// Send data from socket to all open tabs.
socket.addEventListener('message', ({ data }) => {
const package = JSON.parse(data);
connectedPorts.forEach(port => port.postMessage(package));
});
/**
* When a new thread is connected to the shared worker,
* start listening for messages from the new thread.
*/
self.addEventListener('connect', ({ ports }) => {
const port = ports[0];
// Add this new port to the list of connected ports.
connectedPorts.push(port);
/**
* Receive data from main thread and determine which
* actions it should take based on the received data.
*/
port.addEventListener('message', ({ data }) => {
const { action, value } = data;
// Send message to socket.
if (action === 'send') {
socket.send(JSON.stringify(value));
// Remove port from connected ports list.
} else if (action === 'unload') {
const index = connectedPorts.indexOf(port);
connectedPorts.splice(index, 1);
}
});
Main.js This is only part of filter_json_run_all which continues on for about 6 or 7 Tabulator & Dygraph objects. I wante to give an idea of some of the operations called with SetTimeout() etc
function filter_json_run_all(json_str){
const startTime = performance.now();
const data_in_array = json_str //JSON.parse(json_str.data);
// if ('DATETIME' in data_in_array){
// var milliseconds = (new Date()).getTime() - Date.parse(data_in_array['DATETIME']);
// console.log("milliseconds: " + milliseconds);
// }
if (summary in data_in_array){
if("DATETIME" in data_in_array){
var time_str = data_in_array["DATETIME"];
element_time.innerHTML = time_str;
}
// summary Data
const summary_array = data_in_array[summary];
var old_sum_arr_krw = [];
var old_sum_arr_irn = [];
var old_sum_arr_ntn = [];
var old_sum_arr_ccn = [];
var old_sum_arr_ihn = [];
var old_sum_arr_ppn = [];
var filtered_array_krw_summary = filterByProperty_summary(summary_array, "KWN")
old_sum_arr_krw.unshift(Table_summary_krw.getData());
Table_summary_krw.replaceData(filtered_array_krw_summary);
//Colour table
color_table(filtered_array_krw_summary, old_sum_arr_krw, Table_summary_krw);
var filtered_array_irn_summary = filterByProperty_summary(summary_array, "IRN")
old_sum_arr_irn.unshift(Table_summary_inr.getData());
Table_summary_inr.replaceData(filtered_array_irn_summary);
//Colour table
color_table(filtered_array_irn_summary, old_sum_arr_irn, Table_summary_inr);
var filtered_array_ntn_summary = filterByProperty_summary(summary_array, "NTN")
old_sum_arr_ntn.unshift(Table_summary_twd.getData());
Table_summary_twd.replaceData(filtered_array_ntn_summary);
//Colour table
color_table(filtered_array_ntn_summary, old_sum_arr_ntn, Table_summary_twd);
// remove formatting on fwds curves
setTimeout(() => {g_fwd_curve_krw.updateOptions({
'file': dataFwdKRW,
'labels': ['Time', 'Bid', 'Ask'],
strokeWidth: 1,
}); }, 200);
setTimeout(() => {g_fwd_curve_inr.updateOptions({
'file': dataFwdINR,
'labels': ['Time', 'Bid', 'Ask'],
strokeWidth: 1,
}); }, 200);
// remove_colors //([askTable_krw, askTable_inr, askTable_twd, askTable_cny, askTable_idr, askTable_php])
setTimeout(() => { askTable_krw.getRows().forEach(function (item, index) {
row = item.getCells();
row.forEach(function (value_tmp){value_tmp.getElement().style.backgroundColor = '';}
)}); }, 200);
setTimeout(() => { askTable_inr.getRows().forEach(function (item, index) {
row = item.getCells();
row.forEach(function (value_tmp){value_tmp.getElement().style.backgroundColor = '';}
)}); }, 200);
color_table Function
function color_table(new_arr, old_array, table_obj){
// If length is not equal
if(new_arr.length!=old_array[0].length)
console.log("Diff length");
else
{
// Comparing each element of array
for(var i=0;i<new_arr.length;i++)
//iterate old dict dict
for (const [key, value] of Object.entries(old_array[0][i])) {
if(value == new_arr[i][key])
{}
else{
// console.log("Different element");
if(key!="TENOR")
// console.log(table_obj)
table_obj.getRows()[i].getCell(key).getElement().style.backgroundColor = 'yellow';
if(key!="TIME")
if(value < new_arr[i][key])
//green going up
//text_to_speech(new_arr[i]['CCY'] + ' ' +new_arr[i]['TENOR']+ ' getting bid')
table_obj.getRows()[i].getCell(key).getElement().style.backgroundColor = 'Chartreuse';
if(key!="TIME")
if(value > new_arr[i][key])
//red going down
table_obj.getRows()[i].getCell(key).getElement().style.backgroundColor = 'Crimson';
}
}
}
}
Potential fudge / solution, thanks Aaron :):
function limiter(fn, wait){
let isCalled = false,
calls = [];
let caller = function(){
if (calls.length && !isCalled){
isCalled = true;
if (calls.length >2){
calls.splice(0,calls.length-1)
//remove zero' upto n-1 function calls from array/ queue
}
calls.shift().call();
setTimeout(function(){
isCalled = false;
caller();
}, wait);
}
};
return function(){
calls.push(fn.bind(this, ...arguments));
// let args = Array.prototype.slice.call(arguments);
// calls.push(fn.bind.apply(fn, [this].concat(args)));
caller();
};
}
This is then defined as a constant for a web worker to call:
const filter_json_run_allLimited = limiter(data => { filter_json_run_all(data); }, 300); // 300ms for examples
Web worker calls the limited function when new web socket data arrives:
// Event to listen for incoming data from the worker and update the DOM.
webSocketWorker.port.addEventListener('message', ({ data }) => {
// Limited function
filter_json_run_allLimited(data);
});
Please if anyone knows how websites like tradingview or real time high performance data streaming sites allow for low latency visualisation updates, please may you comment, reply below :)
I'm reticent to take a stab at answering this for real without knowing what's going on in color_table. My hunch, based on the behavior you're describing is that filter_json_run_all is being forced to wait on a congested DOM manipulation/render pipeline as HTML is being updated to achieve the color-coding for your updated table elements.
I see you're already taking some measures to prevent some of these DOM manipulations from blocking this function's execution (via setTimeout). If color_table isn't already employing a similar strategy, that'd be the first thing I'd focus on refactoring to unclog things here.
It might also be worth throwing these DOM updates for processed events into a simple queue, so that if slow browser behavior creates a rendering backlog, the function actually responsible for invoking pending DOM manipulations can elect to skip outdated render operations to keep the UI acceptably snappy.
Edit: a basic queueing system might involve the following components:
The queue, itself (this can be a simple array, it just needs to be accessible to both of the components below).
A queue appender, which runs during filter_json_run_all, simply adding objects to the end of the queue representing each DOM manipulation job you plan to complete using color_table or one of your setTimeout` callbacks. These objects should contain the operation to performed (i.e: the function definition, uninvoked), and the parameters for that operation (i.e: the arguments you're passing into each function).
A queue runner, which runs on its own interval, and invokes pending DOM manipulation tasks from the front of the queue, removing them as it goes. Since this operation has access to all of the objects in the queue, it can also take steps to optimize/combine similar operations to minimize the amount of repainting it's asking the browser to do before subsequent code can be executed. For example, if you've got several color_table operations that coloring the same cell multiple times, you can simply perform this operation once with the data from the last color_table item in the queue involving that cell. Additionally, you can further optimize your interaction with the DOM by invoking the aggregated DOM manipulation operations, themselves, inside a requestAnimationFrame callback, which will ensure that scheduled reflows/repaints happen only when the browser is ready, and is preferable from a performance perspective to DOM manipulation queueing via setTimeout/setInterval.
I'd like to know when an operation has finished - It's an iteration where we don't know the size of the list be iterated over (it could be any length.)
Here's my code:
var array = [];
stripe.charges.list().autoPagingEach(function(charge) {
var post = {
chargeId: charge.id,
customerId: charge.customer,
sourceId:charge.source.id,
amount:(charge.amount/100).toFixed(2),
description:charge.description,
dateAndTime:moment(charge.created*1000).format('YYYY-MM-DD HH:mm:ss'),
chargeStatus:charge.status,
failureMessage:charge.failure_message
};
array.push(post)
});
How can I console.log(array.length) once the iteration has finished?
I have seen some examples that use a callback with Done() which would appear to be what I'm after - But I can't figure how to factor it into this code.
As per Stripe's documentation, stripe.charges.list() returns an object with the list of charges in the data property.
You can do:
let charges = stripe.charges.list();
let pending_charges = charges.data.length;
charges.autoPagingEach(function(charge) {
// do your thing
pending_charges -= 1;
if ( pending_charges == 0 ) {
// call a function after the last charge has been processed
// and be careful with failing autoPagingEach() executions
}
});
I'm building a web crawler in node.js and Electron.
Essentially, the program takes in a starting URL, and crawls through to a certain depth, reporting back where it found certain keywords.
This is working aso far, but I can't figure out how to actually tell when it's done. Given a depth of 3-4, this program seems to run forever. with a lower depth, the only way to really tell if it's still crawling is to look at the amount of CPU/memory it's using.
Here is the function that does the crawling:
function crawl(startingSite, depth) {
if (depth < maxDepth) {
getLinks(startingSite, function (sites) { //pulls all the links from a specific page and returns them as an array of strings
for (var i = 0; i < sites.length; i++) { //for each string we got from the page
findTarget(sites[i], depth); //find any of the keywords we want on the page, print out if so
crawl(sites[i], depth + 1); //crawl all the pages on that page, and increase the depth
}
});
}
}
My problem is, I can't figure out how to get this function to report back when it's done.
I tried something like this:
function crawl(startingSite, depth, callback) {
if (depth < maxDepth) {
getLinks(startingSite, function (sites) { //pulls all the links from a specific page and returns them as an array of strings
for (var i = 0; i < sites.length; i++) { //for each string we got from the page
findTarget(sites[i], depth); //find any of the keywords we want on the page, print out if so
crawl(sites[i], depth + 1); //crawl all the pages on that page, and increase the depth
}
});
}
else
{
callback();
}
}
but obviously, the callback() gets called immediately, because the crawler hits the depth quickly and exits the if statement.
All I need is for this function to print out (to the console.log for example) as soon as all the recursive instances of it are done crawling and have reached the maximum depth.
Any ideas?
You could use promises:
const links = (start) =>
new Promise(res => getLinks(start, res));
async function crawl(startingSite, depth) {
if (depth >= maxDepth)
return;
const sites = await links(startingSite);
for (const site of sites) {
await findTarget(site, depth);
await crawl(site, depth + 1);
}
}
Then just do:
crawl(something, 0).then(() => console.log("done"));
I have a for loop like so:
var currentLargest
for (var site in sites) {
// Each site is a website (www.mysite1.com, www.mysite2.com, etc.)
$.getJSON(site).then(function(data) {
if (data > currentLargest) {
currentLargest = data
}
});
}
// Here is where I want to use "currentLargest"
It checks multiple different websites to get the largest value. However, I can't extract the data I need after the for loop. I can only interact with currentLargest from within the promise, but that data is only relevant for ONE of the sites (it hasn't finished looping through all them until after the for loop).
How can I access the currentLargest value after the for loop containing the getJSON promises?
// Your code:
/*
var currentLargest
for (var site in sites) {
// Each site is a website (www.mysite1.com, www.mysite2.com, etc.)
$.getJSON(site).then(function(data) {
if (data > currentLargest) {
currentLargest = data
}
});
}
*/
// Updated:
const sitePromisesArr = sites.map((site) => $.getJSON(site));
$.when(sitePromisesArr)
.then((valArr, index) => {
// The "valArr" here will be an Array with the responses
const largest = valArr.reduce((current, next) => next > current ? next : current, 0);
// This will be the largest value.
console.log(largest);
// This will be the site that contains the largest value.
console.log(sitePromisesArr[index]);
});
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
You need to use a async loop and then execute a callback once all the async getJSON requests are finished. You could use http://caolan.github.io/async/docs.html#each to help with this.
I have only recently started developing for node.js, so forgive me if this is a stupid question - I come from Javaland, where objects still live happily sequentially and synchronous. ;)
I have a key generator object that issues keys for database inserts using a variant of the high-low algorithm. Here's my code:
function KeyGenerator() {
var nextKey;
var upperBound;
this.generateKey = function(table, done) {
if (nextKey > upperBound) {
require("../sync/key-series-request").requestKeys(function(err,nextKey,upperBound) {
if (err) { return done(err); }
this.nextKey = nextKey;
this.upperBound = upperBound;
done(nextKey++);
});
} else {
done(nextKey++);
}
}
}
Obviously, when I ask it for a key, I must ensure that it never, ever issues the same key twice. In Java, if I wanted to enable concurrent access, I would make make this synchronized.
In node.js, is there any similar concept, or is it unnecessary? I intend to ask the generator for a bunch of keys for a bulk insert using async.parallel. My expectation is that since node is single-threaded, I need not worry about the same key ever being issued more than once, can someone please confirm this is correct?
Obtaining a new series involves an asynchronous database operation, so if I do 20 simultaneous key requests, but the series has only two keys left, won't I end up with 18 requests for a new series? What can I do to avoid that?
UPDATE
This is the code for requestKeys:
exports.requestKeys = function (done) {
var db = require("../storage/db");
db.query("select next_key, upper_bound from key_generation where type='issue'", function(err,results) {
if (err) { done(err); } else {
if (results.length === 0) {
// Somehow we lost the "issue" row - this should never have happened
done (new Error("Could not find 'issue' row in key generation table"));
} else {
var nextKey = results[0].next_key;
var upperBound = results[0].upper_bound;
db.query("update key_generation set next_key=?, upper_bound=? where type='issue'",
[ nextKey + KEY_SERIES_WIDTH, upperBound + KEY_SERIES_WIDTH],
function (err,results) {
if (err) { done(err); } else {
done(null, nextKey, upperBound);
}
});
}
}
});
}
UPDATE 2
I should probably mention that consuming a key requires db access even if a new series doesn't have to be requested, because the consumed key will have to be marked as used in the database. The code doesn't reflect this because I ran into trouble before I got around to implementing that part.
UPDATE 3
I think I got it using event emitting:
function KeyGenerator() {
var nextKey;
var upperBound;
var emitter = new events.EventEmitter();
var requesting = true;
// Initialize the generator with the stored values
db.query("select * from key_generation where type='use'", function(err, results)
if (err) { throw err; }
if (results.length === 0) {
throw new Error("Could not get key generation parameters: Row is missing");
}
nextKey = results[0].next_key;
upperBound = results[0].upper_bound;
console.log("Setting requesting = false, emitting event");
requesting = false;
emitter.emit("KeysAvailable");
});
this.generateKey = function(table, done) {
console.log("generateKey, state is:\n nextKey: " + nextKey + "\n upperBound:" + upperBound + "\n requesting:" + requesting + " ");
if (nextKey > upperBound) {
if (!requesting) {
requesting = true;
console.log("Requesting new series");
require("../sync/key-series-request").requestSeries(function(err,newNextKey,newUpperBound) {
if (err) { return done(err); }
console.log("New series available:\n nextKey: " + newNextKey + "\n upperBound: " + newUpperBound);
nextKey = newNextKey;
upperBound = newUpperBound;
requesting = false;
emitter.emit("KeysAvailable");
done(null,nextKey++);
});
} else {
console.log("Key request is already underway, deferring");
var that = this;
emitter.once("KeysAvailable", function() { console.log("Executing deferred call"); that.generateKey(table,done); });
}
} else {
done(null,nextKey++);
}
}
}
I've peppered it with logging outputs, and it does do what I want it to.
As another answer mentions, you will potentially end up with results different from what you want. Taking things in order:
function KeyGenerator() {
// at first I was thinking you wanted these as 'class' properties
// and thus would want to proceed them with this. rather than as vars
// but I think you want them as 'private' members variables of the
// class instance. That's dandy, you'll just want to do things differently
// down below
var nextKey;
var upperBound;
this.generateKey = function (table, done) {
if (nextKey > upperBound) {
// truncated the require path below for readability.
// more importantly, renamed parameters to function
require("key-series-request").requestKeys(function(err,nKey,uBound) {
if (err) { return done(err); }
// note that thanks to the miracle of closures, you have access to
// the nextKey and upperBound variables from the enclosing scope
// but I needed to rename the parameters or else they would shadow/
// obscure the variables with the same name.
nextKey = nKey;
upperBound = uBound;
done(nextKey++);
});
} else {
done(nextKey++);
}
}
}
Regarding the .requestKeys function, you will need to somehow introduce some kind of synchronization. This isn't actually terrible in one way because with only one thread of execution, you don't need to sweat the challenge of setting your semaphore in a single operation, but it is challenging to deal with the multiple callers because you will want other callers to effectively (but not really) block waiting for the first call to requestKeys() which is going to the DB to return.
I need to think about this part a bit more. I had a basic solution in mind which involved setting a simple semaphore and queuing the callbacks, but when I was typing it up I realized I was actually introducing a more subtle potential synchronization bug when processing the queued callbacks.
UPDATE:
I was just finishing up one approach as you were writing about your EventEmitter approach, which seems reasonable. See this gist which illustrates the approach. I took. Just run it and you'll see the behavior. It has some console logging to see which calls are getting deferred for a new key block or which can be handled immediately. The primary moving part of the solution is (note that the keyManager provides the stubbed out implementation of your require('key-series-request'):
function KeyGenerator(km) {
this.nextKey = undefined;
this.upperBound = undefined;
this.imWorkingOnIt = false;
this.queuedCallbacks = [];
this.keyManager = km;
this.generateKey = function(table, done) {
if (this.imWorkingOnIt){
this.queuedCallbacks.push(done);
console.log('KG deferred call. Pending CBs: '+this.queuedCallbacks.length);
return;
};
var self=this;
if ((typeof(this.nextKey) ==='undefined') || (this.nextKey > this.upperBound) ){
// set a semaphore & add the callback to the queued callback list
this.imWorkingOnIt = true;
this.queuedCallbacks.push(done);
this.keyManager.requestKeys(function(err,nKey,uBound) {
if (err) { return done(err); }
self.nextKey = nKey;
self.upperBound = uBound;
var theCallbackList = self.queuedCallbacks;
self.queuedCallbacks = [];
self.imWorkingOnIt = false;
theCallbackList.forEach(function(f){
// rather than making the final callback directly,
// call KeyGenerator.generateKey() with the original
// callback
setImmediate(function(){self.generateKey(table,f);});
});
});
} else {
console.log('KG immediate call',self.nextKey);
var z= self.nextKey++;
setImmediate(function(){done(z);});
}
}
};
If your Node.js code to calculate the next key didn't need to execute an async operation then you wouldn't run into synchronization issues because there is only one JavaScript thread executing code. Access to the nextKey/upperBound variables will be done in sequence by only one thread (i.e. request 1 will access first, then request 2, then request 3 et cetera.) In the Java-world you will always need synchronization because multiple threads will be executing even if you didn't make a DB call.
However, in your Node.js code since you are making an async call to get the nextKey you could get strange results. There is still only one JavaScript thread executing your code, but it would be possible for request 1 to make the call to the DB, then Node.js might accept request 2 (while request 1 is getting data from the DB) and this second request will also make a request to the DB to get keys. Let's say that request 2 gets data from the DB quicker than request 1 and update nextKey/upperBound variables with values 100/150. Once request 1 gets its data (say values 50/100) then it will update nextKey/upperBound. This scenario wouldn't result in duplicate keys, but you might see gaps in your keys (for example, not all keys 100 to 150 will be used because request 1 eventually reset the values to 50/100)
This makes me think that you will need a way to sync access, but I am not exactly sure what will be the best way to achieve this.