We Keep Coding

nodejs write file frequently

I have a listener to listen for the change of content, once the content modified, it will emit the handler function:
$('#editor').on('onchange', () => changeHandler('...','...'));
function changeHandler(filePath, content){
var ws = fs.createWriteStream(filePath, 'utf8');
ws.write(content);
}
My problem is that the 'onchange' occurs too often, so 'write file' too often handles, it may lost data during the period.
Can someone give any suggestion?
Update
Now I've changed code according the answers below looks like:
this.buffer = null; //used to cache
// once content changed, maybe too often
changeHandler() {
if (this.editor.curOp && this.editor.curOp.command.name) {
var id = $('.nav-items li.active .lk-hosts').attr('data-hosts-id');
var content = this.editor.getValue();
// cache data, not immediately write to file
this.buffer = {id: id, content: content};
}
}
setInterval(()=> {
// means there's data in cache
if (this.buffer !== null) {
let id = this.buffer.id;
let content = this.buffer.content;
// reset cache to null
this.buffer = null;
// write file
this.writeContent(id, content, (err)=> {
})
}
}, 800);
Thanks all answers!

Why not simply build a buffer to collect written text then write to file only when you have a certain number of writes:
$('#editor').on('onchange', () => changeHandler('...','...'));
var writeBuffer = ''; // can also make this an array
var writeBufferSize = 0;
var filePath = 'path_to_file';
var ws = fs.createWriteStream(filePath, 'utf8');
function changeHandler(content){
if (writeBufferSize == SOME_THRESHOLD) {
ws.write(writeBuffer);
writeBuffer = '';
writeBufferSize = 0;
} else {
writeBuffer += content + '\n';
writeBufferSize++;
}
}
If you choose a write buffer threshold that's too big, you might want to delegate the write to some worker thread to be done in parallel, and in this case you can create another temporary write buffer to fill out while the original is being written, then switch the two.

This sample below shows how to make debounced event handling, although it's not node.js code it's same in concept.
// eventEmitter variable to use
var emitter = new EventEmitter();
// dom element change event
$('#editor').on('input', function(event) {
emitter.emit('changeEvent', event.target.value);
});
// event listener, which debounces change event of input
emitter.on('changeEvent', debounce(function(data) {
writeFile('li', data);
}, 1000)); // <== debounce for 1second
// sample emitter, for demo
// we don't have access to nodejs EventEmitter class in Stackoverflow
// don't use in production
function EventEmitter() {
var callbacks = [];
return {
on: function(eventName, fn) {
callbacks.push({
eventName: eventName,
callback: fn
})
},
emit: function(eventName, payload) {
var fn = callbacks.find(function(item) {
return item.eventName === eventName;
});
if (fn) {
fn.callback(payload);
}
}
}
}
// simple logger for demo purpose
// emulates write file
function writeFile(name, content) {
var $elem = $(document.createElement(name));
$elem.text(content);
$('#logger').append($elem);
}
// throttle function - reduces fn call with timeout
// credits: https://remysharp.com/2010/07/21/throttling-function-calls
function debounce(fn, delay) {
var timer = null;
return function() {
var context = this,
args = arguments;
clearTimeout(timer);
timer = setTimeout(function() {
fn.apply(context, args);
}, delay);
};
}
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<textarea id="editor" placeholder="Enter text, this will emit change event"></textarea>
<p>
Notice the 1sec throttle (write something, pause for 1sec, write again)
</p>
<ul id="logger"></ul>
The debounce function can be also used on textarea change event
// debounce emitting
$('#editor').on('input', debounce(function(event) {
emitter.emit('changeEvent', event.target.value);
}, 1000));
// write file when received event without debounce
emitter.on('changeEvent', function(data){
logElement('li', data);
});

The Underscore library has _.throttle() and _.debounce() functions.

Unreliable behaviour in Node.js

I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22

}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.

Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};

constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.

How to set callback order invocation same as target function invocation

I have a problem in my project.
To describe this issue I have wrote simplified code snippet:
function waitFor(fnReady, fnCallback) {
var check = function() {
if (fnReady()) {
fnCallback();
}
else {
setTimeout(check, 100); // wait another 100ms, and try again
}
};
check();
}
var result = 0;
var flag = true;
function ajaxRequest() {
setTimeout(
function() { flag = false;
console.log('ping');
},3000
);
}
function ajaxRequestHandler() {
setTimeout(
function() { flag = true;
console.log('pong');
}, 200
);
}
for(var i =0;i<10; i++){
waitFor(function() { return flag; }, ajaxRequest);
waitFor(function() { return !flag; }, ajaxRequestHandler);
}
it returns:
ping - 10 times
pong - 10 times
desired result:
ping
3 second timeout
ping
---------------------
ping
3 second timeout
pong
--------------------
.....
Can you help correct my code?
UPDATE
Actual problem:
I have a google map.
I have a lot of places when I should to redraw it.
For application logic very important that If I send
request1
request2
request3
request4
I should handle responses in the this order
handle response of request1
handle response of request2
handle response of request3
handle response of request4
Problem that I don't know order of requests.
In different places of file I see following code rows:
google.maps.event.addListener(searchBox, 'bounds_changed', renderTerminalsOnMapAndFitBounds);
...
$.getJSON('getAllTerminals.json', renderTerminalsOnMapAndFitBounds);
.....
$.getJSON('getAllTerminalsInsideRectangle.json', renderTerminalsOnMapAndFitBounds);
...
$.getJSON('getAllTerminalsInsideCircle.json', renderTerminalsOnMapAndFitBounds);
...
$.getJSON('getBigTerminals.json', renderTerminalsOnMapAndFitBounds);
........
renderTerminalsOnMapAndFitBounds method sends request to server and in succes alternative render result on map. But this event happens very often

Try this pattern
var map = "abcdefghi".split("");
var responses = []; // collect responses
$.ajaxSetup({
beforeSend : function(jqxhr, settings) {
jqxhr.id = Number(settings.data.split(/id=/)[1]); // add `id` to `request`
console.log(settings.data.split(/id=/)[1]);
}
});
var request = function(id, data) {
// append `id` to `id` data
return $.post("/echo/json/", {json:JSON.stringify([data]), id:id})
};
$.each(map, function(k, v) {
setTimeout(function() {
request(k + 1, v)
.done(function(data) {
// do stuff at each response
console.log(data); // note return values
})
.always(function(data, textStatus, jqxhr) {
// do stuff at each response
responses.push([jqxhr.id, data[0]]);
// do stuff when all requests completed , results items in `responses`
if (responses.length === map.length) {
responses.sort(); // sort `responses` based on `id`
// do stuff with `responses`
console.log(responses);
}
});
},1 + Math.random() * 1000) // async
});
jsfiddle http://jsfiddle.net/guest271314/g254bbjg/

my variant:
var index = 0;
// callback function
function tryMe (param1) {
waitFor(function(){return param1 == index},
function(){console.log(param1);
index++;
}
)
}
// callback executer
function callbackTester (callback,i) {
setTimeout( function(){callback(i);}, 20000 - i*1000);
}
// test function
for(var i=0 ; i<10 ; i++){
callbackTester ( tryMe,i );
}
function waitFor(fnReady, fnCallback) {
var check = function() {
if (fnReady()) {
fnCallback();
}
else {
setTimeout(check, 100); // wait another 100ms, and try again
}
};
check();
}
http://jsfiddle.net/x061dx75/17/

I personally would use promises for this, but you've said no promises (not sure why), so here's a generic sequencer algorithm in plain javascript (tested in the jsFiddle linked below):
function sequence(fn) {
// initialize sequence data upon first use
if (typeof sequence.low === "undefined") {
sequence.low = sequence.high = 0;
sequence.results = {};
}
// save id in local variable so we can reference it in the closure from the function below
var id = sequence.high;
// advance to next sequence number
++sequence.high;
// initialize the result value for this sequence callback
sequence.results[id] = {fn: fn, args: [], ready: false, context: null};
return function(/* args */) {
// save args and context and mark it ready
var args = Array.prototype.slice.call(arguments, 0);
// get the results object for this callback and save info in it
var thisResult = sequence.results[id];
thisResult.args = args;
thisResult.context = this;
thisResult.ready = true;
// now process any requests in order that are ready
for (var i = sequence.low; i < sequence.high; i++) {
var result = sequence.results[i];
// if this one is ready, process it
if (result.ready) {
// increment counter past this result
++sequence.low;
// remove this stored result
delete sequence.results[i];
// process this result
result.fn.apply(result.context, result.args);
} else {
// if this one not ready, then nothing to do yet
break;
}
}
};
}
// your usage:
google.maps.event.addListener(searchBox, 'bounds_changed', sequence(renderTerminalsOnMapAndFitBounds));
...
$.getJSON('getAllTerminals.json', sequence(renderTerminalsOnMapAndFitBounds));
.....
$.getJSON('getAllTerminalsInsideRectangle.json', sequence(renderTerminalsOnMapAndFitBounds));
...
$.getJSON('getAllTerminalsInsideCircle.json', sequence(renderTerminalsOnMapAndFitBounds));
...
$.getJSON('getBigTerminals.json', sequence(renderTerminalsOnMapAndFitBounds));
........
Working demo: http://jsfiddle.net/jfriend00/aqugm1fs/
Conceptually, what this does is as follows:
Pass a substitute completion handler in place of the normal completion callback.
This substitute function marks each response with a sequence id and saved the original completion handler.
If a response comes back while another response with a lower sequence id is still pending, then the result is just stored and saved for later.
As each response comes in, it processes as many responses in sequence as are ready
Note: while all the examples you have use the same callback function, this will work with any callback function so it would work with a mix of different types of operations.

breeze/durandal: configureBreezeManager().saveChanges() doesn't trigger function subscribed hasChangesChanged

I'm been creating a web site following Hottowl's example. However, the value of hasChanges is not changed after saveChanges is called - even the data is actually saved in database table. I use hottowel durandal 2.0.1. I found the durandal version is different from the CCJS-Scratch example. Did I miss anything?
datacontext.js
var manager = configureBreezeManager();
var hasChanges = ko.observable(false);
manager.hasChangesChanged.subscribe(function (eventArgs) {
hasChanges(eventArgs.hasChanges); // The function isn't called when saveChanges is called
});
var cancelChanges = function () {
manager.rejectChanges();
log('Canceled changes', null, true);
};
var saveChanges = function () {
return manager.saveChanges()
.then(saveSucceeded)
.fail(saveFailed);
function saveSucceeded(saveResult) {
log('Saved data successfully', saveResult, true);
}
function saveFailed(error) {
var msg = 'Save failed: ' + getErrorMessages(error);
logError(msg, error);
error.message = msg;
throw error;
}
};
The following code exist in the view model.
var hasChanges = ko.computed(function () {
return datacontext.hasChanges();
});
var cancel = function () {
datacontext.cancelChanges();
};
var canSave = ko.computed(function () {
return hasChanges() && !isSaving();
});
var save = function () {
isSaving(true);
return datacontext.saveChanges().fin(complete);
function complete() {
isSaving(false);
}
};

Maybe you have a timing issue? You have a lot of nested observables and computed observables that I have trouble aligning at this late hour.
What I do know is that the Breeze DocCode:saveTodoTests.js show that hasChangesChanged is called with the correct eventArgs.hasChanges value when both saving changes and reverting (rejectChanges). Here is a passing test from that suite that confirms it.
test("hasChangesChanged event raised after saveChanges", 4, function () {
var em = newTodosEm();
var hasChangesChangedRaised = [];
em.hasChangesChanged.subscribe(
function(eventArgs) {
hasChangesChangedRaised.push(eventArgs.hasChanges);
}
);
// add a Todo (and forget about it)
em.createEntity('TodoItem',{ Description: "Learn to save in breeze" });
stop();
em.saveChanges()
.then ( function() {
equal(hasChangesChangedRaised.length, 2,
"hasChangesChanged should have been raised twice");
ok(hasChangesChangedRaised[0]===true,
"first hasChangesChanged is true after create");
ok(hasChangesChangedRaised[1]===false,
"second hasChangesChanged is false after save");
ok(!em.hasChanges(),
"manager should not have pending changes after save");
})
.fail(handleSaveFailed)
.fin(start);
});
Note that the hasChangesChanged event is raised twice in this test.
Please try debugging again and make sure that the event you're looking at is the one raised after the save, not during some other manipulation.
Also watch out for the nesting of KO events. Race conditions are possible.

How does jQuery do async:false in its $.ajax method?

I have a similar question here, but I thought I'd ask it a different way to cast a wider net. I haven't come across a workable solution yet (that I know of).
I'd like for XCode to issue a JavaScript command and get a return value back from an executeSql callback.
From the research that I've been reading, I can't issue a synchronous executeSql command. The closest I came was trying to Spin Lock until I got the callback. But that hasn't worked yet either. Maybe my spinning isn't giving the callback chance to come back (See code below).
Q: How can jQuery have an async=false argument when it comes to Ajax? Is there something different about XHR than there is about the executeSql command?
Here is my proof-of-concept so far: (Please don't laugh)
// First define any dom elements that are referenced more than once.
var dom = {};
dom.TestID = $('#TestID'); // <input id="TestID">
dom.msg = $('#msg'); // <div id="msg"></div>
window.dbo = openDatabase('POC','1.0','Proof-Of-Concept', 1024*1024); // 1MB
!function($, window, undefined) {
var Variables = {}; // Variables that are to be passed from one function to another.
Variables.Ready = new $.Deferred();
Variables.DropTableDeferred = new $.Deferred();
Variables.CreateTableDeferred = new $.Deferred();
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'drop table Test;',
[],
Variables.DropTableDeferred.resolve()
// ,WebSqlError
);
});
$.when(Variables.DropTableDeferred).done(function() {
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'CREATE TABLE IF NOT EXISTS Test'
+ '(TestID Integer NOT NULL PRIMARY KEY'
+ ',TestSort Int'
+ ');',
[],
Variables.CreateTableDeferred.resolve(),
WebSqlError
);
});
});
$.when(Variables.CreateTableDeferred).done(function() {
for (var i=0;i < 10;i++) {
myFunction(i);
};
Variables.Ready.resolve();
function myFunction(i) {
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'INSERT INTO Test(TestID,TestSort) VALUES(?,?)',
[
i
,i+100000
]
,function() {}
,WebSqlError
)
});
};
});
$.when(Variables.Ready).done(function() {
$('#Save').removeAttr('disabled');
});
}(jQuery, window);
!function($, window, undefined) {
var Variables = {};
$(document).on('click','#Save',function() {
var local = {};
local.result = barcode.Scan(dom.TestID.val());
console.log(local.result);
});
var mySuccess = function(transaction, argument) {
var local = {};
for (local.i=0; local.i < argument.rows.length; local.i++) {
local.qry = argument.rows.item(local.i);
Variables.result = local.qry.TestSort;
}
Variables.Return = true;
};
var myError = function(transaction, argument) {
dom.msg.text(argument.message);
Variables.result = '';
Variables.Return = true;
}
var barcode = {};
barcode.Scan = function(argument) {
var local = {};
Variables.result = '';
Variables.Return = false;
window.dbo.transaction(function(myTrans) {
myTrans.executeSql(
'SELECT * FROM Test WHERE TestID=?'
,[argument]
,mySuccess
,myError
)
});
for (local.I = 0;local.I < 3; local.I++) { // Try a bunch of times.
if (Variables.Return) break; // Gets set in mySuccess and myError
SpinLock(250);
}
return Variables.result;
}
var SpinLock = function(milliseconds) {
var local = {};
local.StartTime = Date.now();
do {
} while (Date.now() < local.StartTime + milliseconds);
}
function WebSqlError(tx,result) {
if (dom.msg.text()) {
dom.msg.append('<br>');
}
dom.msg.append(result.message);
}
}(jQuery, window);

Is there something different about XHR than there is about the executeSql command?
Kind of.
How can jQuery have an async=false argument when it comes to Ajax?
Ajax, or rather XMLHttpRequest, isn't strictly limited to being asynchronous -- though, as the original acronym suggested, it is preferred.
jQuery.ajax()'s async option is tied to the boolean async argument of xhr.open():
void open(
DOMString method,
DOMString url,
optional boolean async, // <---
optional DOMString user,
optional DOMString password
);
The Web SQL Database spec does also define a Synchronous database API. However, it's only available to implementations of the WorkerUtils interface, defined primarily for Web Workers:
window.dbo = openDatabaseSync('POC','1.0','Proof-Of-Concept', 1024*1024);
var results;
window.dbo.transaction(function (trans) {
results = trans.executeSql('...');
});
If the environment running the script hasn't implemented this interface, then you're stuck with the asynchronous API and returning the result will not be feasible. You can't force blocking/waiting of asynchronous tasks for the reason you suspected:
Maybe my spinning isn't giving the callback chance to come back (See code below).