We Keep Coding

Two Node.JS scripts read/write on a single .JSON file but the first script cant modify the file

I have two discord bots(app.js and activity.js) running in Node.JS on a Ubuntu server. The problem i am facing is that when both of them are running, only the activity.js is able to modify the file. The file is users.JSON . Code of the app.js:
const Discord = require("discord.js")
var moment = require("moment")
var data_users = fs.readFileSync('/home/discord/activity_bot/users.json', 'utf-8')
var arxeio = JSON.parse(data_users)
...
for (var i in duos) {
if (arxeio[duos[i].username]) {
console.log(`before: ` + arxeio[duos[i].username])
arxeio[duos[i].username]+=15
console.log(`after: ` + arxeio[duos[i].username])
} else {
arxeio[duos[i].username]=15
}
}
fs.writeFile('/home/discord/activity_bot/users.json', JSON.stringify(arxeio, null, 2), 'utf-8', function(err) {
if (err) throw err
console.log('entered')
})
And the code for the Activity.js is:
const Discord = require("discord.js");
var fs = require('fs');
var data = fs.readFileSync('/home/discord/activity_bot/users.json', 'utf-8')
var arxeio = JSON.parse(data)
...
var kuklos = setInterval(function(done) {
client.guilds.get('323921290543235073').channels.forEach(function(kanali, kanaliID) {
if (kanali.type === 'voice' && !kanali.name.includes("AFK")) {
kanali.members.forEach(function(melos, melosID) {
let xristis = melos.user.username;
if (arxeio[xristis]) {
arxeio[xristis]++;
} else {
arxeio[xristis] = 1
}
fs.writeFile('/home/discord/activity_bot/users.json', JSON.stringify(arxeio, null, 2), 'utf-8', function(err) {
if (err) throw err
})
})
}
})
}, 60*1000);
Where, duos is a table of members.
I have concluded that the problem is the fs.writeFile in the App.js because when activity.js is not running it works. When activity.js is runnning "entered" is indeed logged in the app.js but the file is not modified. Also both commands above and below the += command show it is modified but not saved in the users.Json file. Any idea what is to blame? (except for my skillz :P )

I think that the problem is in your app design. You can absolutely share a file with two processes, but you'll always have concurrency problems.
For these kind of things you have to use a database that locks rows/tables.

Typeerror when including a module in node.js

I'm following the nodeschool learnyounode tutorial and I'm having some trouble with the module lesson. In short I'm supposed to make a module which I pass a directory to and a file extension and then use a callback to print the list of files with that extension in the folder.
I have two files, my module (mymodule.js):
var module = require('module');
var fs = require('fs');
var path = require('path');
module.exports = function(testDirectory, testExtension, callback)
{
fs.readdir(testDirectory, function(error, folderContents)
{
if (error) return callback(error);
for (i = 0; i < folderContents.length; i++)
{
var fileExtension = path.extname(folderContents[i]);
if(fileExtension === "." + testExtension)
{
callback(null, folderContents[i]);
}
}
});
};
and my app file (program.js):
var mymodule = require('./mymodule.js');
mymodule(process.argv[2], process.argv[3], function(error, data)
{
console.log(data);
});
Whenever I try test my app file, I get TypeError: mymodule is not a function
I've done lots of reading, but the more I read the more confused I become. Someone please help?

Is it okay to use synchronous code to load data into the memory at startup?

In my routes.js file, I've this:
var pages = require('./pages')();
...
app.get('/:page', function(req, res, next) {
var p = req.params.page;
if (p in pages) {
res.render('page', pages[p]);
} else {
next();
}
});
pages.js:
module.exports = function() {
var fs = require('fs'),
ret = [],
dir = './pages',
files = fs.readdirSync(dir);
files.forEach(function(file) {
var text = fs.readFileSync(dir + '/' + file, 'utf-8'),
fileName = file.substr(0, file.lastIndexOf('.'));
ret[fileName] = {content: text};
});
return ret;
};
This code runs only one, when I run node. This is how I can make it async:
require('./pages')(function(pages) {
app.get('/:page', function(req, res, next) {
var p = req.params.page;
if (p in pages) {
res.render('page', pages[p]);
} else {
next();
}
});
});
pages.js:
module.exports = function(callback) {
var fs = require('fs'),
ret = [],
dir = './pages';
fs.readdir(dir, function(err, files) {
if (err) throw err;
files.forEach(function(file, i) {
fs.readFile(dir + '/' + file, 'utf-8', function(err, text) {
if (err) throw err;
var fileName = file.substr(0, file.lastIndexOf('.'));
ret[fileName] = {content: text};
if ( i === (files.length - 1) ) callback(ret);
});
});
});
};
Assuming the total pages will not exceed more than 1 MB in size, I can cache the text into memory indefinitely without getting node crashed due to out of memory.
Should I be using the async code?
According to what I've learnt, the async version will make node start listening on localhost faster, but /:page URLs will only work when the files have been loaded into memory.
Is the async code in the right pattern?
What if I need to reuse the pages object in another file? Right now it is only accessible in routes.js.
Can I rewrite pages.js to execute only once like this:
var ret = [];
module.exports = function(callback) {
var fs = require('fs'),
dir = './pages';
if (ret.length < 1) {
fs.readdir(dir, function(err, files) {
if (err) throw err;
files.forEach(function(file) {
fs.readFile(dir + '/' + file, 'utf-8', function(err, text) {
if (err) throw err;
var fileName = file.substr(0, file.lastIndexOf('.'));
ret[fileName] = {content: text};
if ( i === (files.length - 1) ) callback(ret);
});
});
});
} else {
callback(ret);
}
};
What if require('./pages')(function(pages) {}) is called multiple times together? Is there a chance of the if condition failing? I can't wrap my mind around this.

Should I be using the async code?
If you want, why not. But there's no real need for it, synchronous IO on startup is fine. require does it as well.
Is the async code in the right pattern?
No. It does invoke callback once for each directory. Calling app.get('/:page', …) multiple times is not what you want.
What if I need to reuse the pages object in another file? Right now it is only accessible in routes.js.
You could pass it from routes.js to the other modules. Or just rewrite pages.js to store it statically and execute the async things only once, so that you can require it multiple times.
What if require('./pages')(function(pages) {}) is called multiple times together? Is there a chance of the if condition failing?
Yes, it will most certainly fail, because you are populating ret only asynchronously.
I can't wrap my mind around this.
Use promises. The act as asynchronous, unmutable values, just what you need here. They will guarantee that callbacks are only invoked once, that every callback is invoked with the same ret value, and provide many more useful things (like managing the parallel file reads for you).
You'll want to export a promise from pages.js.

Mocha test is failing but implementation works

I am trying to write a newer watch module that uses the fs.watch method instead of the watchFile approach.
So far, it works beautifully, but only when I run it outside of mocha. I can't figure out why my unit test is throwing a tantrum, maybe someone here can?
Here is my class code:
/**
* requirements
*/
var fs, path, events;
fs = require('fs');
path = require('path');
events = require('events');
/**
* private
*/
var Monitor = function(directory, options) {
this.directory = directory;
this.options = options || {};
(this.options.lazy && this.empty()) || this.walk(this.directory);
this.watch(this.directory);
};
Monitor.prototype = new events.EventEmitter();
Monitor.prototype.watch = function(directory, stats) {
var stats = stats || {};
if (!this.directories[directory]) {
var w = fs.watch(directory, this.options, this.justlookatit.bind(this));
}
this.directories[directory] = { 'stats': stats, 'w': w };
};
Monitor.prototype.directories = function() {
if (!Object.keys(this.directories).length) {
this.walk(this.directory);
}
return this.directories;
};
Monitor.prototype.files = function() {
if (!Object.keys(this.files).length) {
this.walk(this.directory);
}
return this.files;
};
Monitor.prototype.unwatch = function() {
if (!Object.keys(this.directories).length) {
for (var dir in this.directories) {
dir.w.close();
}
}
};
Monitor.prototype.empty = function() {
this.unwatch();
this.files = {};
this.directories = {};
};
Monitor.prototype.walk = function(directory) {
var monitor = this;
this.empty();
fs.readdir(directory, function(err, files) {
if (err) return;
for (var file in files) {
var fullname = path.resolve(files[file]);
if (!monitor.options.filter || monitor.options.filter(fullname)) {
fs.stat(fullname, function(err, stats) {
if (err) return;
if (stats.isDirectory()) {
monitor.walk(fullname);
monitor.watch(fullname, stats);
} else {
monitor.files[fullname] = stats;
}
});
}
}
});
};
Monitor.prototype.justlookatit = function(action, file) {
var monitor = this;
var fullname = path.resolve(file);
if (this.options.filter && !this.options.filer(fullname)) return;
fs.exists(fullname, function(exists) {
if (exists) {
fs.stat(fullname, function(err, stats) {
if (stats.isDirectory()) {
monitor.watch(fullname, stats);
} else {
if (monitor.files[fullname]) {
if (stats.mtime.getTime() > monitor.files[fullname].mtime.getTime()) {
monitor.emit('modified', fullname, stats);
}
} else {
monitor.emit('added', fullname, stats);
}
monitor.files[fullname] = stats;
}
});
} else {
if (monitor.files[fullname]) {
delete monitor.files[fullname];
monitor.emit('deleted', fullname);
} else if (monitor.directories[fullname]) {
monitor.directories[fullname].w.close();
delete monitor.directories[fullname];
}
}
});
};
/**
* exports
*/
exports.start = function(directory, options) {
return new Monitor(directory, options);
};
Here is my Working external test code:
var watch = require("./watch.js");
var fs = require('fs');
monitor = watch.start(__dirname);
monitor.on('added', function(file, stats) {
console.log("Caught Added: " + file);
});
monitor.on('modified', function(file, stats) {
console.log("Caught Modified: " + file);
});
monitor.on('deleted', function(file) {
console.log("Caught deleted: " + file);
});
// try creating a file immediately
fs.openSync('v.md', 'w');
The first test file runs perfectly fine, and I've tried both openSync and open. Finally, here is a version of the same test code, wrapped in a mocha unit test which is timing out:
/**
* requirements
*/
var watch, Q, fs, path, mocha, chai, assert;
watch = require('../lib/watch.js');
Q = require('q');
fs = require('fs');
path = require('path');
mocha = require('mocha');
chai = require('chai');
assert = chai.assert;
/**
* variables
*/
var watch_directory = path.join(__dirname, './watch');
/**
* tests
*/
describe('test watch', function() {
it('should create a monitor and run callbacks after fs changes', function(done) {
// I had planned to implement promises that chained the three callbacks
// but couldn't get one of them working in general
var added = function(file, stats) {
console.log("added");
done();
};
var modified = function(file, stats) {
console.log("modified");
};
var deleted = function(file, stats) {
console.log("deleted");
};
// create our service
var monitor = watch.start(watch_directory);
// assert it is defined
assert.isDefined(monitor);
// establish a listener
monitor.on('added', added);
monitor.on('modified', modified);
monitor.on('deleted', deleted);
// here is a file name using the current date to prevent duplication during tests
var file = path.join(watch_directory, (new Date()).getTime() + '.md');
// let's create the file, then delete it
fs.open(file, 'w+', function(err, fileDescriptor) {
// this prints before console output from the watch.js's `justlookatit` method
console.log(err);
console.log("writing to file");
// we probably don't want to try closing the fileDescriptor if the open failed
if (err) return;
// close the file descriptor
fs.close(fileDescriptor, function() {
// delete the file we just created
// fs.unlink(file, function() { /* not a big deal */ });
});
});
// modify a known-existing test file
fs.open('test.md', 'w+', function() {/* we don't care about this */});
})
});
I checked with console.log(fullname) inside the justlookatit method on the watch code, and it spits out the correct file name, matching the one generated by the unit test.
However, it then proceeds to return false when I run fs.exists. As I undestand it, that means the file system is notifying me that a file exists before it exists, which doesn't make sense really. So I tried adding an additional delay by wrapping my fs.exists method in a setTimeout, and that didn't change the results. I have also tried using both openSync and existsSync, and that made no difference.
I'm stumped, does anyone have any ideas why the mocha code isn't working?

So, the solution was to go for a walk. I came back, looked at the code again and figured out the cause of the problem with mocha, and also identified many other bugs.
The problem was the lack of context. The justlookatit method does not have a context, and in the test.js scenario it is watching the current directory, while the mocha test is watching a sub-directory.
The path.resolve was receiving only the file name, not the directory, and therefore merged it with the default (executables) directory, so the level of test.js, or watch_test.js for mocha. It proceeded to fail to locate any of the files in the mocha test case because they were all one level below the executable.
I won't go into detail about all the other bugs, but I may come back and post the repository link when I get to a point that I want to push it online.

You're missing the callback return(done); at the end of your test. Unless you call that callback, Mocha will time out every time.

Require in nodejs

The argument of require(...) in node.js is a filename. If I had a module source code in a string code, could I somehow call require(code) and load functions from that string?

I put this into a function for reuse. It creates a file in the os temp directory based on a random hash, requires it and then deletes it.
var fs = require('fs'),
os = require('os'),
crypto = require('crypto');
function requireString(moduleString) {
var token = crypto.randomBytes(20).toString('hex'),
filename = os.tmpdir() + '/' + token + '.js',
requiredModule = false;
// write, require, delete
fs.writeFileSync(filename, moduleString);
requiredModule = require(filename);
fs.unlinkSync(filename);
return requiredModule;
}
Then you can do:
var carString = "exports.start = function(){ console.log('start'); };",
car = requireString(carString);
console.log("Car:", car);
This is still more of a workaround, but more convenient to use, I think.

A work around could be to write the module source code to a temporary file ./tmp-file.js and then require('./tmp-file'), and then remove the file.
This is probably not optimal because you would either have to block and write the file synchronously, or put everything requiring that module in the callback to the async write.
A working example for async file write (gist - also includes sync file write):
var http = require('http');
var fs = require('fs');
var helloModuleString = "exports.world = function() { return 'Hello World\\n'; }";
fs.writeFile('./hello.js', helloModuleString, function (err) {
if (err) return console.log(err);
var hello = require('./hello');
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end(hello.world());
}).listen(1337, '127.0.0.1');
console.log('Server running at http://127.0.0.1:1337/');
});
Results in:
$ curl 127.0.0.1:1337
> Hello World