We Keep Coding

Browserify require - inline way. How?

Is there any possibility to include one file into another with Browserify?
I mean not standard Browserify behavior, but pasting one file into another in a specific place.
file1.js
console.log("abc");
file2.js
requirePaste("file1.js");
console.log("def");
output.js
console.log("abc");
console.log("def");
I need it for spreading an ES6 class into multiple files like this pattern:
ObjectManager.js
ObjectManager_Events.js
ObjectManager_Rendering.js
These files are about one class. So I can make something like this:
class ObjectManager {
constructor() {
}
requirePaste("./ObjectManager_Events");
requirePaste("./ObjectManager_Rendering");
}
EDIT:
I made a simple transform plugin for Browserify and it works great. There is one problem though it won't work with Watchify. this is because inline-required files aren't counted as being watched. Any idea on how to fix this?
const through = require('through2');
const fs = require('fs');
const path = require('path');
const regex = /require\([\s]*\/\*inline\*\/[\s]*"(.+)"\)/g;
function process(pathToFile, contents) {
while ( (occurence = regex.exec(contents)) ) {
contents = processOne(pathToFile, contents, occurence);
}
return contents;
}
function processOne(pathToFile, contents, occurence) {
const dir = path.dirname(pathToFile);
const includePath = occurence[1] + ".js";
const range = [occurence.index, occurence.index+occurence[0].length-1];
const pathToInnerFile = dir + "/" + includePath;
var innerContents = fs.readFileSync(pathToInnerFile, 'utf8');
innerContents = process(pathToInnerFile, innerContents);
var output = "";
output += contents.substring(0, range[0]);
output += innerContents;
output += contents.substring(range[1]+1);
return output;
}
module.exports = function(pathToFile) {
return through(
function(buf, enc, next) {
var result = process(pathToFile, buf.toString('utf8'));
this.push(result);
next();
}
);
};

That did the trick:
Basically we inform Watchify about file by emiting 'file' event.
const includedFiles = [];
function process(pathToFile, contents) {
includedFiles.push(pathToFile);
while ( (occurence = regex.exec(contents)) ) {
contents = processOne(pathToFile, contents, occurence);
}
return contents;
}
...
module.exports = function(pathToFile) {
return through(
function(buf, enc, next) {
var result = process(pathToFile, buf.toString('utf8'));
this.push(result);
includedFiles.forEach((filePath) => this.emit("file", filePath));
includedFiles.length = 0;
next();
}
);
};

Javascript,Nodejs: search for a specific word string in files

i'm trying to make an app that searches for all files
contains a specified string under the current directory/subdirectory.
as i understand it means i need to create a read stream, loop it, load the read data to an array, if the word found give __filename, dirname and if ! not found message.
unfortunately, i could not make it work...
any clue?
var path = require('path'),
fs=require('fs');
function fromDir(startPath,filter,ext){
if (!fs.existsSync(startPath)){
console.log("no dir ",startPath);
return;
};
var files=fs.readdirSync(startPath);
let found = files.find((file) => {
let thisFilename = path.join(startPath, file);
let stat = fs.lstatSync(thisFilename);
var readStream = fs.createReadStream(fs);
var readline = require('readline');
if (stat.isDirectory()) {
fromDir(thisFilename, filename,readline, ext);
} else {
if (path.extname(createReadStream) === ext && path.basename(thisFilename, ext) === filename) {
return true;
}
}
});
console.log('-- your word has found on : ',filename,__dirname);
}
if (!found) {
console.log("Sorry, we didn't find your term");
}
}
fromDir('./', process.argv[3], process.argv[2]);

Because not everything was included in the question, I made an assumption:
We are looking for full words (if that's not the case, replace the regex with a simple indexOf()).
Now, I've split the code into two functions - to make it both more readable and easier to recursively find the files.
Synchronous version:
const path = require('path');
const fs = require('fs');
function searchFilesInDirectory(dir, filter, ext) {
if (!fs.existsSync(dir)) {
console.log(`Specified directory: ${dir} does not exist`);
return;
}
const files = getFilesInDirectory(dir, ext);
files.forEach(file => {
const fileContent = fs.readFileSync(file);
// We want full words, so we use full word boundary in regex.
const regex = new RegExp('\\b' + filter + '\\b');
if (regex.test(fileContent)) {
console.log(`Your word was found in file: ${file}`);
}
});
}
// Using recursion, we find every file with the desired extention, even if its deeply nested in subfolders.
function getFilesInDirectory(dir, ext) {
if (!fs.existsSync(dir)) {
console.log(`Specified directory: ${dir} does not exist`);
return;
}
let files = [];
fs.readdirSync(dir).forEach(file => {
const filePath = path.join(dir, file);
const stat = fs.lstatSync(filePath);
// If we hit a directory, apply our function to that dir. If we hit a file, add it to the array of files.
if (stat.isDirectory()) {
const nestedFiles = getFilesInDirectory(filePath, ext);
files = files.concat(nestedFiles);
} else {
if (path.extname(file) === ext) {
files.push(filePath);
}
}
});
return files;
}
Async version - because async is cool:
const path = require('path');
const fs = require('fs');
const util = require('util');
const fsReaddir = util.promisify(fs.readdir);
const fsReadFile = util.promisify(fs.readFile);
const fsLstat = util.promisify(fs.lstat);
async function searchFilesInDirectoryAsync(dir, filter, ext) {
const found = await getFilesInDirectoryAsync(dir, ext);
for (file of found) {
const fileContent = await fsReadFile(file);
// We want full words, so we use full word boundary in regex.
const regex = new RegExp('\\b' + filter + '\\b');
if (regex.test(fileContent)) {
console.log(`Your word was found in file: ${file}`);
}
};
}
// Using recursion, we find every file with the desired extention, even if its deeply nested in subfolders.
async function getFilesInDirectoryAsync(dir, ext) {
let files = [];
const filesFromDirectory = await fsReaddir(dir).catch(err => {
throw new Error(err.message);
});
for (let file of filesFromDirectory) {
const filePath = path.join(dir, file);
const stat = await fsLstat(filePath);
// If we hit a directory, apply our function to that dir. If we hit a file, add it to the array of files.
if (stat.isDirectory()) {
const nestedFiles = await getFilesInDirectoryAsync(filePath, ext);
files = files.concat(nestedFiles);
} else {
if (path.extname(file) === ext) {
files.push(filePath);
}
}
};
return files;
}
If you have not worked with / understand async/await yet, it is a great step to take and learn it as soon as possible. Trust me, you will love not seeing those ugly callbacks again!
UPDATE:
As you pointed in comments, you want it to execute the function after running node process on the file. You also want to pass the function parameters as node's arguments.
To do that, at the end of your file, you need to add:
searchFilesInDirectory(process.argv[2], process.argv[3], process.argv[4]);
This extracts our arguments and passes them to the function.
With that, you can call our process like so (example arguments):
node yourscriptname.js ./ james .txt
Personally, if I were to write this, I would leverage the beauty of asynchronous code, and Node.js's async / await.
As a very side note:
You can easily improve readability of your code, if you add proper formatting to it. Don't get me wrong, it's not terrible - but it can be improved:
Use spaces OR newlines after commas.
Use spaces around equality operators and arithmetic operators.
As long as you are consistent with formatting, everything looks much better.

node.js display virtual hosts and current git branch async issue

Ok, i am just starting to learn node.js and i am having a little difficulty getting a good grasp on the async nature of it and when/how to use callbacks to get data passed along like i need it.
The concept of what i am trying to build is this. I have a small node.js app that uses the FS package - defined as var fs = require("fs"). The app is responding to localhost:4000 right now. When i hit that url, the app will use fs.readdir() to get all of the virtual host files in the directory that i pass to readdir().
Next, the app loops through those files and parses each one line by line and word by word (quick and dirty for now). I am using fs.readFile() to read the file and then literally doing lines = data.toString().split("\n") and then var words = lines[l].split(" ") to get to the data in the file i need. Each virtual host file looks something like this:
<VirtualHost *:80>
ServerName some-site.local
DocumentRoot "/path/to/the/docroot"
<Directory "/path/to/the/docroot">
Options Indexes FollowSymLinks Includes ExecCGI
AllowOverride All
Require all granted
</Directory>
ErrorLog "logs/some-site.local-error_log"
</VirtualHost>
Here is my main.js file (routes file):
var express = require("express"),
router = express.Router(),
async = require("async"),
directoryReader = require(__dirname + "/../lib/directoryReader");
fileReader = require(__dirname + "/../lib/fileReader");
router.get("/", function(req, res) {
var response = [];
directoryReader.read(function(files) {
async.each(files, function(file, callback) {
fileReader.read(file, function(data) {
if (data.length > 0) {
response.push(data);
}
callback();
});
}, function(err){
if (err) throw err;
res.json(response);
});
});
});
module.exports = router;
My directoryReader.js file:
var fs = require("fs"),
fileReader = require(__dirname + "/../lib/fileReader");
var directoryReader = {
read: function(callback) {
var self = this;
fs.readdir("/etc/apache2/sites-available", function (err, files) {
if (err) throw err;
var output = [];
for(f in files) {
output.push(files[f]);
}
callback(output);
});
}
};
module.exports = directoryReader;
And my fileReader.js file:
var fs = require("fs");
var fileReader = {
read: function(file, callback) {
fs.readFile("/etc/apache2/sites-available/" + file, { encoding: "utf8" }, function (err, data) {
if (err) throw err;
var vHostStats = ["servername", "documentroot", "errorlog"],
lines = data.toString().split("\n"),
output = [];
for(l in lines) {
var words = lines[l].split(" ");
for(w in words) {
if (words[w].toLowerCase() == "<virtualhost") {
var site = {
"servername" : "",
"documentroot" : "",
"errorlog" : "",
"gitbranch" : ""
}
w++;
}
if (vHostStats.indexOf(words[w].toLowerCase()) !== -1) {
var key = words[w].toLowerCase();
w++;
site[key] = words[w];
}
if (words[w].toLowerCase() == "</virtualhost>" && site.documentroot != "") {
w++;
output.push(site);
var cmd = "cd " + site["documentroot"] + " && git rev-parse --abbrev-ref HEAD";
var branch = ...; // get Git branch based on the above command
site["gitbranch"] = branch;
}
}
}
callback(output);
});
}
};
module.exports = fileReader;
All of this code will spit out json. This all works fine, expect for one part. The line in the fileReader.js file:
var branch = ...; // get Git branch based on the above command
I am trying to get this code to run a shell command and get the Git branch based on the document root directory. I then want to take the branch returned and add the value to the gitbranch proptery of the current site object during the loop. Hope this makes sense. I know there are probably questions on SO that cover something similar to this and i have looked at many of them. I fear i am just not educated enough in node.js yet to apply the answers to those SO questions to my particular use case.
Please let me know if there's anything i can add that can help anyoe answer this question. I should note that this app is for personal uses only, so the solution really just has to work, not be super elegant.
UPDATE: (5/1/2015)
Probably not the best solution but i got what i wanted by using the new execSync added to v0.12.x
if (words[w].toLowerCase() == "</virtualhost>" && site.documentroot != "") {
var cmd = "cd " + site["documentroot"] + " && git rev-parse --abbrev-ref HEAD";
var branch = sh(cmd, { encoding: "utf8" });
site["gitbranch"] = branch.toString().trim();
w++;
output.push(site);
}

PhantomJS copy file with overwrite

Reading from their document:
copy(source, destination)
Currently, the way I cope with this is to check fs.exist('myfile') and manual delete it prior to copy:
var fs = require('fs');
var fileName = 'myfile-backup.txt';
if (fs.exists(fileName)) {
fs.remove(fileName);
}
fs.copy('myfile.txt', fileName);
phantom.exit();
I don't know if there is any better way to overwrite the file. Checking for existing file may have a potential problem when I can't remove the file. I will probably need to do more error handling with this approach. It seems to be a common task, so I would like to know what solution people come up with.

I have written a small extension to the fs module. If you try to overwrite a file with fs.copy it will throw an exception which you can catch to do some error handling like removing an existing file.
I also added an optional maxTrials argument if there is a problem that the file is created every time in between the copy trial and the remove.
var fs = require('fs');
fs.overwrite = function(source, destination, maxTrials){
var overwritten = false;
var trials = 0;
maxTrials = parseInt(maxTrials)
maxTrials = !!maxTrials ? maxTrials : null;
while(!overwritten) {
if (maxTrials && trials > maxTrials) {
return -1;
}
try {
this.copy(source, destination);
overwritten = true;
} catch(e) {
if (fs.exists(destination)) {
fs.remove(destination);
} else {
return -2;
}
}
trials++;
}
return trials;
};

Mocha test is failing but implementation works

I am trying to write a newer watch module that uses the fs.watch method instead of the watchFile approach.
So far, it works beautifully, but only when I run it outside of mocha. I can't figure out why my unit test is throwing a tantrum, maybe someone here can?
Here is my class code:
/**
* requirements
*/
var fs, path, events;
fs = require('fs');
path = require('path');
events = require('events');
/**
* private
*/
var Monitor = function(directory, options) {
this.directory = directory;
this.options = options || {};
(this.options.lazy && this.empty()) || this.walk(this.directory);
this.watch(this.directory);
};
Monitor.prototype = new events.EventEmitter();
Monitor.prototype.watch = function(directory, stats) {
var stats = stats || {};
if (!this.directories[directory]) {
var w = fs.watch(directory, this.options, this.justlookatit.bind(this));
}
this.directories[directory] = { 'stats': stats, 'w': w };
};
Monitor.prototype.directories = function() {
if (!Object.keys(this.directories).length) {
this.walk(this.directory);
}
return this.directories;
};
Monitor.prototype.files = function() {
if (!Object.keys(this.files).length) {
this.walk(this.directory);
}
return this.files;
};
Monitor.prototype.unwatch = function() {
if (!Object.keys(this.directories).length) {
for (var dir in this.directories) {
dir.w.close();
}
}
};
Monitor.prototype.empty = function() {
this.unwatch();
this.files = {};
this.directories = {};
};
Monitor.prototype.walk = function(directory) {
var monitor = this;
this.empty();
fs.readdir(directory, function(err, files) {
if (err) return;
for (var file in files) {
var fullname = path.resolve(files[file]);
if (!monitor.options.filter || monitor.options.filter(fullname)) {
fs.stat(fullname, function(err, stats) {
if (err) return;
if (stats.isDirectory()) {
monitor.walk(fullname);
monitor.watch(fullname, stats);
} else {
monitor.files[fullname] = stats;
}
});
}
}
});
};
Monitor.prototype.justlookatit = function(action, file) {
var monitor = this;
var fullname = path.resolve(file);
if (this.options.filter && !this.options.filer(fullname)) return;
fs.exists(fullname, function(exists) {
if (exists) {
fs.stat(fullname, function(err, stats) {
if (stats.isDirectory()) {
monitor.watch(fullname, stats);
} else {
if (monitor.files[fullname]) {
if (stats.mtime.getTime() > monitor.files[fullname].mtime.getTime()) {
monitor.emit('modified', fullname, stats);
}
} else {
monitor.emit('added', fullname, stats);
}
monitor.files[fullname] = stats;
}
});
} else {
if (monitor.files[fullname]) {
delete monitor.files[fullname];
monitor.emit('deleted', fullname);
} else if (monitor.directories[fullname]) {
monitor.directories[fullname].w.close();
delete monitor.directories[fullname];
}
}
});
};
/**
* exports
*/
exports.start = function(directory, options) {
return new Monitor(directory, options);
};
Here is my Working external test code:
var watch = require("./watch.js");
var fs = require('fs');
monitor = watch.start(__dirname);
monitor.on('added', function(file, stats) {
console.log("Caught Added: " + file);
});
monitor.on('modified', function(file, stats) {
console.log("Caught Modified: " + file);
});
monitor.on('deleted', function(file) {
console.log("Caught deleted: " + file);
});
// try creating a file immediately
fs.openSync('v.md', 'w');
The first test file runs perfectly fine, and I've tried both openSync and open. Finally, here is a version of the same test code, wrapped in a mocha unit test which is timing out:
/**
* requirements
*/
var watch, Q, fs, path, mocha, chai, assert;
watch = require('../lib/watch.js');
Q = require('q');
fs = require('fs');
path = require('path');
mocha = require('mocha');
chai = require('chai');
assert = chai.assert;
/**
* variables
*/
var watch_directory = path.join(__dirname, './watch');
/**
* tests
*/
describe('test watch', function() {
it('should create a monitor and run callbacks after fs changes', function(done) {
// I had planned to implement promises that chained the three callbacks
// but couldn't get one of them working in general
var added = function(file, stats) {
console.log("added");
done();
};
var modified = function(file, stats) {
console.log("modified");
};
var deleted = function(file, stats) {
console.log("deleted");
};
// create our service
var monitor = watch.start(watch_directory);
// assert it is defined
assert.isDefined(monitor);
// establish a listener
monitor.on('added', added);
monitor.on('modified', modified);
monitor.on('deleted', deleted);
// here is a file name using the current date to prevent duplication during tests
var file = path.join(watch_directory, (new Date()).getTime() + '.md');
// let's create the file, then delete it
fs.open(file, 'w+', function(err, fileDescriptor) {
// this prints before console output from the watch.js's `justlookatit` method
console.log(err);
console.log("writing to file");
// we probably don't want to try closing the fileDescriptor if the open failed
if (err) return;
// close the file descriptor
fs.close(fileDescriptor, function() {
// delete the file we just created
// fs.unlink(file, function() { /* not a big deal */ });
});
});
// modify a known-existing test file
fs.open('test.md', 'w+', function() {/* we don't care about this */});
})
});
I checked with console.log(fullname) inside the justlookatit method on the watch code, and it spits out the correct file name, matching the one generated by the unit test.
However, it then proceeds to return false when I run fs.exists. As I undestand it, that means the file system is notifying me that a file exists before it exists, which doesn't make sense really. So I tried adding an additional delay by wrapping my fs.exists method in a setTimeout, and that didn't change the results. I have also tried using both openSync and existsSync, and that made no difference.
I'm stumped, does anyone have any ideas why the mocha code isn't working?

So, the solution was to go for a walk. I came back, looked at the code again and figured out the cause of the problem with mocha, and also identified many other bugs.
The problem was the lack of context. The justlookatit method does not have a context, and in the test.js scenario it is watching the current directory, while the mocha test is watching a sub-directory.
The path.resolve was receiving only the file name, not the directory, and therefore merged it with the default (executables) directory, so the level of test.js, or watch_test.js for mocha. It proceeded to fail to locate any of the files in the mocha test case because they were all one level below the executable.
I won't go into detail about all the other bugs, but I may come back and post the repository link when I get to a point that I want to push it online.

You're missing the callback return(done); at the end of your test. Unless you call that callback, Mocha will time out every time.