Browserify + gulp becoming slow - javascript

We've been using gulp and browserify to create builds of our project whenever a js file changes. As the project has grown this process has become incredibly slow, from 200ms -> ~5s. The project has 69 directories, 173 files, and a max depth of 4 folders. We are applying a few transforms. Here's our build code.
var buildJS = function (entryPoint, name, cb) {
var browserify = require('browserify');
var uglify = require('gulp-uglify');
var source = require('vinyl-source-stream');
var buffer = require('vinyl-buffer');
var reactify = require('reactify');
var literalify = require('literalify');
var brfs = require('brfs');
browserify()
.require(entryPoint + '/' + name + '.jsx')
.transform({
global: true
}, reactify)
.transform({
global: true
}, brfs)
.transform({
global: true
}, literalify.configure(literalifyConfig))
.external(config.libs)
.bundle({
debug: config.DEV,
//detectGlobals : false
})
.on('error', handleError)
.pipe(source('bundle.js'))
.pipe(buffer())
.pipe(gulpIf(!config.DEV, uglify()))
.pipe(gulp.dest(config.buildPath + '/' + name))
.on('finish', cb)
};
Is this just normal behavior based on our project size? Or are we doing something wrong?

That's because browserify always recompile everything. You should be using incremental builds instead if you want good perfs.
I've developed a gulp plugin precisely for that: https://github.com/ngryman/gulp-bro.

Related

Detect what file change with gulp

I have this gulpfile:
var gulp = require('gulp'),
concat = require('gulp-concat'),
uglify = require('gulp-uglify');
gulp.task('minifyJS', function() {
return gulp.src(['src/*.js'])
.pipe(uglify())
.pipe(gulp.dest('min'));
});
gulp.task('watch', function() {
gulp.watch(['src/*.js'], ['minifyJS']);
});
I want to know what file trigger the watcher and his absolute path.
For example: if my project is placed in /myCode and I change the file src/main.js, I want to see /myCode/src/main.js inside minifyJS task. Is there a way to do it?
Thank you for your time.
You can do it by using gulp-ng-annotate and gulp-changed:
var gulp = require('gulp');
var changed = require('gulp-changed');
var rename = require('gulp-rename');
var ngAnnotate = require('gulp-ng-annotate'); // just as an example
var SRC = 'src/*.js';
var DEST = 'src/';
//Function to get the path from the file name
function createPath(file) {
var stringArray = file.split('/');
var path = '';
var name = stringArray[1].split('.');
stringArray = name[0].split(/(?=[A-Z])/);
if (stringArray.length>1) {stringArray.pop()};
return {folder: stringArray[0], name: name[0]}
}
gulp.task('default', function () {
return gulp.src(SRC)
.pipe(changed(DEST))
// ngAnnotate will only get the files that
// changed since the last time it was run
.pipe(ngAnnotate())
.pipe(rename(function (path) {
var createdPath = createPath(path);
path.dirname = createdPath.folder;
path.basename: createdPath.name,
path.prefix: "",
path.suffix: "",
path.extname: ".min.js"
}))
.pipe(gulp.dest(DEST));
});
Result:
Use gulp-changed npm package.
$ npm install --save-dev gulp-changed
Try the below in gulp file, (I haven't tried)
var gulp = require('gulp'),
concat = require('gulp-concat'),
uglify = require('gulp-uglify'),
changed = require('gulp-changed');
gulp.task('minifyJS', function() {
return gulp.src(['src/*.js'])
.pipe(changed('min'))
.pipe(uglify())
.pipe(gulp.dest('min'));
});
gulp.task('watch', function() {
gulp.watch(['src/*.js'], ['minifyJS']);
});
see the documentation of this package https://www.npmjs.com/package/gulp-changed
Based on your comment to Julien's answer this should be fairly close to what you want, or at least get you going in the right direction:
var gulp = require('gulp'),
concat = require('gulp-concat'),
uglify = require('gulp-uglify'),
cache = require('gulp-cached'),
rename = require('gulp-rename'),
path = require('path');
function fileName(file) {
return file.dirname + path.sep + file.basename + file.extname;
}
gulp.task('minifyJS', function() {
return gulp.src(['src/*.js'])
.pipe(cache('minifyJS'))
.pipe(rename(function(file) {
var nameOfChangedFile = fileName(file);
if (nameOfChangedFile == './main.js') {
file.basename = 'main.min'
}
if (nameOfChangedFile == './userView.js') {
file.basename = 'user/userView.min'
}
console.log(nameOfChangedFile + ' -> ' + fileName(file));
}))
.pipe(uglify())
.pipe(gulp.dest('min'));
});
gulp.task('watch', function() {
gulp.watch(['src/*.js'], ['minifyJS']);
});
This uses gulp-cached to keep an in-memory cache of all the files in your src/ folder that have passed through the stream. Only files that have changed since the last invocation of minifyJS are passed down to the gulp-rename plugin.
The gulp-rename plugin itself is then used to alter the destination path of the changed files.
Note: the cache is empty on first run, since no files have passed through the gulp-cached plugin yet. This means that the first time you change a file all files in src/ will be written to the destination folder. On subsequent changes only the changed files will be written.

"Possible EventEmitter memory leak detected" with Gulp + Watchify + Factor bundle

I am using gulp, browserify, watchify and factor bundle to build several javascript files in development. Everything works fine, excepts after some time I start seeing this warning:
Trace
at Browserify.addListener (events.js:179:15)
at f (/Users/benoit/git/figure/web/node_modules/factor-bundle/index.js:55:7)
at Browserify.plugin (/Users/benoit/git/figure/web/node_modules/browserify/index.js:345:9)
at Browserify.bundle (/Users/benoit/git/figure/web/gulpfile.js:46:13)
at Browserify.emit (events.js:107:17)
at null._onTimeout (/Users/benoit/git/figure/web/node_modules/watchify/index.js:126:15)
at Timer.listOnTimeout (timers.js:110:15)
(node) warning: possible EventEmitter memory leak detected. 11 finish listeners added. Use emitter.setMaxListeners() to increase limit.
Trace
at ConcatStream.addListener (events.js:179:15)
at ConcatStream.once (events.js:204:8)
at Labeled.Readable.pipe (/Users/benoit/git/figure/web/node_modules/factor-bundle/node_modules/labeled-stream-splicer/node_modules/stream-splicer/node_modules/readable-stream/lib/_stream_readable.js:612:8)
at /Users/benoit/git/figure/web/node_modules/factor-bundle/index.js:73:43
at Array.reduce (native)
at Transform._flush (/Users/benoit/git/figure/web/node_modules/factor-bundle/index.js:65:35)
at Transform.<anonymous> (/Users/benoit/git/figure/web/node_modules/factor-bundle/node_modules/through2/node_modules/readable-stream/lib/_stream_transform.js:135:12)
at Transform.g (events.js:199:16)
at Transform.emit (events.js:129:20)
at finishMaybe (/Users/benoit/git/figure/web/node_modules/factor-bundle/node_modules/through2/node_modules/readable-stream/lib/_stream_writable.js:371:12)
at endWritable (/Users/benoit/git/figure/web/node_modules/factor-bundle/node_modules/through2/node_modules/readable-stream/lib/_stream_writable.js:378:3)
at Transform.Writable.end (/Users/benoit/git/figure/web/node_modules/factor-bundle/node_modules/through2/node_modules/readable-stream/lib/_stream_writable.js:356:5)
(node) warning: possible EventEmitter memory leak detected. 11 finish listeners added. Use emitter.setMaxListeners() to increase limit.
Below is my gulpfile
var gulp = require('gulp');
var gutil = require('gulp-util');
var source = require('vinyl-source-stream');
var browserify = require('browserify');
var reactify = require('reactify');
var watchify = require('watchify');
var factor = require('factor-bundle');
var uglify = require('gulp-uglify');
var fs = require('fs');
var concat = require('concat-stream');
var file = require('gulp-file');
gulp.task('watch', bundle)
function bundle () {
// react components
var files = [
'/path/to/file1.jsx',
'/path/to/file2.jsx',
'/path/to/file3.jsx'
];
var bundler = watchify(browserify(watchify.args))
bundler.add(files);
bundler.add('./lib/api.js', {expose: 'api'});
bundler.require('./lib/api.js', {expose: 'api'});
bundler.transform('reactify');
bundler.on('update', rebundle);
function rebundle() {
bundler.plugin('factor-bundle', {
outputs: [
write('/path/to/file1.js'),
write('/path/to/file2.js'),
write('/path/to/file3.js'),
]
});
bundler.bundle()
.on('error', gutil.log.bind(gutil, 'Browserify Error'))
.pipe(write('shared.js'));
};
return rebundle();
}
function write (name) {
return concat(function (content) {
// create new vinyl file from content and use the basename of the
// filepath in scope as its basename.
return file(name, content, { src: true })
// uglify content
.pipe(uglify())
// write content to build directory
.pipe(gulp.dest('./public/bundles/'))
});
}
I read I should set max listeners somewhere but I am afraid this might be a geniune memory leak.
My initial solution didn't work, and it looks like it really is a bug. I think I've found a temporary hacky fix though.
Edit node_modules/factor-bundle/index.js and change
From
b.on('reset', addHooks);
to
b.once('reset', addHooks);
Your original code should work.
Here's the GitHub issue for anyone who's keeping score :D

Get Gulp watch to perform function only on changed file

I am new to Gulp and have the following Gulpfile
var gulp = require('gulp');
var jshint = require('gulp-jshint');
var concat = require('gulp-concat');
var rename = require('gulp-rename');
var uglify = require('gulp-uglify');
gulp.task('compress', function () {
return gulp.src('js/*.js') // read all of the files that are in js with a .js extension
.pipe(uglify()) // run uglify (for minification)
.pipe(gulp.dest('dist/js')); // write to the dist/js file
});
// default gulp task
gulp.task('default', function () {
// watch for JS changes
gulp.watch('js/*.js', function () {
gulp.run('compress');
});
});
I would like to configure this to rename, minify and save only my changed file to the dist folder. What is the best way to do this?
This is how:
// Watch for file updates
gulp.task('watch', function () {
livereload.listen();
// Javascript change + prints log in console
gulp.watch('js/*.js').on('change', function(file) {
livereload.changed(file.path);
gutil.log(gutil.colors.yellow('JS changed' + ' (' + file.path + ')'));
});
// SASS/CSS change + prints log in console
// On SASS change, call and run task 'sass'
gulp.watch('sass/*.scss', ['sass']).on('change', function(file) {
livereload.changed(file.path);
gutil.log(gutil.colors.yellow('CSS changed' + ' (' + file.path + ')'));
});
});
Also great to use gulp-livereload with it, you need to install the Chrome plugin for it to work btw.
See incremental builds on the Gulp docs.
You can filter out unchanged files between runs of a task using the gulp.src function's since option and gulp.lastRun

Set working directory in gulpfile.js?

Is there a way to set the working directory for Gulp within a gulpfile, so that I can run a gulp command from a subdirectory without running into any issues? I ran a search for this and didn't find what I was looking for.
To clarify, I'm aware of adding a prefix to the files I'm using. However, instead of this -
var gulp = require('gulp');
var jshint = require('gulp-jshint');
...
var paths = {
js: [__dirname + 'app/*/*.js', __dirname + '!app/lib/**'],
css: __dirname + 'app/*/*.styl',
img: __dirname + 'app/img/*',
index: __dirname + '*.html',
dist: __dirname + 'dist'
};
I'd like to do something like this:
var gulp = require('gulp');
var jshint = require('gulp-jshint');
...
gulp.cwd(__dirname); // This would be much easier to understand, and would make future edits a bit safer.
var paths = {
js: ['app/*/*.js', '!app/lib/**'],
css: 'app/*/*.styl',
img: 'app/img/*',
index: '*.html',
dist: 'dist'
};
I'm wondering if Gulp exposes this functionality. Perhaps node itself allows this.
(I realize that there is likely a way to do command line itself when I run the command, but I would like to include it in the gulp file, especially for distribution purposes. I want the working directory for gulp to match the directory in which the gulpfile resides.)
Thanks!
Besides option.cwd, you can also use process.chdir(yourDir)
it could be used anywhere in a gulpfile. e.g.
process.chdir(yourDir);
var gulp = require('gulp');
Make sure your gulp is up-to-date( > 3.8.10), this may not work in older gulp.
Instead of concatenating strings by yourself, you should be using path.join since it will take care of the proper slash, and following that path you can add a shorcut:
var path = require('path'),
p = function () {
Array
.prototype
.unshift
.call(arguments, __dirname);
return path.join.apply(path, arguments);
};
console.log(p('a', 'b', 'c'));
Or, well, you can just:
gulp.src(..., {cwd: __dirname})
gulp.dest(..., {cwd: __dirname})
Something like:
var src = function (globs, options) {
options = options || {};
options.cwd = __dirname;
return gulp.src(globs, options);
};
var dest = function (folder, options) {
options = options || {};
options.cwd = __dirname;
return gulp.dest(folder, options);
};
Look here and here.

How to uglify output with Browserify in Gulp?

I tried to uglify output of Browserify in Gulp, but it doesn't work.
gulpfile.js
var browserify = require('browserify');
var gulp = require('gulp');
var uglify = require('gulp-uglify');
var source = require('vinyl-source-stream');
gulp.task('browserify', function() {
return browserify('./source/scripts/app.js')
.bundle()
.pipe(source('bundle.js'))
.pipe(uglify()) // ???
.pipe(gulp.dest('./build/scripts'));
});
As I understand I cannot make it in steps as below. Do I need to make in one pipe to preserve the sequence?
gulp.task('browserify', function() {
return browserify('./source/scripts/app.js')
.bundle()
.pipe(source('bundle.js'))
.pipe(uglify()) // ???
.pipe(gulp.dest('./source/scripts'));
});
gulp.task('scripts', function() {
return grunt.src('./source/scripts/budle.js')
.pipe(uglify())
.pipe(gulp.dest('./build/scripts'));
});
gulp.task('default', function(){
gulp.start('browserify', 'scripts');
});
You actually got pretty close, except for one thing:
you need to convert the streaming vinyl file object given by source() with vinyl-buffer because gulp-uglify (and most gulp plugins) works on buffered vinyl file objects
So you'd have this instead
var browserify = require('browserify');
var gulp = require('gulp');
var uglify = require('gulp-uglify');
var source = require('vinyl-source-stream');
var buffer = require('vinyl-buffer');
gulp.task('browserify', function() {
return browserify('./source/scripts/app.js')
.bundle()
.pipe(source('bundle.js')) // gives streaming vinyl file object
.pipe(buffer()) // <----- convert from streaming to buffered vinyl file object
.pipe(uglify()) // now gulp-uglify works
.pipe(gulp.dest('./build/scripts'));
});
Or, you can choose to use vinyl-transform instead which takes care of both streaming and buffered vinyl file objects for you, like so
var gulp = require('gulp');
var browserify = require('browserify');
var transform = require('vinyl-transform');
var uglify = require('gulp-uglify');
gulp.task('build', function () {
// use `vinyl-transform` to wrap the regular ReadableStream returned by `b.bundle();` with vinyl file object
// so that we can use it down a vinyl pipeline
// while taking care of both streaming and buffered vinyl file objects
var browserified = transform(function(filename) {
// filename = './source/scripts/app.js' in this case
return browserify(filename)
.bundle();
});
return gulp.src(['./source/scripts/app.js']) // you can also use glob patterns here to browserify->uglify multiple files
.pipe(browserified)
.pipe(uglify())
.pipe(gulp.dest('./build/scripts'));
});
Both of the above recipes will achieve the same thing.
Its just about how you want to manage your pipes (converting between regular NodeJS Streams and streaming vinyl file objects and buffered vinyl file objects)
Edit:
I've written a longer article regarding using gulp + browserify and different approaches at: https://medium.com/#sogko/gulp-browserify-the-gulp-y-way-bb359b3f9623
Two additional approaches, taken from the vinyl-source-stream NPM page:
Given:
var source = require('vinyl-source-stream');
var streamify = require('gulp-streamify');
var browserify = require('browserify');
var uglify = require('gulp-uglify');
var gulpify = require('gulpify');
var gulp = require('gulp');
Approach 1 Using gulpify (deprecated)
gulp.task('gulpify', function() {
gulp.src('index.js')
.pipe(gulpify())
.pipe(uglify())
.pipe(gulp.dest('./bundle.js'));
});
Approach 2 Using vinyl-source-stream
gulp.task('browserify', function() {
var bundleStream = browserify('index.js').bundle();
bundleStream
.pipe(source('index.js'))
.pipe(streamify(uglify()))
.pipe(gulp.dest('./bundle.js'));
});
One benefit of the second approach is that it uses the Browserify API directly, meaning that you don't have to wait for the authors of gulpify to update the library before you can.
you may try browserify transform uglifyify.

Categories