We Keep Coding

Webpack: Infinite watch loop and auto-generated files

I have a script which generates a file, call it auto.js. This file contains some dynamically generated imports and is being used within a VueJS project.
// auto.js
import { apple, strawberry, orange } from 'delicious-fruits';
import { carrot, cucumber, celery } from 'delicious-vegetables';
While using Webpacks dev server, should any project file change, my goal is to have this script re-generate my auto.js file, and then have that included in the re-compiled project.
I have turned this script into a Webpack plugin, whereby I'm listening for the watchRun compiler hook. This seems like the ideal hook, per its description:
Executes a plugin during watch mode after a new compilation is triggered but before the compilation is actually started.
class AutoGenerate {
constructor(options) {
this.options = options;
}
apply(compiler) {
compiler.hooks.watchRun.tap('AutoGenerate', () => {
generateFile()
})
}
}
function generateFile () {
// generate auto.js and write to disk
}
I always wind up with an infinite loop situation. I have tried approaching the problem by using various life cycles events (hooks), as well ignoring the auto-generated file. Of course, by ignoring it, those changes are not included in the re-compiled project.
const webpack = require('webpack');
const AutoGenerate = require("./auto.plugin");
module.exports = {
configureWebpack: {
plugins: [
new webpack.WatchIgnorePlugin([/.*auto\.js/]),
new AutoGenerate()
]
}
}
I've also tried tapping into the compilation, and adding a new asset to the compilation. While the process does not error out, the generated asset is not a part of the final compilation.
// auto.plugin.js
class AutoGenerate {
static defaultOptions = {
outputFile: 'auto.js',
};
constructor(options = {}) {
this.options = { ...AutoGenerate.defaultOptions, ...options };
}
apply(compiler) {
compiler.hooks.thisCompilation.tap('AutoGenerate', (compilation) => {
const path = require("path")
const filePath = path.resolve(__dirname, `src/plugins/${this.options.outputFile}`)
const { RawSource } = require('webpack-sources')
const fileContent = new RawSource(generateFile())
compilation.emitAsset(
filePath,
fileContent
)
});
}
}
function generateFile() {
// generate file content & return as string
}
module.exports = { AutoGenerate };
// vue.config.js
const AutoGenerate = require("./auto.plugin");
module.exports = {
configureWebpack: {
plugins: [
new AutoGenerate()
]
}
}
How can I trigger my logic for auto-generating this file, while having this file be included as part of any re-compilation, while at the same time avoiding an infinite loop?

I have not been able to identify a direct solution to the problem posed above. However, for anyone reading, I've come to discover that this can be accomplished by utilizing a package called before-build-webpack, notably by including the watch-run trigger.
// vue.config.js
const WebpackBeforeBuildPlugin = require('before-build-webpack')
module.exports = {
configureWebpack: {
plugins: [
new WebpackBeforeBuildPlugin(function(stats, callback) {
// ...
}, ['run', 'watch-run'])
]
}
}

Processing arrays within Gulp to create bundles in each directory with ordered files

Like most I have been concatenating my JS/CSS files into one big file within Gulp, however, with HTTP2 becoming the norm nowadays I thought I would change my gulp file instead to make related "bundles" instead, however am wondering how to manage this in a clean fashion within Gulp.
My prior gulpfile.js:
var gulp = require('gulp');
var browserSync = require('browser-sync').create();
var concat = require('gulp-concat');
var rename = require('gulp-rename');
var sass = require('gulp-sass');
var postcss = require('gulp-postcss');
var cleanCSS = require('gulp-clean-css');
var uglify = require('gulp-uglify');
var gulpif = require('gulp-if');
var sourcemaps = require('gulp-sourcemaps');
var autoprefixer = require('autoprefixer');
var site_url = 'xxxxxxxxxxxxx.local';
// Set some paths
var js_scripts = [
'js/dev/lib/**/*.js',
'js/dev/plugins/**/*.js',
// We have to set the bootstrap lines separately as some need to go before others
'js/dev/bootstrap/alert.js',
'js/dev/bootstrap/collapse.js',
'js/dev/bootstrap/tooltip.js',
'js/dev/bootstrap/popover.js',
'js/dev/bootstrap/tab.js',
'js/dev/bootstrap/transition.js',
'js/dev/custom.js'
];
gulp.task('scripts', function() {
return gulp.src(js_scripts)
.pipe(sourcemaps.init())
.pipe(concat('scripts.js'))
.pipe(sourcemaps.write('../maps'))
.pipe(gulp.dest('./js'));
});
gulp.task('uglify', gulp.series('scripts', function() {
return gulp.src(js_scripts)
.pipe(gulpif('!**/*.min.js', uglify({mangle: false})))
.pipe(concat('scripts.min.js'))
.pipe(gulp.dest('./js'));
}));
// create a task that ensures the `uglify` task is complete before
// reloading browsers
gulp.task('js-watch', gulp.series('uglify', function (done) {
browserSync.reload();
done();
}));
/* Creates the standard version */
gulp.task('styles', function() {
return gulp.src('scss/**/*.scss')
.pipe(sourcemaps.init())
.pipe(sass().on('error', sass.logError))
.pipe(sourcemaps.write('../maps'))
.pipe(gulp.dest('./css/'))
.pipe(browserSync.stream());
});
/* Creates the minified version */
gulp.task('css-minify', gulp.series('styles', function() {
return gulp.src('scss/**/*.scss')
.pipe(sourcemaps.init())
.pipe(sass({
outputStyle: 'compact' // Options: nested, expanded, compact, compressed
}).on('error', sass.logError))
.pipe(postcss([
autoprefixer({
cascade: false
}),
]))
.pipe(cleanCSS({
advanced: false,
aggressiveMerging: false
}))
.pipe(rename({suffix: '.min'}))
.pipe(sourcemaps.write('../maps'))
.pipe(gulp.dest('./css/'));
}));
gulp.task('browser-sync', function(done) {
browserSync.init({
open: 'external',
proxy: site_url,
host: site_url,
// port: 5000,
browser: "chrome",
});
done();
});
gulp.task('watch', gulp.series('browser-sync', function() {
gulp.watch('scss/**/*.scss', gulp.series('css-minify'));
gulp.watch('js/dev/**/*.js', gulp.series('js-watch'));
}));
gulp.task('default', gulp.series('js-watch', 'css-minify'));
Now, to help turn the JS files into bundles I have made a change to the js_scripts array, to:
var js_scripts = [
[
'lib',
[
'js/dev/lib/**/*.js'
],
['lib.js', 'lib.min.js']
],
[
'plugins',
[
'js/dev/plugins/**/*.js'
],
['plugins.js', 'plugins.min.js']
],
[
'bootstrap',
[
// We have to set the bootstrap lines separately as some need to go before others
'js/dev/bootstrap/alert.js',
'js/dev/bootstrap/collapse.js',
'js/dev/bootstrap/tooltip.js',
'js/dev/bootstrap/popover.js',
'js/dev/bootstrap/tab.js',
'js/dev/bootstrap/transition.js',
],
['bootstrap.js', 'bootstrap.min.js']
],
[
'custom',
[
'js/dev/custom.js'
],
['custom.js', 'custom.min.js']
],
];
With the idea being that we will loop through this array and create a separate JS + min.js file for each.
Now, the problem is I'm not sure how to achieve this in Gulp in a clean way.
Take this for example:
gulp.task('scripts', function() {
return gulp.src(js_scripts)
.pipe(sourcemaps.init())
.pipe(concat('scripts.js'))
.pipe(sourcemaps.write('../maps'))
.pipe(gulp.dest('./js'));
});
Ideally it would be good to loop through the array in here; but I'm not sure how to handle doing this multiple times, because once you return the first result then obviously the loop is going to end.
Does Gulp need to return each time? ...if not, then what do you return once you have finished processing the loop?

I tried using gulp-order to force ordering of the bootstrap files but it was unreliable. Here I use merge2 instead - it seems to work much better. Assumes you now have a custom folder.
[I didn't incorporate your sourcemaps pipes and browserSync reloads for brevity.]
const gulp = require('gulp');
const concat = require('gulp-concat');
// const rename = require('gulp-rename'); // not needed
// var gulpif = require('gulp-if'); // not needed
var terser = require('gulp-terser'); // better than uglify, works with es6
const merge2 = require('merge2') // does the ordering of source files
const glob = require('glob');
const path = require('path');
// glob.sync will build your array for you, so you don't need your 'js_scripts' array
const bundleFolders = glob.sync('js/dev/*/'); // returns an array of folders
console.log(bundleFolders);
gulp.task('scripts', () => {
let stream;
for (const bundle of bundleFolders) {
// get just the last directory of 'js/dev/bootstrap', 'js/dev/lib`, etc.
let thisBundle = path.basename(bundle);
console.log('thisBundle = ' + thisBundle);
if (thisBundle === 'bootstrap') {
stream = merge2(
gulp.src([
'js/dev/bootstrap/alert.js',
'js/dev/bootstrap/collapse.js',
'js/dev/bootstrap/tooltip.js',
'js/dev/bootstrap/popover.js',
'js/dev/bootstrap/tab.js',
'js/dev/bootstrap/transition.js',
])
// your other stuff - sourcemaps, etc.
.pipe(concat(thisBundle + '.js'))
// your other stuff - sourcemaps, etc.
.pipe(gulp.dest('./js')));
}
else { // not 'bootstrap' directory
stream = gulp.src(bundle + "**/*.js")
// your other stuff - sourcemaps, etc.
.pipe(concat(thisBundle + '.js'))
// your other stuff - sourcemaps, etc.
.pipe(gulp.dest('./js'));
}
}
return stream;
});
// gulp.task('uglify', gulp.series('scripts', function () { // not needed
gulp.task('terser', () => {
let bundles= [];
for (const bundle of bundleFolders) {
// get `js/dev/custom/custom.js`,'js/dev/bootstrap/bootstrap.js' etc.
bundles.push(bundle + path.basename(bundle) + '.js');
}
console.log(bundles);
return gulp.src(bundles)
// .pipe(gulpif('!**/*.min.js', uglify({mangle: false})))
// assumes what you want to uglify and concat are the 'lib.js', bootstrap.js', etc. files
.pipe(terser({ mangle: false }))
.pipe(concat('scripts.min.js'))
.pipe(gulp.dest('./js'));
});
// create a task that ensures the `terser` task is complete before
// reloading browsers
// gulp.task('js-watch', gulp.series('terser', function (done) {
// browserSync.reload();
// done();
// }));
gulp.task('watch', gulp.series('browser-sync', function () {
gulp.watch('scss/**/*.scss', gulp.series('css-minify'));
// gulp.watch('js/dev/**/*.js', gulp.series('js-watch')); // replaced by below
gulp.watch('js/dev/**/*.js', gulp.series('scripts', 'terser', browserSync.reload));
}));
exports.terser = gulp.series('terser'); // useful for testing
exports.scripts = gulp.series('scripts'); // useful for testing
exports.default = gulp.series('scripts'); // just for testing

How to compile SASS and minify CSS and create its map with gulp 4 in same task

How to compile SASS and minify CSS and create its map with gulp 4 in same task
Im using Gulp 4, i wonder if there is a way to put the css with its map and also put the css minified with its map, But in the same task, i mean something like this:
- css
- main.css
- main.css.map
- main.min.css
- main.min.css.map
My current code actually does it but i have two task
const gulp = require('gulp');
const autoprefixer = require('gulp-autoprefixer');
const cleanCSS = require('gulp-clean-css');
const sass = require('gulp-sass');
const sourcemaps = require('gulp-sourcemaps');
const rename = require('gulp-rename');
//declare the scr folder
let root = '../src' + '/';
let scssFolder = root + 'scss/';
//declare the build folder
let build = '../build/' + '/';
let cssFolder = build + 'css';
// Compile scss into css
function css() {
return gulp
.src(scssFolder + 'main.scss')
.pipe(sourcemaps.init({ loadMaps: true }))
.pipe(
sass({
outputStyle: 'expanded',
}).on('error', sass.logError)
)
.pipe(autoprefixer('last 2 versions'))
.pipe(sourcemaps.write('./'))
.pipe(gulp.dest(cssFolder));
}
//minify css
function minCSS() {
return gulp
.src(scssFolder + 'main.scss')
.pipe(sourcemaps.init({ loadMaps: true }))
.pipe(
sass({
outputStyle: 'compressed',
}).on('error', sass.logError)
)
.pipe(autoprefixer('last 2 versions'))
.pipe(rename({ suffix: '.min' }))
.pipe(sourcemaps.write('./'))
.pipe(gulp.dest(cssFolder));
}
exports.css = css;
exports.minCSS = minCSS;
and id like to know either if i can put in one task or how can i call them in one task for example:
function css() {
return gulp
.src(scssFolder + 'main.scss')
.pipe(sourcemaps.init({ loadMaps: true }))
.pipe(
sass({
outputStyle: 'expanded',
}).on('error', sass.logError)
)
.pipe(autoprefixer('last 2 versions'))
.pipe(sourcemaps.write('./'))
.pipe(gulp.dest(cssFolder))
//Put here the minify code
.pipe(cleanCSS())
.pipe(sourcemaps.write('./'))
.pipe(gulp.dest(cssFolder));
}
but the previous code doesn´t work because it creates main.css and main.css.map

Create new function where you run both functions in series from your first code.
Example
function compileAndMinify(){
return gulp.series(css(),minCss());
}

Ok, so I have what I think might be the ultimate solution for this problem if you are using gulp 4. Also I am using babel to write my gulp file in es6 via "gulpfile.babel.js" so pardon if this example looks weird (I split my gulp 4 builds up into multiple js modules).
This is a non question specific answer, but it illustrates how I output min and non min css and source maps in the same task, by having my gulp task run it twice via gulp 4 parallel.
'use strict';
import config from './config';
import yargs from 'yargs';
import { src, dest, parallel, series } from 'gulp';
import concat from 'gulp-concat';
import rename from 'gulp-rename';
import csso from 'gulp-csso';
import sass from 'gulp-sass';
import tildeImporter from 'node-sass-tilde-importer';
import sassVar from 'gulp-sass-variables';
import sourcemaps from 'gulp-sourcemaps';
sass.compiler = require('node-sass');
var args = yargs.argv;
class isolatedVendorBuild {
static build(done, destination, fileName) {
//this is the actual gulp-sass build function, that will be wrapped by two other gulp4 tasks
let internalBuild = ((shouldMin) => {
//store the stream to a variable before returning it, so we can conditionally add things to it
let ret = src(config.paths.srcSharedIsolatedVendorScss)
.pipe(sourcemaps.init())
.pipe(concat(fileName))
.pipe(sassVar({
$env: args.prod ? 'prod' : 'dev'
}))
.pipe(sass({
importer: tildeImporter,
outputStyle: 'nested'
}).on('error', sass.logError));
if (shouldMin) {
//if the function was called with shouldMin true, then reset the stream from the previous stream but with the rename .min and csso call added to it
ret = ret.pipe(rename({ suffix: '.min' }));
ret.pipe(csso({ sourceMap: true }));
}
//reset the stream to the previous ret and add sourcemaps.write and destination output to it
ret = ret.pipe(sourcemaps.write('.'))
.pipe(dest(destination));
//return the complete stream
return ret;
});
//create two wrapper functions for the internal build to be called in gulp since gulp can't pass arguments to functions treated as gulp tasks
function buildStylesUnMinified() {
return internalBuild(false); //pass false for shouldMin and it will output the unminified css and source map
}
function buildStylesMinified() {
return internalBuild(true); //pass true for shouldMin and it will output the minified css and source map with the .min suffix added to the file name.
}
//the magic, we use gulp parallel to run the unminified version and minified version of this sass build at the same time calculating two separate streams of css output at the same time.
return parallel(buildStylesUnMinified, buildStylesMinified)(done);
}
}
export default isolatedVendorBuild;
I've seen other solutions that involve outputting the non minified css first, then using that as an input for the minification task. That works, but it forces synchronous dependencies and that get's build times crawling to a snails pace in complex builds.
I came up with this method just recently solving for this in a new project with multiple scss output files. I like it because both minified and unminified tasks run at the same time and I was able to get my main build to let the entire scss output process be async, as in, not depend on it completing before doing the scripts and stuff.

Gulp 4 - splitting up main gulpfile.js

I am trying to use Gulp4 API and writing a very basic and simple tasks to start with and am having some issues with splitting up the gulp file.
Basically I have two standalone tasks, one for minifying CSS and another for minifying JS which I am trying to import into the main gulpfile.js and calling them as part of the default task.
Below is a sample of one of the standalone gulp tasks which mainly cleans up minified css and minifies and concats the css again:
"use strict";
const { src, dest, series, task } = require( 'gulp' );
const concat = require( 'gulp-concat' );
const prefix = require( 'gulp-autoprefixer' );
const cleanCSS = require( 'gulp-clean-css' );
const rimraf = require( 'gulp-rimraf' );
const baseURL = './src/main/resources/';
const minifiedCssSources = [ baseURL + '**/*min.css' ];
const cssSources = [ baseURL + '**/*.css' ];
module.exports = function() {
const _cleanupMinifiedCss = function() {
return src(minifiedCssSources
, { allowEmpty: true }
, { read: false })
.pipe(rimraf({ force: true }));
}
const _minifyConcatCss = function() {
return src(cssSources)
.pipe(concat('css.min.css'))
.pipe(cleanCSS())
.pipe(prefix('last 2 versions'))
.pipe(dest(baseURL + 'css/'));
}
task("cssMinify", series(_cleanupMinifiedCss, _minifyConcatCss))
}
And below is a sample of my main gulpfile.js:
"use strict";
const { task, parallel } = require( 'gulp' );
const jsMinify = require("./gulp/tasks/minifyJs");
const cssMinify = require("./gulp/tasks/minifyStyles");
function defaultTask(done) {
parallel(cssMinify, jsMinify)
done();
}
task('default', defaultTask);
The issue I am having is that the default task is starting and finishing fine, however no css is being cleaned up or minified/concatenated. It's like the standalone tasks are being ignored completely.
I have been trying various ways of exporting and importing the standalone tasks however haven't managed to get this working. Unfortunately the documentation on their website is pretty minimal: "Each task can be split into its own file, then imported into your gulpfile for composition. Not only does this keep things organized, but it allows you to test each task independently or vary composition based on conditions."
Anyone has any ideas what I could try or maybe I'm doing wrong with Gulp4 API?

For the benefit of everyone, I seem to have found a way to do this after a number of tries! :)
Pasting an example below. The key is in how you export the task in your standalone task (last 2 lines).
Standalone task - say gulp\css.js
"use strict";
const { src, dest, series, task } = require( 'gulp' );
const concat = require( 'gulp-concat' );
const prefix = require( 'gulp-autoprefixer' );
const cleanCSS = require( 'gulp-clean-css' );
const rimraf = require( 'gulp-rimraf' );
const baseURL = './src/main/resources/';
const minifiedCssSources = [ baseURL + '**/*min.css' ];
const cssSources = [ baseURL + '**/*.css' ];
function _cleanupMinifiedCss() {
return src(minifiedCssSources
, { allowEmpty: true }
, { read: false })
.pipe(rimraf({ force: true }));
}
function _minifyConcatCss() {
return src(cssSources)
.pipe(concat('css.min.css'))
.pipe(cleanCSS())
.pipe(prefix('last 2 versions'))
.pipe(dest(baseURL + 'css/'));
}
const cssTasks = series(_cleanupMinifiedCss, _minifyConcatCss);
exports.cssTasks = cssTasks;
Main gulp file:
"use strict";
const { task, parallel } = require( 'gulp' );
const jsMinify = require("./gulp/tasks/minifyJs");
const cssMinify = require("./gulp/tasks/minifyStyles");
task('default', parallel(jsMinify.jsTasks, cssMinify.cssTasks));

make some operations on factor-bundle's partial bundles

I am using gulp with browserify and factor-bundle.
I have the following code:
b = browserify({
entries: [ 'a.js', 'b.js'],
plugin: [ [ 'factor-bundle', { outputs: [ 'build/a.js', 'build/b.js' ] } ] ]
})
.bundle()
.pipe(source('bundle.js'))
.pipe(buffer())
.pipe(gulp.dest('/build/common'));
I want to pipe some actions (like uglify, bundle-collapser or other job) on the parial bundles ('build/a.js' and 'build/b.js'). I tried to use the method described on the factor-bundle's page:
b.plugin('factor-bundle', { outputs: [ write('x'), write('y') ] });
function write (name) {
return concat(function (body) {
console.log('// ----- ' + name + ' -----');
console.log(body.toString('utf8'));
});
}
But I don't understand the write() method and don't know how to perform uglification and how to gulp.dest the result.
Any idea? explanation?

The write() method returns a writable stream that allows you to pipe bundles
generated by the factor-bundle plugin through further downstream transformations.
For instance, your write() method may look something like this:
var path = require('path');
var file = require('gulp-file');
var sourcemaps = require('gulp-sourcemaps');
function write (filepath) {
return concat(function (content) {
// create new vinyl file from content and use the basename of the
// filepath in scope as its basename.
return file(path.basename(filepath), content, { src: true })
// uglify content
.pipe(uglify())
// write content to build directory
.pipe(gulp.dest('./build/scripts'))
});
}
And you would use it like this:
browserify({
entries: [ 'a.js', 'b.js'],
plugin: [ [ 'factor-bundle', { outputs: [ write('a.js'), write('b.js') ] } ] ]
})
.bundle()
.pipe(write('common.js'))
// Could have use these instead, but it wouldn't be as DRY.
// .pipe(source('common.js'))
// .pipe(uglify())
// .pipe(gulp.dest('./build/scripts'))
Using the factor-bundle plugin affects the output of browserify after
.bundle() is called. Normally, it would generate bundles as readable streams
mapping to each of your entry files, then you would be able to apply further
transformations to them.
Instead you will get a single readable stream that contains a bundle with the
shared common modules from the supplied entry files, which I have called
common.js on the example above. Then you need to handle the transfomations
of the readable streams that map to each entry file separately.
In the example above I have added writable streams to the outputs array, arranged
in the same order as my entry files, which receive their respective bundle as
readable stream and apply further transformations to them
You could also leverage the factor.pipeline event:
var b = browserify({ ... });
b.on('factor.pipeline', function (id, pipeline) {
pipeline.get('wrap').push(write(id));
});
b.plugin(factor);
return b.bundle().pipe(write('common.js'));
I think it is worth noting that applying further downstream work to the outputs
is completely detached from the pipeline. So if you were using gulp and returned
the stream from browserify, the task would have completed prematurely because
it would still be performing operations on the entry files. I haven't run into
issues with this yet.
Hope this helps.

This is a bit old, but it might be usefull to someone else.
The answer above from #Christian helped me, but i had to solve the issue of the task completion. I did it by adding a counter for opened streams, and calling the task callback once they are all closed.
gulp.task('build:js:compile', function(cb) {
const apps = getAllJavascriptFilesPaths(); // this returns an array of full path to the js files i want to bundle
const dist = 'dist'; // the output path
const files = [];
const streams = [];
let openedStreams = 0;
// We use browserify factor-bundle to get the shared code in a separated js file, and not in all apps files
// The write function here handles the post processing of each browserified app by returning a writable stream
// We check the number of opened streams, and call the callback once they are all closed, to be sure the task is
// complete
function write(filepath) {
openedStreams++;
return concat(function (content) {
// create new vinyl file from content and use the basename of the
// filepath in scope as its basename.
return file(path.basename(filepath), content, { src: true })
.pipe(uglify())
.pipe(gulp.dest(dist))
.on('finish', function () {
openedStreams--;
if (openedStreams == 0) {
cb();
}
});
});
}
apps.forEach(function (file) {
files.push(file);
streams.push(write(file)));
});
browserify(files)
.plugin(factor, { outputs: streams })
.transform("babelify", {presets: 'babel-preset-env'})
.bundle()
.pipe(write('common.js'));
});