We Keep Coding

Why is merge-stream requiring async completion signal?

I'm in the process of migrating from gulp#3.9.1 to gulp#4.0.2 and upgrading my gulp dependencies in the process. I have the following task in my gulpfile, where you can assume directories is just an array of directories I want to perform this operation on:
var gulp = require('gulp');
var ngAnnotate = require('gulp-ng-annotate'); //annotates dependencies in Angular components
var rev = require('gulp-rev'); //appends a hash to the end of file names to eliminate stale cached files
var revReplace = require('gulp-rev-replace');
var uglify = require('gulp-uglify'); // minimizes javascript files
var compressCss = require('gulp-minify-css');
var useref = require('gulp-useref'); // replaces style and script blocks in HTML files
var filter = require('gulp-filter');
var merge = require('merge-stream');
var sourcemaps = require('gulp-sourcemaps');
function minify() {
var tasks = directories.map(function (directory) {
var cssFilter = filter("**/all.min.css", {restore:true});
var jsAppFilter = filter("**/app.min.js", {restore:true});
var jsFilter = filter("**/*.js", {restore:true});
return gulp.src(dstBasePath + directory + "index.html", {allowEmpty: true})
.pipe(useref())
.pipe(cssFilter)
.pipe(compressCss({keepSpecialComments:false}))
.pipe(rev())
.pipe(cssFilter.restore)
.pipe(jsAppFilter)
.pipe(sourcemaps.init())
.pipe(ngAnnotate({add:true, single_quotes:true}))
.pipe(jsAppFilter.restore)
.pipe(jsFilter)
.pipe(uglify())
.pipe(rev())
.pipe(jsFilter.restore)
.pipe(revReplace())
.pipe(sourcemaps.write('.')) // sourcemaps need to be written to same folder for Datadog upload to work
.pipe(gulp.dest(dstBasePath + directory))
});
return merge(tasks);
}
Why would this result in the error "Did you forget to signal async completion?" from Gulp when running the task? Note that I'm using Gulp 4. I've tried passing a callback done to this task, and adding .addListener('end', done) to the final pipe, but this causes my merged stream to end prematurely (presumably when the first one ends). So perhaps one of these plugins is not signaling when it's completed, but how would you even get this to work otherwise? Thanks for any insight you can provide.

return merge(folders.map(function (folder) { // this has worked for me in the past
as has this form without merge
gulp.task('init', function (done) {
var zips = getZips(zipsPath);
var tasks = zips.map(function (zip) {
return gulp.src(zipsPath + "/" + zip)
.pipe(unzip({ keepEmpty: true }))
.pipe(gulp.dest(path.join("src", path.basename(zip, ".zip"))))
.on('end', function() { // this bit is necessary
done();
});
});
return tasks;
});

Gulp 4 requires that you signal async completion. There's some good information about it in this answer to a similar question:
Gulp error: The following tasks did not complete: Did you forget to signal async completion?
I had a similar case where I was returning a merged set of tasks, and I was able to resolve the error by making the function async and awaiting the merge. My case looked something like this:
gulp.task("build", async function () {
...
return await merge(tasks);
});
so I think you should be able to do something like
async function minify(){
...
return await merge(tasks);
}

task that uses forEach is not completing

Rather new to both gulp and javascript. The script I wrote is returning an error (appended at the end of the post.)
This is the minimal example file:
const {src, dest, series, parallel }= require('gulp');
const sourceList = ['about', 'contact', 'projects'];
// Next 2 functions take each html file and move them to respective dist folder.
function eachHtml(){
sourceList.forEach(function(htmlFile){
cphtmlTask(`source/${htmlFile}/${htmlFile}.html`, `dist/${htmlFile}/`)
});
//cphtmlTask('source/index.html', 'dist/');
}
function cphtmlTask(i,o){
return src(i)
.pipe(dest(o));
}
exports.default = series(eachHtml);
Output
[17:55:19] Starting 'default'...
[17:55:19] Starting 'eachHtml'...
[17:55:19] The following tasks did not complete: default, eachHtml
[17:55:19] Did you forget to signal async completion?
Any help please?

I think all you need is:
function eachHtml(cb){
sourceList.forEach(function(htmlFile){
cphtmlTask(`source/${htmlFile}/${htmlFile}.html`, `dist/${htmlFile}/`)
});
cb();
//cphtmlTask('source/index.html', 'dist/');
}
That cb is a callback function which will signal to gulp that the eachHtml task has completed.

How to sync a local folder with gulp, with update for deleted files and folders?

I am working on a WordPress plugin and have all the files in my working directory and run gulp in that project folder. Now, I'd like to have a watch task that copies all the changes to my local WP installation for testing.
Therefore I am looking for a way to sync (only in one direction) the project folder with the plugin folder of WP.
I managed to get it to work with gulp-directory-sync
...
var dirSync = require("gulp-directory-sync");
var localDir = "../newDir/";
var buildDir = "./buildDir/";
...
function copy_to_local_folder() {
return pipeline(
gulp.src(buildDir+'**/*'),
dirSync( buildDir, localDir, { printSummary: true } )
);
}
function watch_local() {
gulp.watch(buildDir+'**/*', copy_to_local_folder);
exports.default = watch_local;
However, the plugin hasn't been updated in 4 years and according to this answer, it is not doing it the proper "gulp way" (e.g. not using gulp-src) and this task should be possible with other basic gulp functions.
Copying changed files is pretty easy, but also keeping track of deleted files is more complicated. I also would prefer to only update changed/deleted/new files and not clearing the folder every time before coping all files.
Starting with the updated code in the aforementioned answer, I tried to implement it and made changes to make it work.
...
var newer = require("gulp-newer");
var pipeline = require("readable-stream").pipeline;
var del = require("del");
var localDir = "../newDir/";
var buildDir = "./buildDir/";
function copy_to_local_folder() {
return pipeline(
gulp.src([buildDir+'**/*']),
newer(localDir),
gulp.dest(localDir),
);
}
function watch_local() {
var watcher = gulp.watch(buildDir + '**/*', copy_to_local_folder );
watcher.on('unlink', function(path) {
console.log(path);
var newPath = './'+path;
newPath = newPath.replace(buildDir, localDir);
console.log(newPath);
(async () => {
const deletedPaths = await del(newPath, {dryRun: true, force: true});
console.log('Deleted files and directories:\n', deletedPaths.join('\n'));
})();
});
}
exports.default = watch_local;
With this code, the folder gets updated when I change or delete files, but it does not trigger when I delete an entire folder. Which is probably because I use unlink and not unlinkDir. But even if I use the version of the function below, it doesn't get triggered by deleting a folder (with containing files).
watcher.on('unlinkDir', function(path) {
console.log('folder deleted');
console.log(path);
var newPath = './'+path;
newPath = newPath.replace(buildDir, localDir);
console.log(newPath);
});
What am I doing wrong?
Or is there in general a better way to achieve this?
PS: I'm using
node v11.15.0
gulp v4.0.2
on Linux
deleting files and folders in VS Code
Update:
When I run it with:
watcher.on('unlink', ... and delete a file:
it works
with the console.log output and the ( async () => ...
and Starting and Finished for copy_to_local_folder
watcher.on('unlinkDir', ... and delete a folder:
it works not
nothing happens in the console output
(not even Starting)
watcher.on('unlinkDir', ... and delete a file:
Starting and Finished for copy_to_local_folder
but not the console.log and ( async () => ...
watcher.on('add', ... and watcher.on('addDir', ...
work both
Seems to me that the watcher.on('unlinkDir', ... does never get triggered ... is unlinkDir not supported by gulp-watch?

How to pass object to gulp task to be able to run that task synchronously?

I have two gulp tasks
gulp.task('build:dev', ['task1', task2', task3'], () => {
doCssmin({'destination': ['file1', 'file2']});
});
gulp.task('build:prod', ['task1', task2', task3'], () => {
doCssmin({'destination': ['file3', 'file4']});
});
Now I have cssMin task which is shared between build:dev and build:prod so I created that as a function to share with both of them.
function doCssmin(files) {
_.each(files, function(val, key) {
gulp.src(val)
.pipe(minifyCss({compatibility: 'ie8'}))
.pipe(rename(basename(key)))
.pipe(gulp.dest(dirname(key)));
});
}
and doCssmin accept files parameter as build:dev and build:prod they have different files to minify css. If I run build:dev or build:prod doCssmin will be run asynchronously. But doCssmin depends on task2.
My question is how do I extract doCssmin to a gulp task and accept files parameter so it can be shared with build:dev and build:prod tasks?
Not sure if it's too confusing?

Yesterday, I answered a question on making your Gulp tasks more DRY. I believe a good chunk of that answer applies here as well.
To support loops with that, you want to merge the streams that you receive from gulp.src. Using the example here:
gulp.task('test', function() {
var bootstrap = gulp.src('bootstrap/js/*.js')
.pipe(gulp.dest('public/bootstrap'));
var jquery = gulp.src('jquery.cookie/jquery.cookie.js')
.pipe(gulp.dest('public/jquery'));
return merge(bootstrap, jquery);
});
For an array, you can use Array.prototype.map to convert the items into streams, then merge all of those:
function minifyCss(paths) {
return merge.apply(this, paths.map(function (path) {
gulp.src(path)
.pipe(minifyCss({compatibility: 'ie8'}))
.pipe(rename(basename(path)))
.pipe(gulp.dest(dirname(path)));
}));
}
gulp.task('test', function () {
return minifyCss(['foo', 'bar']);
});

Gulps gulp.watch not triggered for new or deleted files?

The following Gulpjs task works fine when editing files in the glob match:
// watch task.
gulp.task('watch', ['build'], function () {
gulp.watch(src + '/js/**/*.js', ['scripts']);
gulp.watch(src + '/img//**/*.{jpg,jpeg,png,gif}', ['copy:images']);
gulp.watch(src + '/less/*.less', ['styles']);
gulp.watch(src + '/templates/**/*.{swig,json}', ['html']);
});
// build task.
gulp.task('build', ['clean'], function() {
return gulp.start('copy', 'scripts', 'less', 'htmlmin');
});
However it doesn't work (it's not triggered) for new or deleted files. Is there something I'm missing?
EDIT: even using grunt-watch plugin it seems not working:
gulp.task('scripts', function() {
return streamqueue(
{ objectMode: true },
gulp.src([
vendor + '/jquery/dist/jquery.min.js',
vendor + '/bootstrap/dist/js/bootstrap.min.js'
]),
gulp.src([
src + '/js/**/*.js'
]).pipe(plugins.uglify())
)
.pipe(plugins.concat(pkg.name + '.min.js'))
.pipe(gulp.dest(dest + '/js/'));
});
gulp.task('watch', ['build'], function () {
plugins.watch({glob: src + '/js/**/*.js'}, function () {
gulp.start('scripts');
});
});
EDIT: Solved, it was this issue. Globs starting with ./ (that was the value of src) seems not working ATM.

Edit: Apparently gulp.watch does work with new or deleted files now. It did not when the question was asked.
The rest of my answer still stands: gulp-watch is usually a better solution because it lets you perform specific actions only on the files that have been modified, while gulp.watch only lets you run complete tasks. For a project of a reasonable size, this will quickly become too slow to be useful.
You aren't missing anything. gulp.watch does not work with new or deleted files. It's a simple solution designed for simple projects.
To get file watching that can look for new files, use the gulp-watch plugin, which is much more powerful. Usage looks like this:
var watch = require('gulp-watch');
// in a task
watch({glob: <<glob or array of globs>> })
.pipe( << add per-file tasks here>> );
// if you'd rather rerun the whole task, you can do this:
watch({glob: <<glob or array of globs>>}, function() {
gulp.start( <<task name>> );
});
Personally, I recommend the first option. This allows for much faster, per-file processes. It works great during development with livereload as long as you aren't concatenating any files.
You can wrap up your streams either using my lazypipe library, or simply using a function and stream-combiner like this:
var combine = require('stream-combiner');
function scriptsPipeline() {
return combine(coffeeescript(), uglify(), gulp.dest('/path/to/dest'));
}
watch({glob: 'src/scripts/**/*.js' })
.pipe(scriptsPipeline());
UPDATE October 15, 2014
As pointed out by #pkyeck below, apparently the 1.0 release of gulp-watch changed the format slightly, so the above examples should now be:
var watch = require('gulp-watch');
// in a task
watch(<<glob or array of globs>>)
.pipe( << add per-file tasks here>> );
// if you'd rather rerun the whole task, you can do this:
watch(<<glob or array of globs>>, function() {
gulp.start( <<task name>> );
});
and
var combine = require('stream-combiner');
function scriptsPipeline() {
return combine(coffeeescript(), uglify(), gulp.dest('/path/to/dest'));
}
watch('src/scripts/**/*.js')
.pipe(scriptsPipeline());

Both gulp.watch() and require('gulp-watch')() will trigger for new/deleted files however not if you use absolute directories. In my tests I did not use "./" for relative directories BTW.
Both won't trigger if whole directories are deleted though.
var watch = require('gulp-watch');
//Wont work for new files until gaze is fixed if using absolute dirs. It won't trigger if whole directories are deleted though.
//gulp.watch(config.localDeploy.path + '/reports/**/*', function (event) {
//gulp.watch('src/app1/reports/**/*', function (event) {
// console.log('*************************** Event received in gulp.watch');
// console.log(event);
// gulp.start('localDeployApp');
});
//Won't work for new files until gaze is fixed if using absolute dirs. It won't trigger if whole directories are deleted though. See https://github.com/floatdrop/gulp-watch/issues/104
//watch(config.localDeploy.path + '/reports/**/*', function() {
watch('src/krfs-app/reports/**/*', function(event) {
console.log("watch triggered");
console.log(event);
gulp.start('localDeployApp');
//});

If src is an absolute path (starting with /), your code is not going to detect new or deleted files. However there's still a way:
Instead of:
gulp.watch(src + '/js/**/*.js', ['scripts']);
write:
gulp.watch('js/**/*.js', {cwd: src}, ['scripts']);
and it will work!

Globs must have a separate base directory specified and that base location must not be specified in the glob itself.
If you have lib/*.js, it'll look under the current working dir which is process.cwd()
Gulp uses Gaze to watch files and in the Gulp API doc we see that we can pass Gaze specific options to the watch function: gulp.watch(glob[, opts], tasks)
Now in the Gaze doc we can find that the current working dir (glob base dir) is the cwd option.
Which leads us to alexk's answer:
gulp.watch('js/**/*.js', {cwd: src}, ['scripts']);

I know this is an older question, but I thought I'd throw the solution I came up with. None of the gulp plugins I found would notify me of new or renamed files. So I ended up wrapping monocle in a convenience function.
Here's an example of how that function is used:
watch({
root: config.src.root,
match: [{
when: 'js/**',
then: gulpStart('js')
}, {
when: '+(scss|css)/**',
then: gulpStart('css')
}, {
when: '+(fonts|img)/**',
then: gulpStart('assets')
}, {
when: '*.+(html|ejs)',
then: gulpStart('html')
}]
});
I should note that gulpStart is also a convenience function I made.
And here is the actual watch module.
module.exports = function (options) {
var path = require('path'),
monocle = require('monocle'),
minimatch = require('minimatch');
var fullRoot = path.resolve(options.root);
function onFileChange (e) {
var relativePath = path.relative(fullRoot, e.fullPath);
options.match.some(function (match) {
var isMatch = minimatch(relativePath, match.when);
isMatch && match.then();
return isMatch;
});
}
monocle().watchDirectory({
root: options.root,
listener: onFileChange
});
};
Pretty simple, eh? The whole thing can be found over at my gulp starter kit: https://github.com/chrisdavies/gulp_starter_kit

It is important to note that it looks like gulp.watch only reports changed and deleted files on Windows but listens for new and deleted files by default on OSX:
https://github.com/gulpjs/gulp/issues/675

You should use 'gulp-watch' for new/renamed/deleted files instead of gulp.watch
var gulpwatch = require('gulp-watch');
var source = './assets',
destination = './dest';
gulp.task('copy-changed-assets', function() {
gulpwatch(source+'/**/*', function(obj){
gulp.src( obj.path, { "base": source})
.pipe(gulp.dest(destination));
});
});