We Keep Coding

Why is merge-stream requiring async completion signal?

I'm in the process of migrating from gulp#3.9.1 to gulp#4.0.2 and upgrading my gulp dependencies in the process. I have the following task in my gulpfile, where you can assume directories is just an array of directories I want to perform this operation on:
var gulp = require('gulp');
var ngAnnotate = require('gulp-ng-annotate'); //annotates dependencies in Angular components
var rev = require('gulp-rev'); //appends a hash to the end of file names to eliminate stale cached files
var revReplace = require('gulp-rev-replace');
var uglify = require('gulp-uglify'); // minimizes javascript files
var compressCss = require('gulp-minify-css');
var useref = require('gulp-useref'); // replaces style and script blocks in HTML files
var filter = require('gulp-filter');
var merge = require('merge-stream');
var sourcemaps = require('gulp-sourcemaps');
function minify() {
var tasks = directories.map(function (directory) {
var cssFilter = filter("**/all.min.css", {restore:true});
var jsAppFilter = filter("**/app.min.js", {restore:true});
var jsFilter = filter("**/*.js", {restore:true});
return gulp.src(dstBasePath + directory + "index.html", {allowEmpty: true})
.pipe(useref())
.pipe(cssFilter)
.pipe(compressCss({keepSpecialComments:false}))
.pipe(rev())
.pipe(cssFilter.restore)
.pipe(jsAppFilter)
.pipe(sourcemaps.init())
.pipe(ngAnnotate({add:true, single_quotes:true}))
.pipe(jsAppFilter.restore)
.pipe(jsFilter)
.pipe(uglify())
.pipe(rev())
.pipe(jsFilter.restore)
.pipe(revReplace())
.pipe(sourcemaps.write('.')) // sourcemaps need to be written to same folder for Datadog upload to work
.pipe(gulp.dest(dstBasePath + directory))
});
return merge(tasks);
}
Why would this result in the error "Did you forget to signal async completion?" from Gulp when running the task? Note that I'm using Gulp 4. I've tried passing a callback done to this task, and adding .addListener('end', done) to the final pipe, but this causes my merged stream to end prematurely (presumably when the first one ends). So perhaps one of these plugins is not signaling when it's completed, but how would you even get this to work otherwise? Thanks for any insight you can provide.

return merge(folders.map(function (folder) { // this has worked for me in the past
as has this form without merge
gulp.task('init', function (done) {
var zips = getZips(zipsPath);
var tasks = zips.map(function (zip) {
return gulp.src(zipsPath + "/" + zip)
.pipe(unzip({ keepEmpty: true }))
.pipe(gulp.dest(path.join("src", path.basename(zip, ".zip"))))
.on('end', function() { // this bit is necessary
done();
});
});
return tasks;
});

Gulp 4 requires that you signal async completion. There's some good information about it in this answer to a similar question:
Gulp error: The following tasks did not complete: Did you forget to signal async completion?
I had a similar case where I was returning a merged set of tasks, and I was able to resolve the error by making the function async and awaiting the merge. My case looked something like this:
gulp.task("build", async function () {
...
return await merge(tasks);
});
so I think you should be able to do something like
async function minify(){
...
return await merge(tasks);
}

How to sync a local folder with gulp, with update for deleted files and folders?

I am working on a WordPress plugin and have all the files in my working directory and run gulp in that project folder. Now, I'd like to have a watch task that copies all the changes to my local WP installation for testing.
Therefore I am looking for a way to sync (only in one direction) the project folder with the plugin folder of WP.
I managed to get it to work with gulp-directory-sync
...
var dirSync = require("gulp-directory-sync");
var localDir = "../newDir/";
var buildDir = "./buildDir/";
...
function copy_to_local_folder() {
return pipeline(
gulp.src(buildDir+'**/*'),
dirSync( buildDir, localDir, { printSummary: true } )
);
}
function watch_local() {
gulp.watch(buildDir+'**/*', copy_to_local_folder);
exports.default = watch_local;
However, the plugin hasn't been updated in 4 years and according to this answer, it is not doing it the proper "gulp way" (e.g. not using gulp-src) and this task should be possible with other basic gulp functions.
Copying changed files is pretty easy, but also keeping track of deleted files is more complicated. I also would prefer to only update changed/deleted/new files and not clearing the folder every time before coping all files.
Starting with the updated code in the aforementioned answer, I tried to implement it and made changes to make it work.
...
var newer = require("gulp-newer");
var pipeline = require("readable-stream").pipeline;
var del = require("del");
var localDir = "../newDir/";
var buildDir = "./buildDir/";
function copy_to_local_folder() {
return pipeline(
gulp.src([buildDir+'**/*']),
newer(localDir),
gulp.dest(localDir),
);
}
function watch_local() {
var watcher = gulp.watch(buildDir + '**/*', copy_to_local_folder );
watcher.on('unlink', function(path) {
console.log(path);
var newPath = './'+path;
newPath = newPath.replace(buildDir, localDir);
console.log(newPath);
(async () => {
const deletedPaths = await del(newPath, {dryRun: true, force: true});
console.log('Deleted files and directories:\n', deletedPaths.join('\n'));
})();
});
}
exports.default = watch_local;
With this code, the folder gets updated when I change or delete files, but it does not trigger when I delete an entire folder. Which is probably because I use unlink and not unlinkDir. But even if I use the version of the function below, it doesn't get triggered by deleting a folder (with containing files).
watcher.on('unlinkDir', function(path) {
console.log('folder deleted');
console.log(path);
var newPath = './'+path;
newPath = newPath.replace(buildDir, localDir);
console.log(newPath);
});
What am I doing wrong?
Or is there in general a better way to achieve this?
PS: I'm using
node v11.15.0
gulp v4.0.2
on Linux
deleting files and folders in VS Code
Update:
When I run it with:
watcher.on('unlink', ... and delete a file:
it works
with the console.log output and the ( async () => ...
and Starting and Finished for copy_to_local_folder
watcher.on('unlinkDir', ... and delete a folder:
it works not
nothing happens in the console output
(not even Starting)
watcher.on('unlinkDir', ... and delete a file:
Starting and Finished for copy_to_local_folder
but not the console.log and ( async () => ...
watcher.on('add', ... and watcher.on('addDir', ...
work both
Seems to me that the watcher.on('unlinkDir', ... does never get triggered ... is unlinkDir not supported by gulp-watch?

Ordering of tasks in a gulpfile

I want to implement very simple chain of tasks in my project by Gulp:
Copy all files;
Replace some placeholders with values;
Minify some
files;
And for these purposes I have created gulpfile with such main task:
gulp.task(tasks.build, [
tasks.simplyCopy,
tasks.minifyXml,
tasks.minifyJs,
tasks.subst]);
It's a pretty simple and self describable.
Below I wrote full gulpfile.js:
var gulp = require('gulp');
var prettyData = require('gulp-pretty-data');
var uglify = require('gulp-uglify');
var renvy = require('gulp-renvy');
var tasks = {
simplyCopy: "simply-copy",
minifyXml: "minify-xml",
minifyJs: "minify-js",
subst: "renvy-subst",
build: "build"
};
// Collection of tasks
gulp.task(tasks.build, [tasks.simplyCopy, tasks.minifyXml, tasks.minifyJs,
tasks.subst]);
// By this task sources simply copy to the destination
var destination = 'dist/';
gulp.task(tasks.simplyCopy, function () {
gulp.src(['Source/**/*.*', '!Source/www/res/strings/*.*'], {base:
'Source/www'})
.pipe(gulp.dest(destination));
});
var stringsDestPath = 'dist/res/strings/';
var stringSrcPath = 'Source/www/res/strings/';
// By this task some xml files minify
gulp.task(tasks.minifyXml, [tasks.simplyCopy], function() {
gulp.src(stringSrcPath + '*.xml')
.pipe(prettyData({
type: 'minify',
preserveComments: true,
extensions: {
'xlf': 'xml',
'svg': 'xml'
}
}))
.pipe(gulp.dest(stringsDestPath))
});
var placeholder = {
'%version%': {'prod':'010.00', 'dev':'010.00'}
};
// By this task in some files placeholders replaces with value
gulp.task(tasks.subst, [tasks.minifyXml, tasks.minifyJs], function(){
return gulp.src(stringsDestPath + '*.*')
.pipe(renvy(placeholder, 'dev'))
.pipe(gulp.dest(stringsDestPath));
});
// By this task some js files minify
gulp.task(tasks.minifyJs, [tasks.simplyCopy], function () {
gulp.src(stringSrcPath + '*.js')
.pipe(uglify())
.pipe(gulp.dest(stringsDestPath))
});
But I have such unexpected behavior:
replacing of placeholders is not happening, but it's executes.
[16:29:51] Using gulpfile C:\PDDirectory\Workspace\src\some_workbench\User_Part\gulpfile.js
[16:29:51] Starting 'simply-copy'...
[16:29:51] Finished 'simply-copy' after 17 ms
[16:29:51] Starting 'minify-xml'...
[16:29:51] Finished 'minify-xml' after 7.49 ms
[16:29:51] Starting 'minify-js'...
[16:29:51] Finished 'minify-js' after 5.84 ms
[16:29:51] Starting 'renvy-subst'...
[16:29:51] Finished 'renvy-subst' after 28 ms
[16:29:51] Starting 'build'...
[16:29:51] Finished 'build' after 5.66 ?s
Task tasks.subst executed sepparetly works fine, but in a chain with other tasks, I see results of executing copy and minify only.
Why so?

Place tasks.subst has the sole dependency for tasks.build
gulp.task(tasks.build, [tasks.subst]);
Since tasks.subst requires all the others, the ordering should be correct and adding all the other tasks may lead to ordering problems.
From the gulp.task documentation:
Note: Are your tasks running before the dependencies are complete?
Make sure your dependency tasks are correctly using the async run
hints: take in a callback or return a promise or event stream.
To ensure that a task dependencies are fulfilled before executing it, Gulp needs each task to return a stream or a promise, or call the task function callback parameter.
In your case, the following task should just return the stream:
tasks.minifyXml
tasks.minifyJs
tasks.simplyCopy
For example:
gulp.task(tasks.minifyXml, [tasks.simplyCopy], function(done) {
// just return the task stream here
return gulp.src(stringSrcPath + '*.xml')
.pipe(prettyData({
// ...
}))
.pipe(gulp.dest(stringsDestPath));
});
or using the callback when it's not possible to return a stream:
gulp.task('somename', function(done) {
// async function which does not return a stream like other gulp functions
getFilesAsync(function(err, res) {
// pass any errors to the callback
if (err) return done(err);
var stream = gulp.src(res)
.pipe(minify())
.pipe(gulp.dest('build'))
.on('end', done); // use the callback when it's done
});
});

Gulp 4 watch runs only 1 time

As I mentioned in the title there is a problem running gulp.watch. It runs watch only after first change in the file, when I changing second, third and etc it doesn't run task.
Below is my gulpfile.js:
var gulp = require('gulp');
var babel = require('gulp-babel');
var rename = require("gulp-rename");
var del = require('del');
var less = require('gulp-less');
gulp.task('es6', function () {
return gulp.src('./test.js')
.pipe(rename(function (path) {
path.basename += "-es6";
return path;
}))
.pipe(babel())
.pipe(gulp.dest('./'))
});
gulp.task('clean', function () {
return del('./test-es6.js');
});
gulp.task('watch', function () {
gulp.watch( './test.js', gulp.series('es6') );
console.log('Running watch...');
});
gulp.task('default', gulp.series('clean', 'es6', gulp.parallel('watch') ));
And some logs :
$: gulp
[14:22:40] Using gulpfile /var/www/html/es2015/gulpfile.js
[14:22:40] Starting 'default'...
[14:22:40] Starting 'clean'...
[14:22:40] Finished 'clean' after 11 ms
[14:22:40] Starting 'es6'...
[14:22:43] Finished 'es6' after 2.11 s
[14:22:43] Starting '<parallel>'...
[14:22:43] Starting 'watch'...
Running watch...
[14:22:55] Starting '<series>'...
[14:22:55] Starting 'es6'...
[14:22:55] Finished 'es6' after 42 ms
[14:22:55] Finished '<series>' after 43 ms << first change, but no second third and etc.
I used similar configuration in couple of projects and it was fine, everything worked.
I don't know if this information is important, but I'm using Ubuntu 14.04

After long trying I found a solution for this problem.
I don't know why but this configuration works on Windows, but as appeared on Ubuntu I had to add ** to path of the watched file this part of code:
gulp.task('watch', function () {
gulp.watch( './**/test.js', gulp.series('es6') );
console.log('Running watch...');
});

Gulps gulp.watch not triggered for new or deleted files?

The following Gulpjs task works fine when editing files in the glob match:
// watch task.
gulp.task('watch', ['build'], function () {
gulp.watch(src + '/js/**/*.js', ['scripts']);
gulp.watch(src + '/img//**/*.{jpg,jpeg,png,gif}', ['copy:images']);
gulp.watch(src + '/less/*.less', ['styles']);
gulp.watch(src + '/templates/**/*.{swig,json}', ['html']);
});
// build task.
gulp.task('build', ['clean'], function() {
return gulp.start('copy', 'scripts', 'less', 'htmlmin');
});
However it doesn't work (it's not triggered) for new or deleted files. Is there something I'm missing?
EDIT: even using grunt-watch plugin it seems not working:
gulp.task('scripts', function() {
return streamqueue(
{ objectMode: true },
gulp.src([
vendor + '/jquery/dist/jquery.min.js',
vendor + '/bootstrap/dist/js/bootstrap.min.js'
]),
gulp.src([
src + '/js/**/*.js'
]).pipe(plugins.uglify())
)
.pipe(plugins.concat(pkg.name + '.min.js'))
.pipe(gulp.dest(dest + '/js/'));
});
gulp.task('watch', ['build'], function () {
plugins.watch({glob: src + '/js/**/*.js'}, function () {
gulp.start('scripts');
});
});
EDIT: Solved, it was this issue. Globs starting with ./ (that was the value of src) seems not working ATM.

Edit: Apparently gulp.watch does work with new or deleted files now. It did not when the question was asked.
The rest of my answer still stands: gulp-watch is usually a better solution because it lets you perform specific actions only on the files that have been modified, while gulp.watch only lets you run complete tasks. For a project of a reasonable size, this will quickly become too slow to be useful.
You aren't missing anything. gulp.watch does not work with new or deleted files. It's a simple solution designed for simple projects.
To get file watching that can look for new files, use the gulp-watch plugin, which is much more powerful. Usage looks like this:
var watch = require('gulp-watch');
// in a task
watch({glob: <<glob or array of globs>> })
.pipe( << add per-file tasks here>> );
// if you'd rather rerun the whole task, you can do this:
watch({glob: <<glob or array of globs>>}, function() {
gulp.start( <<task name>> );
});
Personally, I recommend the first option. This allows for much faster, per-file processes. It works great during development with livereload as long as you aren't concatenating any files.
You can wrap up your streams either using my lazypipe library, or simply using a function and stream-combiner like this:
var combine = require('stream-combiner');
function scriptsPipeline() {
return combine(coffeeescript(), uglify(), gulp.dest('/path/to/dest'));
}
watch({glob: 'src/scripts/**/*.js' })
.pipe(scriptsPipeline());
UPDATE October 15, 2014
As pointed out by #pkyeck below, apparently the 1.0 release of gulp-watch changed the format slightly, so the above examples should now be:
var watch = require('gulp-watch');
// in a task
watch(<<glob or array of globs>>)
.pipe( << add per-file tasks here>> );
// if you'd rather rerun the whole task, you can do this:
watch(<<glob or array of globs>>, function() {
gulp.start( <<task name>> );
});
and
var combine = require('stream-combiner');
function scriptsPipeline() {
return combine(coffeeescript(), uglify(), gulp.dest('/path/to/dest'));
}
watch('src/scripts/**/*.js')
.pipe(scriptsPipeline());

Both gulp.watch() and require('gulp-watch')() will trigger for new/deleted files however not if you use absolute directories. In my tests I did not use "./" for relative directories BTW.
Both won't trigger if whole directories are deleted though.
var watch = require('gulp-watch');
//Wont work for new files until gaze is fixed if using absolute dirs. It won't trigger if whole directories are deleted though.
//gulp.watch(config.localDeploy.path + '/reports/**/*', function (event) {
//gulp.watch('src/app1/reports/**/*', function (event) {
// console.log('*************************** Event received in gulp.watch');
// console.log(event);
// gulp.start('localDeployApp');
});
//Won't work for new files until gaze is fixed if using absolute dirs. It won't trigger if whole directories are deleted though. See https://github.com/floatdrop/gulp-watch/issues/104
//watch(config.localDeploy.path + '/reports/**/*', function() {
watch('src/krfs-app/reports/**/*', function(event) {
console.log("watch triggered");
console.log(event);
gulp.start('localDeployApp');
//});

If src is an absolute path (starting with /), your code is not going to detect new or deleted files. However there's still a way:
Instead of:
gulp.watch(src + '/js/**/*.js', ['scripts']);
write:
gulp.watch('js/**/*.js', {cwd: src}, ['scripts']);
and it will work!

Globs must have a separate base directory specified and that base location must not be specified in the glob itself.
If you have lib/*.js, it'll look under the current working dir which is process.cwd()
Gulp uses Gaze to watch files and in the Gulp API doc we see that we can pass Gaze specific options to the watch function: gulp.watch(glob[, opts], tasks)
Now in the Gaze doc we can find that the current working dir (glob base dir) is the cwd option.
Which leads us to alexk's answer:
gulp.watch('js/**/*.js', {cwd: src}, ['scripts']);

I know this is an older question, but I thought I'd throw the solution I came up with. None of the gulp plugins I found would notify me of new or renamed files. So I ended up wrapping monocle in a convenience function.
Here's an example of how that function is used:
watch({
root: config.src.root,
match: [{
when: 'js/**',
then: gulpStart('js')
}, {
when: '+(scss|css)/**',
then: gulpStart('css')
}, {
when: '+(fonts|img)/**',
then: gulpStart('assets')
}, {
when: '*.+(html|ejs)',
then: gulpStart('html')
}]
});
I should note that gulpStart is also a convenience function I made.
And here is the actual watch module.
module.exports = function (options) {
var path = require('path'),
monocle = require('monocle'),
minimatch = require('minimatch');
var fullRoot = path.resolve(options.root);
function onFileChange (e) {
var relativePath = path.relative(fullRoot, e.fullPath);
options.match.some(function (match) {
var isMatch = minimatch(relativePath, match.when);
isMatch && match.then();
return isMatch;
});
}
monocle().watchDirectory({
root: options.root,
listener: onFileChange
});
};
Pretty simple, eh? The whole thing can be found over at my gulp starter kit: https://github.com/chrisdavies/gulp_starter_kit

It is important to note that it looks like gulp.watch only reports changed and deleted files on Windows but listens for new and deleted files by default on OSX:
https://github.com/gulpjs/gulp/issues/675

You should use 'gulp-watch' for new/renamed/deleted files instead of gulp.watch
var gulpwatch = require('gulp-watch');
var source = './assets',
destination = './dest';
gulp.task('copy-changed-assets', function() {
gulpwatch(source+'/**/*', function(obj){
gulp.src( obj.path, { "base": source})
.pipe(gulp.dest(destination));
});
});