Grunt: Loading dependencies takes several minutes - javascript

I've put together a few pretty standard grunt tasks with a reasonable number of dependencies, but for some reason it takes 1-3 minutes to actually load the dependencies. I'm moderately new to grunt, but since most of the other questions about extreme slowness loading grunt dependencies cite times of a few seconds, I'm guessing I'm doing something very wrong.
Here's what my gruntfile looks like:
const path = require("path");
module.exports = function(grunt) {
require('jit-grunt')(grunt);
require('time-grunt')(grunt);
const ignoredSourceScriptPatterns = ['!**/*.debug.js', '!**/*.min.js', '!scripts/*', '!**/*.map'],
baseUIPath = 'presentation/ui',
scripts = grunt.file.expand({filter: 'isFile',
matchBase: true,
cwd: baseUIPath},
['*.js', ...ignoredSourceScriptPatterns]);
// Project configuration.
grunt.initConfig({
pkg: grunt.file.readJSON('package.json'),
copy: {
build: {
cwd: 'presentation/',
src: 'ui/**',
dest: path.join('presentation', 'static'),
expand: true
}
},
uglify: {
options: {
sourceMap: true,
compress: false
},
build: {
cwd: baseUIPath,
files: function() {
var modules = grunt.file.expand({
filter: 'isDirectory',
expand: true,
cwd: 'presentation/ui/modules'
}, ['**', '!**/css', '!**/scripts', ...ignoredSourceScriptPatterns]),
components = grunt.file.expand({
filter: 'isFile',
expand: true,
cwd: 'presentation/ui'
}, ['components/!*.js', ...ignoredSourceScriptPatterns]),
files, componentFiles;
files = modules.map(function(path) {
var modulePath = `modules/${path}/scripts`,
moduleName = modulePath.split('/').reduce(function(result, next) {
var nameFragment;
switch(next) {
case "modules":
nameFragment = "Poptart.";
break;
case "scripts":
nameFragment = "min.js";
break;
default:
nameFragment = `${next.charAt(0).toUpperCase() + next.slice(1)}.`;
break;
}
return result + nameFragment;
}, "");
return {
src: grunt.file.expand({
filter: 'isFile'
}, [`${baseUIPath}/${modulePath}/*.js`, ...ignoredSourceScriptPatterns]).sort(
function(a, b) {
return a.length - b.length;
}
),
dest: `${baseUIPath}/${modulePath}/${moduleName}`
};
});
componentFiles = components.map(function(componentPath) {
return {
src: `${baseUIPath}/${componentPath}`,
dest: `${baseUIPath}/${componentPath.replace('.js', '.min.js')}`
};
});
files = [...files, ...componentFiles];
files.push({
src: grunt.file.expand({
filter: 'isFile'
}, [`${baseUIPath}/*.js`, ...ignoredSourceScriptPatterns]),
dest: `${baseUIPath}/poptart.min.js`
});
return files;
}(),
extDot: 'last',
expand: true
}
},
cssmin: {
options: {
sourceMap: true
},
build: {
cwd: 'presentation/static/ui/',
src: ['**/*.css', '!css/jquery-ui/**', '!css/ionicons/**', '!**/*.min.js'],
dest: 'presentation/static/ui/',
ext: '.min.css',
expand: true
}
},
eslint: {
options: {
configFile: 'presentation/build/eslint.json',
ignorePath: 'presentation/build/.eslintignore'
},
target: ['presentation/**/*.js', '!presentation/ui/scripts/*', '!presentation/static/**']
},
shell: {
test: {
command: 'python manage.py test -p "*tests.py"',
options: {
stdout: true,
failOnError: true
}
}
},
karma: {
unit: {
configFile: 'karma.conf.js',
singleRun: true
}
},
mochaTest: {
unit: {
options: {
reporter: 'spec'
},
src: ['presentation/test/server/*.js']
}
}
});
/*grunt.loadNpmTasks('grunt-shell');
grunt.loadNpmTasks('grunt-eslint');
grunt.loadNpmTasks('grunt-karma');
grunt.loadNpmTasks('grunt-mocha-test');
grunt.loadNpmTasks('grunt-contrib-uglify');
grunt.loadNpmTasks('grunt-contrib-cssmin');
grunt.loadNpmTasks('grunt-contrib-concat');
grunt.loadNpmTasks('grunt-newer');*/
grunt.registerTask('default', ['lint', 'test']);
grunt.registerTask('test', ['shell:test', 'mochaTest:unit', 'karma:unit']);
grunt.registerTask('lint', ['eslint']);
grunt.registerTask('build-static', ['uglify', 'copy', 'cssmin']);
};
And my dev dependencies from package.json:
"devDependencies": {
"chai": "^3.5.0",
"chai-jquery": "^2.0.0",
"eslint": "^3.7.1",
"grunt": "^1.0.1",
"grunt-contrib-copy": "^1.0.0",
"grunt-contrib-cssmin": "^1.0.2",
"grunt-contrib-uglify": "^2.0.0",
"grunt-eslint": "^19.0.0",
"grunt-karma": "^2.0.0",
"grunt-mocha-test": "^0.13.2",
"grunt-newer": "^1.2.0",
"grunt-shell": "^1.3.1",
"jit-grunt": "^0.10.0",
"jquery": "^3.1.1",
"karma": "^1.3.0",
"karma-chai": "^0.1.0",
"karma-chrome-launcher": "^2.0.0",
"karma-cli": "^1.0.1",
"karma-jquery-chai": "^0.1.3",
"karma-mocha": "^1.3.0",
"karma-phantomjs-launcher": "^1.0.2",
"karma-sinon": "^1.0.5",
"mocha": "^3.2.0",
"sinon": "^1.17.6",
"sinon-chai": "^2.8.0",
"time-grunt": "^1.4.0"
}
Here's what I've tried so far:
Using time-grunt to verify that loading tasks was actually the problem. It was. actually running tasks took very reasonable amounts of time.
using jit-grunt. This made no noticeable difference.
npm prune. This actually did make a fairly big (~30 seconds) difference, but still leaves me at a completely unreasonable time for task loading.
Removing and unloading some of the dependencies (uglify, copy, cssmin) I had added around the time that I started seeing this problem (originally, when I just had the testing/linting tasks, everything worked nice and fast). This also made no noticeable difference.
So. What else could be causing slowness on this scale?
Thanks for your help!

So it looks like the root cause of this was some inefficient file searching in the uglify task. I'd completely forgotten that I had made that giant blob of crazy and IEF, which explains why I was seeing the slowness even when I wasn't running the uglify task. I suppose it also explains why the time for the file search was getting lumped in with the loading tasks.
In any event, the main culprit was (predictably) where I was returning the source/dest paths for the dynamically found script modules, here:
return {
src: grunt.file.expand({
filter: 'isFile'
}, [`${baseUIPath}/${modulePath}/*.js`, ...ignoredSourceScriptPatterns]).sort(
function(a, b) {
return a.length - b.length;
}
),
dest: `${baseUIPath}/${modulePath}/${moduleName}`
};
Unspurprising because it is in a loop. In any event, I noticed it was missing a cwd. Adding that cut the time down to 7s. Still needs improving, but at least now I know what to improve.
Morals of this story:
add cwd's when search for files
Maybe just don't search for files if you don't have to

I tried running your Gruntfile and installed the dependencies manually, and the loading time of the Gruntfile was ~4 secs. Not fast, but not the minutes you talk about.
My guess is grunt.file.expand is working hard. Maybe presentation/ui folder is too big? What do you get by running tree presentation/ui | wc -l ? That shows the number of files you have there.
Otherwise, it would be helpful to also see your package.json and npm ls output.

Related

Karma not running tests that have "import" statements in karma-webpack

I have some test files with tests I'd like to run against my app.
I am attempting to use karma, karma-webpack, karma-babel-preprocessor, karma-chrome-launcher, and jasmine in my testing. My app depends on many things, including backbone, marionette, etc. My app is built using webpack, and I am attempting to use webpack to bundle my files together for testing. (I initially wanted to see if I could skip this step, i.e. simply import a file to be tested, but it seems this is impossible.)
My test script looks like
package.json (scripts section)
"test": "./node_modules/karma/bin/karma start",
The rest of the files:
karma.conf.js
var webpackConfig = require('./config/webpack/webpack.test.conf.js');
module.exports = function(config) {
config.set({
basePath: '',
frameworks: ['jasmine'],
files: [
{ pattern: 'test/**/*.spec.js', watched: true },
{ pattern: 'test/*.spec.js', watched: true }
],
exclude: [
],
preprocessors: {
'test/**/*.spec.js': ['webpack'],
'test/*.spec.js': ['webpack']
},
webpack: webpackConfig,
webpackMiddleware: {
stats: 'errors-only'
},
reporters: ['progress'],
port: 9876,
colors: true,
logLevel: config.LOG_INFO,
autoWatch: true,
browsers: ['Chrome'],
singleRun: false,
concurrency: Infinity
})
}
test/test.spec.js This file is seen
describe("A suite", function () {
it("contains spec with an expectation", function () {
expect(true).toBe(true);
});
});
describe("Another suite", function () {
it("contains another spec with an expectation", function () {
expect(true).toBe(false);
});
});
test/models/devicegroup.spec.js This file is not seen
import backbone from 'backbone';
describe("backbone", function () {
it("containsasdfasdfasdfasdfspec with an expectation", function ()
{
expect(true).toBe(false);
});
});
My folder structure is:
- karma.conf.js
- test/
- - test.spec.js
- - models/
- - - devicegroup.spec.js
- public/
- - js/
- - - app.js
When my files don't have import statements at the top, karma will run and pass/fail as expected. Putting an import statement at the top will cause karma to ignore the file. No errors are thrown.
How can I make karma / karma-webpack run my tests that have import statements / what is the karma-safe way to import modules into my tests?
When test/models/devicegroup.spec.js does not have an import statement:
// import backbone from 'backbone';
describe("backbone", function () {
it("contains with an expectation", function () {
expect(true).toBe(false);
});
});
the terminal output is: (notice one less test is run)
When test/models/devicegroup.spec.js does have an import statement:
import backbone from 'backbone';
describe("backbone", function () {
it("contains with an expectation", function () {
expect(true).toBe(false);
});
});
the terminal output is:
I see no errors in the browser Karma opens.
EDIT:
I have experimented by adding my source files to the files and preprocessors attributes in my karma.conf.js file, as per this repo example. There was no change in behavior other than a massively increased testing time.
karma.conf.js
files: [
{ pattern: 'public/js/**/*.js', watched: true},
{ pattern: 'test/**/*.spec.js', watched: true },
// each file acts as entry point for the webpack configuration
],
preprocessors: {
// add webpack as preprocessor
'public/js/**/*.js': ['webpack'],
'test/**/*.spec.js': ['webpack'],
},
EDIT2:
For the sake of experimentation (and based off this person's struggles), I have tried the above karma.conf.js in every possible combination - only test files in files and preprocessors, only source files, test files in one but not the other, source files in one but not the other, none, both. No good results, though occasionally new errors.
Little late, but I ran into the same problem, and was searching for hours, why my imports prevent the test suite from being executed. karma-webpack-4.0.0-rc.2 brought the enlightenment by providing error messages!!
I my case a couple of modules where not found, angular-mock, jquery, angular and more.
How to fix
Put there modules into the files array in your karma.config like:
files = [
"node_modules/jquery/dist/jquery.js",
"node_modules/angular/angular.js",
"node_modules/angular-mocks/angular-mocks.js",
{ pattern: "test/**/*.ts", watched: false }
I hope, this helps someone.
EDIT
My current versions of the testing related packages:
"#types/jasmine": "^2.8.8",
"jasmine": "^3.2.0",
"jasmine-core": "^3.2.1",
"jasmine-reporters": "2.3.2",
"jasmine-ts": "^0.2.1",
"karma": "3.0.0",
"karma-chrome-launcher": "2.2.0",
"karma-jasmine": "1.1.2",
"karma-junit-reporter": "1.2.0",
"karma-phantomjs-launcher": "1.0.4",
"karma-sourcemap-loader": "^0.3.7",
"karma-spec-reporter": "0.0.32",
"karma-webpack": "^4.0.0-rc.2",
"typescript": "3.0.3",
"webpack": "4.17.2",
"webpack-cli": "^3.1.0",
"webpack-dev-server": "3.1.8"

Grunt not defined

I am new to Grunt and have been struggling all day to make this work:
This is my Gulpfile.js:
'use strict';
// Time how long tasks take. Can help when optimizing build times
require('time-grunt')(grunt);
// Automatically load required Grunt tasks
require('jit-grunt')(grunt);
module.exports = function(grunt) {
// Define the configuration for all the tasks
grunt.initConfig({
pkg: grunt.file.readJSON('package.json'),
// Make sure code styles are up to par and there are no obvious mistakes
jshint: {
options: {
jshintrc: '.jshintrc',
reporter: require('jshint-stylish')
},
all: {
src: [
'Gruntfile.js',
'app/scripts/{,*/}*.js'
]
}
}
});
grunt.registerTask('build', [
'jshint'
]);
grunt.registerTask('default', ['build']);
};
This my package.json:
{
"name": "conFusion",
"private": true,
"devDependencies": {
"grunt": "^1.0.1",
"grunt-contrib-jshint": "^1.0.0",
"jit-grunt": "^0.10.0",
"jshint-stylish": "^2.2.1",
"time-grunt": "^1.4.0"
},
"engines": {
"node": ">=0.10.0"
}
}
And this is the error message i get:
grunt build
Loading "Gruntfile.js"
tasks...ERROR >>
ReferenceError: grunt is not defined
Warning: Task "build" not found.Use--force to continue.
Aborted due to warnings.
Pls guys, I need help. I am working on windows 10, so am using the command line there.
module.exports = function(grunt) {
That is where you define a grunt variable. It is an argument to the function you created.
But you try to use it here:
require('time-grunt')(grunt);
and here:
require('jit-grunt')(grunt);
This is outside the function where the variable does not exist.
Move those lines inside the function.

Failing to run tests using Karma

I am setting up Karma to simplify unit tests in a legacy project. The problem is, I am getting an error You need to include some adapter that implements __karma__.start method!. As I have found, this is a very non-specific error, so I am at a loss of how to debug it further. I have reduced my setup to the bare minimum, but the error still persists.
Here is the karma config file:
module.exports = function (config) {
config.set({
browsers: [ 'PhantomJS' ], //run in Phantom
autoWatch: false,
singleRun: true, //just run once by default
frameworks: [ 'mocha', 'chai', 'sinon', 'chai-sinon' ], // test frameworks
port: 9876,
colors: true,
logLevel: config.LOG_INFO,
files: [
'./js/test/tests.webpack.js' //just load this file
],
preprocessors: {
'./js/test/tests.webpack.js': [ 'webpack', 'sourcemap' ] //preprocess with webpack and our sourcemap loader
},
reporters: [ 'mocha' ], //report results in this format
webpack: { // webpack settings
devtool: 'inline-source-map',
module: {
loaders: [
{ test: /\.js$/, loader: 'babel-loader' }
]
}
},
webpackServer: {
noInfo: true
},
plugins: [
'karma-mocha',
'karma-webpack',
'karma-sourcemap-loader',
'karma-mocha-reporter',
'karma-phantomjs-launcher',
'karma-chai',
'karma-sinon',
'karma-chai-sinon'
]
});
};
Here are the packages I have installed (from package.json):
"babel-core": "^6.9.1",
"babel-loader": "^6.2.4",
"babel-plugin-module-alias": "^1.5.0",
"babel-preset-es2015": "^6.9.0",
"chai": "^3.5.0",
"karma": "^1.0.0",
"karma-chai": "^0.1.0",
"karma-chai-sinon": "^0.1.5",
"karma-chrome-launcher": "^1.0.1",
"karma-mocha": "^1.0.1",
"karma-mocha-reporter": "^2.0.4",
"karma-phantomjs-launcher": "^1.0.1",
"karma-sinon": "^1.0.5",
"karma-sourcemap-loader": "^0.3.7",
"karma-webpack": "^1.7.0",
"mocha": "^2.5.3",
"phantomjs": "^2.1.7",
"phantomjs-polyfill": "0.0.2",
"sinon": "^1.17.4",
"sinon-chai": "^2.8.0",
"webpack": "^1.13.1"
Here is .babelrc:
{
"presets": ["es2015"]
}
And here is the test file that I am referencing in karma.conf.js (tests.webpack.js). My initial idea was to require all spec files in it, but now I have changed it to run at least a single test. Still no luck:
import chai from 'chai';
var expect = chai.expect;
console.log('I was found');
it('works', function () {
expect(true).to.equal(true);
})
Could you please advise how to debug this problem further?
UPDATE: Here's a gist with a minimal reproducible case. I must be doing something wrong there, because I am still receiving the error You need to include some adapter that implements __karma__.start method!, but I can't figure out what my mistake is.
Try removing the reference to chai-sinon in karma config. When I get rid of it, the test runs. I can't exactly explain why, but perhaps there is version incompatibility going on.

Grunt watch: livereload reloads 1 step behind…

here's my gruntfile:
module.exports = function(grunt) {
// Project configuration.
grunt.initConfig({
hologram: {
generate: {
options: {
config: 'config.yml'
}
}
},
libsass: {
files: {
src: 'src/scss/style.scss',
dest: 'templates/static/css/style.css'
}
},
connect: {
server: {
options: {
port: 8000,
hostname: 'localhost',
base: 'docs/',
livereload: 35729
}
}
},
watch: {
scss: {
files: ['src/scss/**/*.scss', 'templates/static/css/*.css'],
tasks: ['libsass','hologram'],
options: {
livereload: true
}
}
}
});
// Load plugins.
grunt.loadNpmTasks('grunt-libsass');
grunt.loadNpmTasks('grunt-hologram');
grunt.loadNpmTasks('grunt-contrib-connect');
grunt.loadNpmTasks('grunt-contrib-watch');
// Default task(s).
grunt.registerTask('default', ['connect','libsass','hologram','watch']);
};
And here's my package file:
{
"name": "...",
"version": "1.0.0",
"description": "...",
"dependencies": {
"grunt": "^0.4.5"
},
"devDependencies": {
"connect-livereload": "^0.5.2",
"grunt": "^0.4.5",
"grunt-contrib-connect": "^0.9.0",
"grunt-contrib-watch": "^0.6.1",
"grunt-hologram": "0.0.4",
"grunt-libsass": "^0.2.0"
},
"repository": {
"type": "git",
"url": "..."
},
"author": "Yann Bettremieux",
"homepage": "..."
}
Everything seems to be working fine. When I go to http://localhost:8000/ I see my site and when I save my watched files the page reloads etc. But it doesn't actually reloads the previous changes. Meaning, the first time I edit a SCSS file to say color: blue, I see in the inspector that some CSS livereload files are loaded but there is no change on the page. If I change the CSS to color: red the page reload but show everything in blue… If I change it to green, it reloads and shows me the everything in red, etc. etc.
I tried tu use the chrome livereload extension insrtead but it didn't change anything.
I tried grunt-sass instead of libsass. Same behavior.
Not sure what else to try to resolve this issue. Any pointer in thr right direction much appreciated!
Livereload Readme already addresses the issue. See live-reload-with-preprocessors:
Any time a watched file is edited with the livereload option enabled,
the file will be sent to the live reload server. Some edited files you
may desire to have sent to the live reload server, such as when
preprocessing (sass, less, coffeescript, etc). As any file not
recognized will reload the entire page as opposed to just the css or
javascript.
The solution is to point a livereload watch target to your destination files.
You should enable livereload only for css files.
watch: {
scss: {
files: ['src/scss/**/*.scss'],
tasks: ['libsass','hologram']
},
css: {
files: ['templates/static/css/*.css'],
options: {
livereload: true
}
}
}

Grunt and Bourbon - Cannot Find Module 'Bourbon'

I'm trying to use GruntJS to make some improvements to my workflow - I've been using Compass for a while but I'm wanting to try out Bourbon and so I've been trying to get this working but failing.
I'm getting the following error when I run 'grunt':
ERROR: Cannot find module 'bourbon'
I've installed this through Node using 'npm install' with the following 'package.json' file:
{
"name" : "project",
"description": "description",
"version" : "0.0.1",
"dependencies" : {
"node-sass": "~0.8.4",
"node-bourbon": "~1.0.0",
"grunt": "~0.4.4",
"grunt-contrib-watch": "~0.6.1",
"grunt-sass": "~0.12.0",
"grunt-contrib-uglify": "~0.4.0",
"matchdep": "~0.3.0"
}
}
My grunt file looks like this:
module.exports = function(grunt) {
grunt.loadNpmTasks('grunt-sass');
grunt.loadNpmTasks('grunt-contrib-watch');
grunt.loadNpmTasks('grunt-contrib-uglify');
grunt.initConfig({
uglify: {
my_target: {
files: {
'assets/js/main.js': ['_/js/scripts.js']
} // files
} // my_target
}, // uglify
sass: {
dist: {
options: {
includePaths: require('bourbon').includePaths,
outputStyle: 'compressed'
},
files: {
'assets/css/main.css': '_/stylesheets/**/*.scss'
}
}
},
watch: {
options: { livereload: true },
grunt: { files: ['gruntfile.js'] },
scripts: {
files: ['_/js/scripts.js'],
tasks: ['uglify']
}, //script
sass: {
files: ['_/stylesheets/**/*.scss'],
tasks: ['sass']
}, //sass
php: {
files: ['**/*.php']
}
} //watch
}) //initConfig
grunt.registerTask('default', 'watch');
} //exports
I've also imported the file in the top of my SCSS stylesheets using:
#import 'bourbon';
Not sure what I'm doing wrong here, any help would be greatly appreciated!
Please let me know if you need any more info.
You are requiring the wrong package name. It should be:
includePaths: require('node-bourbon').includePaths
but you have:
includePaths: require('bourbon').includePaths
Install Bourbon manually:
Install the gem:
$ gem install bourbon
Install Bourbon into your project's stylesheets directory by generating the bourbon folder:
$ bourbon install

Categories