We Keep Coding

Where does this "_.startsWith" come from?

I am trying to understand the following code from the browserify-css repo:
var gulp = require('gulp');
var gutil = require('gulp-util');
var path = require('path');
var browserify = require('browserify');
var sourceStream = require('vinyl-source-stream');
var fse = require('fs-extra');
var bundleStream = browserify()
.add('src/index.js')
.transform(require('browserify-css'), {
rootDir: 'src',
processRelativeUrl: function(relativeUrl) {
var stripQueryStringAndHashFromPath = function(url) {
return url.split('?')[0].split('#')[0];
};
var rootDir = path.resolve(process.cwd(), 'src');
var relativePath = stripQueryStringAndHashFromPath(relativeUrl);
var queryStringAndHash = relativeUrl.substring(relativePath.length);
//
// Copying files from '../node_modules/bootstrap/' to 'dist/vendor/bootstrap/'
//
var prefix = '../node_modules/';
if (_.startsWith(relativePath, prefix)) {
var vendorPath = 'vendor/' + relativePath.substring(prefix.length);
var source = path.join(rootDir, relativePath);
var target = path.join(rootDir, vendorPath);
gutil.log('Copying file from ' + JSON.stringify(source) + ' to ' + JSON.stringify(target));
fse.copySync(source, target);
// Returns a new path string with original query string and hash fragments
return vendorPath + queryStringAndHash;
}
return relativeUrl;
}
})
.bundle();
bundleStream
.pipe(sourceStream(bundleFile))
.pipe(gulp.dest(browserifyConfig.dest));
I don't understand the part
_.startsWith(relativePath, prefix)
Where is the underscore coming from? It's supposed to be javascript executed by a task runner. I've found that in the NodeJS REPL the underscore character outputs the result of the last executed expression, but that functionality can't be used inside scripts. It's also not an underscore.js instance because it is not being declared anywhere. startsWith is a String method.
So what am I missing?

That code is using the lodash library. You can see in this section of the readme that they're importing lodash with var _ = require('lodash');

Pass data between gulp 4 tasks, pipes or commandline

I'm trying to do in my opinion a simple task but can't figure it out. Can't find it on google or here. Maybe i'm doing it completely wrong.
I need to change a property in a json file with gulp 4. To get the new value of the property I have another task or pipe (I don't care :-)) and want to pass it along.
Also, how to pass variable from command line? so in example: gulp release -branchname "foobar"
How can I do this. I know there are similar question here but nothing is working for me...
var gulp = require('gulp');
var git = require('gulp-git');
var jeditor = require("gulp-json-editor");
var paths = {
manifest: "vss-extension.json"
}
function getBranchName(){
console.log("Get the current branch name");
return git.revParse({args:'--abbrev-ref HEAD'}, function (err, branch) {
console.log('current git branch: ' + branch);
var currentBranch = branch;
return currentBranch;
});
// console.log("Current branch is: "+ branchName);
// return branchName;
}
function updateExtensionManifestVersion(foo){
var x = 10;
gulp.src(paths.manifest)
.pipe(jeditor(function(json) {
console.log("Append current branchname " + currentBranch + " to extension manifest id");
var currentId = json.id;
console.log("Current manifest id: "+ currentId);
// var branchName = git.revParse({args:'--abbrev-ref HEAD'}, function (err, branch) {
// console.log('current git branch: ' + branch);
// return branch;
// });
// console.log(branchName.branch);
var newId = currentId + "-" + branchName;
console.log("New manifest id: " + newId);
json.id = newId;
return json;
}))
.pipe(gulp.dest(paths.manifest));
}
gulp.task('release', gulp.series(getBranchName, updateExtensionManifestVersion, function(done) {
console.log("release is done");
done();
}));

To pass variables from command line you can use yargs as explained in this answer Pass Parameter to Gulp Task.
If you want to pass data between tasks set a global variable that will be accessible to both.

gulp-concat : from unknown source to the same destination

I recently started to use gulp to keep my dev project organized and I've run into a little something that I cant figure out. So this is my task :
gulp.task('jsassemble', function () {
return gulp
.src('vendor/proj/**/**/src/assets/js/*.js')
.pipe(concat('all.js'))
.pipe(gulp.dest('public/js'));
});
As you can see, it'll fetch every js file in vendor/proj/anyFolder/anySubFolder/src/assets/js, put them together, rename the newly created js 'all.js' and then put it in public/js. The problem is that I would like to have gulp to keep the folder hierarchy, for example :
Source = vendor/proj/anyFolder1/anySubFolder1/src/assets/js/*.js
Destination = public/js/anyFolder1/anySubFolder1/src/assets/js/all.js
Source = vendor/proj/anyFolder1/anySubFolder2/src/assets/js/*.js
Destination = public/js/anyFolder1/anySubFolder2/src/assets/js/all.js
Instead of simply having everything on those folder into a 1 and only public/js/all.js
Is there anyway to do it ? I've tried to google it first but I wasn't able to properly formulate my question in a few words and was given not-wanted results :/

You could create the function which keep your folder hierarchy. In this page (http://www.jamescrowley.co.uk/2014/02/17/using-gulp-packaging-files-by-folder/) you find the solution.
var fs = require('fs');
var path = require('path');
var es = require('event-stream');
var gulp = require('gulp');
var concat = require('gulp-concat');
var rename = require('gulp-rename');
var uglify = require('gulp-uglify');
var scriptsPath = './src/scripts/';
function getFolders(dir){
return fs.readdirSync(dir)
.filter(function(file){
return fs.statSync(path.join(dir, file)).isDirectory();
});
}
gulp.task('scripts', function() {
var folders = getFolders(scriptsPath);
var tasks = folders.map(function(folder) {
return gulp.src(path.join(scriptsPath, folder, '/*.js'))
.pipe(concat(folder + '.js'))
.pipe(gulp.dest(scriptsPath))
.pipe(uglify())
.pipe(rename(folder + '.min.js'))
.pipe(gulp.dest(scriptsPath));
});
return es.concat.apply(null, tasks);
});

Thanks to #caballerog who puts my on the right path, here's the explained code :
//get every folder from a 'pathTo/Something'
function getFolders(dir){
return fs.readdirSync(dir)
return fs.statSync(path.join(dir, file)).isDirectory();
}
var projectsRoot = 'vendor/proj/';
var pathToJsFiles = '/src/assets/js/';
var pathToPublic = 'public/js/';
gulp.task('scripts', function() {
var sites = [];
var pathToProjects = [];
// Fetching every folders in vendor/proj
projects = getFolders(projectsRoot);
// Fetching every subfolder in vendor/proj/something
for(index in projects){
sites.push(getFolders(projectsRoot + '/' + projects[index]));
}
// Pushing every projects/site that exists into an array
for(var i=0;i<projects.length;i++){
for(var j=0; j<sites.length; j++)
if(sites[i][j] != null)
pathToProjects.push(projects[i] + '/' + sites[i][j]);
}
// Fetching every JS on vendor/proj/pathToAProject/pathToJsFiles
// concatenate them together
// and sending them to pathToPublic/pathToAProject/all.js
var tasks = pathToProjects.map(function(pathToAProject) {
return gulp.src( projectsRoot + pathToAProject + pathToJsFiles + '/*.js')
.pipe(concat('all.js'))
.pipe(gulp.dest(pathToPublic + pathToAProject));
});
return es.concat.apply(null, tasks);
});
TL:DR = get every JS file in pathToPublic/someFolder/someFolder/PathToJS.

node.js display virtual hosts and current git branch async issue

Ok, i am just starting to learn node.js and i am having a little difficulty getting a good grasp on the async nature of it and when/how to use callbacks to get data passed along like i need it.
The concept of what i am trying to build is this. I have a small node.js app that uses the FS package - defined as var fs = require("fs"). The app is responding to localhost:4000 right now. When i hit that url, the app will use fs.readdir() to get all of the virtual host files in the directory that i pass to readdir().
Next, the app loops through those files and parses each one line by line and word by word (quick and dirty for now). I am using fs.readFile() to read the file and then literally doing lines = data.toString().split("\n") and then var words = lines[l].split(" ") to get to the data in the file i need. Each virtual host file looks something like this:
<VirtualHost *:80>
ServerName some-site.local
DocumentRoot "/path/to/the/docroot"
<Directory "/path/to/the/docroot">
Options Indexes FollowSymLinks Includes ExecCGI
AllowOverride All
Require all granted
</Directory>
ErrorLog "logs/some-site.local-error_log"
</VirtualHost>
Here is my main.js file (routes file):
var express = require("express"),
router = express.Router(),
async = require("async"),
directoryReader = require(__dirname + "/../lib/directoryReader");
fileReader = require(__dirname + "/../lib/fileReader");
router.get("/", function(req, res) {
var response = [];
directoryReader.read(function(files) {
async.each(files, function(file, callback) {
fileReader.read(file, function(data) {
if (data.length > 0) {
response.push(data);
}
callback();
});
}, function(err){
if (err) throw err;
res.json(response);
});
});
});
module.exports = router;
My directoryReader.js file:
var fs = require("fs"),
fileReader = require(__dirname + "/../lib/fileReader");
var directoryReader = {
read: function(callback) {
var self = this;
fs.readdir("/etc/apache2/sites-available", function (err, files) {
if (err) throw err;
var output = [];
for(f in files) {
output.push(files[f]);
}
callback(output);
});
}
};
module.exports = directoryReader;
And my fileReader.js file:
var fs = require("fs");
var fileReader = {
read: function(file, callback) {
fs.readFile("/etc/apache2/sites-available/" + file, { encoding: "utf8" }, function (err, data) {
if (err) throw err;
var vHostStats = ["servername", "documentroot", "errorlog"],
lines = data.toString().split("\n"),
output = [];
for(l in lines) {
var words = lines[l].split(" ");
for(w in words) {
if (words[w].toLowerCase() == "<virtualhost") {
var site = {
"servername" : "",
"documentroot" : "",
"errorlog" : "",
"gitbranch" : ""
}
w++;
}
if (vHostStats.indexOf(words[w].toLowerCase()) !== -1) {
var key = words[w].toLowerCase();
w++;
site[key] = words[w];
}
if (words[w].toLowerCase() == "</virtualhost>" && site.documentroot != "") {
w++;
output.push(site);
var cmd = "cd " + site["documentroot"] + " && git rev-parse --abbrev-ref HEAD";
var branch = ...; // get Git branch based on the above command
site["gitbranch"] = branch;
}
}
}
callback(output);
});
}
};
module.exports = fileReader;
All of this code will spit out json. This all works fine, expect for one part. The line in the fileReader.js file:
var branch = ...; // get Git branch based on the above command
I am trying to get this code to run a shell command and get the Git branch based on the document root directory. I then want to take the branch returned and add the value to the gitbranch proptery of the current site object during the loop. Hope this makes sense. I know there are probably questions on SO that cover something similar to this and i have looked at many of them. I fear i am just not educated enough in node.js yet to apply the answers to those SO questions to my particular use case.
Please let me know if there's anything i can add that can help anyoe answer this question. I should note that this app is for personal uses only, so the solution really just has to work, not be super elegant.
UPDATE: (5/1/2015)
Probably not the best solution but i got what i wanted by using the new execSync added to v0.12.x
if (words[w].toLowerCase() == "</virtualhost>" && site.documentroot != "") {
var cmd = "cd " + site["documentroot"] + " && git rev-parse --abbrev-ref HEAD";
var branch = sh(cmd, { encoding: "utf8" });
site["gitbranch"] = branch.toString().trim();
w++;
output.push(site);
}

Node.js script fails due to proxy problems

I am trying to install protractor in my Mac. The installation is done via command line but one of the scripts is failing due to connectivity problems. The main problem is that I am behind a corporate proxy server.
I set the proxy server in my console and npm is also configured with the correct proxy settings.
The script that fails is here
and it contains the following
#!/usr/bin/env node
var fs = require('fs');
var os = require('os');
var url = require('url');
var http = require('http');
var AdmZip = require('adm-zip')
// Download the Selenium Standalone jar and the ChromeDriver binary to
// ./selenium/
// Thanks to http://www.hacksparrow.com/using-node-js-to-download-files.html
// for the outline of this code.
var SELENIUM_URL =
'http://selenium.googlecode.com/files/selenium-server-standalone-2.35.0.jar';
var CHROMEDRIVER_URL_MAC =
'https://chromedriver.googlecode.com/files/chromedriver_mac32_2.2.zip';
var CHROMEDRIVER_URL_LINUX32 =
'https://chromedriver.googlecode.com/files/chromedriver_linux32_2.2.zip';
var CHROMEDRIVER_URL_LINUX64 =
'https://chromedriver.googlecode.com/files/chromedriver_linux64_2.2.zip';
var CHROMEDRIVER_URL_WINDOWS =
'https://chromedriver.googlecode.com/files/chromedriver_win32_2.2.zip';
var DOWNLOAD_DIR = './selenium/';
var START_SCRIPT_FILENAME = DOWNLOAD_DIR + 'start';
var chromedriver_url = '';
var start_script = 'java -jar selenium/selenium-server-standalone-2.35.0.jar';
if (!fs.existsSync(DOWNLOAD_DIR) || !fs.statSync(DOWNLOAD_DIR).isDirectory()) {
fs.mkdirSync(DOWNLOAD_DIR);
}
console.log(
'When finished, start the Selenium Standalone Server with ./selenium/start \n');
// Function to download file using HTTP.get
var download_file_httpget = function(file_url, callback) {
console.log('downloading ' + file_url + '...');
var options = {
host: url.parse(file_url).host,
port: 80,
path: url.parse(file_url).pathname
};
var file_name = url.parse(file_url).pathname.split('/').pop();
var file_path = DOWNLOAD_DIR + file_name;
var file = fs.createWriteStream(file_path);
http.get(options, function(res) {
res.on('data', function(data) {
file.write(data);
}).on('end', function() {
file.end(function() {
console.log(file_name + ' downloaded to ' + file_path);
if (callback) {
callback(file_path);
}
});
});
});
};
download_file_httpget(SELENIUM_URL);
if (!(process.argv[2] == '--nocd')) {
if (os.type() == 'Darwin') {
chromedriver_url = CHROMEDRIVER_URL_MAC;
} else if (os.type() == 'Linux') {
if (os.arch() == 'x64') {
chromedriver_url = CHROMEDRIVER_URL_LINUX64;
} else {
chromedriver_url = CHROMEDRIVER_URL_LINUX32;
}
} else if (os.type() == 'Windows_NT') {
chromedriver_url = CHROMEDRIVER_URL_WINDOWS;
}
var chromedriver_zip = chromedriver_url.split('/').pop();
start_script += ' -Dwebdriver.chrome.driver=./selenium/chromedriver';
download_file_httpget(chromedriver_url, function(file_name) {
var zip = new AdmZip(file_name);
zip.extractAllTo(DOWNLOAD_DIR);
if (os.type() != 'Windows_NT') {
fs.chmod(DOWNLOAD_DIR + 'chromedriver', 0755);
}
});
}
var start_script_file = fs.createWriteStream(START_SCRIPT_FILENAME);
start_script_file.write(start_script);
start_script_file.end(function() {
fs.chmod(START_SCRIPT_FILENAME, 0755);
});
How can we modify this script in order to resolve the connectivity issue?

In download_file_httpget, you're calling http.get(...). That makes a direct connection to the target server.
Configuring a proxy in npm only affects npm. Node's http module has no concept of proxy servers.
If you need to make a request through a proxy, consider using the request module, which does support proxies.