We Keep Coding

For loop that gets file names from specified folder and all subfolders - js

I have two js scripts that I would like to merge into one, but I do not know how.
Script one, uploads all files inside specified folder into virustotal, scans them, and returns the result of the scan.
Script two, lists all files inside the specified folder and all of its subfolders.
I would like to make a script that uploads all files inside specified folder and all of its subfolders into virustotal, scans them, and returns the result of the scan.
How would I go about doing that?
Script one:
/*jshint esversion: 8 */
const path = require('path');
const fsp = require('fs').promises;
const VirusTotalApi = require("virustotal-api");
const virusTotal = new VirusTotalApi('<YOUR API KEY>');
const basePath = '/home/username/Desktop/TEST/';
const wait = (time) => new Promise((resolve) => setTimeout(resolve, time));
async function scan() {
const files = await fsp.readdir(basePath);
let errors = [];
for (let file of files) {
const fullPath = path.join(basePath, file);
console.log(file);
try {
const data = await fsp.readFile(fullPath);
const response = await virusTotal.fileScan(data, file);
const resource = response.resource;
const result = await virusTotal.fileReport(resource);
const resultLine = `${file}: ${JSON.stringify(result, ["verbose_msg","total","positives"])}\n`;
await fsp.appendFile('Result.txt', resultLine);
console.log(`${file}: Saved!`);
} catch (e) {
// collect the error, log the error and continue the loop
e.fullPath = fullPath;
errors.push(e);
console.log(`Error processing ${fullPath}`, e);
continue;
}
// Wait for 30 seconds
await wait(30000);
}
// if there was an error, then reject with all the errors we got
if (errors.length) {
let e = new Error("Problems scanning files");
e.allErrors = errors;
throw e;
}
}
scan().then(() => {
console.log("all done scanning - no errors");
}).catch(err => {
console.log(err);
});
Script two:
const { promisify } = require('util');
const { resolve } = require('path');
const fs = require('fs');
const readdir = promisify(fs.readdir);
const stat = promisify(fs.stat);
async function getFiles(dir) {
const subdirs = await readdir(dir);
const files = await Promise.all(subdirs.map(async (subdir) => {
const res = resolve(dir, subdir);
return (await stat(res)).isDirectory() ? getFiles(res) : res;
}));
return files.reduce((a, f) => a.concat(f), []);
}
getFiles('/home/username/Desktop/TEST')
.then(files => console.log(files))
.catch(e => console.error(e));

You have quite a few options to get to a result here. The quick and dirty approach is to:
eliminate naming conflicts (make sure nothing is named the same between the two files
Copy over the consts and the function in file B into file A.
Copy the getFiles call in right after the scan().then... call
There are other cleaner approaches. But this should get you to a proof of concept that it is possible to have both scripts function together in a single script.

How can I force a one block of code with nested if statements to finish before subsequent code in node.js?

In the following block of code, I want all the statements below readdir to execute before the final statement, so that the final console log includes the entire database. I've tried running with readdir sync, and that doesn't seem to work either. Is it something to do with the nested if statements? The last line executes before any of the other code, and I'd like to circumvent that.
// Load node.js filesystem module
var fs = require('fs');
// Load cheerio module for HTML scraping
const cheerio = require('cheerio');
// Load find and replace module
const replace = require('replace-in-file');
// Make for loop that cycles through all files in folder
var stream = fs.createWriteStream("my_file.txt");
var database = {};
const CurrentFolder = './';
fs.readdir(CurrentFolder, (err, files) => {
files.forEach(file => {
// only modify files that contain 'page' and end in .html
if (file.includes('.html')) {
if (file.includes('page')) {
console.log(file + ' includes html');
fs.readFile(file, 'utf8', function (err, contents) {
if (err) console.log('ERROR: ' + err);
// only modify files that contain a '.page-title-lvl-cover' heading
if (contents.includes('page-title-lvl-cover')) {
var x = contents;
// console.log(x);
const $ = cheerio.load(contents)
// Extract page title (found within page-title-lvl-cover class)
const result = $('.page-title-lvl-cover').text()
database[file] = result;
console.log(database);
console.log(result)
}
});
}
}
});
});
console.log('Last ' + database)

Just make everything synchronize, including readdir and readFile.
// Load node.js filesystem module
var fs = require('fs');
// Load cheerio module for HTML scraping
const cheerio = require('cheerio');
// Load find and replace module
const replace = require('replace-in-file');
// Make for loop that cycles through all files in folder
var stream = fs.createWriteStream("my_file.txt");
var database = {};
const CurrentFolder = './';
const files = fs.readdirSync(CurrentFolder);
files.forEach(file => {
// only modify files that contain 'page' and end in .html
if (file.includes('.html')) {
if (file.includes('page')) {
console.log(file + ' includes html');
const contents = fs.readFileSync(file, 'utf8');
// only modify files that contain a '.page-title-lvl-cover' heading
if (contents.includes('page-title-lvl-cover')) {
var x = contents;
// console.log(x);
const $ = cheerio.load(contents)
// Extract page title (found within page-title-lvl-cover class)
const result = $('.page-title-lvl-cover').text()
database[file] = result;
console.log(database);
console.log(result)
}
}
}
});
console.log('Last ' + database)

Array doesnt have expected value - Nodejs

I am running a script which looks into a directory and lists files, then checks for the file type to process, if the extension matches then the file is read and each line of the file (.col which is just a txt file renamed) is inserted into an array.
Now after the file is read and the array is populated I would like to use the array and do some further processing, e.g create a db record. I am missing something really basic here because on each console log I do as below I always get the full items (in my array) of the contents of all files.
So to make it a bit simpler:
array is empty.
Then file is read and processed and array now has
array[0]=line 0 of file
array[0]=line 1 of file etc
const fs = require('fs');
const readline =require('readline');
var files = fs.readdirSync('/home/proj/data');
var path = require('path');
var model=[];
var lineReader=[];
for(var i=0; i<files.length; i++) {
if(path.extname(files[i]) === ".col") {
lineReader[i] = readline.createInterface({
input: require('fs').createReadStream(files[i])
});
lineReader[i].on('line', function (line) {
model.push(line);
}).on('close', async function() {
console.log(model);
});
}
}
Instead the script is run and array[] holds all lines of all files that match the extension.
Your help is greatly appreciated and anyone is allowed to scorch my JS as I am pretty sure I am missing something basic here.

So, you want to read the files in parallel (because that's what your program does) and put it in an array of arrays?
You can make the reading file mechanism a promise and use it using Promise.all. Here is an example to get you started.
const fs = require('fs');
const readline = require('readline');
var files = fs.readdirSync('./');
var path = require('path');
function readFile(fileName) {
return new Promise(resolve => {
const array = [];
const lineReader = readline.createInterface({
input: fs.createReadStream(files[i])
});
lineReader.on('line', function (line) {
array.push(line);
}).on('close', async function () {
//do some proc
console.log(array);
resolve(array);
});
});
}
const readFilePromises = [];
for (var i = 0; i < files.length; i++) {
if (path.extname(files[i]) === ".js") {
readFilePromises.push(readFile(files[i]));
}
}
Promise.all(readFilePromises) //or await Promise.all([..])
.then(data => {
console.log(data);//will be array of arrays
})
If you want a single Array you can always flatten the result using data.flat()

If your files are not very big and sync methods are OK, you can simplify the code this way:
'use strict';
const fs = require('fs');
const path = require('path');
const model = [];
fs.readdirSync('/home/proj/data')
.filter(name => path.extname(name) === '.col')
.forEach((name) => {
model.push(...fs.readFileSync(name, 'utf8').split('\n'));
});
console.log(model);

Javascript,Nodejs: search for a specific word string in files

i'm trying to make an app that searches for all files
contains a specified string under the current directory/subdirectory.
as i understand it means i need to create a read stream, loop it, load the read data to an array, if the word found give __filename, dirname and if ! not found message.
unfortunately, i could not make it work...
any clue?
var path = require('path'),
fs=require('fs');
function fromDir(startPath,filter,ext){
if (!fs.existsSync(startPath)){
console.log("no dir ",startPath);
return;
};
var files=fs.readdirSync(startPath);
let found = files.find((file) => {
let thisFilename = path.join(startPath, file);
let stat = fs.lstatSync(thisFilename);
var readStream = fs.createReadStream(fs);
var readline = require('readline');
if (stat.isDirectory()) {
fromDir(thisFilename, filename,readline, ext);
} else {
if (path.extname(createReadStream) === ext && path.basename(thisFilename, ext) === filename) {
return true;
}
}
});
console.log('-- your word has found on : ',filename,__dirname);
}
if (!found) {
console.log("Sorry, we didn't find your term");
}
}
fromDir('./', process.argv[3], process.argv[2]);

Because not everything was included in the question, I made an assumption:
We are looking for full words (if that's not the case, replace the regex with a simple indexOf()).
Now, I've split the code into two functions - to make it both more readable and easier to recursively find the files.
Synchronous version:
const path = require('path');
const fs = require('fs');
function searchFilesInDirectory(dir, filter, ext) {
if (!fs.existsSync(dir)) {
console.log(`Specified directory: ${dir} does not exist`);
return;
}
const files = getFilesInDirectory(dir, ext);
files.forEach(file => {
const fileContent = fs.readFileSync(file);
// We want full words, so we use full word boundary in regex.
const regex = new RegExp('\\b' + filter + '\\b');
if (regex.test(fileContent)) {
console.log(`Your word was found in file: ${file}`);
}
});
}
// Using recursion, we find every file with the desired extention, even if its deeply nested in subfolders.
function getFilesInDirectory(dir, ext) {
if (!fs.existsSync(dir)) {
console.log(`Specified directory: ${dir} does not exist`);
return;
}
let files = [];
fs.readdirSync(dir).forEach(file => {
const filePath = path.join(dir, file);
const stat = fs.lstatSync(filePath);
// If we hit a directory, apply our function to that dir. If we hit a file, add it to the array of files.
if (stat.isDirectory()) {
const nestedFiles = getFilesInDirectory(filePath, ext);
files = files.concat(nestedFiles);
} else {
if (path.extname(file) === ext) {
files.push(filePath);
}
}
});
return files;
}
Async version - because async is cool:
const path = require('path');
const fs = require('fs');
const util = require('util');
const fsReaddir = util.promisify(fs.readdir);
const fsReadFile = util.promisify(fs.readFile);
const fsLstat = util.promisify(fs.lstat);
async function searchFilesInDirectoryAsync(dir, filter, ext) {
const found = await getFilesInDirectoryAsync(dir, ext);
for (file of found) {
const fileContent = await fsReadFile(file);
// We want full words, so we use full word boundary in regex.
const regex = new RegExp('\\b' + filter + '\\b');
if (regex.test(fileContent)) {
console.log(`Your word was found in file: ${file}`);
}
};
}
// Using recursion, we find every file with the desired extention, even if its deeply nested in subfolders.
async function getFilesInDirectoryAsync(dir, ext) {
let files = [];
const filesFromDirectory = await fsReaddir(dir).catch(err => {
throw new Error(err.message);
});
for (let file of filesFromDirectory) {
const filePath = path.join(dir, file);
const stat = await fsLstat(filePath);
// If we hit a directory, apply our function to that dir. If we hit a file, add it to the array of files.
if (stat.isDirectory()) {
const nestedFiles = await getFilesInDirectoryAsync(filePath, ext);
files = files.concat(nestedFiles);
} else {
if (path.extname(file) === ext) {
files.push(filePath);
}
}
};
return files;
}
If you have not worked with / understand async/await yet, it is a great step to take and learn it as soon as possible. Trust me, you will love not seeing those ugly callbacks again!
UPDATE:
As you pointed in comments, you want it to execute the function after running node process on the file. You also want to pass the function parameters as node's arguments.
To do that, at the end of your file, you need to add:
searchFilesInDirectory(process.argv[2], process.argv[3], process.argv[4]);
This extracts our arguments and passes them to the function.
With that, you can call our process like so (example arguments):
node yourscriptname.js ./ james .txt
Personally, if I were to write this, I would leverage the beauty of asynchronous code, and Node.js's async / await.
As a very side note:
You can easily improve readability of your code, if you add proper formatting to it. Don't get me wrong, it's not terrible - but it can be improved:
Use spaces OR newlines after commas.
Use spaces around equality operators and arithmetic operators.
As long as you are consistent with formatting, everything looks much better.

What is the most efficient way to read only the first line of a file in Node JS?

Imagine you have many long text files, and you need to only extract data from the first line of each one (without reading any further content). What is the best way in Node JS to do it?
Thanks!

I ended up adopting this solution, which seems the most performant I've seen so far:
var fs = require('fs');
var Q = require('q');
function readFirstLine (path) {
return Q.promise(function (resolve, reject) {
var rs = fs.createReadStream(path, {encoding: 'utf8'});
var acc = '';
var pos = 0;
var index;
rs
.on('data', function (chunk) {
index = chunk.indexOf('\n');
acc += chunk;
index !== -1 ? rs.close() : pos += chunk.length;
})
.on('close', function () {
resolve(acc.slice(0, pos + index));
})
.on('error', function (err) {
reject(err);
})
});
}
I created a npm module for convenience, named "firstline".
Thanks to #dandavis for the suggestion to use String.prototype.slice()!

There's a built-in module almost for this case - readline. It avoids messing with chunks and so forth. The code would look like the following:
const fs = require('fs');
const readline = require('readline');
async function getFirstLine(pathToFile) {
const readable = fs.createReadStream(pathToFile);
const reader = readline.createInterface({ input: readable });
const line = await new Promise((resolve) => {
reader.on('line', (line) => {
reader.close();
resolve(line);
});
});
readable.close();
return line;
}

I know this doesn't exactly answer the question but for those who are looking for a READABLE and simple way to do so:
const fs = require('fs').promises;
async function getFirstLine(filePath) {
const fileContent = await fs.readFile(filePath, 'utf-8');
return (fileContent.match(/(^.*)/) || [])[1] || '';
}
NOTE:
naturaly, this will only work with text files, which I assumed you used from your description
this will work with empty files and will return an empty string
this regexp is very performant since it is simple (no OR conditions`or complex matches) and only reads the first line

Please try this:
https://github.com/yinrong/node-line-stream-util#get-head-lines
It unpipe the upstream once got the head lines.

Node.js >= 16
In all current versions of Node.js, readline.createInterface can be used as an async iterable, to read a file line by line - or just for the first line. This is also safe to use with empty files.
Unfortunately, the error handling logic is broken in versions of Node.js before 16, where certain file system errors may go uncaught even if the code is wrapped in a try-catch block because of the way asynchronous errors are propagated in streams. So I recommend using this method only in Node.js >= 16.
import { createReadStream } from "fs";
import { createInterface } from "readline";
async function readFirstLine(path) {
const inputStream = createReadStream(path);
try {
for await (const line of createInterface(inputStream)) return line;
return ''; // If the file is empty.
}
finally {
inputStream.destroy(); // Destroy file stream.
}
}
const firstLine = await readFirstLine("path/to/file");

//Here you go;
var lineReader = require('line-reader');
var async = require('async');
exports.readManyFiles = function(files) {
async.map(files,
function(file, callback))
lineReader.open(file, function(reader) {
if (reader.hasNextLine()) {
reader.nextLine(function(line) {
callback(null,line);
});
}
});
},
function(err, allLines) {
//do whatever you want to with the lines
})
}