We Keep Coding

Ignore errors returned from API calls nodeJS

I have a NodeJS script which is making thousands of REST API GET calls to SharePoint to retrieve all folders and files recursively. A problem I have encountered is once in a while I am receiving ECONNRESET errors and 404 Not Found for some folders with unsupported characters. This causes my code to stop and I lose my progress in retrieving all the files and folders. Is there something I can do to prevent these errors from stopping my code and just have it ignore the error and continue onto the next folder?
I thought catching the error and console logging it would suffice but this still ends the execution of the code. Thanks
//Imports and Declarations
const prompt = require("prompt-sync")({ sigint: true });
const { getCreds, pullFromSharePoint } = require('./helper');
const sprequest = require('sp-request');
var path = require('path');
let creds = getCreds();
let spr = sprequest.create(creds)
let SPUrl = 'url to sharepoint site';
let files = [];
let folders = [];
let scannedFolders = [];
let reiteration = false;
let processingReq = false;
//Prompt user for which folder to copy
let targetFolder = prompt("Folder to copy: ");
//Call function to get list of files and folders in specified folder
// getFolderContents(targetFolder);
processInfo([], targetFolder)
//Gets list of files/folders inside folder
function getFolderContents(targetFolder) {
return new Promise((resolve, reject) => {
logger.info('Scanning this folder: ' + targetFolder)
//Skip if folder name contains chars we dont support
if (targetFolder.includes("'")) {
targetFolder = targetFolder.replace(/'/g, "''")
}
//If reiteration = true, format the target folder
if (reiteration === true) {
targetFolder = targetFolder.replace('/sites/sharepointsite', '');
}
console.log('Scanning this folder: ' + targetFolder)
targetFolder = encodeURIComponent(targetFolder)
var url = `${SPUrl}/_api/web/GetFolderByServerRelativeUrl('Shared%20Documents/${targetFolder}')/Files?$filter=TimeLastModified gt datetime'2021-07-01T00:00:00'`;
var url2 = `${SPUrl}/_api/web/GetFolderByServerRelativeUrl('Shared%20Documents/${targetFolder}')?$expand=Folders`;
//THESE SPR CALLS ARE THE REST API CALLS TO SHAREPOINT RESULTING IN THE MOST 404 ERRORS, HOW TO IGNORE ERRORS FROM THESE?
// Call to get list of files
spr.get(url).then(response => {
console.log('Calling SP API to get files')
//Adds files to array
let fileRes = response.body.d.results;
for (let i = 0; i < fileRes.length; i++) {
files.push(fileRes[i].ServerRelativeUrl)
}
console.log(files.length)
}).catch(function (err) {
console.log('Fetch Error :-S', err);
});
//Call to get list of folders
spr.get(url2).then(response => {
//Adds folders to array
let folderRes = response.body.d.Folders.results;
for (let j = 0; j < folderRes.length; j++) {
folders.push(folderRes[j].ServerRelativeUrl)
}
//Push current folder read through to another array so we dont scan it multiple times
scannedFolders.push('/sites/sharepointsite/Shared Documents' + targetFolder);
resolve(folders);
}).catch(function (err) {
console.log('Fetch Error :-S', err);
});
})
}
//If folders exist in folders array scan them; once all folders have been scanned, send files to be copied
async function processInfo(folders, firstFolder) {
let firstRun = await getFolderContents(firstFolder);
let firstGone = false;
if (firstRun) {
if (firstRun.length > 0) {
for (let k = 0; k < firstRun.length; k++) {
await sleep(500);
//If folder has already been scanned, remove from array and skip to next iteration
if (scannedFolders.includes(firstRun[k])) {
if (k === 0) {
firstRun.splice(k, 1)
}
firstRun.splice(k, k)
continue;
}
//Else if folder has not been scanned, send it to be scanned
else {
if (firstRun[k].includes('%') || firstRun[k].includes('%')) {
console.log('skipping')
continue;
}
reiteration = true;
let foldersScanned = await getFolderContents(firstRun[k]).catch(function (err) {
console.log('Fetch Error :-S', err);
});;
//Send each file to pullFile() function to be downloaded
// if (foldersScanned && k == firstRun.length - 1) {
if (firstRun[k] == 10) {
do {
await pullFile(files[0]);
files.shift();
} while (files.length > 0)
}
}
}
}
}
console.log(files.length)
}
//Manipulate file string to get SP folder and File Name, then call the helper function to download the file
async function pullFile(file) {
let filename = file.replace(/^.*[\\\/]/, '')
let spFolder = file.replace('/sites/sharepointsite/', '')
spFolder = spFolder.replace(filename, '');
let downloadPath = path.join('./testfolder', spFolder)
const sppullContext = {
siteUrl: SPUrl, //SharePoint URL
creds: creds //Credentials
};
const sppullOptions = {
spRootFolder: spFolder, // The folder path for the file in SharePoint
dlRootFolder: downloadPath, // Where the file is saved locally
strictObjects: [filename], // Only download the filename specified as a query parameter
muteConsole: true
};
pullFromSharePoint(sppullContext, sppullOptions) //Pull file with filename from SharePoint
.then(() => {
return true;
}).catch(err => {
return err;
});
}
//Timer to wait specified amount of time
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
Code from helper.js
//Imports and Declarations
var fs = require('fs');
const { SPPull } = require('sppull');
const winston = require('winston');
//Winston logger
const logger = winston.createLogger({
level: 'info',
format: winston.format.json(),
defaultMeta: { service: 'user-service' },
transports: [
new winston.transports.File({ filename: 'error.log', level: 'error' }),
new winston.transports.File({ filename: 'info.log' }),
],
});
/**
* Returns the SharePoint credentials object to use.
*/
function getCreds() {
return {
username: 'UN',
password: 'PW'
};
}
module.exports.getCreds = getCreds;
/**
* Pulls a file from SharePoint and then checks for errors.
*
* #param sppullContext context object for sppull
* #param sppullOptions options object for sppull
*/
async function pullFromSharePoint(sppullContext, sppullOptions) {
SPPull.download(sppullContext, sppullOptions)
.then((res) => {
logger.info(res)
return res
})
.catch((err) => {
logger.error(err)
return err
});
}

To count files present in a directory with specific extension in node.js

In a given task,I need to count file with extensions such as '.js','.txt' and other remaining extensions by using promise.
const fs = require("fs");
const path = require("path");
function fun(pathTodirectory) {
return new Promise(resolve => {
fs.readdir(pathTodirectory, (error, files) => {
if (error) {
reject("Error occured");
}
else {
var obj = {
countJs: function () {
var count = 0;
for (var i = 0; i < files.length; i++) {
if (path.extname(files[i]) === ".js") {
count++;
}
}
return count;
},
countTxt: function () {
var count = 0;
for (var i = 0; i < files.length; i++) {
if (path.extname(files[i]) === ".txt") {
count++;
}
}
return count;
},
count: function () {
var count = 0;
for (var i = 0; i < files.length; i++) {
if (path.extname(files[i]) !== ".js" && path.extname(files[i] !== ".txt")) {
count++;
}
}
return count;
},
files: files //will print list of all files in form of array.
}
resolve(obj);
}
});
});
}
//then part for promise.
This is my code so far,I'm getting problem in my code.The count part in not working and giving assertion error in console. I need to resolve in form of object only. How do i effectively do this?

You never define path anywhere in the function. Unless you're defining it somewhere else that'll cause your error.

Async function: for-Loop object empty after await statement

tldr at the bottom:
I don't really know how to explain my problem so I start with an example.
I have this async function (in reactJS but I think this is a JS related issue).
onUploadDrop = async (e, folderId) => {
e.preventDefault();
// check if the user uploaded files or folders
var uploadedItems = e.dataTransfer.items;
let files = [];
for (var i = 0; i < uploadedItems.length; i++) {
let item = uploadedItems[i].webkitGetAsEntry();
if (item.isDirectory) {
alert("is directory")
} else {
var file = await this.getFileByWebkitEntry(item);
files.push(file);
}
console.log(i);
}
// do something with files[]
}
This function is calling another async function:
getFileByWebkitEntry = async (item) => {
return new Promise(resolve => {
item.file(function (file) {
resolve(file);
}, function (err) {
console.log(err);
resolve("");
});
});
}
I'm looping through e.datatransfer.files which are basically some uploaded files or folders. Unfortunately this for-loop gets only executed once.
I did some debugging and found out that if I place a console.log before and after this line: var file = await ... This comes out:
tldr: After the await statement uploadedItems is empty thus ending the loop. Why is this happening?

I solved this by not using async - await but Promises instead.
It looks like this:
onUploadDrop = (e, folderId) => {
e.preventDefault();
// check if the user uploaded files or folders
var uploadedItems = e.dataTransfer.items;
let promises = [];
for (var i = 0; i < uploadedItems.length; i++) {
let item = uploadedItems[i].webkitGetAsEntry();
if (item.isDirectory) {
alert("is directory")
} else {
promises.push(this.getFileByWebkitEntry(item));
}
console.log(i);
}
Promise.all(promises).then(result => {
// do something with result (result = files)
});

Count the number of files in a directory using JavaScript/nodejs?

How can I count the number of files in a directory using nodejs with just plain JavaScript or packages? I want to do something like this:
How to count the number of files in a directory using Python
Or in bash script I'd do this:
getLength() {
DIRLENGTH=1
until [ ! -d "DIR-$((DIRLENGTH+1))" ]; do
DIRLENGTH=$((DIRLENGTH+1))
done
}

Using fs, I found retrieving the directory file count to be straightforward.
const fs = require('fs');
const dir = './directory';
fs.readdir(dir, (err, files) => {
console.log(files.length);
});

const fs = require('fs')
const length = fs.readdirSync('/home/directory').length

1) Download shell.js and node.js (if you don't have it)
2) Go where you download it and create there a file named countFiles.js
var sh = require('shelljs');
var count = 0;
function annotateFolder (folderPath) {
sh.cd(folderPath);
var files = sh.ls() || [];
for (var i=0; i<files.length; i++) {
var file = files[i];
if (!file.match(/.*\..*/)) {
annotateFolder(file);
sh.cd('../');
} else {
count++;
}
}
}
if (process.argv.slice(2)[0])
annotateFolder(process.argv.slice(2)[0]);
else {
console.log('There is no folder');
}
console.log(count);
3) Open the command promt in the shelljs folder (where countFiles.js is) and write node countFiles "DESTINATION_FOLDER" (e.g. node countFiles "C:\Users\MyUser\Desktop\testFolder")

Alternative solution without external module, maybe not the most efficient code, but will do the trick without external dependency:
var fs = require('fs');
function sortDirectory(path, files, callback, i, dir) {
if (!i) {i = 0;} //Init
if (!dir) {dir = [];}
if(i < files.length) { //For all files
fs.lstat(path + '\\' + files[i], function (err, stat) { //Get stats of the file
if(err) {
console.log(err);
}
if(stat.isDirectory()) { //Check if directory
dir.push(files[i]); //If so, ad it to the list
}
sortDirectory(callback, i + 1, dir); //Iterate
});
} else {
callback(dir); //Once all files have been tested, return
}
}
function listDirectory(path, callback) {
fs.readdir(path, function (err, files) { //List all files in the target directory
if(err) {
callback(err); //Abort if error
} else {
sortDirectory(path, files, function (dir) { //Get only directory
callback(dir);
});
}
})
}
listDirectory('C:\\My\\Test\\Directory', function (dir) {
console.log('There is ' + dir.length + ' directories: ' + dir);
});

Here the simple code,
import RNFS from 'react-native-fs';
RNFS.readDir(dirPath)
.then((result) => {
console.log(result.length);
});

Okay, I got a bash script like approach for this:
const shell = require('shelljs')
const path = require('path')
module.exports.count = () => shell.exec(`cd ${path.join('path', 'to', 'folder')} || exit; ls -d -- */ | grep 'page-*' | wc -l`, { silent:true }).output
That's it.

const readdir = (path) => {
return new Promise((resolve, reject) => {
fs.readdir(path, (error, files) => {
error ? reject(error) : resolve(files);
});
});
};s
readdir("---path to directory---").then((files) => {
console.log(files.length);
});

I think many people look for function like this:
const countFiles = (dir: string): number =>
fs.readdirSync(dir).reduce((acc: number, file: string) => {
const fileDir = `${dir}/${file}`;
if (fs.lstatSync(fileDir).isDirectory()) {
return (acc += countFiles(fileDir));
}
if (fs.lstatSync(fileDir).isFile()) {
return ++acc;
}
return acc;
}, 0);
They count all files in the entire file tree.

NodeJS recursively list files in directory

I am trying to list all files in a directory (and files within any subdirectories) with the following code:
var fs = require('fs')
var walk = function(directoryName) {
fs.readdir(directoryName, function(e, files) {
files.forEach(function(file) {
fs.stat(file, function(e, f) {
if (f.isDirectory()) {
walk(file)
} else {
console.log('- ' + file)
}
})
})
})
}
walk(__dirname)
However, when my code attempts to invoke walk(file) on line 8 I get the following error:
TypeError: Cannot call method 'isDirectory' of undefined
at program.js:7:15
at Object.oncomplete (fs.js:107:15)
Why is f undefined? If I have the directory structure below, shouldn't the code identify aaa.txt and bbb.txt as files, my_dir as a directory at which point it recursively calls walk and begins the process again (with zzz.txt being the value of f)?
- aaa.txt
- bbb.txt
+ my_dir
- zzz.txt

Function fs.readdir lists the simple file names in that directory, not their absolute path. This is why the program failed to find them, thus leading to an error in fs.stat.
Here's the solution: concatenate the directory path name to the file.
var fs = require('fs');
var path = require('path');
var walk = function(directoryName) {
fs.readdir(directoryName, function(e, files) {
if (e) {
console.log('Error: ', e);
return;
}
files.forEach(function(file) {
var fullPath = path.join(directoryName,file);
fs.stat(fullPath, function(e, f) {
if (e) {
console.log('Error: ', e);
return;
}
if (f.isDirectory()) {
walk(fullPath);
} else {
console.log('- ' + fullPath);
}
});
});
});
};

var fs = require('fs');
var path = require('path');
var walk = function(directoryName) {
fs.readdir(directoryName, function(e, files) {
files.forEach(function(file) {
fs.stat(directoryName + path.sep + file, function(e, f) {
if (f.isDirectory()) {
walk(directoryName + path.sep + file)
} else {
console.log(' - ' + file)
}
})
})
})
}
walk(__dirname)

A fully synchronous version, for those situations where you cannot use async:
const walk = (dir, files = []) => {
const dirFiles = fs.readdirSync(dir)
for (const f of dirFiles) {
const stat = fs.lstatSync(dir + path.sep + f)
if (stat.isDirectory()) {
walk(dir + path.sep + f, files)
} else {
files.push(dir + path.sep + f)
}
}
return files
}
const allFiles = walk(someDir)

Here's a version for async/await:
const { promises: fs } = require("fs");
const path = require("path");
async function walk(dir) {
const entries = await fs.readdir(dir);
let ret = [];
for (const entry of entries) {
const fullpath = path.resolve(dir, entry);
const info = await fs.stat(fullpath);
if (info.isDirectory()) {
ret = [...ret, ...(await walk(fullpath))];
} else {
ret = [...ret, fullpath];
}
}
return ret;
}
(async function () {
console.log(await walk("/path/to/some/dir"));
})();