We Keep Coding

Creating An Object From A Directory

Im just wondering how to create a JavaScript Object from a directory using node js.
Here's What I Mean:
If the root folder was "test" and the tree would look like this
So the object structure would look like this:
{ test: { joe: {mama: "(file contents)"}, pp: {help: "very mature example\r\ni'm aware"} } }
edit: here's my attempt
const fs = require("fs");
const path = require("path");
const root = "test";
const dir = __dirname + "\\" + root + "\\";
var currentFolder = dir.replace(__dirname + "\\", "");
var data = {};
const getFileSize = function (dirPath) {
files = fs.readdirSync(dirPath);
var length = 0;
files.forEach(function (file) {
length++;
});
return length;
};
fs.readdirSync(dir).forEach((rootFile, rootIndex) => {
fs.readdirSync(currentFolder).forEach((file, index) => {
if (getFileSize(currentFolder) - 1 == index) {
console.log(index, file, rootFile, currentFolder);
currentFolder = currentFolder.replace(`\\file`, "");
index++;
}
// if (file.includes("."))
// data[currentFolder + file] = fs.readFileSync(currentFolder + file, "utf8");
// else currentFolder = currentFolder + file + "\\";
});
});
console.log(data);

Yeh after hours of googling and stack overflow I found this:
var fs = require("fs");
var path = require("path");
var walk = function (dir, done) {
var results = [];
fs.readdir(dir, function (err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = path.resolve(dir, file);
fs.stat(file, function (err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function (err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
};
var root = "test";
var data = {};
walk("test", function (err, results) {
if (err) throw err;
for (i in results) {
data[
results[i].replace(__dirname + "\\" + root + "\\", "")
] = fs.readFileSync(results[i], "utf8");
}
console.log(data);
});

Why is the for loop ending very early?

I am using Node js (I am running the server with nodemon) to upload an excel file, parse it and send each row to a Mongodb database, the length of the complete array of rows is 476 but it ends the loop at number 31 or 95 and never finishes all the rows. I want to get all the rows but I really dont understand why is not completing the loop.
Here is the console.log of the iterations:
Here is the Node js server file:
app.post('/upload', function(req, res){
var exceltojson;
upload(req, res, function(err){
if (err) {
res.json({error_code:1,err_desc:err});
return;
}
if(!req.file){
res.json({error_code:1, err_desc:"No file passed"});
return;
}
if(req.file.originalname.split('.')[req.file.originalname.split('.').length-1] === 'xlsx'){
exceltojson = xlsxtojson;
} else {
exceltojson = xlstojson;
}
try {
exceltojson({
input: req.file.path,
output: "./outPutJSON/output.json",
lowerCaseHeaders: true
}, function(err, result){
if(err){
return res.json({error_code:1, err_desc:err, data: null});
}
res.json({datos:"Data was succesfully added "});
fs.readFile("./outPutJSON/output.json", 'utf8', async (err, fileContents) => {
if (err) {
console.error(err);
return;
}
try {
let data = JSON.parse(fileContents); // this variable has all the cases
console.log(data + 'the cases');
console.log(data.length + ' length of the cases')
for(let cantidad = 0; cantidad < data.length; cantidad++){ //here is the loop
var document = data[cantidad];
console.log(cantidad + ' - ' + data.length + ' - ' + document);
if(document.nombre === '' || document.cedula === '' || document.direccion === '') {
console.log('Could not save cases');
} else {
var mostrar = await increment();
document.caso = mostrar;
document.montoApoderado = data[cantidad]['monto apoderado'];
document.numeroCliente = data[cantidad]['no.cliente'];
document.numeroProducto = data[cantidad]['no.producto'];
let today = moment().format('YYYY M D');
document.fechaCreado = today;
var mod = new model(document);
await mod.save(document);
}
}
} catch(err) {
console.error(err);
}
})
});
var fs = require('fs');
try {
fs.unlinkSync(req.file.path)
}catch(e){
}
} catch (e) {
res.json({error_code:1, err_desc:"Corrupted excel file"});
}
});
});

I believe you made be writing the modified files to a directory that is being included in nodemon to monitor for changes. This is triggering a restart. Double-check that and adding paths to ignore in your config file:
Example:
"ignore": ["*.test.js", "fixtures/*"],

You could try debugging it at the points of failure(i.e start your loop at i=31. Also, it looks like a case of a timeout to me. You could also check if that is the case.

ReplaceAll causing issues in array.reduce

I am still pretty new to this, so forgive me if I dont' say this correctly. We have an array.reduce that calls a method with a returning promise that iterates through a list of files and post results to the db. Everything was working great, until it ran into a field that had an apostrophe in it and then the db insert fails. This is the field value. 'Expected 100002822' to be 100002822.'
I tried adding a replaceAll on the field and now get an error in the array.reduce.
Here is the .reduce
console.log('Found test results in ' + matches.length + ' files. Parsing and posting to the database now...');
var startTime = moment();
var parser = new Parser();
matches.reduce(function (p, val) {
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
var endTime = moment();
var testDuration = moment.duration(endTime.diff(startTime));
console.log(chalk.blue('*** File parsing time: ' + testDuration.humanize() + ' ***'));
if (finalResult.insertSuccess == matches.length) {
var publishOut = {
totalFiles: matches.length,
totalTests: 0,
totalTestsSuccess: 0,
totalTestsFailed: 0
}
publishOut.totalTests += finalResult.totalTests;
publishOut.totalTestsSuccess += finalResult.testPassedCount;
publishOut.totalTestsFailed += finalResult.testFailedCount;
console.log(`Successfully inserted ${finalResult.insertSuccess} of ${publishOut.totalTests} test results.`);
// for (var i = 0; i < matches.length; i++) {
// var currentFile = `./testing/results/${matches[i]}`;
// fs.unlinkSync(currentFile);
// }
resolve(publishOut);
} else {
reject('Only ' + finalResult.insertSuccess + ' of ' + matches.length + ' successfully posted to the database');
}
}, function (err) {
reject('error in reduce', err);
});
I have tried several different ways of using the replaceAll with the same failure. It hits this code from the array.reduce
}, function (err) {
reject('error in reduce', err);
});
And this is the called method. The added code causing the failure in the .reduce is this Message = expectation.message.replaceAll("'", "");
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
//console.log('In parseresults', fileName);
var currentFile = './testing/results/' + fileName
json.readFile(currentFile, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function (obj) {
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
testResults.totalTests++;
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length - 1];
Passed = 0;
Message = expectation.message.replaceAll("'", "");
Stack = expectation.stack.split('\n')[1].trim();
testResults.testFailedCount++
} else {
testResults.testPassedCount++
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.then(() => {
//fs.unlinkSync(currentFile);
testResults.insertSuccess++;
//console.log('insertSuccess', testResults.insertSuccess);
},
err => { console.log('… failed', err); throw err; }
));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results).then(() => testResults);
});
};

How to set object array back to iterated collection from nested find() in MongoDB?

I develop application using AngularJS , NodeJS and MongoDB. I'd like to load Product with classified by ProductCategoryCode sending from
AngualrJS to NodeJS. First, I need to find Products by ProductCategoryCode and then iterate for each product to find Uoms by UomCode and ContainUomCode
which each product should has 2 uoms. How can I set uom object docUom back to product document doc[i] and update to product collection doc?
For following code line
doc[i].Uom = docUom;
The system throw error cannot set property 'Uom' of undefined.
Here is product.js snippet code.
router.get("/LoadProductByProductCategoryCode/:productCategoryCode", function (req, res) {
console.log('user.js -> /users ');
var productCategoryCode = req.params.productCategoryCode;
console.log(productCategoryCode );
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://localhost:27017/NodeDB', function (err, db) {
if (err) throw err;
var query = { ProductCategoryCode : productCategoryCode}
var new_product = [];
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
if (doc) {
console.log("Found Product..."+doc.length);
　 for (var i = 0; i < doc.length; i++) {
console.log(doc[i].ProductCode + " each document " + doc[i].UomCode + " " + doc[i].ContainUomCode);
var qUom = {
$or: [ { UomCode: doc[i].UomCode}, { UomCode: doc[i].ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
doc[i].Uom = docUom;
console.dir(product);
}
});
}
res.json(doc);
} else {
console.log('something happen');
}
}); //End
}); // MongoClient
var findProduct = function (db, query, callback) {
db.collection('Product').find(query).toArray(function (err, doc) {
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
var findUom = function(db, queryUom, callback) {
db.collection('Uom').find(queryUom).toArray(function (err, doc) {
// db.close();
if(err) {
callback(err);
}
else {
callback(null, doc);
}
});
}
});
Any idea? THANKS

Because of the asynchronous nature of the Node.js MongoDB driver, both the findProduct() and findUom() methods start, but don't necessarily complete by the time you reach res.json(doc) meaning doc will still be empty. You are expecting this to work in a linear fashion, but node works differently.
Instead, you should send your response back once all asynchronous calls complete meaning you could try something like:
findProduct(db, query, function (err, doc) {
if(err) {
// something went wrong
console.log(err);
return;
}
var processedProduct = function (item) {
console.log(item.ProductCode + " each document " + item.UomCode + " " + item.ContainUomCode);
var qUom = {
$or: [ { UomCode: item.UomCode}, { UomCode: item.ContainUomCode } ]
}
// Find uom
findUom(db, qUom, function(errUom, docUom) {
if(errUom) {
console.log("error " + errUom);
return;
}
if (docUom) {
item.Uom = docUom;
console.dir(product);
return item;
}
});
} 　
if (doc) {
var productsToFind = doc.length;
var products = [];
console.log("Found Products..." + productsToFind);
for (var i = 0; i < doc.length; i++) {
product = doc[i];
product = processedProduct(product);
products.push(product);
productsToFind -= 1;
if(productsToFind === 0){
res.json(products);
}
}
} else {
console.log('something happen');
}
}); //End
I could explain better about asynchronous calls and callbacks as this topic is a bit broad but from the above you can get the idea that I have used a counter productsToFind for all of the inner async calls that once each findUom() call completes this counter decrements and once it reaches 0 it means that all callbacks have fired.

NodeJS recursively list files in directory

I am trying to list all files in a directory (and files within any subdirectories) with the following code:
var fs = require('fs')
var walk = function(directoryName) {
fs.readdir(directoryName, function(e, files) {
files.forEach(function(file) {
fs.stat(file, function(e, f) {
if (f.isDirectory()) {
walk(file)
} else {
console.log('- ' + file)
}
})
})
})
}
walk(__dirname)
However, when my code attempts to invoke walk(file) on line 8 I get the following error:
TypeError: Cannot call method 'isDirectory' of undefined
at program.js:7:15
at Object.oncomplete (fs.js:107:15)
Why is f undefined? If I have the directory structure below, shouldn't the code identify aaa.txt and bbb.txt as files, my_dir as a directory at which point it recursively calls walk and begins the process again (with zzz.txt being the value of f)?
- aaa.txt
- bbb.txt
+ my_dir
- zzz.txt

Function fs.readdir lists the simple file names in that directory, not their absolute path. This is why the program failed to find them, thus leading to an error in fs.stat.
Here's the solution: concatenate the directory path name to the file.
var fs = require('fs');
var path = require('path');
var walk = function(directoryName) {
fs.readdir(directoryName, function(e, files) {
if (e) {
console.log('Error: ', e);
return;
}
files.forEach(function(file) {
var fullPath = path.join(directoryName,file);
fs.stat(fullPath, function(e, f) {
if (e) {
console.log('Error: ', e);
return;
}
if (f.isDirectory()) {
walk(fullPath);
} else {
console.log('- ' + fullPath);
}
});
});
});
};

var fs = require('fs');
var path = require('path');
var walk = function(directoryName) {
fs.readdir(directoryName, function(e, files) {
files.forEach(function(file) {
fs.stat(directoryName + path.sep + file, function(e, f) {
if (f.isDirectory()) {
walk(directoryName + path.sep + file)
} else {
console.log(' - ' + file)
}
})
})
})
}
walk(__dirname)

A fully synchronous version, for those situations where you cannot use async:
const walk = (dir, files = []) => {
const dirFiles = fs.readdirSync(dir)
for (const f of dirFiles) {
const stat = fs.lstatSync(dir + path.sep + f)
if (stat.isDirectory()) {
walk(dir + path.sep + f, files)
} else {
files.push(dir + path.sep + f)
}
}
return files
}
const allFiles = walk(someDir)

Here's a version for async/await:
const { promises: fs } = require("fs");
const path = require("path");
async function walk(dir) {
const entries = await fs.readdir(dir);
let ret = [];
for (const entry of entries) {
const fullpath = path.resolve(dir, entry);
const info = await fs.stat(fullpath);
if (info.isDirectory()) {
ret = [...ret, ...(await walk(fullpath))];
} else {
ret = [...ret, fullpath];
}
}
return ret;
}
(async function () {
console.log(await walk("/path/to/some/dir"));
})();