there is a function that I use to read all files in a directory and then sent an object with emitter to the client.
this is my code that works fine,
const getFilesList = (path, emitter) => {
fs.readdir(path, (err, files) => {
emitter('getFileList', files);
});
};
but when I want to filter hidden files with this code, the 'standardFolders' will send empty in the emitter.
const getFilesList = (path, emitter) => {
let standardFolders = [];
fs.readdir(path, (err, files) => {
if (files) {
files.map((file) => {
winattr.get(path + file, function (err, attrs) {
if (err == null && attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
});
});
} else {
standardFolders = null;
}
emitter('getFileList', standardFolders);
});
};
what is wrong with my code in the second part?
winattr.get(filepath,callback) is asynchronous, so imagine your code "starts" the file.map() line and then immediately skips to emitter('getFileList',standardFolders) --- which standardFolders is empty because it hasn't finished yet!
You can use a library like async.io to handle your callback functions, or you can use a counter and keep track of when all of the callbacks (for each file) has finished yourself.
Example:
// an asynchronous function because setTimeout
function processor(v,cb){
let delay = Math.random()*2000+500;
console.log('delay',delay);
setTimeout(function(){
console.log('val',v);
cb(null,v);
},delay);
}
const main = function(){
const list = ['a','b','c','d'];
let processed = [];
let count = 0;
console.log('starting');
list.map(function(v,i,a){
console.log('calling processor');
processor(v,function(err,value){
processed.push(v);
count+=1;
console.log('count',count);
if(count>=list.length){
// all are finished, continue on here.
console.log('done');
}
})
})
console.log('not done yet!');
};
main();
Similarly, for your code:
const getFilesList = (path, emitter) => {
let standardFolders = [];
fs.readdir(path, (err, files) => {
if (files) {
let count = 0;
files.map((file) => {
winattr.get(path + file, function (err, attrs) {
if (err == null && attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
count+=1;
if(count>=files.length){
// finally done
emitter('getFileList', standardFolders);
}
});
});
} else {
standardFolders = null;
emitter('getFileList', standardFolders);
}
});
};
As already sayed in the other answer winattr.get is async, so the loop finishes before any of the callbacks of winattr.get is called.
You could convert your code using async/await and primitify into a code that looks almost like a sync version, and you can completely get rid of the callbacks or counters
const {promisify} = require('util')
const readdir = promisify(require('fs').readdir)
const winattrget = promisify(require('winattr').get)
const getFilesList = async (path, emitter) => {
let standardFolders = [];
try {
let files = await readdir(path);
for (let file of files) {
try {
let attrs = await winattrget(path + file)
if (attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
} catch (err) {
// do nothing if an error occurs
}
}
} catch (err) {
standardFolders = null;
}
emitter('getFileList', standardFolders);
};
An additional note: In your code you write files.map, but mapping is use to transform the values of a given array and store them in a new one, and this is not done in your current code, so in the given case you should use a forEach loop instead of map.
Related
When the code below runs I expect score to have a value (lets say: {"A": 1, "B": 2}), but when I print it I get an empty dict ({}).
I have tried to use promises but the result is the same.
driversBySeason(season) {
var query = `SELECT raceId FROM races WHERE year = ${season}`;
var score = {};
this.con.query(query, (err, drop) => {
if (err) {
console.error(err);
}
drop.forEach((element) => {
var raceId = element["raceId"];
query = `SELECT driverId, points FROM results WHERE raceId = ${raceId}`;
this.con.query(query, (err, drop) => {
if (err) {
console.error(err);
}
drop.forEach((element) => {
if (score[element["driverId"]] == undefined) {
score[element["driverId"]] = 0;
} else if (score[element["points"]] != undefined) {
score[element["driverId"]] += element["points"];
}
});
});
});
console.log(score);
});
}
First of all you need to change your .forEach loop to for const of loop or just a regular for loop because it just fires the code inside the callback and never waits for it. And then you need to change your this.con.query(...) function which has callback to promises too. Your code should be like this:
const asyncQuery = (query) => new Promise((resolve, reject) => {
this.con.query(query, (err, drop) => {
if (!!err) {
console.log(error);
return reject(err)
}
return resolve(drop);
})
});
async function driversBySeason(season) {
var query = `SELECT raceId FROM races WHERE year = ${season}`;
var score = {};
const drop = await asyncQuery(query).catch(err => err /* some error handling */);
for (const element of drop) {
var raceId = element["raceId"];
query = `SELECT driverId, points FROM results WHERE raceId = ${raceId}`;
const drop2 = await asyncQuery(query).catch(err => err /* some error handling */);
drop2.forEach((element) => {
if (score[element["driverId"]] == undefined) {
score[element["driverId"]] = 0;
} else if (score[element["points"]] != undefined) {
score[element["driverId"]] += element["points"];
}
});
}
console.log(score);
}
You are outside of your callback. When this executes the code continues to run past "this.con.query". You are seeing your empty object that was assigned at the top do to this. Go inside the callback after the drop.forEach where you assign the values, or convert to an async/await approach.
when you call driversBySeason your score variable has value of {}
and that is value for console.log() when you call it because of how closers are works and you updating score with callback function that happen later in time...
you get better understanding if you use promise...not callback
I tried and have looked at StackOverflow and the other posts are not answering my questions. This is a unique question. How can I get the path of the most recently uploaded file saved to a variable so it can be used later?
Code:
var pathtocheck = "C:\Users\user1\Downloads";
var path = require('path');
var fs = require('fs');
var getMostRecent = function (dir, cb) {
var dir = path.resolve(dir);
var files = fs.readdir(dir, function (err, files) {
var sorted = files.map(function(v) {
var filepath = path.resolve(dir, v);
return {
name:v,
time:fs.statSync(filepath).mtime.getTime()
};
})
.sort(function(a, b) { return b.time - a.time; })
.map(function(v) { return v.name; });
if (sorted.length > 0) {
cb(null, sorted[0]);
} else {
cb('Y U NO have files in this dir?');
}
})
}
await getMostRecent(pathtocheck, function (err, recent) {
if (err) console.error(err);
console.log(recent);
});
var lastdownloadedimage = ;
With callbacks, you have to write your code in the callback passed to getMostRecent, so
// Note: awaiting this takes no effect unless getMostRecent returns a promise.
getMostRecent(pathtocheck, function (err, recent) {
if (err) console.error(err);
console.log(recent);
var lastdownloadedimage = recent;
// Or just directly use `recent`
});
Or,
Async-await and Promise can also solve your issue, though I'm not sure how much you're familiar with them.
You can use Promisified version of file system API in Node.js v10 or above, by require('fs').promises instead of require('fs') (Documentation here)
Decalration of the functions like this:
// Also it's a good practice to use `const` or `let`, instead of `var`
const pathtocheck = "C:\Users\user1\Downloads";
const path = require('path');
const fs = require('fs');
const fsp = require('fs').promises;
// Decalre the function with `async` to use `await` in it
const getMostRecent = async function (dir) {
dir = path.resolve(dir);
const files = await fsp.readdir(dir)
const sorted = files.map(function(v) {
const filepath = path.resolve(dir, v);
return {
name:v,
time:fs.statSync(filepath).mtime.getTime()
// maybe you can use fsPromises.stat here
};
})
.sort(function(a, b) { return b.time - a.time; })
.map(function(v) { return v.name; });
if (sorted.length > 0) {
return sorted[0];
} else {
// Now you have no callbacks, so there are two options to return the error state.
// 1. Throw an Error with an error message
// 2. Return a special value such as `null` or `false`, which you can track it.
}
}; // <-- perhaps you need place a semicolon here.
And you call the function in async IIFE, wrapping anonymous async function to use await
(async function() {
const lastdownloadedimage = await getMostRecent(pathtocheck);
console.log(lastdownloadedimage)
})();
Or use Promise.then:
getMostRecent(pathtocheck).then(function(recent) {
var lastdownloadedimage = recent;
// Again, you can just directly use `recent`
})
I want to return the files of a directory.
I need it to pass the route to another function.
In other words, how can I return the files of a directory using JavaScript/Node.js?
const fs = require('fs');
const path = require('path');
const mdLinks = require('../index');
exports.extension = (route) => {
return new Promise((resolve, reject) => {
try {
recursive(route);
} catch (e) {
reject(e);
}
});
}
const recursive = (route) => {
const extMd = ".md";
let extName = path.extname(route);
let files = [];
fs.stat(route, (err, stats) => {
if (stats && stats.isDirectory()) {
fs.readdir(route, (err, files) => {
files.forEach(file => {
let reFile = path.join(route, file);
if (file !== '.git') {
recursive(reFile);
}
});
})
}
else if (stats.isFile() && (extMd == extName)) {
files.push(route);
}
})
return files;
}
There are multiple problems.
First off, your function is asynchronous so it cannot just return the files value because your function returns long before anything is added to the files array (that's one reason that it's empty). It will have to either call a callback when its done or return a promise that the caller can use. You will have to fix that in both the top level and when you call yourself recursively.
Second, you are redefining files at each recursive step so you have no way to collect them all. You can either pass in the array to add to or you can define the files array at a higher level where everyone refers to the same one or you can have the call to recursive concat the files that are returned to your current array.
Third, you haven't implemented any error handling on any of your asynchronous file I/O calls.
Here's my recommended way of solving all these issue:
exports.extension = (route) => {
return recursive(route);
}
const util = require('util');
const stat = util.promisify(fs.stat);
const readdir = util.promisify(fs.readdir);
// returns a promise that resolves to an array of files
async function recursive(route) {
const extMd = ".md";
let extName = path.extname(route);
let files = [];
let stats = await stat(route);
if (stats.isDirectory()) {
let dirList = await readdir(route);
for (const file of dirList) {
if (file !== '.git') {
let reFile = path.join(route, file);
let newFiles = await recursive(reFile);
// add files onto the end of our list
files.push(...newFiles);
}
}
} else if (stats.isFile() && extMd === extName) {
files.push(route);
}
// make the files array be the resolved value
return files;
});
I am reading all the json files in the directory and performing CRUD operations on them in Couchbase in a cyclic manner for some performance benchmarks.
function readFiles(dirName,crudOps, onError){
fs.readdir(dirName,function(err,filenames){
if (err) {
onError(err);
return;
}
var circularIterator = cyclicIterator(filenames);
while(1){
fname = circularIterator.next().value;
fs.readFile(dirName + fname, function(err, content) {
console.log(fname) // NEVER REACHES HERE
if (err) {
console.log(err);
return;
}
crudOps(fname, content);
});
});
}
})
}
However, it does not seem to be executing the fs.readFile function. How can I make it 'circular' iterate through the list of filenames and use the file content for my crudOps function?
EDIT:
Per the suggestion by Ry, I have used a promise to readFile.
const readFile = util.promisify(fs.readFile);
async function getStuff(filename) {
return await readFile(filename);
}
function readFiles(dirName,onFileContent, onError){
fs.readdir(dirName,function(err,filenames){
if (err) {
onError(err);
return;
}
var iterator = circularIterator(filenames);
//filenames.forEach(function(filename) {
while(1){
fname = iterator.next().value;
//iterator.getNext(function(filename){
//
getStuff(dirName + fname).then(data => {console.log(data); onFileContent(fname, data)})
}
})
}
This is hanging and causing my Chrome browser to crash and with the below error on my console:
FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
Is there a way to make this better?
You cannot have limitless amounts of file reads, you can find the maximum your computer can handle and limit the program to that.
To limit activity you can recursively call a function that will (if not maximum active processes are started) read file and do stuff with it and if it's finished then do it again.
function readFiles(dirName, onFileContent, onError) {
fs.readdir(dirName, function (err, filenames) {
if (err) {
onError(err);
return;
}
const recur = (iterator) => {
const fname = iterator.next().value;
getStuff(dirName + fname)
.then(
data => {
console.log(data);
//if the following is an asynchronous function then
// you should wait for it to finish before calling recur
onFileContent(fname, data);
//recursively call itself
recur(iterator);
}
);
};
const iterator = circularIterator(filenames);
var i = 0;
while(i++<1000){//maximum amount of active tasks
recur(iterator);
}
})
}
Working example:
var later = time => value =>
new Promise(
(resolve,reject)=>
setTimeout(
()=>resolve(value),
time
)
)
;
var waitTwoSeconds = later(2000);
function *myIterator(){
var i = 1;
while(i<=15){
yield i++;
}
}
function readFiles() {
const recur = (iterator) => {
const value = iterator.next().value;
if(value===undefined){
console.log("----- Done")
return;
}
waitTwoSeconds(value)
.then(
data => {
console.log("returned",data);
recur(iterator);
}
);
};
const iterator = myIterator();
var i = 0;
while(i++<5){
recur(iterator);
}
}
readFiles();
I've used the following code to call two modules, but the invoke action is called before the validate file (I saw in debug). What I should do to verify that validateFile is called before appHandler.invokeAction? Should I use a promise?
var validator = require('../uti/valid').validateFile();
var appHandler = require('../contr/Handler');
appHandler.invokeAction(req, res);
Update
this is the validate file code
var called = false;
var glob = require('glob'),
fs = require('fs');
module.exports = {
validateFile: function () {
glob("myfolder/*.json", function (err, files) {
var stack = [];
files.forEach(function (file) {
fs.readFile(file, 'utf8', function (err, data) { // Read each file
if (err) {
console.log("cannot read the file", err);
}
var obj = JSON.parse(data);
obj.action.forEach(function (crud) {
for (var k in crud) {
if (_inArray(crud[k].path, stack)) {
console.log("duplicate founded!" + crud[k].path);
break;
}
stack.push(crud[k].path);
}
})
});
});
});
}
};
Because glob and fs.readFile are async functions and appHandler.invokeAction is invoked during i/o from disk.
Promise is a good solution to solve this but an old school callback could do the job.
validator.validateFile().then(function() {
appHandler.invokeAction(req, res);
});
and for validate
var Promise = require("bluebird"), // not required if you are using iojs or running node with `--harmony`
glob = require('mz/glob'),
fs = require('mz/fs');
module.exports = {
validateFile: function () {
return glob("myfolder/*.json").then(function(files) {
return Promise.all(files.map(function(file) {
// will return an array of promises, if any of them
// is rejected, validateFile promise will be rejected
return fs.readFile(file).then(function (content) {
// throw new Error(''); if content is not valid
});
}));
})
}
};
If you want working with promise mz could help :)
As the fs.fileRead is async, you should put the code that you want to execute after validateFile to its callback.
The origin could be:
var validator = require('../uti/valid').validateFile();
var appHandler = require('../contr/Handler');
// create a new function that when execute, will call appHandler.invokeAction with req and res given to its arguments.
validator.validateFile(appHandler.invokeAction.bind(null, req, res));
The validator part should be:
var called = false;
var glob = require('glob'),
fs = require('fs');
module.exports = {
validateFile: function (callback) {
glob("myfolder/*.json", function (err, files) {
var stack = [];
// Use it to decide whether all files processed or not.
var filesToLoad = files.length;
files.forEach(function (file) {
fs.readFile(file, 'utf8', function (err, data) { // Read each file
--filesToLoad;
if (err) {
console.log("cannot read the file", err);
// If the invoke action doesn't depend on the result. You may want to call it here too.
}
var obj = JSON.parse(data);
obj.action.forEach(function (crud) {
for (var k in crud) {
if (_inArray(crud[k].path, stack)) {
console.log("duplicate founded!" + crud[k].path);
break;
}
stack.push(crud[k].path);
}
})
// Only called the callback after all files processed.
if (filesToLoad === 0) {
callback();
}
});
});
});
}
};
Edit: Thanks for Bergi's remind that there's the files are an array and you have to call the callback when all files is processed, so we have to further use a variable to decide how many files are not processed yet.