The code below prints [ ] in the console, filenames Array is actually being changed, but the print occurs before it happens. How to get the final valeu of filenames array, after all changes occurs?
function search(directoryPath, searchString, filenames) {
fs.readdir(directoryPath, { withFileTypes: true }, function (err, files) {
if (err) {
return console.log('Unable to scan directory: ' + err);
}
files.forEach(function (file) {
if (file.isDirectory()) {
if (file.name === searchString) {
filenames.push(file.name)
}
search(directoryPath + "/" + file.name, searchString, filenames);
}
});
});
}
const filenames = []
search(directoryPath, "1048594132", filenames);
console.log(filenames);
The simplest solution is to only use synchronous functions. You can do this by replacing the asynchronous fs.readdir() with the synchronous fs.readdirSync().
Related
Im trying to read multiple xml files and parse data from them and i managed to do that but now new problem appeared.
allData variable is never changed, no matter what i do. What am i supposed to do here?
I dont know what to do or what to try, this is my first time working with files and im honestly surprised ive managed to come this far.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
fs.readFile(__dirname + '\\parse\\' + filename, 'utf-8', function (err, content) {
if (err) {
console.log(err)
return;
}
parseString(content, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
})
});
console.log("All data: ",allData)
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
SOLVED
adjusted code to use.readFileSync()(removed callback function) and now it works.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
let file = fs.readFileSync(__dirname + '\\parse\\' + filename, 'utf-8')
parseString(file, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
console.log("All data: ",allData)
});
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
The .readdir() and .readFile() methods are async, so in fact the console.log() is executed before all of the readFile operations.
In order to access the allData variable after these operations are complete, you have to either make them sync using .readFileSync() instead or you need to promisify the .readFile() method and wait for all of the promises to resolve.
the following array list I need to get all the price one by one.
this returns the full json object console.log('File data:', jsonString); but the for loop never seems to get called , it never enters it. I need to loop through a json file but its in different folder the json file is under menu folder called list.json menu-> projectName\menu\list.json the file looks like this
The data:
[
{
"code": "ZC",
"price": "1"
},
{
"code": "ZS",
"price": "3"
},
{
"code": "VC",
"price": "4"
},
...]
JS:
const jsonList = fs.readFile("../menu/list.json", "utf8", (err, jsonString) => {
if (err) {
console.log("File read failed:", err);
return;
}
console.log("File data:", jsonString);
console.log("File data:", jsonString.url);
for (var key in jsonString) {
if (jsonString.hasOwnProperty(key)) {
console.log("===>", jsonString[key].price);
}
return jsonString;
}
});
There are two ways to fix the issue you are facing, one is to have your code run inside the callback:
const jsonList = fs.readFile("../menu/list.json", "utf8", (err, jsonString) => {
if (err) {
console.log("File read failed:", err);
return;
}
console.log("File data:", jsonString);
for (var key in JSON.parse(jsonString)) {
if (jsonList.hasOwnProperty(key)) {
console.log("===>", jsonList[key].price); // This is never called
}
}
});
or by using sync function to read file:
const jsonString = fs.readFileSync("../menu/list.json", "utf8");
console.log("File data:", jsonString);
const jsonList = JSON.parse(jsonString);
for (var key in jsonList) {
if (jsonList.hasOwnProperty(key)) {
console.log("===>", jsonList[key].price); // This is never called
}
}
I think you need to loop in the callback as it is async and so jsonList is not the object you expect when you access it. See Get data from fs.readFile
I am writing this code as a project for a customer
and when i go to a show route i got this 500 internal server error
http.get('/files/:id', function(req, res) {
var vid;
var pap;
Videos.find({}, function(err, videos) {
if (err) {
console.log(err);
} else {
vid = videos;
}
});
Papers.find({}, function(err, file) {
if (err) {
console.log(err);
} else {
pap = file;
}
});
Material.findById(req.params.id, function(err, found) {
if (err) {
console.log(err);
} else {
res.render('files', {
file: pap,
video: vid,
current: found
});
}
});
});
this is my show route code.
Note : if i reload the page the error is gone and the page open.
The reason is you need to wait for all the database queries to finish before rendering. In your code, it is possible for the page to render before the other two queries have completed and returned their data. The good news is that Mongoose supports Promises for asynchronous functions.
http.get('/files/:id', function(req, res) {
Promise.all([
Videos.find({}).exec(),
Papers.find({}).exec(),
Material.findById(req.params.id).exec()
]).then( ([video, paper, material]) => {
res.render('files', {
file: paper,
video: video,
current: material
});
}).catch( error => console.log(error) );
});
The functions you're using with Mongoose are asynchronous in nature; the variables vid and pap are not initialized when you run res.render. When you attempt to use those variables in your frontend (template like Jade, Handlebars EJS, I don't know what you're using), they are undefined, and subsequently cause the 500 error. You'll need to run the functions such that the results of all Mongoose queries are available to res.render when it runs; either using an async NodeJS library, or calling each function within one another and then calling res.render at the end.
Solution 1: Using async Node module
var async = require('async');
async.parallel([
// Each function in this array will execute in parallel
// The callback function is executed once all functions in the array complete
function (cb) {
Videos.find({}, function(err, videos) {
if (err) {
return cb(err);
} else {
return cb(null, videos);
}
});
},
function (cb) {
Papers.find({}, function(err, papers) {
if (err) {
return cb(err);
} else {
return cb(null, papers);
}
});
},
function (cb) {
Material.findById(req.params.id, function(err, found) {
if (err) {
return cb(err);
} else {
return cb(null, found);
}
});
}
], function (err, results) {
if (err) {
// If any function returns an error
// (first argument), it will be here
console.log(err);
}
else {
// Even though the functions complete asynchronously,
// the order in which they are declared in the array
// will correspond to the position in the array
// if it returns anything as a second argument.
var videos = results[0];
var files = results[1];
var found = results[2];
res.render('files', {
file: files,
video: videos,
current: found
});
}
});
Solution 2: Nested Callbacks
Videos.find({}, function(err, videos) {
var vid = videos;
if (err) {
console.log(err);
} else {
Papers.find({}, function(err, file) {
var pap = file;
if (err) {
console.log(err);
} else {
Material.findById(req.params.id, function(err, found) {
if (err) {
console.log(err);
} else {
res.render('files', {
file: pap,
video: vid,
current: found
});
}
});
}
});
}
});
I am trying to delete several objects after copying them to a different folder.
My code is like:
var deleteParam = {
Bucket: 'frontpass-test',
Delete: {
Objects: [
{Key: '1.txt'},
{Key: '2.txt'},
{Key: '3.txt'}
]
}
};
s3.deleteObjects(deleteParam, function(err, data) {
if (err) console.log(err, err.stack);
else console.log('delete', data);
});
and the returned data is:
delete { Deleted: [ { Key: '1.txt' }, { Key: '3.txt' }, { Key: '2.txt' } ],
Errors: [] }
so I assume the deletion is completed. But the objects are still exist in the folder, is there something wrong with my code?
I also tried to delete objects using for loop and s3.deleteObject, but it only delete the last object in my list of files.
for (var i = 0; i < files.length; i++) {
var copyParams = {
Bucket: 'frontpass-test',
CopySource: 'frontpass-test/unsold/' + files[i].filename,
Key: 'sold/' + files[i].filename
};
var deleteParam = {
Bucket: 'frontpass-test',
Key: 'unsold/' + files[i].filename
};
s3.copyObject(copyParams, function(err, data) {
if (err) console.log(err, err.stack);
else {
s3.deleteObject(deleteParam, function(err, data) {
if (err) console.log(err, err.stack);
else console.log('delete', data);
});
}
});
}
Any idea on how to delete objects in my case? Thanks in advance.
Well the first example looks good. Do you have object versioning turned on in the bucket? That would keep a copy of a file even after you delete it.
The second example actually contains some bugs that would explain why only the last one gets deleted. Because Node.js is asynchronous, when you hit the copyObject function call, the loop iteration ends and goes to the next iteration, not waiting for the callback on copyObject to be called. You try to define the params variables for each iteration of the loop with the var keyword, but because Javascript has function level scope not block level scope, you aren't actually creating new variables on each iteration. You only have one instance of copyParmas and deleteParams. So you quickly run through the loop and deleteParams stays on the value it receives in the last iteration of the loop. Then eventually the callbacks to the copyObject calls start firing, and they all call deleteObject with deleteParams which by now is the last one. In order to make multiple asynchronous calls in a loop, I like to use the async library. Using it, you could do the following:
async.each(files, function iterator(file, callback) {
var copyParams = {
Bucket: 'frontpass-test',
CopySource: 'frontpass-test/unsold/' + file.filename,
Key: 'sold/' + file.filename
};
var deleteParam = {
Bucket: 'frontpass-test',
Key: 'unsold/' + file.filename
};
s3.copyObject(copyParams, function(err, data) {
if (err) callback(err);
else {
s3.deleteObject(deleteParam, function(err, data) {
if (err) callback(err)
else {
console.log('delete', data);
callback();
}
});
}
});
}, function allDone(err) {
//This gets called when all callbacks are called
if (err) console.log(err, err.stack);
});
Just had to implement folder rename on top of s3, I did it as follows: (promise api)
_getDataForItemRename(from, to) {
return s3.listObjectsV2({Bucket: services.conf.workspace, Prefix: from}).promise()
.then((data) => {
const toCopy = [];
const toRemove = [];
const s3Contents = Object.assign([], data.Contents);
// in case of a single dir (with no children)
if (s3Contents.length === 0) {
s3Contents.push({Key: from});
}
s3Contents.forEach((item) => {
const copyPromise = s3.copyObject({
Bucket: services.conf.workspace,
Key: to,
CopySource: services.conf.workspace + '/' + item.Key
}).promise();
const deletePromise = s3.deleteObjects({
Bucket: services.conf.workspace,
Delete: {Objects: [{Key: from}]}
}).promise();
toCopy.push(copyPromise);
toRemove.push(deletePromise);
});
return {copy: toCopy, remove: toRemove};
}).catch((err) => {
return Promise.reject(err);
});
}
return this._getDataForItemRename(_from, _to).then((files) => {
return Promise.all(files.copy).then(() => {
return Promise.all(files.remove).then(result => {
return result;
});
});
}).catch((err) => {
return Promise.reject(err);
});
I'm in deep trouble trying to understand how to make my code asynchronous in Node.js land. Please see my code below, in two varieties.
Ok so here's my first try - I have three functions here. A processing function (iim), a file copy function (fse.copy), and an archive function (da).
I need da to happen after iim, and iim to happen after fse.copy.
This first approach results in archive happening, but it's empty because iim never appears to happen.
da(randomString, function(err) {
if (err) {
log.error(err);
} else {
fse.copy(temp_path, new_location + file_name, function(err) {
if (err) {
log.error(err);
} else {
log.info("File saved to " + new_location + file_name);
var sourceImage = new_location + file_name;
log.debug(sourceImage);
log.debug(randomString);
iim(sourceImage, randomString, function(err) {
if (err) {
log.error(err);
}
});
}
});
}
});
The next block is an alternate approach which results in the da happening before iim is finished.
fse.copy(temp_path, new_location + file_name, function(err) {
if (err) {
log.error(err);
} else {
log.info("File saved to " + new_location + file_name);
var sourceImage = new_location + file_name;
log.debug(sourceImage);
log.debug(randomString);
iim(sourceImage, randomString, function(err) {
if (err) {
log.error(err);
}
});
da(randomString, function(err) {
if (err) {
log.error(err);
}
});
}
});
Here's what I'd recommend -- in your question you say you need to essentially run three functions in series -- correct? Run function A, then function B, and lastly, run function C.
The simplest way to do this is using the asyncjs library.
Here's an example:
var async = require('async');
async.series([
function a(cb) {
// do stuff
cb();
},
function b(cb) {
// do stuff
cb();
},
function c(cb) {
// do stuff
cb();
},
], function() {
// this will run once all three functions above have finished
});
Now, let's say that each of those functions needs to return data to the next function. SO imagine that function B needs input from function A to run. How do you accomplish that? Using async.waterfall!
var async = require('async');
async.waterfall([
function a(cb) {
// do stuff
cb(null, 'value');
},
function b(val, cb) {
// do stuff with val
cb(null, 'woot');
},
function c(val, cb) {
// do stuff with val
cb(null);
},
], function() {
// this will run once all three functions above have finished
});
Not bad right?
Hope this helps!
EDIT: Here's a code block showing your code above refactored using asyncjs:
async.waterfall([
function(cb) {
fse.copy(temp_path, new_location + file_name, function(err) {
if (err) {
log.error(err);
} else {
log.info("File saved to " + new_location + file_name);
var sourceImage = new_location + file_name;
log.debug(sourceImage);
log.debug(randomString);
}
console.log('Finished running fs.copy');
cb(null, sourceImage, randomString);
});
},
function(sourceImage, randomString, cb) {
iim(sourceImage, randomString, function(err) {
if (err) {
log.error(err);
}
console.log('Finished running iim');
cb(null, randomString);
});
},
function(randomString, cb) {
da(randomString, function(err) {
if (err) {
log.error(err);
}
console.log('Finished running da');
cb();
});
}
], function() {
console.log('All done!');
});
So you can either put da into the callback for iim (right now it's not) from your second example:
fse.copy(temp_path, new_location + file_name, function(err) {
if (err) {
log.error(err);
} else {
log.info("File saved to " + new_location + file_name);
var sourceImage = new_location + file_name;
log.debug(sourceImage);
log.debug(randomString);
iim(sourceImage, randomString, function(err) {
if (err) {
log.error(err);
return;
}
da(randomString, function(err) {
if (err) {
log.error(err);
}
});
});
}
});
That said, callback depth can be flattened with the use of a library like async (https://github.com/caolan/async)