Im trying to read multiple xml files and parse data from them and i managed to do that but now new problem appeared.
allData variable is never changed, no matter what i do. What am i supposed to do here?
I dont know what to do or what to try, this is my first time working with files and im honestly surprised ive managed to come this far.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
fs.readFile(__dirname + '\\parse\\' + filename, 'utf-8', function (err, content) {
if (err) {
console.log(err)
return;
}
parseString(content, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
})
});
console.log("All data: ",allData)
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
SOLVED
adjusted code to use.readFileSync()(removed callback function) and now it works.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
let file = fs.readFileSync(__dirname + '\\parse\\' + filename, 'utf-8')
parseString(file, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
console.log("All data: ",allData)
});
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
The .readdir() and .readFile() methods are async, so in fact the console.log() is executed before all of the readFile operations.
In order to access the allData variable after these operations are complete, you have to either make them sync using .readFileSync() instead or you need to promisify the .readFile() method and wait for all of the promises to resolve.
Related
for a school project I have to rewrite a Json Object using node fs. I wrote a module and if I use deleteCity or addCity on their own, they work perfectly fine. But when I call both after another only one works. In my JSON File there is one array with 10 objects. In my main javascript file I require my module and call upon addCity and deleteCity functions.
//Modules
const fs = require("fs");
//Variablen
const citiesPath = "./cities.json";
//Funktionen
const deleteCity = (name) => {
fs.readFile(citiesPath, "utf-8", (err, jstring) => {
if (err) {
console.log(err);
}
try {
let data = JSON.parse(jstring);
for (let i = 0; i < data.length; i++) {
if (data[i].Name == name) {
data.splice(i, 1);
}
}
fs.writeFile(citiesPath, JSON.stringify(data, null, 2), (err) => {
if (err) {
console.log(err);
}
});
} catch (error) {
console.log(error);
}
});
};
const addCity = (obj) => {
fs.readFile(citiesPath, "utf-8", (err, jstring) => {
if (err) {
console.log(err);
}
try {
let data = JSON.parse(jstring);
data.push(obj);
fs.writeFile(citiesPath, JSON.stringify(data, null, 2), (err) => {
if (err) {
console.log(err);
}
});
} catch (error) {
console.log(error);
}
});
};
const showCity = () => {
fs.readFile(citiesPath, "utf-8", (err, jstring) => {
if (err) {
console.log(err);
}
try {
let data = JSON.parse(jstring);
console.log(data);
} catch (error) {
console.log(error);
}
});
};
//Exporte
module.exports = {
deleteCity,
addCity,
showCity
};
I suppose you are calling both function synchronously, i.e.
deleteCity("London");
addCity({ "Name": "Paris" });
The problem here is that the calls are both asynchronous and the second one will start before the first call terminates, so basically before a city has been deleted.
If this is a school project the simplest solution to fix your code is using the synchronous version of the fs calls fs.readFileSync and fs.writeFileSync:
//Modules
const fs = require("fs");
//Variablen
const citiesPath = "./cities.json";
//Funktionen
const deleteCity = (name) => {
const jstring = fs.readFileSync(citiesPath, "utf-8");
let data = JSON.parse(jstring);
for (let i = 0; i < data.length; i++) {
if (data[i].Name == name) {
data.splice(i, 1);
}
}
fs.writeFileSync(citiesPath, JSON.stringify(data, null, 2));
};
const addCity = (obj) => {
const jstring = fs.readFileSync(citiesPath, "utf-8");
let data = JSON.parse(jstring);
data.push(obj);
fs.writeFileSync(citiesPath, JSON.stringify(data, null, 2));
};
const showCity = () => {
const jstring = fs.readFileSync(citiesPath, "utf-8");
let data = JSON.parse(jstring);
console.log(data);
};
//Exporte
module.exports = {
deleteCity,
addCity,
showCity
};
Note that you don't need to catch the errors only to log them inside your synchronous functions. If an error is thrown and not caught, Node.js will log it for your.
Hello I am creating 1 function with dynamic arguments where as I am calling api and on defined route I am calling express middleware function and from there I am calling another dynamic function which will help me to insert data into the database.
I am using Sequalize ORM
Here is code:
var async = require('async');
// Models
var LogSchema = require('../models/Logs')
module.exports = {
insertLog: async (req, res) => {
let result = await insertLogFn('1', 'method_name()', 'module_name_here', 'req.body', '{ api response }', 'action', '24')
console.log("result", result)
res.status(200).json(result)
}
};
function insertLogFn(status, invokedMethodName, moduleName, bodyRequest, apiResponse = null, actionName = null, userId) {
async.waterfall([
(nextCall) => {
let dataToBeInserted = {}
dataToBeInserted.status = status,
dataToBeInserted.invoked_method_name = invokedMethodName,
dataToBeInserted.module_name = moduleName,
dataToBeInserted.body_request = bodyRequest,
dataToBeInserted.api_response = apiResponse
dataToBeInserted.action_name = actionName,
dataToBeInserted.user_id = userId
LogSchema.create(dataToBeInserted).then(res => {
const dataObj = res.get({plain:true})
nextCall(null, {
status: 200,
message: "Log inserted successfully",
data: dataObj
})
}).catch(err => {
})
}
], (err, response) => {
if(err) {
}
return response
})
}
In module.export I have added insertLog function which is getting called in api and from there I am calling insertLogFn() which is declared outside of the module.export.
I am able to get inserted result in function insertLogFn() but the things is await is not working and not waiting for the result.
What I want to do is to wait till insertLogFn gets executed and the returned response has to be stored in the variable and return it as an api response.
You cannot. As per my understanding, IMO, Thumb rule is "Async/Await operation should return a promise"
function insertLogFn(status, invokedMethodName, moduleName, bodyRequest, apiResponse = null, actionName = null, userId) {
async.waterfall([
(nextCall) => {
let dataToBeInserted = {}
dataToBeInserted.status = status,
dataToBeInserted.invoked_method_name = invokedMethodName,
dataToBeInserted.module_name = moduleName,
dataToBeInserted.body_request = bodyRequest,
dataToBeInserted.api_response = apiResponse
dataToBeInserted.action_name = actionName,
dataToBeInserted.user_id = userId
LogSchema.create(dataToBeInserted).then(res => {
const dataObj = res.get({plain:true})
nextCall(null, {
status: 200,
message: "Log inserted successfully",
data: dataObj
})
return ;
console.log("you should return something here<-------");
}).catch(err => {
})
}
], (err, response) => {
if(err) {
}
return response
})
}
Now the answer will be clear if you read this one from Bergi: https://stackoverflow.com/a/40499150/9122159
I know this topic as already asked many times before but I didn't find the right answer to do what I want.
Actually, I try to save two different list of JSON object in MongoDB via Mongoose. To perform both at the same time I use 'async'.
However, when I save it with the command insertMany() I get an error because he calls the callback of async before finishing the insertMany(). Therefore answer[0] is not defined.
What will be the proper way of doing it ?
Here is my code with the async:
const mongoose = require("mongoose");
const async = require("async");
const utils = require("../utils");
const experimentCreate = function(req, res) {
let resData = {};
let experimentList = req.body.experiment;
let datasetList = req.body.datasetList;
async.parallel(
{
dataset: function(callback) {
setTimeout(function() {
answer = utils.createDataset(datasetList);
callback(answer[0], answer[1]);
}, 100);
},
experiment: function(callback) {
setTimeout(function() {
answer = utils.createExp(experimentList);
callback(answer[0], answer[1]);
}, 100);
}
},
function(err, result) {
if (err) {
console.log("Error dataset or metadata creation: " + err);
sendJSONresponse(res, 404, err);
} else {
console.log("Experiment created.");
resData.push(result.dataset);
resData.push(result.experiment);
console.log(resData);
sendJSONresponse(res, 200, resData);
}
}
);
};
Then the two functions called createExp and createDataset are the same in another file. Like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
return [err, null];
} else {
console.log("All dataset created.");
return [null, ds];
}
});
};
There's a few problems with your code. For one, you're not returning anything in your createDataset function. You're returning a value in the callback of insertMany but it doesn't return that value to the caller of createDataset as it's within another scope. To solve this issue, you can wrap your Dataset.insertMany in a promise, and resolve or reject depending on the result of Data.insertMany like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
return new Promise((resolve, reject) => {
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
reject(err);
} else {
console.log("All dataset created.");
resolve(ds);
}
});
});
};
Now your return object is no longer going to be an array so you won't be able to access both the error and the result via answer[0] and answer[1]. You're going to need to chain a then call after you call createDataset and use callback(null, answer) in the then call (as that means createDataset executed successfully) or use callback(err) if createDataset throws an error like below:
dataset: function(callback) {
setTimeout(function() {
utils.createDataset(datasetList).then(answer => {
callback(null, answer);
}).catch(err => callback(err)); // handle error here);
}, 100);
}
Note: You'll most likely need to alter your createExp code to be structurally similar to what I've produced above if it's also utilizing asynchronous functions.
I am trying to get the name and created date of the files. In the code below it throws error when I call the api. It is reading the directory and printing all the file names but it's not sending back to callback. Any idea what is implemented wrong?
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var currentDate = new Date();
var objToReturn = [];
var logsDirectory = './logs'
function readDirectory(env, callback) {
fs.readdir(logsDirectory + '/' + env, function(err, files) {
// loop through each file
async.eachSeries(files, function(file, done) {
var dirPath = logsDirectory + '/' + env;
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't exist");
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
objToReturn.push(fileInfo);
done();
}
});
});
},
function(err) {
if (err) {
console.info('error', err);
return;
}
// when you're done reading all the files, do something...
console.log('before Callback', objToReturn);
callback(objToReturn);
});
}
exports.readDirectory = readDirectory;
app.js
var stDirectory = require('./app/serverfiles/stDir');
app.get('/getAllFiles',function(req,res){
var env = req.query.env
console.log('printing',env);
stDirectory.readDirectory(env,function(files){
res.json(files);
console.log('Api files',files);
});
});
There are a few issues:
instead of passing the "final" handler to async.eachSeries(), you're passing it to fs.readdir(), so callback will never get called;
you're declaring objToReturn outside of the function, which isn't a good idea because multiple requests could be handled in parallel;
you're not handling any errors properly;
you should really use the Node.js callback idiom of calling callbacks with two arguments, the first being errors (if there are any) and the second being the result of the asynchronous operation.
The code below should fix these issues:
function readDirectory(env, callback) {
let objToReturn = [];
fs.readdir(
logsDirectory + "/" + env,
function(err, files) {
if (err) return callback(err);
// loop through each file
async.eachSeries(files, function(file, done) {
var dirPath = logsDirectory + "/" + env;
var filePath = path.join(dirPath, file);
var fileInfo = {};
fs.stat(filePath, function(err, stats) {
if (err) {
console.info("File doesn't exist");
return done(err);
} else {
fileInfo.fileDate = stats.birthtime;
fileInfo.filename = file;
objToReturn.push(fileInfo);
done();
}
});
}, function(err) {
if (err) {
console.info("error", err);
return callback(err);
}
// when you're done reading all the files, do something...
console.log("before Callback", objToReturn);
callback(null, objToReturn);
}
);
}
// To call it:
stDirectory.readDirectory(env, function(err, files) {
if (err) {
res.sendStatus(500);
} else {
res.json(files);
console.log('Api files',files);
}
});
You should also consider using async.mapSeries() instead of async.eachSeries() and using a separate array (objToReturn).
I'm trying to bulk upload attachments to CouchDB using node.js and nano.
First, the walk module is used to find all files in upload folder and create array from them.
Next, each file from the array is supposed to be inserted into CouchDB via pipe and nano module.
However, the final result is that only one attachment has been uploaded.
var nano = require('nano')('http://localhost:5984')
var alice = nano.use('alice');
var fs = require('fs');
var walk = require('walk');
var files = [];
// Walker options
var walker = walk.walk('./uploads', {
followLinks: false
});
// find all files and add to array
walker.on('file', function (root, stat, next) {
files.push(root + '/' + stat.name);
next();
});
walker.on('end', function () {
// files array ["./uploads/2.jpg","./uploads/3.jpg","./uploads/1.jpg"]
files.forEach(function (file) {
//extract file name
fname = file.split("/")[2]
alice.get('rabbit', {revs_info: true}, function (err, body) {
fs.createReadStream(file).pipe(
alice.attachment.insert('rabbit', fname, null, 'image/jpeg', {
rev: body._rev
}, function (err, body) {
if (!err) console.log(body);
})
)
});
});
});
This is because you are mixing an asynchronous api with assumptions of this being synchronous.
After the first request you will get conflicts, cause the rabbit document has changed.
Can you confirm this using NANO_ENV=testing node yourapp.js?
I recommend using async if this is the problem
var nano = require('nano')('http://localhost:5984')
var alice = nano.use('alice');
var fs = require('fs');
var walk = require('walk');
var files = [];
// Walker options
var walker = walk.walk('./uploads', {
followLinks: false
});
walker.on('file', function (root, stat, next) {
files.push(root + '/' + stat.name);
next();
});
walker.on('end', function () {
series(files.shift());
});
function async(arg, callback) {
setTimeout(function () {callback(arg); }, 100);
}
function final() {console.log('Done');}
function series(item) {
if (item) {
async(item, function (result) {
fname = item.split("/")[2]
alice.get('rabbit', { revs_info: true }, function (err, body) {
if (!err) {
fs.createReadStream(item).pipe(
alice.attachment.insert('rabbit', fname, null, 'image/jpeg', {
rev: body._rev
}, function (err, body) {
if (!err) console.log(body);
})
)
}
});
return series(files.shift());
});
}
else {
return final();
}
}