We Keep Coding

Node.JS Javascript fetching file from directory returning same file

I'm trying to read data files from a directory called 'myfiles' using a script 'app.js'.
The file to be read depends on the configurations passed to 'app.js' by another process which generates the files 'log.txt' and 'config.json'.
'app.js' then reads the files 'log.txt' and 'config.json' and fetches a data file from 'myfiles' based on the file name received.
The files in 'myfiles' are labeled my-files1.txt, my-files2.txt, my-files3.txt, ... etc.
The problem is, I keep fetching the same file 'my-files1.txt', even though 'log.txt' and 'config.json' provide new names for files to fetch.
Could you please help me spot where the problem is?
Thanks for your help!
app.js:
const fs = require('fs');
const { resolve } = require('path');
function readLog() {
try {
return fs.readFileSync(__dirname + '/' + 'log.txt', 'utf8');
}
catch (err) {
return '';
}
}
function readConfig() {
try {
return fs.readFileSync(__dirname + '/' + 'config.json', 'utf8');
}
catch (err) {
return '';
}
}
let config = readConfig();
let log = readLog();
let cfg = JSON.parse(config);
let lcg = JSON.parse(log);
var currentPage = lcg.tPartition; // Current page to fetch
const numberOfPages = 10;
if((cfg.running!=true) && (Number(currentPage)>Number(numberOfPages))){
currentPage = 1;
}
function readDataset() {
try {
return fs.readFileSync(resolve(`./app/assets/myfiles/my-files${currentPage}.txt`), 'utf8');
}
catch (err) {
return err;
}
}
let dataset = readDataset();
const data = {
data1: dataset // set dataset as value of data1
}
module.exports={data};

you have written
let lcg = JSON.parse(log);
but your log is a txt file how can you parse it, is this the problem let me know if this worked by any chance;

Create new json from source json file in test project

UPDATE
I have continued to work through this and have the following code - st
async generatejobs() {
const fs = require("fs");
const path = require("path");
const directoryPath = path.join(__dirname, "../data/pjo/in");
fs.readdir(directoryPath, function (err, files) {
if (err) {
console.log("Error getting directory information." + err);
} else {
files.forEach(function (file) {
console.log(file);
fs.readFile(file, (err, data) => {
console.log(file); // this works, if I stop here
// if (err) throw err;
// let newJson = fs.readFileSync(data);
// console.log(newJson);
})
// let data = fs.readFileSync(file);
// let obj = JSON.parse(data);
// let isoUtc = new Date();
// let isoLocal = toISOLocal(isoUtc);
// obj.printingStart = isoLocal;
// obj.printingEnd = isoLocal;
// let updatedFile = JSON.stringify(obj);
// let write = fs.createWriteStream(
// path.join(__dirname, "../data/pjo/out", updatedFile)
// );
// read.pipe(write);
});
}
});
As soon as I try uncomment the line shown below, it fails.
let newJson = fs.readFileSync(data);
The error I am getting is this.
Uncaught ENOENT: no such file or directory, open 'C:\projects\codeceptJs\ipt\80-012345.json'
This is a true statement as the path should be as follows.
'C:\projects\codeceptJs\ipt\src\data\pjo\in\80-012345.json'
I do not understand why it is looking for the file here given that earlier in the code the path is set and seems to work correctly for finding the file via this.
const directoryPath = path.join(__dirname, "../data/pjo/in");
The remainder of the code which is currently commented out is where I am attempting to do the following.
Grab each file from source dir
put into json object
Update the json object to change two date entries
Save to a new json file / new location in my project
Original Post
I have a codeceptjs test project and would like to include a set of existing json files in my project (src/data/jsondata/in) and then update the date attribute within each and write them to an output location in my project (src/data/jsondata/out). I need to change the date and then get it back into a very specific string format, which I have done and then insert this back into the new json being created. I got this about 80% of the way there and then ran into issues when trying to get the files from one folder within my project to another.
I broke this up in to two parts.
function to take a date and convert it to the date string I need
function to grab the source json, update the date, and make a new json at a new folder location
Number 1 is working as it should. Number 2 is not.
If there is a better way to accomplish this, I am very much open to that.
Here is the code where I'm trying to update the json. The main issue here is I'm not understanding and / or handling correctly the join path stuff.
generatePressJobs() {
//requiring path and fs modules
const path = require('path');
const fs = require('fs');
//joining path of directory
const directoryPath = path.join(__dirname, '../', 'data/pjo/in/');
//passsing directoryPath and callback function
fs.readdir(directoryPath, function (err, files) {
//handling error
if (err) {
I.say('unable to scan directory: ' + err);
return console.log('Unable to scan directory: ' + err);
}
//listing all files using forEach
files.forEach(function (file) {
// Update each file with new print dates
let data = fs.readFileSync(file);
let obj = JSON.parse(data);
let isoUtc = new Date();
let isoLocal = toISOLocal(isoUtc);
obj.printingStart = isoLocal;
obj.printingEnd = isoLocal;
let updatedFile = JSON.stringify(obj);
fs.writeFile(`C:\\projects\\csPptr\\ipt\\src\\data\\pjo\\out\\${file}`, updatedFile, (err) => {
if (err) {
throw err;
}
});
});
});
},
Error received
Uncaught ENOENT: no such file or directory, open '80-003599.json'
at Object.openSync (fs.js:462:3)
at Object.readFileSync (fs.js:364:35)
at C:\projects\codeceptJs\ipt\src\pages\Base.js:86:23
at Array.forEach (<anonymous>)
at C:\projects\codeceptJs\ipt\src\pages\Base.js:84:13
at FSReqCallback.oncomplete (fs.js:156:23)
The function to generate the json is located in src/pages/basePage.js
The folder structure I've built for the json file is located in
src/data/jsondata/in --> for original source files
src/data/jsondata/out --> for resulting json after change
Any insight or suggestions would be hugely appreciated.
Thank you,
Bob

My approach / resolution
Passing along the final approach I took in the event this is helpful to anyone else. The data in the middle was specific to my requirements, but left in to show the process I took to do what I needed to do.
async generatePressjobs(count) {
const fs = require("fs");
const path = require("path");
const sourceDirectoryPath = path.join(__dirname, "../data/pjo/in/");
const destDirectoryPath = path.join(__dirname, "../data/pjo/out/");
for (i = 0; i < count; i++) {
// read file and make object
let content = JSON.parse(
fs.readFileSync(sourceDirectoryPath + "source.json")
);
// Get current date and convert to required format for json file
let isoUtc = new Date();
let isoLocal = await this.toISOLocal(isoUtc);
let fileNameTimeStamp = await this.getFileNameDate(isoUtc);
// Get current hour and minute for DPI time stamp
let dpiDate = new Date;
let hour = dpiDate.getHours();
let minute = dpiDate.getMinutes();
dpiStamp = hour + '' + minute;
// update attributes in the json obj
content.batchid = `80-0000${i}`;
content.id = `80-0000${i}-10035-tcard-${dpiStamp}-0101010000_.pdf`
content.name = `80-0000${i}-8.5x11CALJEF-CalBody-${dpiStamp}-01010100${i}_.pdf`;
content.printingStart = isoLocal;
content.printingEnd = isoLocal;
// write the file
fs.writeFileSync(
destDirectoryPath + `80-0000${i}-SOME-JOB-NAME-${dpiStamp}.pdf_Press Job printing end_${fileNameTimeStamp}.json`,
JSON.stringify(content)
);
}
},

Can't read file after writing it with fs

I want to read an image ,write it in a folder and read it again to get it's base64
I get the following error:
Error: ENOENT: no such file or directory, access 'C:\Workspace\Project\upload_storage\image.jpg'
at Object.accessSync (fs.js:192:3)
My code:
const FS = require("fs");
var multiparty = require('multiparty');
var path = require('path');
function readAndWriteFile(file , newPath){
FS.readFileSync(file.path, (err, data)=>{
FS.writeFileSync(newPath, data, (err)=>{
});
});
}
function base64Encode(path,filemime) {
FS.readFileSync(path, {encoding: 'base64'}, (err, data)=>{
if (err) {
throw err;
}
return `data:${filemime};base64,${data}`;
});
}
...
var form = new multiparty.Form()
//retrieve files using multiparty form
form.parse(req, function(err, fields, files) {
var document;
const documents = files.file;
for(i=0; i<documents.length; i++){
document=documents[i];
const contentType = String(document.headers["content-type"]);
filePath = path.join(process.cwd(),'/upload_storage/',document.originalFilename);
readAndWriteFile(document,filePath);
// // convert image to base64 encoded string
const base64str = base64Encode(filePath, contentType);
console.log(base64str);
}
}
if I comment the base64Encode function call the files get created.
What am I doing wrong?

Don't use callbacks with _fileSync. But it looks like you want copyFileSync followed by unlinkSync anyway:
const fs = require('fs');
function readAndWriteFile(file , newPath){
fs.copyFileSync(file.path, newPath);
fs.unlinkSync(file.path)
}
Did you try reading the documentation for fs?
More reading and examples in this question,

Javascript Node Js & Cheerio web scraper creating object with data

I'm building a simple web scraper using cheerio, this is my code :
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/', function(req, res){
url = 'http://www.gazzetta.it/calcio/fantanews/voti/serie-a-2016-17/giornata-32';
request(url, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
var json = {
teamsList : {}
};
$('.listView .magicTeamList').each(function(){
var teamName = $(this).find('.teamNameIn').text();
var playerName = $(this).find('.playerNameIn').text();
var playerGrade = $(this).find('.fvParameter').not('.head').text();
json.teamsList[teamName] = {};
json.teamsList[teamName][playerName] = playerGrade;
})
} else {
console.log('error happened :' + error);
}
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err){
console.log('File successfully written! - Check your project directory for the output.json file');
})
// Finally, we'll just send out a message to the browser reminding you that this app does not have a UI.
res.send('Check your console!')
});
})
app.listen(8081);
console.log('Magic happens on port 8081');
exports = module.exports = app;
I want to push data inside the json object but I'm not having the desired effect, the output.json i get is this (i'll just paste an excerpt of the result):
{
"teamsList": {
"atalanta": {
"Gollini P.Masiello A.Conti A.Hateboer H.Caldara M.Toloi R.Kurtic J.Cristante B.Freuler R.Kessie F.Petagna A.Dalessandro M.Cabezas B.Paloschi A.": "4.56.57.55.5779667666-"
},
"bologna": {
"Mirante A.Torosidis V.Maietta D.Masina A.Gastaldello D.Pulgar E.Taider S.Dzemaili B.Verdi S.Di Francesco F.Petkovic B.Krafth E.Krejci L.Sadiq U.": "5.5665.5636.565.556--5.5"
}
}
}
But what i want is an output like this:
{
"teamsList": {
"atalanta": {
"Gollini P." : 4.5,
"Masiello A." : 6.5,
...
}
}
}
I've searched for answers but I couldn't find any for my specific problem, or maybe I'm just missing something very stupid.. btw any help would be appreciated, thx guys

On each loop you have this
json.teamsList[teamName] = {};
That is going to remove any existing players from the team object. You need to have IF statement to check if it already exists.

Writing to file only writes last item, not all items, why?

i'm trying to write a feed to a file using node.js. the problem is, it doesn't write all the feeds, only the last 1.
var fs = require('fs');
var feedParser = require('ortoo-feedparser')
var url = "http://iwnsvg.com/feed";
feedParser.parseUrl(url).on('article', function(article) {
console.log('title; ', article.title);
fs.writeFile("articles.json", JSON.stringify(article.title), function(err) {
if(err) {
console.log(err);
}
});
});
Why?

Just change fs.writeFile( to fs.appendFile( and you're fine.
fs.writeFile overwrites your file each time you call it whereas fs.appendFile adds to a file.

As #Robert says you should use appendFile, but also note that that change won't write out valid json. I'm not sure what output you're trying to achieve - it you just want the titles you could write out a txt file with a title on each line like so:
var fs = require('fs');
var feedParser = require('ortoo-feedparser')
var url = "http://iwnsvg.com/feed";
feedParser.parseUrl(url).on('article', function(article) {
console.log('title; ', article.title);
fs.appendFile("articles.txt", article.title + "\n", function(err) {
if(err) {
console.log(err);
}
});
});
To write out json you can do:
var fs = require('fs');
var feedParser = require('ortoo-feedparser')
var url = "http://iwnsvg.com/feed";
let titles = [];
feedParser.parseUrl(url)
.on('article', function (article) {
console.log('title; ', article.title);
titles.push(article.title);
})
.on('end', function () {
fs.writeFile('articles.json', JSON.stringify({ titles }), function (err) {
if (err) {
console.log(err);
}
});
});

fs.writeFile comes with some options like flag. Default value of flag is w for write, so your data are replaced by the new one.
Use 'a' instead
{flag:'a'}
and you'll be fine.
But don't forget that WriteFile or AppendFile are upper layer in fs library which open and close file each time you need to add data.
Preferably, use fs.createWriteStream which returns a writable stream (writable file handle in other languages). Then use and reuse this stream when you need to write data in your file.