I had this code working earlier, but made some changes and I'm not sure what I did to break it. The path to the .csv file is correct, and the code seems correct, but the array raw_data is empty after the function call.
require('./trip.js');
const parser = require('csv-parser');
const fs = require('fs');
let raw_data = [];
function readFile() {
fs.createReadStream('./trips.csv')
.pipe(parser())
.on('data', (data) => raw_data.push(data))
.on('end', () => console.log('CSV has been piped into an array'));
}
const trips = async () => {
await readFile();
console.log(raw_data.length)
};
I expect the raw_data array to contain 9999 items. It contains zero. I am also not getting the console.log statement to execute on 'end'.
readFile must return Promise like this
require('./trip.js');
const parser = require('csv-parser');
const fs = require('fs');
let raw_data = [];
function readFile() {
return new Promise(resolve =>
fs.createReadStream('./trips.csv')
.pipe(parser())
.on('data', (data) => raw_data.push(data))
.on('end', resolve)
);
}
const trips = async () => {
await readFile();
console.log(raw_data.length)
};
Related
I have some csv data I need to parse into array of objects for an API I am building.
I am able to see that the parser I am using is working as expected this way:
const csv = require('fast-csv')
const fs = require('fs');
const path = require('path');
let results = [];
async function parseCSVFromCSV(_sourceCSVFilePath){
return new Promise((resolve, reject) => {
fs.createReadStream(_sourceCSVFilePath)
.pipe(csv.parse({ headers: true }))
.on('error', error => reject(error))
.on('data', row => {
console.log(row);
results.push(row);
})
resolve(results);
})
}
so the console.log(row) prints out the parsed object as expected but the results object is empty. I am needing this parsed data but no matter what I try I can't get results to populate with parsed csv.
Any guidance on what I a missing here?
So I figured out what I need to do:
const fs = require('fs');
const path = require('path');
const csv = require('fast-csv');
async function getData(){
let arrOfObj = [];
fs.createReadStream(path.resolve(__dirname, 'downloads', 'sampleData.csv'))
.pipe(csv.parse({ headers: true }))
.on('error', error => console.error(error))
.on('data', row =>
{arrOfObj.push(row);
//console.log(row);
})
.on('end', rowCount => { printData(arrOfObj);
});
}
async function printData(_arrOfObj){
console.log(_arrOfObj);
}
getData();
Basically the finished array will not be accessible expect inside of the end event. This allows it be exposed to outside the event. Hope this helps others that were needing to do this same thing.
I have 3 files:
Ingredients.js
const fs = require("fs");
const readline = require('readline');
const stream = require('stream');
const ingredients = () => {
const instream = fs.createReadStream('ingredients.txt');
const outstream = new stream;
const rl = readline.createInterface(instream, outstream);
const listIngredients = {};
rl.on('line', function (line) {
let lower = line.toLowerCase();
listIngredients[lower] = 0;
});
rl.on('close', function () {
console.log('listIngredients', listIngredients);
});
}
module.exports = ingredients;
cookbook.js:
let fs = require("fs");
const book = () => {
const regex = /\b(\w+)\b/g;
fs.readFile('cook-book.txt', 'utf8', function (err, data) {
let book = data;
let lower = book.toLowerCase();
let split = lower.match(regex);
console.log(split);
});
}
module.exports = book;
compare.js
const ingredients = require('./ingredients');
const book = require('./book');
I'm trying to increase the key values of ingredients every time they are mentioned in the cookbook. I think this should go into a different js file to make it cleaner.
Whilst i can console.log out the information from the above files, I cannot figure out how to actually access the data and make changes to the ingredients object in compare.js?
as others noticed your ingredients and book variables are functions having required information inside their scope and not returning it outside. to fix it, you have to return values.
as you're working with asynchronous stuff, your functions should be wrapped into Promise's to handle the flow correctly.
this code should help you:
const fs = require('fs');
const readline = require('readline');
const { Writable } = require('stream');
const fsp = fs.promises;
// ingredients.js
const getIngredients = async () => new Promise((resolve, reject) => {
const instream = fs.createReadStream('ingredients.txt');
const outstream = new Writable();
const rl = readline.createInterface(instream, outstream);
const listIngredients = {};
rl.on('line', line => {
const lower = line.toLowerCase();
listIngredients[lower] = 0;
});
rl.on('error', reject);
rl.on('close', () => resolve(listIngredients));
});
// cookbook.js
const getBookContent = async () => new Promise(async (resolve, reject) => {
try {
const wordRegEx = /\b(\w+)\b/g;
const book = await fsp.readFile('cook-book.txt', 'utf8')
const lower = book.toLowerCase();
return resolve(lower.match(wordRegEx));
} catch (error) {
return reject(error);
}
});
// compare.js
(async () => {
const ingredients = await getIngredients();
const words = await getBookContent();
console.log(ingredients);
console.log(words);
})();
the names of functions have been change for better representations of their instances.
i've also used an async iife to use async/await syntax, however you can still work with Promises themselves
Have tried PapaParse without success, how would one get the first column value of a CSV file?
const csv = require('csv-parser');
const fs = require('fs');
(async () => {
try {
fs.createReadStream('test.csv')
.pipe(csv())
.on('data', (row) => {
console.log(row);
})
.on('end', () => {
console.log('CSV file successfully processed');
});
} catch (err) {
console.log(error(err));
await browser.close();
console.log(error("Browser Closed"));
}
})();
For anyone in the future, set a function then set as a const to your CSV list of URLs, the number [1] represents the column.
function readURLFile(path) {
return fs.readFileSync(path, 'utf-8')
.split('\n')
.map((elt) => {
const url = elt.split(',')[1].replace('\r', '');
return `http://${url.toLowerCase()}`;
});
}
Inside a function, I would like to set the value of a variable (foldersInDir) to the results of getting the contents of a directory using fs.readdir();
I thought using await would force the console.log line to wait for a response, but it's not.
How can I set foldersInDir = the return value?
/*Begin function*/
const listContents = async (myPath) => {
var fs = require('fs');
let foldersInDir = await fs.readdir(myPath, function(err, items) {
console.log(items); //works
return items;
});
console.log(foldersInDir); //does not work, undefined
}
You need to convert readdir to a promise, e.g.:
const foldersPromised = (path) =>
new Promise((resolve, reject) =>
fs.readdir(path, (err, items) =>
err !== undefined ? reject(err) : resolve(items)
)
);
try {
let foldersInDir = await foldersPromised(myPath);
} catch(err) {
console.log(err);
}
const fs = require('fs');
const test = () => {
let folders = fs.readdirSync('.');
return folders;
}
console.log(test());
Edit: sorry, need to promisify() the function
const fs = require('fs');
const { promisify } = require('util') // available in node v8 onwards
const readdir = promisify(fs.readdir)
async function listContents() {
try { // wrap in try-catch in lieu of .then().catch() syntax
const foldersInDir = await readdir(myPath) // call promised function
console.log('OK, folders:', foldersInDir) // success
} catch (e) {
console.log('FAIL reading dir:', e) // fail
}
}
listContents('path/to/folder') // run test
I recommend using the promisify function provided by Node.js to fix the problem. This function will convert a callback-based function to a promise-based function, which can then be used using the await keyword.
const fs = require('fs');
const {
promisify
} = require('util');
const readdirAsync = promisify(fs.readdir);
/*Begin function*/
const listContents = async(myPath) => {
let foldersInDir = await readdirAsync(myPath);
console.log(foldersInDir);
}
I have a function, startSurvey, which, when run, checks if there are questions in a .json file. If there are no questions, it fetches some questions from Typeform and writes them to the .json file using saveForm. After it writes, I would like to continue executing some code that reads the .json file and logs its contents. Right now, await saveForm() never resolves.
I have promisified the fs.readFile and fs.writeFile functions.
//typeform-getter.js
const fs = require('fs')
const util = require('util')
const fetch = require('cross-fetch')
require('dotenv').config()
const conf = require('../../private/conf.json')
const typeformToken = conf.tokens.typeform
const writeFile = util.promisify(fs.writeFile)
const getForm = async () => {
const form = await fetch(`https://api.typeform.com/forms/${process.env.FORM_ID}`, {
headers: {
"Authorization": `bearer ${typeformToken}`
}
}).then(res => res.json())
const fields = form.fields
return fields
}
const saveForm = async () => {
const form = await getForm()
return writeFile(__dirname + '/../data/questions.json', JSON.stringify(form))
.then((e) => {
if (e) console.error(e)
else console.log('questions saved')
return
})
}
module.exports = saveForm
//controller.js
const fs = require('fs')
const util = require('util')
const request = require('request')
require('dotenv').config()
const typeformGetter = require('./functions/typeform-getter')
const readFile = util.promisify(fs.readFile)
const saveForm = util.promisify(typeformGetter)
let counter = 1
const data = []
const getQuestions = async() => {
console.log('called')
try {
let data = await readFile(__dirname + '/data/questions.json')
data = JSON.parse(data)
return data
} catch (e) {
console.error('error getting questions from read file', e)
}
}
const startSurvey = async (ctx) => {
try {
const questions = await getQuestions()
if (!questions) await saveForm()
console.log(questions) //NEVER LOGS
} catch (error) {
console.error('error: ', error)
}
}
startSurvey() //function called
I don't know your exact error, but there are multiple things wrong with your code:
You're using incorrectly the promisified version of fs.writeFile, if an error occurs, the promise will be rejected, you won't get a resolved promise with an error as the resolved value, which is what you're doing.
Use path.join instead of concatenating paths.
In startSurvey, you're using console.log(questions) but that wont have any data if questions.json doesn't exists, which should happen the first time you run the program, since it's filled by saveForm, so you probably want to return the questions in saveForm
So saveForm should look something like this:
const saveForm = async () => {
const form = await getForm();
const filePath = path.join(path.__dirname, '..', 'data', 'questions.json');
await writeFile(filePath, JSON.stringify(form));
console.log('questions saved');
return form;
}
And startSurvey
const startSurvey = async (ctx) => {
try {
const questions = await getQuestions() || await saveForm();
// This will be logged, unless saveForm rejects
// In your code getQuestions always resolves
console.log(questions);
} catch (error) {
console.error('error: ', error)
}
}
In your controller.js you're using util.promisify on saveForm when it is already a promise.
So it should be:
const saveForm = require('./functions/typeform-getter')