I'm newbie in JS and I need to create simple CSV parser for test data. And I have a very strange problem.
This is code of my test:
'use strict';
const assert = require('assert');
const HomePage = require('../pages/HomePage');
const csv = require('../tools/CsvReader');
describe('Check if values are correct', () => {
let dataProvider = csv.readFromCsv("test/resources/places.csv");
function valuesTest(city, expectedLat, expectedLong) {
it('Data must match', () => {
let searchPlace = HomePage
.open()
.findByPlaceName(city)
.getSearchPlace();
assert.strictEqual(searchPlace.getLatitude(), expectedLat);
assert.strictEqual(searchPlace.getLongtitude(), expectedLong);
console.log(dataProvider[0].CITY);
});
}
for (let i = 0; i < dataProvider.length; i++) {
valuesTest(dataProvider[i].CITY, dataProvider[i].LAT, dataProvider[i].LONG)
}
});
And code of my CSV-reader:
'use strict';
const csv = require('csv-parser');
const fs = require('fs');
class CsvReader {
readFromCsv(path) {
let results = [];
fs.createReadStream(path)
.pipe(csv())
.on('data', (data) => results.push(data));
return results;
}
}
module.exports = new CsvReader();
And this is my CSV:
CITY,LAT,LONG
Kyiv,50.447731,30.542721
Lviv,49.839684,24.029716
Ivano-Frankivsk,48.922634,24.711117
The problem is as follows: Why can I use the variable "dataProvider" in the "valuesTest" block and it works correctly and returns the value of the CITY variable, but in the "for" loop I can't use it (variable "dataProvider", "CITY", etc. is unavailable there), although they are located in the same block "describe".
Your CSV Reader is an asynchronous operation. I suspect your for loop gets executed even before the csv is parsed and value is returned. Try putting your for loop in a function and passing to the readFromCsv function as a callback. Call this function on the data event, where you will be sure to get the data.
You should pass a callback to readFromCsv that will be executed when the createReadStream is complete. You can determine when createReadStream is complete by listening to its end event. (See csv-parser example code for instance).
In your case, you could do something like:
readFromCsv(path, onEnd) {
let results = [];
fs.createReadStream(path)
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => onEnd(results));
}
csv.readFromCsv("test/resources/places.csv", function onData(data) {
for (let i = 0; i < data.length; i++) {
valuesTest(data[i].CITY, data[i].LAT, data[i].LONG);
}
});
You should use the end event instead of the data event to determine when the stream is complete. If you returned after the first data event, you might not get all of the data.
The data event is fired once per data chunk. If your data has more than one "chunk", you'll truncate it if you return in the data callback. See nodeJS docs for details on the different event types. You might not notice a difference with small test files, but make a larger file and you'll see the difference.
Related
I have a folder that has some json files in it, so I want to access them through my backend dynamically without the hassle to hard code changing data.
The Code works except one thing
localhost/Foo.json
Prints {Name:Bar} => expected Foo
localhost/Bar.json
Prints {Name:Bar} => expected Bar
so always the last entry in the data array is shown
interface filesPath{
name:string,
ext:string, //here .json
filepath:string,
}
var generateJson = (files:filesPath[]) => {
var dataFiles = [];
var dataNames = [];
for(var x in files){
dataFiles[x] = require(files[x].filepath);
dataNames[x] = files[x].name;
app.get('/'+dataNames[x]+files[x].ext,(req, res) => {
res.json(dataFiles[x]) //Returns always last entry in array
});
}
}
Is there a simple fix?
Thanks in advance
You've stumbled upon the concept of closures. What is happening is that the x you are passing to your function is not a copy of the value of x but the variable x itself (kind of like a pointer). This means all functions share the same variable thus one variable can only have one value.
The classical way to solve this is to pass x as an argument, thus breaking the closure:
for(var x in files){
dataFiles[x] = require(files[x].filepath);
dataNames[x] = files[x].name;
app.get('/'+dataNames[x]+files[x].ext,((new_x) => (req, res) => {
res.json(dataFiles[new_x])
})(x)); // <-- pass in x here to become new_x
}
ES6 introduced the concept of let which does not create a closure. So a more modern solution is to simply use let instead of var:
for(let x in files){ // <-- this fixes it!!
dataFiles[x] = require(files[x].filepath);
dataNames[x] = files[x].name;
app.get('/'+dataNames[x]+files[x].ext,(req, res) => {
res.json(dataFiles[x])
});
}
I'm having some problems getting the asynchronous nature of node to co-operate with me, and after hours of callbacks and googling; I finally turn to you guys.
I have a program that needs to read in lines from a file using the readline module of node. This file contains data that is passed to some asynchronous functions defined within my node program. Once all the data is successfully read and processed, this data needs to be parsed into JSON format, and then outputted.
My problem here is that when I call: readLine.on('close', function() { ...... }, this is run before the asynchronous functions finish running, and therefore I am left with an output of nothing, but the program keeps running the asynchronous functions.
I've created a simple skeleton of functions that should explain my situation more clearly:
function firstAsyncFunc(dataFromFile) {
//do something asynchronously
return processedData;
}
function secondAsyncFunc(dataFromFile) {
//do something else asynchronously
return processedData;
}
//create readline
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream('data.txt')
});
//array to hold all the data processed
var totalDataStorage;
//read file
lineReader.on('line', function(line) {
var processedData = firstAsyncFunction(line);
var moreProcessedData = secondAsyncFunction(line);
//store processed data and concatenate into one array
var tempDataStorage = [{ 'first': processedData, 'second': moreProcessedData }]
totalDataStorage = totalDataStorage.concat(tempDataStorage);
}).on('close', function() {
var JSONString = JSON.stringify(... //create JSON for totalDataStorage ...);
console.log(JSONString); //DOESN'T OUTPUT ANYTHING!
});
I have tried to add a callback to the first/secondAsynFunction, I have tried to make the reading and parsing bit of the program seperate functions, and create callbacks so that parsing is only called when reading finished, but none of those solutions seemed to be working and i'm really struggling - so any help would be appreciated.
Thanks!
EDIT: The data.txt file is of the form
IPData1 DataCenter1
IPData2 DataCenter2
...
IPDataN DataCenterN
I use str.split(" ") to get the respective values, and then pass them appropriately. IPData is a number, and DataCenter is a string
Asynchronous functions do not return a value, but you must pass a callback function to it instead. Your line
var processedData = firstAsyncFunction(line);
doesn't make sense at all. If your data.txt file looks like this
IPData1 DataCenter1
IPData2 DataCenter2
IPData3 DataCenter3
you can read data as following
var fs = require('fs');
var rl = require('readline').createInterface({
input: fs.createReadStream('data.txt')
});
var arr = [];
rl.on('line', a => {
a = a.split(' ');
arr.push({
first: a[0],
second: a[1]
});
}).on('close', () => {
console.log(JSON.stringify(arr, null, 2));
});
It will log
[
{
"first": "IPData1",
"second": "DataCenter1"
},
{
"first": "IPData2",
"second": "DataCenter2"
},
{
"first": "IPData3",
"second": "DataCenter3"
}
]
I changed the following and its working locally.
use promises to make your life easier.
remove .close since you dont have an output defined in the interface.
The 'close' event is emitted when one of the following occur:
The rl.close() method is called and the readline.Interface instance has relinquished control over the input and output streams;
The input stream receives its 'end' event;
The input stream receives -D to signal end-of-transmission (EOT);
The input stream receives -C to signal SIGINT and there is no SIGINT event listener registered on the readline.Interface instance.
function firstAsyncFunc(dataFromFile) {
return new Promise(function(resolve, reject) {
//do something asynchronously
resolve(result);
})
}
function secondAsyncFunc(dataFromFile) {
return new Promise(function(resolve, reject) {
//do something asynchronously
resolve(result);
})
}
//create readline
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream('data.txt')
});
//array to hold all the data processed
var totalDataStorage;
//read file
lineReader.on('line', function(line) {
Promise.all([
firstAsyncFunc(line),
secondAsyncFunc(line)
])
.then(function(results) {
var tempDataStorage = [{
'first': results[0],
'second': results[1]
}];
// i'd use push instead of concat
totalDataStorage = totalDataStorage.concat(tempDataStorage);
});
})
I'm using fast-csv's fromPath() method to read data from a file. I would like to write this data into an array (which I will subsequently sort). I would expect the code below to work for this purpose, but it does not:
var csv = require('fast-csv');
var dataArr = [];
csv.fromPath("datas.csv", {headers: true})
.on("data", data => {
console.log(data);
// > { num: '4319', year: '1997', month: '4', day: '20', ...
dataArr.push(data);
});
console.log(dataArr);
// > []
I am able to read the data in the file with this code, but the array is not populated.
What is a good way to accomplish this, and why does the code above not work?
Well, I know that this question has been asked a long back but just now I got to work with CSV file for creating API with node js. Being a typical programmer I googled "Reading from a file with fast-csv and writing into an array" well something like this but till date, there isn't any proper response for the question hence I decided to answer this.
Well on is async function and hence execution will be paused in main flow and will be resumed only after nonasync function gets executed.
var queryParameter = ()=> new Promise( resolve =>{
let returnLit = []
csv.fromPath("<fileName>", {headers : true})
.on('data',(data)=>{
returnLit.push(data[<header name>].trim())
})
.on('end',()=>{
resolve(returnLit)
})
})
var mainList = [];
queryParameter().then((res)=>mainList = res)
If you want to validate something pass argument into queryParameter() and uses the argument in validate method.
The "on data" callback is asynchronous, and the commands that follow the callback will run before the callback finishes. This is why the code does not work, and this reasoning has been pointed out by others who have posted answers and comments.
As for a good way to accomplish the task, I have found that using the "on end" callback is a good fit; since the intention here is to "do something" with the whole data, after the file has been read completely.
var dataArr = [];
csv.fromPath("datas.csv", {headers: true})
.on("data", data => {
dataArr.push(data);
})
.on("end", () => {
console.log(dataArr.length);
// > 4187
});
As of "fast-csv": "^4.1.3" the approach by #ChandraKumar no longer works
The fromPath function has been removed in place of "parseFile"
var queryParameter = ()=> new Promise( resolve =>{
let returnLit = []
csv.parseFile("<fileName>", {headers : true})
.on('data',(data)=>{
returnLit.push(data[<header name>].trim())
})
.on('end',()=>{
resolve(returnLit)
})
})
var mainList = [];
queryParameter().then((res)=>mainList = res)
The "on data" callback of the module is asynchronous. Therefore, this line
console.log(dataArr);
will always return empty because it runs before the callback.
To fix this you need to process the array and sort it within the callback. For example:
var dataArr = [];
csv.fromPath("datas.csv", {headers: true})
.on("data", data => {
dataArr.push(data);
var sorted = _.sortBy(dataArr, 'propertyX');
// do something with 'sorted'
});
In a Chrome extension im using the HTML5 FileSytem API.
Im retrieving a list of records in a folder.
var entries = [];
var metadata = [];
listFiles(folder);
function listFiles(fs) {
var dirReader = fs.createReader();
entries = [];
// Call the reader.readEntries() until no more results are returned.
var readEntries = function () {
dirReader.readEntries(function (results) {
if (!results.length) {
addMeta(entries);
} else {
console.log(results);
entries = entries.concat(toArray(results));
readEntries();
}
});
};
readEntries(); // Start reading dirs.
}
The FileEntry object does not contain metadata, I need the last modified date. I'm able to retrieve a object of metadata
function addMeta(entries) {
for (var i = 0; i < entries.length; i++) {
entries[i].getMetadata(function (metadata) {
console.log(entries);
console.log(metadata);
});
}
}
Problem is that i get the metadata in a callback.
How can i join the two object making sure the right match is made?
The simplified result im looking for is:
[
["fileName1", "modifyDate1"],
["fileName2", "modifyDate2"],
]
To get lastModifiedDate, you don't need to use getMetadata, as per the description of this question, just use entry.file.lastModifiedDate, though maybe file() is another callback.
To "join the two object making sure the right match is made", because of Closures, you could use the following code to get the right results. (Assuming the data structure is [[entry, metadata]] as you mentioned)
var ans = [];
function addMeta(entries) {
for (var i = 0; i < entries.length; i++) {
(function(entry) {
entry.getMetadata(function (metadata) {
ans.push([entry, metadata]);
});
}(entries[i]);
}
}
If what you want is to wait for all asynchronous callback ends, see this answer for more details, basically you could adjust your code and use Promise, or use other implementations like setInterval or use a variable to calculate how many callbacks remain.
I suggest to have a look on Promise-based bro-fs implementation of HTML Filesystem API.
To read all entries with metadata you can do something like this:
fs.readdir('dir')
.then(entries => {
const tasks = entries.map(entry => fs.stat(entry.fullPath))
return Promise.all(tasks);
})
.then(results => console.log(results))
It is not a question about how to do post multipart form in nodejs.
But how to do such logic(first do a n times loops(async) then one time function(async)) in callback way?
for example, client will post multipart form with normal form fields:
req.files[n]: contains n images, needs to save to server's local filesystem
req.body: contains post.title, post.content, post.user
In normal way(php, java...), sample code would be
array savedPath = [];
// save images to local filesystem
foreach image in files
savedPath.push(saveImageToLocal(image))
// append saved images path to post
var post = req.body;
post.images = savedPath;
Posts.insert(post)
But in nodejs, callback way, how can i write it?
var savedPath = [];
saveImageToLocal(files[0], function(path) {
savedPath.push(path);
saveImageToLocal(files[1], function(path) {
savedPath.push(path);
//.... its n elements, how can I write it??
var post = req.body;
post.images = savedPath;
Posts.insert(postfunction(err, result) {
res.send(err, result)
});
});
});
Or
var savedPath = [];
for(i=0;i<n;i++) {
savesaveImageToLocalTo(files[i], function(path) {
savedPath.push(path);
});
}
waitSaveToFinished() ??
var post = req.body;
post.images = savedPath;
Posts.insert(postfunction(err, result) {
res.send(err, result)
});
How to do these kind of things in the way of nodejs/callback?
The best way to coordinate multiple asynchronous operation is to use promises. So, if this were my code, I would change or wrap saveImageToLocalTo() and Posts.insert() to return promises and then use promise features for coordinating them. If you're going to be writing much node.js code, I'd suggest you immediately invest in learning how promises work and start using them for all async behavior.
To solve your issue without promises, you'd have to implement a counter and keep track of when all the async operations are done:
var savedPath = [];
var doneCnt = 0;
for(i=0;i<n;i++) {
savesaveImageToLocalTo(files[i], function(path) {
++doneCnt;
savedPath.push(path);
// if all the requests have finished now, then do the next steps
if (doneCnt === n) {
var post = req.body;
post.images = savedPath;
Posts.insert(postfunction(err, result) {
res.send(err, result)
});
}
});
}
This code looks like it missing error handling since most async operations have a possibility of errors and can return an error code.