I would like to start by saying that (coming from c++ and python) I am totally new to JS and so I welcome any wise suggestions regarding my code.
I wish to read a number of files using the HTML5 file API, then open them with JS, perform some manipulation and download the results zipped. My problem is that reading the files seems to be an asynchronous operation, and I don't really see an elegant way to wait for them all to finish and then zip the results.
One possible solution is presented here:
https://stackoverflow.com/a/17491515
but I am wondering if one can do better than using a global flag.
I also have a problem with retrieving the result from the async function as I have no idea how to get back new_file_list in changeCharsInFiles.
Thank you!
Code example:
HTML:
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
</head>
<body >
<div class="container">
<div class="jumbotron">
<h3>Add Files Here</h3>
<input type="file" id="the-file-field" multiple>
</div>
</div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.1.5/jszip.min.js"></script>
<script src="http://cdn.jsdelivr.net/g/filesaver.js"></script>
<script>
########### SEE JS BELOW #####################
</script>
</body>
JS:
if (window.File && window.FileReader && window.FileList && window.Blob) {
//functions
function zipMyFilesAndSave(file_list){
var zip = new JSZip();
for (var i = 0; i<file_list.length; i+=1)
{
zip.file(file_list[i].name, file_list[i] );
}
zip.generateAsync({type:"blob"}).then(
function (blob) {
saveAs(blob, "hello.zip");
},
function (err) {
jQuery("#blob").text(err);
});
}
function changeCharsInFiles(file_list){
var new_file_list = [];
for (var i = 0; i<file_list.length; i+=1)
{
var file = file_list[i]
var reader = new FileReader();
reader.onload = function() {
//alert(reader.result);
var txt = reader.result;
console.log("txt: ",txt)
var new_txt = ""
var allTextLines = txt.split(/\r\n|\n/);
for (var j = 0; j<allTextLines.length; j+=1)
{
var res = allTextLines[j].replace("a", "A");
res = res.replace("b", "B");
res = res.replace("c", "C");
res = res.replace("d", "D");
new_txt += res + "\n"
}
console.log("new_txt: ", new_txt)
var new_file = new Blob([new_txt], {type: "text/plain"});
new_file_list.push(new_file); //<---------------------------how do I get this back?
}
reader.readAsText(file);
}
return new_file_list;
}
//watcher
$( "#the-file-field" ).change(function() {
console.log("files have been chosen")
var file_list = this.files
file_list = changeCharsInFiles(file_list)
zipMyFilesAndSave(file_list)
});
} else {
alert('The File APIs are not fully supported in this browser.');
}
Try reading up on the Promise class, it was developed to make asynchronous operations easier:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
In your case, you could use it like this:
function changeCharsInFiles(file_list){
let promises = [];
for (let file of file_list) {
let filePromise = new Promise(resolve => {
let reader = new FileReader();
reader.readAsText(file);
reader.onload = () => resolve(reader.result);
});
promises.push(filePromise);
}
Promise.all(promises).then(fileContents => {
// fileContents will be an array containing
// the contents of the files, perform the
// character replacements and other transformations
// here as needed
});
}
This is just a rough outline of the solution. I'd suggest you experiment a bit with Promises first (it can be a fairly deep topic) to figure out the basic principles, and then apply something like the above.
Related
I have a large datasets which i am retrieving via tableau API call. im using async await to call the data and storing this as txt extension.
How i am retrieving the data is by using this script below, script is working as expected and the logic i came out with is
Retrieve data records via api call
Append data into div element
Once data is fully loaded to div, use file streamer to save records as txt file
script used -
<!DOCTYPE html>
<html>
<head>
<title>getData() Basic Example</title>
<script type="text/javascript" src="https://public.tableau.com/javascripts/api/tableau-2.min.js"></script>
<script type="text/javascript">
var viz, sheet, table;
function initViz() {
var containerDiv = document.getElementById("vizContainer"),
url = "http://public.tableau.com/views/RegionalSampleWorkbook/Storms",
options = {
hideTabs: true,
hideToolbar: true,
onFirstInteractive: function () {
document.getElementById('getData').disabled = false; // Enable our button
}
};
viz = new tableau.Viz(containerDiv, url, options);
}
async function savefile(data){
const newHandle = await window.showSaveFilePicker();
const writableStream = await newHandle.createWritable();
await writableStream.write(data)
await writableStream.close();
}
function getUnderlyingData(){
sheet = viz.getWorkbook().getActiveSheet().getWorksheets().get("Storm Map Sheet");
sheet.getUnderlyingDataAsync().then(function(dataTable){
let _tmpdata = ''
for(let i = 0; i < dataTable.getData().length;i++){
for(let a = 0; a < dataTable.getColumns().length;a++){
_tmpdata = dataTable.getData()[i][a].formattedValue;
document.getElementById('storage').innerHTML += _tmpdata
}
}
let whatisthis = document.getElementById('storage').innerHTML
savefile(whatisthis)
});
}
</script>
</head>
<body onload="initViz();">
<div class="page-header">
<button id="getData" onclick="getUnderlyingData()" class="btn" disabled>Get Data</button>
<div id="storage"></div>
</div>
<div id="vizContainer" style="width:600px; height:600px;"></div>
<div id="dataTarget"></div>
</body>
</html>
This is working as expected but what worries me is when i have super large volume of data, the alternative logic i have in mind which i tried to implement is as follow
create streamer inside getunderlyingdata function
append data directly in for loop
New logic i tried, ets say saveFile() does not exist and writableStream are directy implemented in getUnderlyingData, this is script i tried
async function getUnderlyingData(){
// save file to location
const newHandle = await window.showSaveFilePicker();
const writableStream = await newHandle.createWritable();
sheet = viz.getWorkbook().getActiveSheet().getWorksheets().get("Storm Map Sheet");
sheet.getUnderlyingDataAsync().then(async function(dataTable){
let _tmpdata = ''
for(let i = 0; i < dataTable.getData().length;i++){
for(let a = 0; a < dataTable.getColumns().length;a++){
_tmpdata = dataTable.getData()[i][a].formattedValue;
// Write data to stream
await writableStream.write(_tmpdata)
}
}
});
// Close Sream
await writableStream.close();
}
It was not able to capture the data is because the page get reloaded as soon as i tried to save to a location . Is it possible to disable the page reload when a location is selected to save the file ?
I am using the LoadBytes function from P5.JS which returns an object but I am unable to access "object" via using ".bytes" or "['bytes']".
For some reason when running a live server it seems to work perfectly.
function get_rom () {
var fileList = document.getElementById("rom").files;
var fileReader = new FileReader();
if (fileReader && fileList && fileList.length) {
url = URL.createObjectURL(fileList[0])
return loadBytes(url)
}
}
function loadarom () {
object = get_rom()
print(object.bytes[1]) // <-- issue here
}
When using loadBytes() the returned object will not immediately have the bytes array populated. You need to use the callback parameter to loadBytes():
let loadBtn;
function setup() {
noCanvas();
noLoop();
loadBtn = createButton("Load Rom");
loadBtn.mousePressed(loadarom);
}
function get_rom(onSuccess) {
let fileList = document.getElementById("rom").files;
// Alternate method using FileReader has been commented out
// let fileReader = new FileReader();
/*
fileReader.addEventListener("load", function () {
// convert image file to base64 string
loadBytes(fileReader.result, onSuccess);
}); */
if (fileList && fileList.length) {
loadBytes(URL.createObjectURL(fileList[0]), onSuccess);
// fileReader.readAsDataURL(fileList[0]);
}
}
function loadarom() {
get_rom(obj => {
print(obj.bytes[0]);
})
}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.3.1/p5.js"></script>
</head>
<body>
<input id="rom" type="file" />
</body>
</html>
Barley don't need p5.js for such a easy task.
use blob.arrayBuffer()
<script type="module">
var file = new File(['abc'], 'abc.txt')
var bytes = new Uint8Array(await file.arrayBuffer())
console.log(bytes)
</script>
(SO! You should support top level await in a async IIFE fn or start adding type="module" to js-code
I'm currently learning JavaScript and I'm struggling to read a txt file and use its contents in the program, what I have so far:
fileBtn.addEventListener("change", function()
{
var content = [];
var file = fileBtn.files[0];
readFile(file, function(data)
{
console.log(JSON.parse(data));
//content.push(JSON.parse(data)) doesn't work, data is undef.
});
});
and a function readFile
function readFile(file, f)
{
var reader = new FileReader();
reader.onload = function(evt)
{
f(evt.target.result);
};
reader.readAsText(file);
}
My txt file is currenty only containing a "1", and it logs this number to the console but I can't work with it, if I try to push it into an array the values is suddenly undefined. My goal is to use the content of the file in the program later on
1 . no need to use JSON.parse if the text file only contain string .
data is containing all the text file content
2 . you need to put var content = [];
globally and not inside the function readFile
follow this snippet of code i think it will solve your problem
<html lang="en">
<head>
<meta charset="utf-8">
</head>
<body>
<label for="file-upload" class="custom-file-upload">
Custom Upload
</label>
<input id="file-upload" type="file" />
<input id="log-content" type="button" value="Click Me"/>
</body>
<script>
var content = [];
function readFile(file, f) {
var reader = new FileReader();
reader.onload = function (evt) {
f(evt.target.result);
};
var text = reader.readAsText(file);
}
var fileBtn = document.getElementById("file-upload");
var logContnt = document.getElementById("log-content");
logContnt.addEventListener("click", function () {
alert(content);
})
fileBtn.addEventListener("change", function () {
var file = fileBtn.files[0];
readFile(file, function (data) {
content.push(data);
});
});
</script>
</html>
I'm new to d3.js so I know this might seem as a silly question to some so please bear with me. I'm trying to parse a csv file which a user uploads and print it's output in the console. I'm able to parse the CSV file when I provide the absolute path of the CSV file but when I try doing the same with file upload functionality I'm not getting any output in the console..
Working Javascript Code..
var dataset = [];
d3.csv("sample.csv", function(data) {
dataset = data.map(function(d) { return [ d["Title"], d["Category"], d["ASIN/ISBN"], d["Item Total"] ]; });
console.log(dataset[0]);
console.log(dataset.length);
});
Console Output...
["Men's Brooks Ghost 8 Running Shoe Black/High Risk Red/Silver Size 11.5 M US", "Shoes", "B00QH1KYV6", "$120.00 "]
8
New HTML code..
<input type="file" id="csvfile" name="uploadCSV"/>
<br/>
<button onclick="howdy()">submit</button>
Modified Javascript Code(not working)..
var myfile = $("#csvfile").prop('files')[0];
var reader = new FileReader();
reader.onload = function(e) {
var text = reader.result;
}
reader.readAsDataURL(myfile);
var dataset = [];
d3.csv(reader.result , function(data) {
dataset = data.map(function(d) { return [ d["Title"], d["Category"], d["ASIN/ISBN"], d["Item Total"] ]; });
console.log(dataset[0]);
console.log(dataset.length);
})
Since there was no official documentation on how to handle user uploaded CSV file I can't figure out where I'm going wrong..Is there a way I can use HTML5 file reader?? Please help..
You are close but you don't need to and can't call d3.csv on a reader.result. d3.csv makes an async AJAX call to retrieve a CSV file from a server. You already have the file contents and just want to parse, so use d3.csv.parse.
Full working example:
<!DOCTYPE html>
<html>
<head>
<script data-require="d3#3.5.3" data-semver="3.5.3" src="//cdnjs.cloudflare.com/ajax/libs/d3/3.5.3/d3.js"></script>
</head>
<body>
<input type="file" onchange="loadFile()" />
<script>
var reader = new FileReader();
function loadFile() {
var file = document.querySelector('input[type=file]').files[0];
reader.addEventListener("load", parseFile, false);
if (file) {
reader.readAsText(file);
}
}
function parseFile(){
var doesColumnExist = false;
var data = d3.csv.parse(reader.result, function(d){
doesColumnExist = d.hasOwnProperty("someColumn");
return d;
});
console.log(doesColumnExist);
}
</script>
</body>
</html>
This is for d3-csv#3
<!-- https://www.jsdelivr.com/package/npm/d3-dsv -->
<script src="https://cdn.jsdelivr.net/npm/d3-dsv#3.0.1/dist/d3-dsv.min.js" integrity="sha256-IrzYc2a3nTkfvgAyowm/WKmIGdVCMCcccPtz+Y2y6VI=" crossorigin="anonymous"></script>
<input type="file" accept=".csv">
<button>test button</button>
<script>
const testData = `owner,repo,"branch name"
foo,demo,master
boo,"js awesome",sha1123456
`
document.querySelector(`input`).onchange = async e => {
const input = e.target
const file = input.files[0]
const reader = new FileReader()
reader.readAsText(new Blob(
[file],
{"type": file.type}
))
const fileContent = await new Promise(resolve => {
reader.onloadend = (event) => {
resolve(event.target.result)
}
})
const csvData = d3.csvParse(fileContent)
console.log(csvData)
}
document.querySelector(`button`).onclick = e => {
const csvData = d3.csvParse(testData)
console.log(csvData)
}
</script>
The below link may help you know the implementation of csvParse
csv.js : The csv, tsv(tab) are dependent by dsv.js
dsv.js
If you just load the CSV only then do not import the whole JS. (instead of the d3-csv.js)
https://cdn.jsdelivr.net/npm/d3#7.0.1/dist/d3.min.js
https://cdn.jsdelivr.net/npm/d3-dsv#3.0.1/dist/d3-dsv.min.js
This is an old question and I think we have to clarify some points.
How to load a local csv file
How to link the loaded file with D3
1. Load a file is very simple just check this example:
const fileInput = document.getElementById('csv')
const readFile = e => {
const reader = new FileReader()
reader.onload = () => {
document.getElementById('out').textContent = reader.result
}
reader.readAsBinaryString(fileInput.files[0])
}
fileInput.onchange = readFile
<div>
<p>Select local CSV File:</p>
<input id="csv" type="file" accept=".csv">
</div>
<pre id="out"><p>File contents will appear here</p></pre>
Here we have a simple input element with type="file" attribute, this lets us to pick a csv file. Then the readFile() function will be triggered whenever a file is selected and will call the onload function after reading the file as a binary string.
2. I recommend to use readAsDataURL() to integrate it with d3 like this:
const fileInput = document.getElementById('csv')
const previewCSVData = async dataurl => {
const d = await d3.csv(dataurl)
console.log(d)
}
const readFile = e => {
const file = fileInput.files[0]
const reader = new FileReader()
reader.onload = () => {
const dataUrl = reader.result;
previewCSVData(dataUrl)
}
reader.readAsDataURL(file)
}
fileInput.onchange = readFile
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>
<div>
<p>Select local CSV File:</p>
<input id="csv" type="file" accept=".csv">
</div>
<pre id="out"><p>File contents will appear here</p></pre>
To integrate the loaded file we call previewCSVData() and pass the file then we parse it using d3.csv() method. Also lets use await because it is an asynchronous call.
Note:
d3.csv internally uses fetch and works for any type of URL, (httpURL, dataURL, blobURL, etc...)
I'm tying to read and output a text file. The Chrome console complains:
caught TypeError: Cannot read property '0' of undefined FinanceDashBoard.html:22"
Not sure what I am doing wrong ?
The code is as follows:
<html>
<head>
<title>Read File (via User Input selection)</title>
</head>
<body>
<main>
<label>Load a text database file: <input type="file" id="txtfile" ></label>
</main>
<script type="text/javascript">
var dbFileElm = document.getElementById('txtfile');
dbFileElm.onchange = function() {
var filePath = dbFileElm.files[0];
var reader = new FileReader();
var output = ""; //placeholder for text output
reader.onload = function (e) {
output = e.target.result;
displayContents(output);
}
reader.readAsText(filePath.files[0]);
}
// Ignore code below it doesn't work yet.
function displayContents(txt) {
var el = document.getElementById('main');
el.innerHTML = txt; //display output in DOM
}
</script>
</body>
</html>
Two mistakes.
1) Change this line:
reader.readAsText(filePath.files[0]);
to this:
reader.readAsText(filePath);
Because filePath is already: dbFileElm.files[0];
2) The main tag has no ID, so getting element by ID main will not work.
Just edit it to:
<main id="main">
You have no elements that have id="main"
Try something like...
<main id="main"> ...
Or if you're trying to populate your text box...
var el = document.getElementById('txtfile');
this post has answer to question
HTML input file selection event not firing upon selecting the same file
and if you set your input value to null, is working for me
dbFileElm.onChange = function() {
this.value = null;
var filePath = dbFileElm.files[0];
var reader = new FileReader();
var output = ""; //placeholder for text output
reader.onload = function (e) {
output = e.target.result;
displayContents(output);
}
reader.readAsText(filePath.files[0]);
}