We Keep Coding

XMLHttpRequest stream crashing when uploading large files (~1 GB)

I'm trying to make an online file manager for another project with friends, and when uploading files bigger than 1GB, the process either crashes (firefox), or succeeds but the received file weighs 0 bytes (chromium).
JS:
function uploadFile(fileInputId, fileIndex) {
//send file name
try {
var fileName = document.getElementById('fileUploader').files[0].name;
}
catch {
document.getElementById('uploadStatus').innerHTML = `<font color="red">Mettre un fichier serait une bonne idée.</font>`;
return false;
}
document.cookie = 'fname=' + fileName;
//take file from input
const file = document.getElementById(fileInputId).files[fileIndex];
const reader = new FileReader();
reader.readAsBinaryString(file);
reader.onloadend = function(event) {
ajax = new XMLHttpRequest();
//send data
ajax.open("POST", 'uploader.php', true);
//all browser supported sendAsBinary
XMLHttpRequest.prototype.mySendAsBinary = function(text) {
var data = new ArrayBuffer(text.length);
var ui8a = new Uint8Array(data, 0)
for (var i = 0; i < text.length; i++) ui8a[i] = (text.charCodeAt(i) & 0xff);
if (typeof window.Blob == "function") {
var blob = new Blob([data]);
}else {
var bb = new (window.MozBlobBuilder || window.WebKitBlobBuilder || window.BlobBuilder)();
bb.append(data);
var blob = bb.getBlob();
}
this.send(blob);
}
//track progress
var eventSource = ajax.upload || ajax;
eventSource.addEventListener('progress', function(e) {
//percentage
var position = e.position || e.loaded;
var total = e.totalSize || e.total;
var percentage = Math.round((position/total)*100);
document.getElementById('uploadStatus').innerHTML = `${percentage}%`;
});
ajax.onreadystatechange = function() {
if(ajax.readyState == 4 && ajax.status == 200) {
document.getElementById('uploadStatus').innerHTML = this.responseText;
}
}
ajax.mySendAsBinary(event.target.result);
}
}
PHP:
//mysql login
$conn = new PDO([Redacted]);
//file info
$fileName = $_COOKIE['fname'];
$targetDir = "uploads/";
$targetFile = $targetDir.$fileName;
$fileNameRaw = explode('.', $fileName)[0]; //file name with no extension
$tempFilePath = $targetDir.$fileNameRaw.'.tmp';
if (file_exists($targetFile)) {
echo '<font color="red">Un fichier du même nom existe déjà.</font>';
exit();
}
//read from stream
$inputHandler = fopen('php://input', 'r');
//create temp file to store data from stream
$fileHandler = fopen($tempFilePath, 'w+');
//store data from stream
while (true) {
$buffer = fgets($inputHandler, 4096);
if (strlen($buffer) == 0) {
fclose($inputHandler);
fclose($fileHandler);
break;
}
fwrite($fileHandler, $buffer);
}
//when finished
rename($tempFilePath, $targetFile);
chmod($targetFile, 0777);
echo 'Fichier envoyé avec succès !';
$bddInsert = $conn->prepare('INSERT INTO files(nom, chemin) VALUES(?,?)');
$bddInsert->execute(array($fileName, $targetFile));
in my php.ini,
max_execution_time is set to 0
max_input_time to -1
and my post max and upload max sizes are at 4G
I'm using apache2

You should not be reading the file with the fileReader if you don't need it.
Just send the file (blob) directly to your ajax request and avoid the FileReader
function uploadFile (fileInputId, fileIndex) {
// Send file name
try {
var fileName = document.getElementById('fileUploader').files[0].name;
}
catch {
document.getElementById('uploadStatus').innerHTML = `<font color="red">Mettre un fichier serait une bonne idée.</font>`;
return false;
}
document.cookie = 'fname=' + fileName;
// Take file from input
const file = document.getElementById(fileInputId).files[fileIndex];
const ajax = new XMLHttpRequest();
// send data
ajax.open("POST", 'uploader.php', true);
// track progress
ajax.upload.addEventListener('progress', function(e) {
// percentage
var position = e.position || e.loaded;
var total = e.totalSize || e.total;
var percentage = Math.round((position/total)*100);
document.getElementById('uploadStatus').innerHTML = `${percentage}%`;
});
ajax.onreadystatechange = function() {
if (ajax.readyState == 4 && ajax.status == 200) {
document.getElementById('uploadStatus').innerHTML = this.responseText;
}
}
ajax.send(file)
}

A image converted with canvas fails to be read by python PIL _io.BytesIO

Python PIL rejects to read an image you have resized with Javascript canvas
I resize an image on the client-side with Javascript:
var reader = new FileReader();
reader.onload = function (e) {
el('image-picked').src = e.target.result;
el('image-picked').className = '';
var image = new Image();
//compress Image
image.onload=function(){
el("image-picked").src=image.src;
var canvas=document.createElement("canvas");
var context=canvas.getContext("2d");
new_size = get_sizes(image.width,image.height,max_side_px)
[canvas.width,canvas.height] = new_size;
context.drawImage(image,
0,
0,
image.width,
image.height,
0,
0,
canvas.width,
canvas.height
);
console.log("Converted");
//el('image-picked').className = 'no-display'
//el('image-picked').src=""
el('upload-Preview').className = ''
el('upload-Preview').src = canvas.toDataURL("image/png", quality);
The result seems ok, less size, seemingly ok:
There are only minor differences on the files with identify:
(base) ➜ Desktop file before.png after.png
before.png: PNG image data, 4048 x 3036, 8-bit/color RGB, non-interlaced
after.png: PNG image data, 500 x 375, 8-bit/color RGBA, non-interlaced
Then I send the file via POST:
var xhr = new XMLHttpRequest();
var loc = window.location
xhr.open('POST', `${loc.protocol}//${loc.hostname}:${loc.port}/analyze`, true);
xhr.onerror = function() {alert (xhr.responseText);}
xhr.onload = function(e) {
if (this.readyState === 4) {
var response = JSON.parse(e.target.responseText);
el('result-label').innerHTML = `Result = ${response['result']}`;
}
}
var fileData = new FormData();
var file = new File([el('upload-Preview').src],
"image.png", { type: "image/png",
lastModified : new Date()});
fileData.append('file', uploadFiles[0]);
xhr.send(fileData);
And then I read on the server with python open_image(BytesIO(img_bytes)) :
#app.route('/analyze', methods=['POST'])
async def analyze(request):
data = await request.form()
img_bytes = await (data['file'].read())
img = open_image(BytesIO(img_bytes))
The above works without problems with any normal image, but it fails with any image that is the result of the resize with js, and the error is
File "/Users/brunosan/anaconda3/envs/fastai/lib/python3.7/site-packages/PIL/Image.py", line 2705, in open
% (filename if filename else fp))
OSError: cannot identify image file <_io.BytesIO object at 0x124ce6360>```
I've tried canvas.toDataURL("image/jpeg", quality) on the JS side, and reading directly with PIL (not fastai, which calls PIL). It's the same error :frowning_face:

Found the answer here.
I was injecting the image as a a DataURL, when the POST expected a binary. I could see the difference using the "Network" tab:
To convert a DataURL into the binary we need to make a Blob, and then put it into a File:
function dataURItoBlob(dataURI) {
// convert base64/URLEncoded data component to raw binary data held in a string
var byteString;
if (dataURI.split(',')[0].indexOf('base64') >= 0)
byteString = atob(dataURI.split(',')[1]);
else
byteString = unescape(dataURI.split(',')[1]);
// separate out the mime component
var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
// write the bytes of the string to a typed array
var ia = new Uint8Array(byteString.length);
for (var i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
return new Blob([ia], {type:mimeString});
}
blob=dataURItoBlob(el('upload-Preview').src)
var file = new File([blob],
"image.png", { type: "image/png",
lastModified : new Date()});
var fileData = new FormData();
fileData.append('file', file);

Getting corrupted file while exporting .xls file using java/javascript and apache POI

I am trying to downlaod a .xls file in browser from a web application. Below is the code for the same.
try(FileInputStream inputStream = new FileInputStream("C:\\Users\\Desktop\\Book1.xls")){
response.setContentType("application/vnd.ms-excel");
//response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
response.setHeader("Content-Disposition", "attachment; filename=Book1.xls");
outputStream = response.getOutputStream();
byte[] buffer = new byte[BUFFERSIZE];
int bytesRead = -1;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
}
Below is the javascript code used to download the file content.
success: function(response, status, xhr) {
let type = xhr.getResponseHeader('Content-Type');
let blob = new Blob([response], { type: type });
if (typeof window.navigator.msSaveBlob !== 'undefined') {
// IE workaround for "HTML7007: One or more blob URLs were revoked by closing the blob for which they were created.
//These URLs will no longer resolve as the data backing the URL has been freed."
window.navigator.msSaveBlob(blob, filename);
} else {
let URL = window.URL || window.webkitURL;
let downloadUrl = URL.createObjectURL(blob);
if (filename) {
// use HTML5 a[download] attribute to specify filename
let a = document.createElement("a");
// safari doesn't support this yet
if (typeof a.download === 'undefined') {
window.location = downloadUrl;
} else {
a.href = downloadUrl;
a.download = filename;
document.body.appendChild(a);
a.click();
}
} else {
window.location = downloadUrl;
}
setTimeout(function () {
URL.revokeObjectURL(downloadUrl);
}, 100); // cleanup
}
}
I am able to download the file, but downloaded file content is not in readble format. If it is csv file I am able to see content in my javascript response object where as for .xls file javascript response object contains unreadable formatted data.
Can somebody help me here?

Posting this solution if anyone else faces the same issue, I resolved this issue via base64 encoding the byte array to a string as below.
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
workbook.write(outputStream);
String res = Base64.getEncoder().encodeToString(outputStream.toByteArray());
In javascript I decoded that string using base64ToBlob method from below link
https://stackoverflow.com/a/20151856/2011294
function base64toBlob(base64Data, contentType) {
contentType = contentType || '';
var sliceSize = 1024;
var byteCharacters = atob(base64Data);
var bytesLength = byteCharacters.length;
var slicesCount = Math.ceil(bytesLength / sliceSize);
var byteArrays = new Array(slicesCount);
for (var sliceIndex = 0; sliceIndex < slicesCount; ++sliceIndex) {
var begin = sliceIndex * sliceSize;
var end = Math.min(begin + sliceSize, bytesLength);
var bytes = new Array(end - begin);
for (var offset = begin, i = 0; offset < end; ++i, ++offset) {
bytes[i] = byteCharacters[offset].charCodeAt(0);
}
byteArrays[sliceIndex] = new Uint8Array(bytes);
}
return new Blob(byteArrays, { type: contentType });
}

Converting image dataUrl to Blob image for AJAX POST with javascript

I have the following code which will take an image, allow the user to crop (with other code not shown or necessary for this question), and then render the image in base64 from canvas.
I need to be able to convert the image to binary, as the API endpoint its being submitted to can't take base64. I have functionality to convert to a Blob, but I'm not sure how to implement it correctly:
$(function () {
var fileInput = document.getElementById("file"),
renderButton = $("#renderButton"),
submit = $(".submit"),
imgly = new ImglyKit({
container: "#container",
ratio: 1 / 1
});
// As soon as the user selects a file...
fileInput.addEventListener("change", function (event) {
var file;
var fileToBlob = event.target.files[0];
var blob = new Blob([fileToBlob], {
"type": fileToBlob.type
});
// do stuff with blob
console.log(blob);
// Find the selected file
if (event.target.files) {
file = event.target.files[0];
} else {
file = event.target.value;
}
// Use FileReader to turn the selected
// file into a data url. ImglyKit needs
// a data url or an image
var reader = new FileReader();
reader.onload = (function (file) {
return function (e) {
data = e.target.result;
// Run ImglyKit with the selected file
try {
imgly.run(data);
} catch (e) {
if (e.name == "NoSupportError") {
alert("Your browser does not support canvas.");
} else if (e.name == "InvalidError") {
alert("The given file is not an image");
}
}
};
})(file);
reader.readAsDataURL(file);
});
// As soon as the user clicks the render button...
// Listen for "Render final image" click
renderButton.click(function (event) {
var dataUrl;
imgly.renderToDataURL("image/jpeg", {
size: "1200"
}, function (err, dataUrl) {
// `dataUrl` now contains a resized rendered image
//Convert DataURL to Blob to send over Ajax
function dataURItoBlob(dataUrl) {
// convert base64 to raw binary data held in a string
var byteString = atob(dataUrl.split(',')[1]);
// separate out the mime component
var mimeString = dataUrl.split(',')[0].split(':')[1].split(';')[0];
// write the bytes of the string to an ArrayBuffer
var ab = new ArrayBuffer(byteString.length);
var ia = new Uint8Array(ab);
for (var i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
// write the ArrayBuffer to a blob, and you're done
//var bb = new BlobBuilder();
//bb.append(ab);
//return bb.getBlob(mimeString);
return new Blob([ab], {
type: 'image/jpeg'
});
}
var blob = dataURItoBlob(dataUrl);
//console.log("var blob: " + blob);
//var fd = new FormData(document.forms[0]);
var image = $("<img><br>").attr({
src: dataUrl
});
image.appendTo($(".result"))
$removeButton = $('<button class="btn btn-default remove">')
.text('Remove ' + imageid.value).appendTo($(".result"))
.on('click', function () {
panel.remove();
$(this).remove();
return false;
});
$submitButton = $('<div class="btn btn-default submit"></div>')
.text('Submit ' + imageid.value).appendTo($(".result"))
.on('click', function () {
var fd = new FormData;
fd.append('file', blob, 'image.png');
//console.log("var fd: " + fd);
var xhr = new XMLHttpRequest();
var saveImage = encodeURIComponent(dataUrl);
//console.log("SAVE IMAGE: " + saveImage);
//console.log(saveImage);
fd.append("myFile", blob);
xhr.open('POST', 'http://url.com/rest/v1/utils/guid/encode?' + saveImage + '&imageid=' + imageid.value, true);
xhr.send(fd);
});
});
});
});
On Submit, I get the following in the console:
http://url.com/rest/v1/utils/guid/encode?data%3Aimage%2Fjpeg%3Bbase64%2C%2F…CiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAP%2FZ
The current version in jsFiddle: LINK

Safari iOS 6 - ajax request blob image

I have a function that mainly download images in a blob object, and it's working fine on chrome, FF, iOS 7+, but not on iOS 6...
downloadImage: function( url ) {
var that = this;
return new Ember.RSVP.Promise(function( resolve, reject ) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.onreadystatechange = function() {
if (this.readyState === this.DONE) {
that.chart.incrementProgress();
if (this.status === 200) {
var blob = this.response;
resolve( that.imageStore.writeImage( that, url, blob ) );
}
else {
resolve();
}
}
};
xhr.responseType = 'blob';
xhr.send();
});
}
In iOS6 in the console debugger, when I want to see my blob object, its seems to be a string with super weird character in it.. I'm not sure if it normal or my request doesn't work properly on this version of iOS.
After that I need to convert it into a base64, so I use FileReader for that like this :
this.writeImage = function( controller, url, blob ) {
var that = this;
return new Ember.RSVP.Promise(function( resolve ) {
var reader = new window.FileReader();
reader.readAsDataURL(blob);
reader.onloadend = function() {
var base64 = reader.result;
var object = { id: url, key: url, base64: base64 };
//controller.store.update('image', object).save();
controller.store.findQuery('image', { key: url })
.then(function( result ) {
var record = result.content[0];
record._data.base64 = base64;
record.save().then( resolve );
})
.catch(function() {
controller.store.createRecord('image', object).save().then( resolve );
});
};
});
};
Don't pay attention to the Promise thing and other arguments, but the blob is the same as the one in the downloadImage function.
And for a mysterious reason, the reader.loadend is never triggered because the state in reader is always at 0.
Should I do something particular for iOS6 or my code is wrong ?
[edit] : It's like the onloadend callback is not triggered ??
[edit2] : After further investigation, it seems that the response from the ajax request is a string instead of a blob... And my responseType is set as "" as well ?

I have found a workaround for now, I convert my binaryString into a blob like this :
function binaryStringToBlob( byteCharacters, contentType ) {
var sliceSize = 1024;
var bytesLength = byteCharacters.length;
var slicesCount = Math.ceil(bytesLength / sliceSize);
var byteArrays = new Array(slicesCount);
for (var sliceIndex = 0; sliceIndex < slicesCount; ++sliceIndex) {
var begin = sliceIndex * sliceSize;
var end = Math.min(begin + sliceSize, bytesLength);
var bytes = new Array(end - begin);
for (var offset = begin, i = 0 ; offset < end; ++i, ++offset) {
bytes[i] = byteCharacters[offset].charCodeAt(0);
}
byteArrays[sliceIndex] = new Uint8Array(bytes);
}
return new Blob(byteArrays, { type: contentType });
}
You just need to get the content-type and here you go !