I am trying to upload large binary files from a web client to a .NET 4.6.1 Framework MVC API. These files could range anywhere from 5GB to 20GB.
I have tried splitting the file into chunks to upload each chunk and merge the results at the end, but the merged file is always corrupted. If I work with small files and don't split, the binary will work correctly. However, when I split and merge the file is "corrupted". It won't load or behave as expected.
I have looked all over and haven't seen a proper solution to this so i'm hoping someone can help me here.
I followed this https://forums.asp.net/t/1742612.aspx?How+to+upload+a+big+file+in+Mvc, but I can't get it to work and the corrected solution was never posted. I am keeping track of the order of files before merging on the server.
Javascript (Call to uploadData is made to initiate)
function uploadComplete(file) {
var formData = new FormData();
formData.append('fileName', file.name);
formData.append('completed', true);
var xhr3 = new XMLHttpRequest();
xhr3.open("POST", "api/CompleteUpload", true); //combine the chunks together
xhr3.send(formData);
return;
}
function uploadData(item) {
var blob = item.zipFile;
var BYTES_PER_CHUNK = 750000000; // sample chunk sizes.
var SIZE = blob.size;
//upload content
var start = 0;
var end = BYTES_PER_CHUNK;
var completed = 0;
var count = SIZE % BYTES_PER_CHUNK == 0 ? SIZE / BYTES_PER_CHUNK : Math.floor(SIZE / BYTES_PER_CHUNK) + 1;
while (start < SIZE) {
var chunk = blob.slice(start, end);
var xhr = new XMLHttpRequest();
xhr.onload = function () {
completed = completed + 1;
if (completed === count) {
uploadComplete(item.zipFile);
}
};
xhr.open("POST", "/api/MultiUpload", true);
xhr.setRequestHeader("contentType", false);
xhr.setRequestHeader("processData", false);
xhr.send(chunk);
start = end;
end = start + BYTES_PER_CHUNK;
}
}
Server Controller
//global vars
public static List<string> myList = new List<string>();
[HttpPost]
[Route("CompleteUpload")]
public string CompleteUpload()
{
var request = HttpContext.Current.Request;
//verify all parameters were defined
var form = request.Form;
string fileName;
bool completed;
if (!string.IsNullOrEmpty(request.Form["fileName"]) &&
!string.IsNullOrEmpty(request.Form["completed"]))
{
fileName = request.Form["fileName"];
completed = bool.Parse(request.Form["completed"]);
}
else
{
return "Invalid upload request";
}
if (completed)
{
string path = HttpContext.Current.Server.MapPath("~/Data/uploads/Tamp");
string newpath = Path.Combine(path, fileName);
string[] filePaths = Directory.GetFiles(path);
foreach (string item in myList)
{
MergeFiles(newpath, item);
}
}
//Remove all items from list after request is done
myList.Clear();
return "success";
}
private static void MergeFiles(string file1, string file2)
{
FileStream fs1 = null;
FileStream fs2 = null;
try
{
fs1 = System.IO.File.Open(file1, FileMode.Append);
fs2 = System.IO.File.Open(file2, FileMode.Open);
byte[] fs2Content = new byte[fs2.Length];
fs2.Read(fs2Content, 0, (int)fs2.Length);
fs1.Write(fs2Content, 0, (int)fs2.Length);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message + " : " + ex.StackTrace + " " + file2);
}
finally
{
if(fs1 != null) fs1.Close();
if (fs2 != null)
{
fs2.Close();
System.IO.File.Delete(file2);
}
}
}
[HttpPost]
[Route("MultiUpload")]
public string MultiUpload()
{
try
{
var request = HttpContext.Current.Request;
var chunks = request.InputStream;
string path = HttpContext.Current.Server.MapPath("~/Data/uploads/Tamp");
string fileName = Path.GetTempFileName();
string newpath = Path.Combine(path, fileName);
myList.Add(newpath);
using (System.IO.FileStream fs = System.IO.File.Create(newpath))
{
byte[] bytes = new byte[77570];
int bytesRead;
while ((bytesRead = request.InputStream.Read(bytes, 0, bytes.Length)) > 0)
{
fs.Write(bytes, 0, bytesRead);
}
}
return "test";
}
catch (Exception exception)
{
return exception.Message;
}
}
Related
i'm trying to do chunk uplod for large file so i slice the file and send each slice in request.
i send the chunk and the chunk number and total chank number withrequest with XmlHttpRequest to the controller.
but in the method on controller i c'ant acces to the chunk index or the total chunk il always 0 .
[HttpPost]
public string UploadChunks( int CHUNK_INDEX, int TOTAL_CHUNK)
{
var chunks = Request.Body;
string path = Path.Combine(_hostEnvironment.WebRootPath, #"file\temp");
string newpath = Path.Combine(path, Guid.NewGuid()+".tmp"+CHUNK_INDEX.ToString());
using (System.IO.FileStream fs = System.IO.File.Create(newpath))
{
byte[] bytes = new byte[775700];
int bytesRead;
while ((bytesRead = chunks.Read(bytes, 0, bytes.Length)) > 0)
{
fs.Write(bytes, 0, bytesRead);
}
}
return "succes" ;
}
the js function
function upload(file) {
var blob = file;
var BYTES_PER_CHUNK = 775700;
var SIZE = file.size;
var start = 0;
var end = BYTES_PER_CHUNK;
var completed = 0;
var count = SIZE % BYTES_PER_CHUNK == 0 ? SIZE / BYTES_PER_CHUNK : Math.floor(SIZE / BYTES_PER_CHUNK) + 1;
var index = 1;
while (start < SIZE) {
var chunk = blob.slice(start, end);
var data = new FormData();
data.append("TOTAL_CHUNK", count);
data.append("CHUNK_INDEX", index++);
data.append("CHUNK", chunk);
var xhr = new XMLHttpRequest();
xhr.open("POST", "UploadChunks", true);
xhr.setRequestHeader("Content-Type", "multipart/form-data");
xhr.send(data);
/* fetch("/UploadFile/MultiUpload", {
method: 'post',
body: data
});*/
start = end
end = start + BYTES_PER_CHUNK;
}
}
Change your js like below:
while (start < SIZE) {
index++;
var xhr = new XMLHttpRequest();
xhr.open("POST", "UploadChunks", true);
xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
xhr.send("TOTAL_CHUNK=" + count + "&CHUNK_INDEX=" + index);
}
My purpose is to display pdf use pdf.js in lazy mode,I have two choice:
Use disableRange=false
It worked fine when use a url in Nginx, but when I use a java servlet url: /dowload/fileid/123，it doesn't load via 206 partial content (range requests) but 200 and then viewed in the viewer.
class Scratch {
public static void main(String[] args) {
public void download (String identNo, HttpServletRequest request, HttpServletResponse response){
File file = getFileFromServer(identNo);
BufferedInputStream bis = null;
OutputStream os = null;
BufferedOutputStream bos = null;
InputStream is = null;
try {
is = new FileInputStream(file);
bis = new BufferedInputStream(is);
os = response.getOutputStream();
bos = new BufferedOutputStream(os);
int startByte, endByte, totalByte;
if (request != null && request.getHeader("range") != null) {
String[] range = request.getHeader("range").replaceAll("[^0-9\\-]", "").split("-");
totalByte = is.available();
startByte = Integer.parseInt(range[0]);
if (range.length > 1) {
endByte = Integer.parseInt(range[1]);
} else {
endByte = totalByte - 1;
}
response.setStatus(206);
} else {
totalByte = is.available();
startByte = 0;
endByte = totalByte - 1;
response.setHeader("Accept-Ranges", "bytes");
response.setStatus(200);
}
int length = endByte - startByte + 1;
response.setHeader("Accept-Ranges", "bytes");
response.setHeader("Content-Range", "bytes " + startByte + "-" + endByte + "/" + totalByte);
response.setContentType("Content-Type: application/octet-stream");
response.setContentLength(length);
bis.skip(startByte);
int len = 0;
byte[] buff = new byte[1024 * 64];
while ((len = bis.read(buff, 0, buff.length)) != -1) {
if (length <= len) {
bos.write(buff, 0, length);
break;
} else {
length -= len;
bos.write(buff, 0, len);
}
}
} catch (IOException e) {
} finally {
FileUtil.closeQuiet(bos);
FileUtil.closeQuiet(os);
FileUtil.closeQuiet(bis);
FileUtil.closeQuiet(is);
}
}
}
}
Split big pdf file in server side，then display multiple pdf documents as one with pdf.js
I found this: Is there a way to combine PDFs in pdf.js?
but it needed to load all file at once, what I need is: when scroll to bottom of file, load next file then merge to current pdf
PDF.js，set the parameter： "disableAutoFetch": true, "disableStream": true
server support accept ranges,range
Then pdf.js whill only fetch data in pages are visible now from server
https://github.com/mozilla/pdf.js/issues/11933
I am making Thumbnail View pages.
I stored thumbnail images external folder(C:/temp/image) and I get them from server as byte data.
now I want to convert this data into image in javascript and show in HTML.
so I tried to make blob URL using these byte data. But I got error "FILE NOT FOUND"
Here is my code. Please let me know what i'm missing
(It is Spring boot and angular.js )
Service
public List<Map<String, Object>> listRecentWithInfo( int userid) throws IOException, SerialException, SQLException {
List<Map<String, Object>> recentList = dashboardDao.listRecentWithInfo( userid);
for (Map<String, Object> map : recentList) {
int dashboardid = (int) map.get( "DASHBOARDID");
FileInputStream is = null;
BufferedImage bi = null;
ByteArrayOutputStream bao= null;
byte[] imageByte = null;
ResponseEntity<byte[]> res = null;
HttpHeaders headers = new HttpHeaders();
headers.setCacheControl(CacheControl.noCache().getHeaderValue());
if (thumbnailDao.findOne( dashboardid) != null) {
String path = thumbnailPath + thumbnailDao.getOne( dashboardid).getFileName();
is = new FileInputStream(path);
bi = ImageIO.read(is);
System.out.println(bi.getType());
bao = new ByteArrayOutputStream();
ImageIO.write(bi, "png", bao);
imageByte = bao.toByteArray();
res = new ResponseEntity<>(imageByte, headers, HttpStatus.OK);
}
map.put("THUMBNAIL", res);
}
return recentList;
}
js
$http.get("app/dashboard/recentWithInfo").then( function( rtn) {
rtn.data.map(item=> {
if (item.THUMBNAIL) {
var bytes = new Uint8Array(item.THUMBNAIL.body / 2);
for (var i = 0; i < item.THUMBNAIL.body; i += 2) {
bytes[i / 2] = parseInt(item.THUMBNAIL.body.substring(i, i + 2), /* base = */ 16);
}
// Make a Blob from the bytes
var blob = new Blob([bytes], {type: 'image/png'});
var imageUrl = URL.createObjectURL(blob);
URL.revokeObjectURL(imageUrl);
item.THUMBNAIL = imageUrl;
}
});
$scope.items = rtn.data;
console.log(rtn.data);
});
});
Thanks in advance!
I Got a solution By myself
I realized that I can convert Byte Data into Base64 just like that
"data:image/png;base64," + item.THUMBNAIL.body;
and this BASE64 encoded data can be used as Image source in HTML!
Can anyone help me with this please?
I have a simple test Java servlet as shown below:
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
byte[] bytes = ReadWaveformAsBinary();
response.setContentType("application/octet-stream");
response.setContentLength(bytes.length);
ServletOutputStream servletOutputStream = response.getOutputStream();
servletOutputStream.write(bytes, 0, bytes.length);
servletOutputStream.flush();
servletOutputStream.close();
}
This function works. It returns a byte array with 10 double precision numbers in it. I know its all working because I can call it from a C# application as below:
public static bool CallWebServiceDownloadEndPoint(string szWebEndPoint, string szRequest, out double[] data)
{
data = null;
bool bSuccess = true;
WebClient webClient = new WebClient();
try
{
byte[] byteData = webClient.DownloadData(szWebEndPoint + "?" + szRequest);
Array.Reverse(byteData);
data = CreateDoubleArrayFromByteArray(byteData);
Array.Reverse(data);
}
catch
{
bSuccess = false;
}
return bSuccess;
}
The resultant byte array has the expected size of 80 bytes (10 * 8 bytes) and the 10 numbers are all as expected.
My question is, how can I call this Java servlet from JavaScript via an AJAX call?
For instance, I tried the following:
function AJAXSendString(ajaxRequestObject, szURL, szParams, OnCallCompleted)
{
if (ajaxRequestObject != null)
{
ajaxRequestObject.open("GET", szURL, true);
ajaxRequestObject.responseType = "arraybuffer";
ajaxRequestObject.onreadystatechange = function ()
{
if (ajaxRequestObject.readyState == 4)
{
if (ajaxRequestObject.status == 200)
{
var arrayBuffer = ajaxRequestObject.response;
if(arrayBuffer)
{
var byteArray = new Uint8Array(arrayBuffer);
alert(byteArray.byteLength);
}
}
}
}
ajaxRequestObject.send(szParams);
}
But the alert box says 19 (not 80 as I hoped it would).
Thanks for any help.
As suggested I try the following but I get the same result :(
function AJAXSendString2(ajaxRequestObject, szURL, szParams, OnCallCompleted)
{
if (ajaxRequestObject != null)
{
ajaxRequestObject.open("GET", szURL, true);
ajaxRequestObject.responseType = "arraybuffer";
ajaxRequestObject.onload = function(oEvent)
{
var arrayBuffer = ajaxRequestObject.response;
if(arrayBuffer)
{
var byteArray = new Uint8Array(arrayBuffer);
alert(byteArray.byteLength);
}
}
/*ajaxRequestObject.onreadystatechange = function ()
{
if (ajaxRequestObject.readyState == 4)
{
if (ajaxRequestObject.status == 200)
{
var arrayBuffer = ajaxRequestObject.response;
if(arrayBuffer)
{
var byteArray = new Uint8Array(arrayBuffer);
alert(byteArray.byteLength);
OnCallCompleted("1,-1,0,0,-1,1");
}
}
}
}*/
ajaxRequestObject.send(szParams);
}
}
I still see 19 and not 80.
You're attempting to send your parameters in the request body, since it is a GET request they should be in the url
ajaxRequestObject.open("GET", szURL+'?'+szParams, true);
You should use the "onload" event as in this example to get the complete payload/response.
https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Sending_and_Receiving_Binary_Data
I am experiencing high memory consumption on my Node.js app, when loading ~100MB zip files one after the other it is keeping them in memory as a "NodeBufferReader". The library I am using is called JSZip and is found here: https://stuk.github.io/jszip/
If I access the same zip file twice then it doesn't increase memory usage but for every 'extra' .zip file I access the memory increases by approx the size of the .zip file. The files I am accessing are all around 100MB or larger so as you can imagine this has the potential to get rather large, rather quickly.
The Node.js application is a websocket server that reads files from within .zip files and returns them back to the requestor as base64 data. The function in question is here:
function handleFileRequest(args, connection_id) {
var zipIndex = 0,
pathLen = 0,
zip_file = "",
zip_subdir = "";
try {
if (args.custom.file.indexOf(".zip") > -1) {
// We have a .zip directory!
zipIndex = args.custom.file.indexOf(".zip") + 4;
pathLen = args.custom.file.length;
zip_file = args.custom.file.substring(0, zipIndex);
zip_subdir = args.custom.file.substring(zipIndex + 1, pathLen);
fs.readFile(zip_file, function (err, data) {
if (!err) {
zipObj.load(data);
if (zipObj.file(zip_subdir)) {
var binary = zipObj.file(zip_subdir).asBinary();
var base64data = btoa(binary);
var extension = args.custom.file.split('.').pop();
var b64Header = "data:" + MIME[extension] + ";base64,";
var tag2 = args.custom.tag2 || "unset";
var tag3 = args.custom.tag3 || "unset";
var rargs = {
action: "getFile",
tag: args.tag,
dialogName: connections[connection_id].dialogName,
custom: {
file: b64Header + base64data,
tag2: tag2,
tag3: tag3
}
};
connections[connection_id].sendUTF(JSON.stringify(rargs));
rargs = null;
binary = null;
base64data = null;
} else {
serverLog(connection_id, "Requested file doesn't exist");
}
} else {
serverLog(connection_id, "There was an error retrieving the zip file data");
}
});
} else {
// File isn't a .zip
}
} catch (e) {
serverLog(connection_id, e);
}
}
Any help would be much appreciated in getting rid of this problem - Thanks!
Working Code Example
function handleFileRequest(args, connection_id) {
var zipIndex = 0,
pathLen = 0,
f = "",
d = "";
try {
if (args.custom.file.indexOf(".zip") > -1) {
// We have a .zip directory!
zipIndex = args.custom.file.indexOf(".zip") + 4;
pathLen = args.custom.file.length;
f = args.custom.file.substring(0, zipIndex);
d = args.custom.file.substring(zipIndex + 1, pathLen);
fs.readFile(f, function (err, data) {
var rargs = null,
binary = null,
base64data = null,
zipObj = null;
if (!err) {
zipObj = new JSZip();
zipObj.load(data);
if (zipObj.file(d)) {
binary = zipObj.file(d).asBinary();
base64data = btoa(binary);
var extension = args.custom.file.split('.').pop();
var b64Header = "data:" + MIME[extension] + ";base64,";
var tag2 = args.custom.tag2 || "unset";
var tag3 = args.custom.tag3 || "unset";
rargs = {
action: "getFile",
tag: args.tag,
dialogName: connections[connection_id].dialogName,
custom: {
file: b64Header + base64data,
tag2: tag2,
tag3: tag3
}
};
connections[connection_id].sendUTF(JSON.stringify(rargs));
} else {
serverLog(connection_id, "Requested file doesn't exist");
}
} else {
serverLog(connection_id, "There was an error retrieving the zip file data");
}
rargs = null;
binary = null;
base64data = null;
zipObj = null;
});
} else {
// Non-Zip file
}
} catch (e) {
serverLog(connection_id, e);
}
}
If you use the same JSZip instance to load each and every file, you will keep everything in memory : the load method doesn't replace the existing content.
Try using a new JSZip instance each time :
var zipObj = new JSZip();
zipObj.load(data);
// or var zipObj = new JSZip(data);