Google Storage resumable upload range header - javascript

I'm uploading files to a GCS bucket with the JSON API and the upload is resumable. I'm doing this from the browser using JS.
The files upload with no problems. If there is a network problem then I try to resume it. Per the documentation I'm sending an empty PUT request to the upload URI. Then parsing the range header of the response. But then I get the error:
Refused to get unsafe header "range"
This is the code:
var xhr = new XMLHttpRequest();
xhr.open("PUT", url, true);
xhr.setRequestHeader("Authorization", token);
xhr.setRequestHeader("content-range", "bytes */" + file_object.size);
xhr.onreadystatechange = function () {
if (xhr.readyState != 4) {
return;
}
if (xhr.status === 308) {
var range = xhr.getResponseHeader("range"); // Error here
// continue with the upload
}
};
xhr.send();
I'm creating the upload URI at the server side with Python. These are the headers included in the request:
headers = {
"Authorization": authorization,
"Content-Type": "application/json; charset=UTF-8",
"origin": 'http://' + self.request.headers.get('HOST'),
"X-Upload-Content-Type": content_type,
"X-Upload-Content-Length": content_length,
}
How can I read the header range?

Related

Strange CORS issue with Google Drive API

Google Drive API, files.update:
https://developers.google.com/drive/api/v3/reference/files/update
This API is giving CORS error, the strange thing is in the same drive app that I'm creating, on the same domain (I'm testing it directly online):
files.create API runs OK
files.update API gives me CORS error (preflight request by browser can't find allow origin header in response)
The documentation on developers.google.com doesn't mention about CORS issue, what could be the problem that one API is ok while the other is not?
The 2 functions to create and update are here but shouldn't be a code issue, since CORS is about the origin of script while the functions are on the same page:
// Create new file in Gdrive
// See: https://developers.google.com/drive/api/v3/reference/files/create
// See: https://tanaikech.github.io/2018/08/13/upload-files-to-google-drive-using-javascript
async function gdrive_create_file(Folder_Id,File_Name,Binobj){
var Metadata = {
"name": File_Name, // Filename at Google Drive
"mimeType": "text/plain", // mimeType at Google Drive
"parents": [Folder_Id], // Folder ID at Google Drive
};
// Here gapi is used for retrieving the access token.
var Access_Token = gapi.auth.getToken().access_token;
var Form = new FormData();
Form.append("metadata", new Blob([JSON.stringify(Metadata)], {type: 'application/json'}));
Form.append("file", Binobj);
// Make request to Gdrive
var [Lock,Unlock] = new_lock();
var File_Id = null;
var Xhr = new XMLHttpRequest();
// Gdrive to return id as indicated in 'fields=id'
Xhr.open(
"post",
"https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=id"
);
Xhr.setRequestHeader("Authorization", "Bearer "+Access_Token);
Xhr.responseType = "json";
Xhr.onload = ()=>{
log("[Dad's TE] Gdrive file id:",Xhr.response.id); // Retrieve uploaded file ID.
File_Id = Xhr.response.id;
if (File_Id==null)
alert("Failed to create file in Gdrive!");
Unlock();
};
Xhr.send(Form);
// Wait to get resulting file id
await Lock;
return File_Id;
}
// Update a file in Gdrive
log("DEBUG HERE: 0");
async function gdrive_update_file(File_Id,File_Name,Binobj){
var Metadata = {
"name": File_Name, // Filename at Google Drive
"mimeType": "text/plain" // mimeType at Google Drive
};
// Here gapi is used for retrieving the access token.
var Access_Token = gapi.auth.getToken().access_token;
var Form = new FormData();
Form.append("metadata", new Blob([JSON.stringify(Metadata)], {type: 'application/json'}));
Form.append("file", Binobj);
// Make request to Gdrive
var [Lock,Unlock] = new_lock();
var Xhr = new XMLHttpRequest();
// Gdrive to return id as indicated in 'fields=id'
Xhr.open(
"patch",
`https://www.googleapis.com/upload/drive/v3/files/${File_Id}?uploadType=multipart&fields=id`
);
Xhr.setRequestHeader("Authorization", "Bearer "+Access_Token);
Xhr.responseType = "json";
Xhr.onload = ()=>{
log("[Dad's TE] Gdrive file id:",Xhr.response.id); // Retrieve uploaded file ID.
File_Id = Xhr.response.id;
if (File_Id==null)
alert("Failed to update file in Gdrive!");
Unlock();
};
alert("DEV ERROR: See CORS error in console log!");
Xhr.send(Form);
// Wait to get resulting file id
await Lock;
return File_Id;
}
The new_lock function:
function new_lock(){
var Unlock,Lock = new Promise((Res,Rej)=>{ Unlock=Res; });
return [Lock,Unlock];
}
In your script, how about modifying as follows? In my environment, when I tested your script, I confirmed the same error. In this case, when "patch" is modified to "PATCH", the error was removed.
Modified script:
From:
Xhr.open(
"patch",
`https://www.googleapis.com/upload/drive/v3/files/${File_Id}?uploadType=multipart&fields=id`
);
To:
Xhr.open(
"PATCH",
`https://www.googleapis.com/upload/drive/v3/files/${File_Id}?uploadType=multipart&fields=id`
);
or
Xhr.open(
"PATCH",
"https://www.googleapis.com/upload/drive/v3/files/" + File_Id + "?uploadType=multipart&fields=id"
);
And also, please remove alert("DEV ERROR: See CORS error in console log!");. When this is used, this is always displayed. Please be careful this.
When console.log(Xhr.response); is put in Xhr.onload = ()=>{,,,}, you can see the response from the API.
Note:
After I updated the file with "PATCH", when I tested the script with "patch", no error occurred. But, when new file is updated with the script with "patch", such error occurred again. So in this case, how about using "PATCH" instead of "patch"?
As the simple test script, how about the following script?
function gdrive_update_file(File_Id,File_Name,Binobj){
var Metadata = { "name": File_Name, "mimeType": "text/plain" };
var Access_Token = gapi.auth.getToken().access_token;
var Form = new FormData();
Form.append("metadata", new Blob([JSON.stringify(Metadata)], { type: 'application/json' }));
Form.append("file", Binobj);
var Xhr = new XMLHttpRequest();
Xhr.open(
"PATCH",
"https://www.googleapis.com/upload/drive/v3/files/" + File_Id + "?uploadType=multipart&fields=id"
);
Xhr.setRequestHeader("Authorization", "Bearer " + Access_Token);
Xhr.responseType = "json";
Xhr.onload = () => {
console.log(Xhr.response);
};
Xhr.send(Form);
}
References:
XMLHttpRequest.open()
HTTP request methods

.each() wait for iteration to finish before proceeding during async requests

I have a form that allows for uploads to Dropbox via the API. It works by breaking the file up into chunks and sending those chunks up to Dropbox.
The chunks have to go up one after the other. So the first is sent and the second can not be sent until the first request is completed. Not to mention that different chunks have to go to different end points.
To this end I went with using syncronus requests in a .each() loop so that the process would wait before sending the next chunk.
My code below.
My problem is that when a large file is sent (150MB +) the browser begins to timneout and then the form is useless.
My question is, how can I make these requests asyncornus and still upload them in the right order and only send a chunk when the previous chunk has been sent.
knpv_chunked_upload(chunks){
var dropbox = this;
//Send each chunk to Dropbox
jQuery(dropbox.chunks).each(function(index, chunk){
//This request is for the first chunk
if (index == 0) {
var xhr = new XMLHttpRequest();
xhr.open('POST', 'https://content.dropboxapi.com/2/files/upload_session/start', false);
xhr.setRequestHeader('Authorization', 'Bearer access_key');
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({close:false}));
}
//This request is for the middle chunks
if (index >= 1 && index < dropbox.chunks.length - 1) {
var xhr = new XMLHttpRequest();
xhr.open('POST', 'https://content.dropboxapi.com/2/files/upload_session/append_v2', false);
xhr.setRequestHeader('Authorization', 'Bearer access_key');
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({
'close':false,
'cursor': {
'session_id': dropbox.session_id,
'offset': dropbox.knpv_offset(index)
}
})
);
}
//This request is for the last chunk
if (index == dropbox.chunks.length - 1) {
var xhr = new XMLHttpRequest();
xhr.open('POST', 'https://content.dropboxapi.com/2/files/upload_session/finish', false);
xhr.setRequestHeader('Authorization', 'Bearer access_key');
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({
'cursor': {
'session_id': dropbox.session_id,
'offset': dropbox.knpv_offset(index)
},
'commit': {
'path': dropbox.folderpath+'/'+chunk.name,
'mode': 'overwrite'
}
})
);
}
//Send the chunk
xhr.send(chunk);
})
}

Bad JSON request using One Drive API

My goal is to create a folder programmatically in OneDrive API using Javascript/Jquery in the application that I'm building. I am not using Node.js or Angular.js. I have registered my application with OneDrive's Application Registration Portal, then used the token flow to get the access token from my web browser url address bar. Now that I have the access token, I'm trying to send it and my request to the API. Below is my code:
var accesshash = window.location.hash.substring(1);
//console.log(url);
console.log(accesshash);
var token = JSON.parse('{' + accesshash.replace(/([^=]+)=([^&]+)&?/g, '"$1":"$2",').slice(0,-1) + '}', function(key, value) { return key === "" ? value : decodeURIComponent(value); });
console.log(token.access_token);
var url = "https://graph.microsoft.com/v1.0/me/drive/root/children/"
var xhr = new XMLHttpRequest();
if(xhr.readyState == 4) {
console.log("success");
}
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json");
xhr.setRequestHeader("Authorization", "Bearer " + token.access_token);
var newfolder = {
"name": "0000000000",
"folder": {}
}
xhr.send(newfolder);
I'm getting this as my JSON response:
{
"error": {
"code": "BadRequest",
"message": "Unable to read JSON request payload. Please ensure Content-Type header is set and payload is of valid JSON format.",
"innerError": {
"request-id": "c8d43cbc-a59b-4244-8c4e-9193295ec7f8",
"date": "2018-06-07T19:42:57"
}
}
}
Does this mean that my access token is at least valid? Or is something wrong with it? Is there something I'm missing? This is my first time attempting to integrate Onedrive API into an application.
You are sending object, but content type is application/json, json is string representation of javascript object
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-type", "application/json;charset=UTF-8"); // added charset
xhr.setRequestHeader("Authorization", "Bearer " + token.access_token);
var newfolder = {
"name": "0000000000",
"folder": {}
}
xhr.send(JSON.stringify(newfolder)); // converted to string
There are many http libraries like fetch, request - that can make your life much easier

Dropbox direct upload files from browser

I am trying to upload files directly to dropbox [from a browser / web application], The "uploadFile" function on the code API needs the file to be uploaded available on the server, this puts me in trouble, because I do not want any files to be uploaded to my server and from there to dropbox.
$f = fopen("test.jpg", "rb"); // requires file on server
$result = $dbxClient->uploadFile("test.jpg", dbx\WriteMode::add(), $f);
fclose($f);
Tried out this https://github.com/dropbox/dropbox-js disappointed to say that there is no clear documentation, many of the links on the documentation part is broken.
I need the files to be uploaded to my account and the clients need not login to dropbox.
Any pointers would be really appreciated. looking for Ajax / JavaScript methods.
Update
I have tried the following, but no response from Dropbox
HTML
<input type="file" name="file" id="file" onchange="doUpload(event)">
JavaScript
var doUpload = function(event){
var input = event.target;
var reader = new FileReader();
reader.onload = function(){
var arrayBuffer = reader.result;
$.ajax({
url: "https://api-content.dropbox.com/1/files_put/auto/uploads/" + input.files[0].name,
headers: {
Authorization: 'Bearer ' + MyAccessToken,
contentLength: file.size
},
crossDomain: true,
crossOrigin: true,
type: 'PUT',
contentType: input.files[0].type,
data: arrayBuffer,
dataType: 'json',
processData: false,
success : function(result) {
$('#uploadResults').html(result);
}
});
}
reader.readAsArrayBuffer(input.files[0]);
}
Dropbox just posted a blog with instructions on how to do this. You can find it at https://blogs.dropbox.com/developers/2016/03/how-formio-uses-dropbox-as-a-file-backend-for-javascript-apps/ (Full disclosure, I wrote the blog post.)
Here is how to upload a file.
/**
* Two variables should already be set.
* dropboxToken = OAuth token received then signing in with OAuth.
* file = file object selected in the file widget.
*/
var xhr = new XMLHttpRequest();
xhr.upload.onprogress = function(evt) {
var percentComplete = parseInt(100.0 * evt.loaded / evt.total);
// Upload in progress. Do something here with the percent complete.
};
xhr.onload = function() {
if (xhr.status === 200) {
var fileInfo = JSON.parse(xhr.response);
// Upload succeeded. Do something here with the file info.
}
else {
var errorMessage = xhr.response || 'Unable to upload file';
// Upload failed. Do something here with the error.
}
};
xhr.open('POST', 'https://content.dropboxapi.com/2/files/upload');
xhr.setRequestHeader('Authorization', 'Bearer ' + dropboxToken);
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({
path: '/' + file.name,
mode: 'add',
autorename: true,
mute: false
}));
xhr.send(file);
Then to download a file from dropbox do this.
var downloadFile = function(evt, file) {
evt.preventDefault();
var xhr = new XMLHttpRequest();
xhr.responseType = 'arraybuffer';
xhr.onload = function() {
if (xhr.status === 200) {
var blob = new Blob([xhr.response], {type: ’application/octet-stream’});
FileSaver.saveAs(blob, file.name, true);
}
else {
var errorMessage = xhr.response || 'Unable to download file';
// Upload failed. Do something here with the error.
}
};
xhr.open('POST', 'https://content.dropboxapi.com/2/files/download');
xhr.setRequestHeader('Authorization', 'Bearer ' + dropboxToken);
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({
path: file.path_lower
}));
xhr.send();
}
FileSaver and Blob will not work on older browsers so you could add a workaround to them.
As other answers have noted, each session uploading or downloading the file will need to have access to a dropbox token. Sending someone else's token to a user is a security issue since having the token will give them complete control over the dropbox account. The only way to make this work is to have each person authenticate with Dropbox and get their own token.
At Form.io we've implemented both the authentication and the upload/download into our platform. This makes it really easy to build web apps with dropbox as a backend for files.
"I need the files to be uploaded to my account and the clients need not login to dropbox."
Then you'll really need to do the upload server-side. To do it client side would mean sending the access token to the browser, at which point any user of your app could use that access token to do whatever they wanted with your account. (E.g. delete all the other files, upload their private DVD collection, etc.)
For security reasons, I would strongly recommend doing the upload server-side where you can keep the access token a secret.
The answers given so far don't utilize the Dropbox javascript SDK which I think would prob be the best way to go about it. Check out this link here:
https://github.com/dropbox/dropbox-sdk-js/blob/master/examples/javascript/upload/index.html
which provides an example which is ofc dependent on having downloaded the SDK. (Edit: after playing with SDK I realize that it creates a POST request similar to the accepted answer in this thread. However something the popular answer omits is the presence of an OPTIONS preflight call that the sdk makes prior to the actual POST)
I might also add that something that is not shown in the dropbox sdk examples is that you can upload a blob object to dropbox; this is useful for instance if you want to dynamically extract images from a canvas and upload them and don't want to upload something that has been selected from the file system via the file uploaded input.
Here is a brief example of the scenario I'm describing:
//code below after having included dropbox-sdk-js in your project.
//Dropbox is in scope!
var dbx = new Dropbox.Dropbox({ accessToken: ACCESS_TOKEN });
//numerous stack overflow examples on creating a blob from data uri
var blob = dataURIToBlob(canvas.toDataUrl());
//the path here is the path of the file as it will exist on dropbox.
//should be unique or you will get a 4xx error
dbx.filesUpload({path: `unq_filename.png`, contents: blob})
Many thanks to #smarx with his pointers I was able to reach the final solution.
Also I have added a few extra features like listening to upload progress so that the users can be showed with the upload progress percentage.
HTML
<input type="file" name="file" id="file" onchange="doUpload(event)">
JavaScript
var doUpload = function(event){
var input = event.target;
var reader = new FileReader();
reader.onload = function(){
var arrayBuffer = reader.result;
var arrayBufferView = new Uint8Array( arrayBuffer );
var blob = new Blob( [ arrayBufferView ], { type: input.files[0].type } );
var urlCreator = window.URL || window.webkitURL;
var imageUrl = urlCreator.createObjectURL( blob );
$.ajax({
url: "https://api-content.dropbox.com/1/files_put/auto/YourDirectory/" + input.files[0].name,
headers: {
'Authorization':'Bearer ' +YourToken,
'Content-Length':input.files[0].size
},
crossDomain: true,
crossOrigin: true,
type: 'PUT',
contentType: input.files[0].type,
data: arrayBuffer,
dataType: 'json',
processData: false,
xhr: function()
{
var xhr = new window.XMLHttpRequest();
//Upload progress, litsens to the upload progress
//and get the upload status
xhr.upload.addEventListener("progress", function(evt){
if (evt.lengthComputable) {
var percentComplete = parseInt( parseFloat(evt.loaded / evt.total) * 100);
//Do something with upload progress
$('#uploadProgress').html(percentComplete);
$('#uploadProgressBar').css('width',percentComplete+'%');
}
}, false);
},
beforeSend: function(){
// Things you do before sending the file
// like showing the loader GIF
},
success : function(result) {
// Display the results from dropbox after upload
// Other stuff on complete
},
});
}
reader.readAsArrayBuffer(input.files[0]);
}
U have used the PUT method as our only objective is to upload files,As per my studies on various resources ( StackOverflow and zacharyvoase ) A put method can stream large files, also its desigend to put files on a specified URI , if file exist the file must be replaced. A PUT method cannot be moved to a different URL other than the URL Specified.
The Risk
You are at risk by using access token at client side, there needs to be high security measures to mask the token. But modern Web dev tools like Browser consoles , Firebug etc can monitor your server requests and can see your access token.
upload.html
Upload
upload.js
$('#form_wizard_1 .button-submit').click(function () {
var ACCESS_TOKEN ="Your token get from dropbox";
var dbx = new Dropbox({ accessToken: ACCESS_TOKEN });
var fileInput = document.getElementById('files1');
var file = fileInput.files[0];
res=dbx.filesUpload({path: '/' + file.name, contents: file})
.then(function(response) {
var results = document.getElementById('results');
results.appendChild(document.createTextNode('File uploaded!'));
res=response;
console.log(response);
})
.catch(function(error) {
console.error(error);
});
}

How to use a Model with octet-stream data in requests?

Give a HTTP Filesystem API which accepts
GET /storage/filename
PUT /storage/filename
Request body: the file contents
and these headers
Content-Length: ...
Content-Type: application/octet-stream
How can one build a backbone.js File Model which works with it? (e.g. get file contents in a variable, put back updated contents)
If backbone.js doesn't support this, how would jQuery ajax requests look like?
While I haven't found support in jQuery/backbone, here's how it can be done:
var xhr = new XMLHttpRequest();
xhr.open('GET', "url", true);
xhr.setRequestHeader("Authorization", token);
xhr.responseType = "text";
xhr.onload = function(e) {
if (this.status == 200) {
var contents = this.response;
console.log("RECEIVED " + contents);
}
};
xhr.send();
More details at http://www.html5rocks.com/en/tutorials/file/xhr2/

Categories