Im working on exporting data from a wordpress environment to a MongoDB using MongooseJS as data model bridges. I've got a JSON with every objects including all required information.
As a example, I've got user item including an avatarpath field pointing to the wordpress server url: (ex: http://url/wp-content/upload/img/avatar.jpg)
What I would like to do it retrieving the image from its url, upload it to my new storage folder, retrieve the new path, and store the new object in my mongodb.
My issue is that I can't manage to find a way to get the file data from a http get or any other way. Usually, I've got a file input in my html, and I start from the file object from this input. How should I proceed to make this work? Am I going into the wrong direction?
I've found this answer but it seems deprecated:
how to upload image file from url using FileReader API?
Here is what I've got for now:
$scope.curateurs_data = {};
$scope.curateurs = [];
$http.get('resources/json_curateurs.json').success(function(data) {
$scope.curateurs_data = data;
console.log(data[0]);
$scope.getPics();
});
//RETRIEVE IMAGE DATA
$scope.getPics = function(data){
console.log("RETRIEVING PICTURE")
var uploadPlace = '/upload/user';
var images;
angular.forEach($scope.curateurs_data, function(item, key) {
$scope.curitem = item;
console.log($scope.curitem);
$http.get(item.avatarpath, {responseType: "arraybuffer"}).success(function(data){
var arrayBufferView = new Uint8Array( data );
var blob = new Blob( [ arrayBufferView ], { type: "image/png" } );
var urlCreator = window.URL || window.webkitURL;
var imageUrl = urlCreator.createObjectURL( blob );
console.log(imageUrl);
console.log(blob);
images = blob;
var pic = {
images: images
};
Upload.upload({
url: uploadPlace,
arrayKey: '',
data: pic,
}).then(function(response) {
// Adding data paths to formData object before creating mood
// MUST respect images array order
$scope.curitem.avatarpath = response.data.files[0].path;
console.log(response.data.files[0].path);
});
}).error(function(err, status){})
$scope.curateurs.push($scope.curitem);
});
}
I've also tried something like this but I can't seems to make it work as well.
$http.get(item.avatarpath,{responseType: "blob"}).
success(function(data, status, headers, config) {
// encode data to base 64 url
fr = new FileReader();
fr.onload = function(){
// this variable holds your base64 image data URI (string)
// use readAsBinary() or readAsBinaryString() below to obtain other data types
console.log( fr.result );
};
fr.readAsDataURL(data);
console.log(fr.readAsDataURL(data));
}).
error(function(data, status, headers, config) {
alert("The url could not be loaded...\n (network error? non-valid url? server offline? etc?)");
});
Use node's http object on the backend to download the image. Something like:
http.request(url, function(response) {
// Your code to write the file out or store it in the database here.
});
Related
I send audio to client using node.js :
res.send(audio); / audio it is the buffer array
And I get object 'arrayBuffer' in data.
And I conver it to Blob and after to file (I want to get dataURL to show it in player)
.then(
(result) => {
result.arrayBuffer().then(function (data) {
const blob = new Blob([data], { type: 'audio/wav' });
var fileReader = new FileReader();
fileReader.readAsDataURL(blob);
fileReader.onload = function (evt) {
// Read out file contents as a Data URL
var url = evt.target.result;
res({blob, url})
};
})
}
)
And it works good. I use this url it as src of my <audio> attribute and it works.
But now I want to send this file from server as a part of json. Now I get audioBuffer from the serve in "audio" property. I chage arrayBuffer method to json in fetch:
result.json().then(function (data) {
const blob = new Blob([data.audio], { type: 'audio/wav' });
...
But now it doesn't work. I tried to use module www.npmjs.com/package/to-array-buffer to convert data.audio to js-arrayBuffer, but it doesn't help.
Maybe you know what's problem here?
Ok, I solved problem. I use module about whick I wrote to-array-buffer, but I use not toArrayBuffer(data.audio) but toArrayBuffer(data.audio.data)
Using dropbox you can create a shortcut by dragging and dropping a URL into your Dropbox folder. This will be saved like this:
Using the /2/files/download HTTP API from dropbox will return an XHR response that looks something like this:
How do you parse this response so that you can grab only the URL and make that a clickable link?
Here is what needs to go into an Angular 1 factory. To use this, you would just call the downloadFile function from a controller and provide the path to the file in your dropbox account.
function downloadFile(filePath) {
if (!filePath) {
console.error('Cannot download file because no file was specified.');
return;
}
return $q(function(fulfill, reject) {
$http({
url: 'https://content.dropboxapi.com/2/files/download',
method: 'POST',
headers: {
'Authorization': 'Bearer {{access-token-goes-here}}',
'Dropbox-API-Arg': `{"path": "${filePath}"}`
},
responseType: 'blob'
}).then(
results => {
// data received from dropbox is binary data saved as a blob
// The FileReader object lets web applications asynchronously read the contents of files
// https://developer.mozilla.org/en-US/docs/Web/API/FileReader
var fileReader = new FileReader();
// function will run after successfully reading the file
fileReader.onload = function() {
var string = this.result; // store the file contents
string = encodeURI(string); // get rid of the paragraph return characters
var endPosition = string.indexOf('%0D%0A', 32); // find the end of the URL, startPosition is 32
var actualURL = string.substring(32, endPosition); // grab only the characters between start and end positions
fulfill(actualURL);
};
fileReader.readAsText(results.data);
},
error => reject(error));
});
}
I have a url which has a image. I want to download that image using chrome api. I am developing a chrome app and not extension.
Can anyone please tell me where i am going wrong??
My code:
service('fileService', function($window){
this.saveUserInfo = function(theFilePath) {
// filepath is the url passed from controller{
// Get the directory entry stored (if any).
var fileUrl = "";
var config = {
type: 'saveFile',
suggestedName: 'TheBee.png',
accepts:[{extensions: ['png', 'jpg', 'jpeg']}]
};
chrome.fileSystem.chooseEntry(config, function(writableEntry) {
chrome.fileSystem.getWritableEntry(writableEntry, function(entry1) {
entry1.getFile(theFilePath, {create:true}, function(entry2) {
entry2.createWriter(function(writer) {
var blob = new Blob({type: 'image/png'});
writer.write(blob);
});
});
});
});
});
});
Change
var blob = new Blob({type: 'image/png'});
To
var blob = new Blob([entry1],{type: 'image/png'});
The Blob contructor acceptes blob parts: MSDN=> Blob(blobParts[, options])
You cannot use getFile on entry1 because it is a file itself. You will need to change config.type to "openDirectory"
You also need to add an empty array as the first input to the blob constructor.
I am trying to upload files directly to dropbox [from a browser / web application], The "uploadFile" function on the code API needs the file to be uploaded available on the server, this puts me in trouble, because I do not want any files to be uploaded to my server and from there to dropbox.
$f = fopen("test.jpg", "rb"); // requires file on server
$result = $dbxClient->uploadFile("test.jpg", dbx\WriteMode::add(), $f);
fclose($f);
Tried out this https://github.com/dropbox/dropbox-js disappointed to say that there is no clear documentation, many of the links on the documentation part is broken.
I need the files to be uploaded to my account and the clients need not login to dropbox.
Any pointers would be really appreciated. looking for Ajax / JavaScript methods.
Update
I have tried the following, but no response from Dropbox
HTML
<input type="file" name="file" id="file" onchange="doUpload(event)">
JavaScript
var doUpload = function(event){
var input = event.target;
var reader = new FileReader();
reader.onload = function(){
var arrayBuffer = reader.result;
$.ajax({
url: "https://api-content.dropbox.com/1/files_put/auto/uploads/" + input.files[0].name,
headers: {
Authorization: 'Bearer ' + MyAccessToken,
contentLength: file.size
},
crossDomain: true,
crossOrigin: true,
type: 'PUT',
contentType: input.files[0].type,
data: arrayBuffer,
dataType: 'json',
processData: false,
success : function(result) {
$('#uploadResults').html(result);
}
});
}
reader.readAsArrayBuffer(input.files[0]);
}
Dropbox just posted a blog with instructions on how to do this. You can find it at https://blogs.dropbox.com/developers/2016/03/how-formio-uses-dropbox-as-a-file-backend-for-javascript-apps/ (Full disclosure, I wrote the blog post.)
Here is how to upload a file.
/**
* Two variables should already be set.
* dropboxToken = OAuth token received then signing in with OAuth.
* file = file object selected in the file widget.
*/
var xhr = new XMLHttpRequest();
xhr.upload.onprogress = function(evt) {
var percentComplete = parseInt(100.0 * evt.loaded / evt.total);
// Upload in progress. Do something here with the percent complete.
};
xhr.onload = function() {
if (xhr.status === 200) {
var fileInfo = JSON.parse(xhr.response);
// Upload succeeded. Do something here with the file info.
}
else {
var errorMessage = xhr.response || 'Unable to upload file';
// Upload failed. Do something here with the error.
}
};
xhr.open('POST', 'https://content.dropboxapi.com/2/files/upload');
xhr.setRequestHeader('Authorization', 'Bearer ' + dropboxToken);
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({
path: '/' + file.name,
mode: 'add',
autorename: true,
mute: false
}));
xhr.send(file);
Then to download a file from dropbox do this.
var downloadFile = function(evt, file) {
evt.preventDefault();
var xhr = new XMLHttpRequest();
xhr.responseType = 'arraybuffer';
xhr.onload = function() {
if (xhr.status === 200) {
var blob = new Blob([xhr.response], {type: ’application/octet-stream’});
FileSaver.saveAs(blob, file.name, true);
}
else {
var errorMessage = xhr.response || 'Unable to download file';
// Upload failed. Do something here with the error.
}
};
xhr.open('POST', 'https://content.dropboxapi.com/2/files/download');
xhr.setRequestHeader('Authorization', 'Bearer ' + dropboxToken);
xhr.setRequestHeader('Dropbox-API-Arg', JSON.stringify({
path: file.path_lower
}));
xhr.send();
}
FileSaver and Blob will not work on older browsers so you could add a workaround to them.
As other answers have noted, each session uploading or downloading the file will need to have access to a dropbox token. Sending someone else's token to a user is a security issue since having the token will give them complete control over the dropbox account. The only way to make this work is to have each person authenticate with Dropbox and get their own token.
At Form.io we've implemented both the authentication and the upload/download into our platform. This makes it really easy to build web apps with dropbox as a backend for files.
"I need the files to be uploaded to my account and the clients need not login to dropbox."
Then you'll really need to do the upload server-side. To do it client side would mean sending the access token to the browser, at which point any user of your app could use that access token to do whatever they wanted with your account. (E.g. delete all the other files, upload their private DVD collection, etc.)
For security reasons, I would strongly recommend doing the upload server-side where you can keep the access token a secret.
The answers given so far don't utilize the Dropbox javascript SDK which I think would prob be the best way to go about it. Check out this link here:
https://github.com/dropbox/dropbox-sdk-js/blob/master/examples/javascript/upload/index.html
which provides an example which is ofc dependent on having downloaded the SDK. (Edit: after playing with SDK I realize that it creates a POST request similar to the accepted answer in this thread. However something the popular answer omits is the presence of an OPTIONS preflight call that the sdk makes prior to the actual POST)
I might also add that something that is not shown in the dropbox sdk examples is that you can upload a blob object to dropbox; this is useful for instance if you want to dynamically extract images from a canvas and upload them and don't want to upload something that has been selected from the file system via the file uploaded input.
Here is a brief example of the scenario I'm describing:
//code below after having included dropbox-sdk-js in your project.
//Dropbox is in scope!
var dbx = new Dropbox.Dropbox({ accessToken: ACCESS_TOKEN });
//numerous stack overflow examples on creating a blob from data uri
var blob = dataURIToBlob(canvas.toDataUrl());
//the path here is the path of the file as it will exist on dropbox.
//should be unique or you will get a 4xx error
dbx.filesUpload({path: `unq_filename.png`, contents: blob})
Many thanks to #smarx with his pointers I was able to reach the final solution.
Also I have added a few extra features like listening to upload progress so that the users can be showed with the upload progress percentage.
HTML
<input type="file" name="file" id="file" onchange="doUpload(event)">
JavaScript
var doUpload = function(event){
var input = event.target;
var reader = new FileReader();
reader.onload = function(){
var arrayBuffer = reader.result;
var arrayBufferView = new Uint8Array( arrayBuffer );
var blob = new Blob( [ arrayBufferView ], { type: input.files[0].type } );
var urlCreator = window.URL || window.webkitURL;
var imageUrl = urlCreator.createObjectURL( blob );
$.ajax({
url: "https://api-content.dropbox.com/1/files_put/auto/YourDirectory/" + input.files[0].name,
headers: {
'Authorization':'Bearer ' +YourToken,
'Content-Length':input.files[0].size
},
crossDomain: true,
crossOrigin: true,
type: 'PUT',
contentType: input.files[0].type,
data: arrayBuffer,
dataType: 'json',
processData: false,
xhr: function()
{
var xhr = new window.XMLHttpRequest();
//Upload progress, litsens to the upload progress
//and get the upload status
xhr.upload.addEventListener("progress", function(evt){
if (evt.lengthComputable) {
var percentComplete = parseInt( parseFloat(evt.loaded / evt.total) * 100);
//Do something with upload progress
$('#uploadProgress').html(percentComplete);
$('#uploadProgressBar').css('width',percentComplete+'%');
}
}, false);
},
beforeSend: function(){
// Things you do before sending the file
// like showing the loader GIF
},
success : function(result) {
// Display the results from dropbox after upload
// Other stuff on complete
},
});
}
reader.readAsArrayBuffer(input.files[0]);
}
U have used the PUT method as our only objective is to upload files,As per my studies on various resources ( StackOverflow and zacharyvoase ) A put method can stream large files, also its desigend to put files on a specified URI , if file exist the file must be replaced. A PUT method cannot be moved to a different URL other than the URL Specified.
The Risk
You are at risk by using access token at client side, there needs to be high security measures to mask the token. But modern Web dev tools like Browser consoles , Firebug etc can monitor your server requests and can see your access token.
upload.html
Upload
upload.js
$('#form_wizard_1 .button-submit').click(function () {
var ACCESS_TOKEN ="Your token get from dropbox";
var dbx = new Dropbox({ accessToken: ACCESS_TOKEN });
var fileInput = document.getElementById('files1');
var file = fileInput.files[0];
res=dbx.filesUpload({path: '/' + file.name, contents: file})
.then(function(response) {
var results = document.getElementById('results');
results.appendChild(document.createTextNode('File uploaded!'));
res=response;
console.log(response);
})
.catch(function(error) {
console.error(error);
});
}
I am allowing the user to load images into a page via drag&drop and other methods. When an image is dropped, I'm using URL.createObjectURL to convert to an object URL to display the image. I am not revoking the url, as I do reuse it.
So, when it comes time to create a FormData object so I can allow them to upload a form with one of those images in it, is there some way I can then reverse that Object URL back into a Blob or File so I can then append it to a FormData object?
Modern solution:
let blob = await fetch(url).then(r => r.blob());
The url can be an object url or a normal url.
As gengkev alludes to in his comment above, it looks like the best/only way to do this is with an async xhr2 call:
var xhr = new XMLHttpRequest();
xhr.open('GET', 'blob:http%3A//your.blob.url.here', true);
xhr.responseType = 'blob';
xhr.onload = function(e) {
if (this.status == 200) {
var myBlob = this.response;
// myBlob is now the blob that the object URL pointed to.
}
};
xhr.send();
Update (2018): For situations where ES5 can safely be used, Joe has a simpler ES5-based answer below.
Maybe someone finds this useful when working with React/Node/Axios. I used this for my Cloudinary image upload feature with react-dropzone on the UI.
axios({
method: 'get',
url: file[0].preview, // blob url eg. blob:http://127.0.0.1:8000/e89c5d87-a634-4540-974c-30dc476825cc
responseType: 'blob'
}).then(function(response){
var reader = new FileReader();
reader.readAsDataURL(response.data);
reader.onloadend = function() {
var base64data = reader.result;
self.props.onMainImageDrop(base64data)
}
})
The problem with fetching the blob URL again is that this will create a full copy of the Blob's data, and so instead of having it only once in memory, you'll have it twice. With big Blobs this can blow your memory usage quite quickly.
It's rather unfortunate that the File API doesn't give us access to the currently linked Blobs, certainly they thought web-authors should store that Blob themselves at creation time anyway, which is true:
The best here is to store the object you used when creating the blob:// URL.
If you are afraid this would prevent the Blob from being Garbage Collected, you're right, but so does the blob:// URL in the first place, until you revoke it. So holding yourself a pointer to that Blob won't change a thing.
But for those who aren't responsible for the creation of the blob:// URI (e.g because a library made it), we can still fill that API hole ourselves by overriding the default URL.createObjectURL and URL.revokeObjectURL methods so that they do store references to the object passed.
Be sure to call this function before the code that does generate the blob:// URI is called.
// Adds an URL.getFromObjectURL( <blob:// URI> ) method
// returns the original object (<Blob> or <MediaSource>) the URI points to or null
(() => {
// overrides URL methods to be able to retrieve the original blobs later on
const old_create = URL.createObjectURL;
const old_revoke = URL.revokeObjectURL;
Object.defineProperty(URL, 'createObjectURL', {
get: () => storeAndCreate
});
Object.defineProperty(URL, 'revokeObjectURL', {
get: () => forgetAndRevoke
});
Object.defineProperty(URL, 'getFromObjectURL', {
get: () => getBlob
});
const dict = {};
function storeAndCreate(blob) {
const url = old_create(blob); // let it throw if it has to
dict[url] = blob;
return url
}
function forgetAndRevoke(url) {
old_revoke(url);
try {
if(new URL(url).protocol === 'blob:') {
delete dict[url];
}
} catch(e){}
}
function getBlob(url) {
return dict[url] || null;
}
})();
// Usage:
const blob = new Blob( ["foo"] );
const url = URL.createObjectURL( blob );
console.log( url );
const retrieved = URL.getFromObjectURL( url );
console.log( "retrieved Blob is Same Object?", retrieved === blob );
fetch( url ).then( (resp) => resp.blob() )
.then( (fetched) => console.log( "fetched Blob is Same Object?", fetched === blob ) );
And an other advantage is that it can even retrieve MediaSource objects, while the fetching solutions would just err in that case.
Using fetch for example like below:
fetch(<"yoururl">, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + <your access token if need>
},
})
.then((response) => response.blob())
.then((blob) => {
// 2. Create blob link to download
const url = window.URL.createObjectURL(new Blob([blob]));
const link = document.createElement('a');
link.href = url;
link.setAttribute('download', `sample.xlsx`);
// 3. Append to html page
document.body.appendChild(link);
// 4. Force download
link.click();
// 5. Clean up and remove the link
link.parentNode.removeChild(link);
})
You can paste in on Chrome console to test. the file with download with 'sample.xlsx' Hope it can help!
See Getting BLOB data from XHR request which points out that BlobBuilder doesn't work in Chrome so you need to use:
xhr.responseType = 'arraybuffer';
Unfortunately #BrianFreud's answer doesn't fit my needs, I had a little different need, and I know that is not the answer for #BrianFreud's question, but I am leaving it here because a lot of persons got here with my same need. I needed something like 'How to get a file or blob from an URL?', and the current correct answer does not fit my needs because its not cross-domain.
I have a website that consumes images from an Amazon S3/Azure Storage, and there I store objects named with uniqueidentifiers:
sample: http://****.blob.core.windows.net/systemimages/bf142dc9-0185-4aee-a3f4-1e5e95a09bcf
Some of this images should be download from our system interface.
To avoid passing this traffic through my HTTP server, since this objects does not require any security to be accessed (except by domain filtering), I decided to make a direct request on user's browser and use local processing to give the file a real name and extension.
To accomplish that I have used this great article from Henry Algus:
http://www.henryalgus.com/reading-binary-files-using-jquery-ajax/
1. First step: Add binary support to jquery
/**
*
* jquery.binarytransport.js
*
* #description. jQuery ajax transport for making binary data type requests.
* #version 1.0
* #author Henry Algus <henryalgus#gmail.com>
*
*/
// use this transport for "binary" data type
$.ajaxTransport("+binary", function (options, originalOptions, jqXHR) {
// check for conditions and support for blob / arraybuffer response type
if (window.FormData && ((options.dataType && (options.dataType == 'binary')) || (options.data && ((window.ArrayBuffer && options.data instanceof ArrayBuffer) || (window.Blob && options.data instanceof Blob))))) {
return {
// create new XMLHttpRequest
send: function (headers, callback) {
// setup all variables
var xhr = new XMLHttpRequest(),
url = options.url,
type = options.type,
async = options.async || true,
// blob or arraybuffer. Default is blob
dataType = options.responseType || "blob",
data = options.data || null,
username = options.username || null,
password = options.password || null;
xhr.addEventListener('load', function () {
var data = {};
data[options.dataType] = xhr.response;
// make callback and send data
callback(xhr.status, xhr.statusText, data, xhr.getAllResponseHeaders());
});
xhr.open(type, url, async, username, password);
// setup custom headers
for (var i in headers) {
xhr.setRequestHeader(i, headers[i]);
}
xhr.responseType = dataType;
xhr.send(data);
},
abort: function () {
jqXHR.abort();
}
};
}
});
2. Second step: Make a request using this transport type.
function downloadArt(url)
{
$.ajax(url, {
dataType: "binary",
processData: false
}).done(function (data) {
// just my logic to name/create files
var filename = url.substr(url.lastIndexOf('/') + 1) + '.png';
var blob = new Blob([data], { type: 'image/png' });
saveAs(blob, filename);
});
}
Now you can use the Blob created as you want to, in my case I want to save it to disk.
3. Optional: Save file on user's computer using FileSaver
I have used FileSaver.js to save to disk the downloaded file, if you need to accomplish that, please use this javascript library:
https://github.com/eligrey/FileSaver.js/
I expect this to help others with more specific needs.
If you show the file in a canvas anyway you can also convert the canvas content to a blob object.
canvas.toBlob(function(my_file){
//.toBlob is only implemented in > FF18 but there is a polyfill
//for other browsers https://github.com/blueimp/JavaScript-Canvas-to-Blob
var myBlob = (my_file);
})
Following #Kaiido answer, another way to overload URL without messing with URL is to extend the URL class like this:
export class URLwithStore extends URL {
static createObjectURL(blob) {
const url = super.createObjectURL(blob);
URLwithStore.store = { ...(URLwithStore.store ?? {}), [url]: blob };
return url;
}
static getFromObjectURL(url) {
return (URLwithStore.store ?? {})[url] ?? null;
}
static revokeObjectURL(url) {
super.revokeObjectURL(url);
if (
new URL(url).protocol === "blob:" &&
URLwithStore.store &&
url in URLwithStore.store
)
delete URLwithStore.store[url];
}
}
Usage
const blob = new Blob( ["foo"] );
const url = URLwithStore.createObjectURL( blob );
const retrieved = URLwithStore.getFromObjectURL( url );
console.log( "retrieved Blob is Same Object?", retrieved === blob );