Framework: node.js/express.js/busboy/gridfs-stream(mongodb)
I am using busboy to upload files and then use gridfs-stream to store files in mongodb gridfs.
req.pipe(req.busboy);
req.busboy.on('file', function (bus_fieldname, bus_file, bus_filename) {
var writestream = gfs.createWriteStream({
filename: bus_filename,
});
bus_file.pipe(writestream);
writestream.on('close', function (file) {
res.redirect('/xxxxx/');
});
});
Download is simple: Use gridfs-stream's createReadStream I read the contents from mongodb and then use the following code to send it to browser.
gfs.findOne({_id: attachmentid}, function (err, file) {
if (err || !file){
res.send(404);
}else{
var fileName = file.filename;
var readstream = gfs.createReadStream({_id: attachmentid});
var buffer = "";
readstream.on("data", function (chunk) {
buffer += chunk;
});
// dump contents to buffer
readstream.on("end", function () {
res.set("Content-Disposition","attachment; filename=" + fileName);
res.send(buffer);
});
}
Problem: When I upload a 90kb pdf file, it uploads fine. I see the size is correct in mongodb. But when I download, the file size of the downloaded file is about 165kb. There is a mismatch. This does not happen with text files. Sure its something to do with data type.
can anyone please help?
Pipe the gfs read stream to the response directly. This is what works for me
res.set("Content-Disposition","attachment; filename=" + fileName);
var readstream = gfs.createReadStream({_id: attachmentid});
readstream.pipe(res);
Like Sasikanth said you can fix the problem with piping and that's the best approach.
But if you are wondering what's wrong with your current approach, then I suggest to read the Buffer documentation.You are naming your variable as buffer but assigning string.
Then in your data callback you are adding string with Buffer. When you do that the chunk buffer is converted to string using it's toString() method. So your binary data is converted to utf8 encoded string and here it goes wrong. Since utf8 is multi-byte encoding the output size becomes large(I don't know the exact mechanism of this conversion).
So the right way to do is to keep it in buffers:
var buffers = [];
readstream.on("data", function (chunk) {
buffers.push(chunk)
});
readstream.on("end", function () {
res.set("Content-Disposition","attachment; filename=" + fileName);
res.send(Buffer.concat(buffers));
});
Related
This question already exists:
Avoid to store duplicates in Mongo GridFs
Closed 3 years ago.
How to get MD5 hash of a file being downloaded from the server before you save the file to Mongo Db using Node JS ?. Because I am download a file from a remote server and then I directly pipe the request stream to the gridfs write stream and now it is stored on the mongo db with its MD5 data and etc. Now I want to avoid saving duplicates so that it will only save files which does not exist.
Now , How to get MD5 hash of a file being downloaded so that i can compare the md5 hash to the once stored in the databse. Cause on my example I just directly pipe the request stream to the gridfs write stream.
Any idea. thanyou.
Sample Code
var gfs = Grid(db, mongo);
var download = function (url, dest, filename callback) {
const file = filename
const fileStorage = gfs.createWriteStream({ filename: file});
request.get(url)
.on('error', function (err) { console.log(err) })
.pipe(fileStorage)
.on('close', callback);
};
final_list.forEach(function (str) {
var filename = str.split('/').pop();
console.log('Downloading ' + filename);
download(str, filename, function () { console.log('Finished Downloading' + "" + filename) });
});
My Express app is receiving a base64-encoded MP4 from the browser and writing it to a file. But the file isn't a valid video file, and the "file" utility simply identifies it as "data".
My Code:
const path = `${config.uploadPath}`;
const filename = `${uniqid()}.mp4`;
let base64Data = req.body.base64.replace(/^data:([A-Za-z-+/]+);base64,/, '');
fs.writeFileSync(`${path}${filename}`, base64Data, 'base64');
Are you sure there is a variable named base64 is request response? If so, please try this code:
req.body.base64 = req.body.base64.replace(/^data:(.*?);base64,/, ""); // <--- make it any type
req.body.base64 = req.body.base64.replace(/ /g, '+'); // <--- this is important
fs.writeFile(`${path}${filename}`, req.body.base64, 'base64', function(err) {
console.log(err);
});
I am a newbie to both Javascript and ipfs and I am trying an experiment to fetch an image buffer from the ipfs hash "QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n" using ipfs-mini node module.
Below is my code
const IPFS = require('ipfs-mini');
const FileReader = require('filereader');
var multer = require('multer');
const ipfs = initialize();
app.post('/upload',function(req,res){
upload(req,res, function(err){
console.log(req.file.originalname);
ipfs.cat('QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n', function(err, data){
if(err) console.log("could not get the image from the ipfs for hash " + ghash);
else {
var wrt = data.toString('base64');
console.log('size ; ' + wrt.length);
fs.writeFile('tryipfsimage.gif',wrt, (err) =>{
if(err)console.log('can not write file');
else {
//console.log(data);
ipfs.stat('QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n', (err, data)=>{
// console.log(hexdump(wrt));
});
console.log("files written successfully");
}
});
}
});
});
});
function initialize() {
console.log('Initializing the ipfs object');
return new IPFS({
host: 'ipfs.infura.io',
protocol: 'https'
});
}
I could view the image properly in the browser using the link below "https://ipfs.io/ipfs/QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n", but if I open the file 'tryipfsimage.gif' in which I dump the return buffer of the cat API in above program, the content of the image seems corrupted. I am not sure what the mistake I am doing in the code. it would be great If someone points me the mistake.
From ipfs docs https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#javascript---ipfsfilescatipfspath-callback
file in the callback is actually a Buffer so by toString('base64')'ing it you are writing actual base64 into the .gif file - no need to do this. you can pass the Buffer directly to the fs.writeFile api with
fs.writeFile('tryipsimage.gif', file, ...
For larger files I would recommend using the ipfs catReadableStream, where you can do something more like:
const stream = ipfs.catReadableStream('QmdD8FL7N3kFnWDcPSVeD9zcq6zCJSUD9rRSdFp9tyxg1n')
// don't forget to add error handlers to stream and whatnot
const fileStream = fs.createWriteStream('tryipsimage.gif')
stream.pipe(fileStream);
I am writing an Express app that takes in a base64 encoded string that represents an image. Right now, i'm not really sure how I can take that string and upload the image to AWS S3, so i'm reading in the encoded image string data, decoding it, writing a file using fs, and then trying to upload. I have this working for an endpoint that just takes in a raw file, and all of its content is correctly uploaded to AWS s3.
Now when I try to do what I described above, i'm able to upload to S3, but the file has 0kb and is empty, and i'm not sure why. I tested just taking the stringData and writing a file to a test file, and it works. However, when I try uploading to s3, the file shows but it's empty. Here is my code:
router.post('/images/tags/nutritionalInformation/image/base64encoded', function (req, res) {
console.log(req.body.imageString);
var base64Stream = req.body.imageString;
var imgDecodedBuffer = decodeBase64Image(base64Stream);
console.log(imgDecodedBuffer);
// write to image file
var prefix = guid().toString() + ".jpg";
var filePath = './uploads/' + prefix;
console.log(filePath);
fs.writeFile(filePath, imgDecodedBuffer.data, function(err) {
console.log(err);
});
var stream = fs.createReadStream(filePath);
console.log(stream);
return s3fsImpl.writeFile(prefix, stream).then(function () {
fs.unlink(filePath, function (err) {
if (err) {
console.error(err);
}
});
});
})
Here are the relevant import statements:
var fs = require('fs');
var s3fs = require('s3fs');
var multiparty = require('connect-multiparty'),
multipartyMidleware = multiparty();
var s3fsImpl = new s3fs('blahblah', {
accessKeyId: 'ACCESS_KEY_ID',
secretAccessKey: 'SECRET'
});
Any help would be greatly appreciated!
If you simply just pass in the buffer, which I presume is in your imgDecodedBuffer.data value, it should work.
I'm trying to insert an image into a pdf I'm creating server-side with PDFkit. I'm using cfs:dropbox to store my files. Before when I was using cvs:filesystem, it was easy to add the images to my pdf's cause they were right there. Now that they're stored remotely, I'm not sure how to add them, since PDFkit does not support adding images with just the url. It will, however, accept a buffer. How can I get a buffer from my CollectionFS files?
So far I have something like this:
var portrait = Portraits.findOne('vS2yFy4gxXdjTtz5d');
readStream = portrait.createReadStream('portraits');
I tried getting the buffer two ways so far:
First using dataMan, but the last command never comes back:
var dataMan = new DataMan.ReadStream(readStream, portrait.type());
var buffer = Meteor.wrapAsync(Function.prototype.bind(dataMan.getBuffer, dataMan))();
Second buffering the stream manually:
var buffer = new Buffer(0);
readStream.on('readable', function() {
buffer = Buffer.concat([buffer, readStream.read()]);
});
readStream.on('end', function() {
console.log(buffer.toString('base64'));
});
That never seems to come back either. I double-checked my doc to make sure it was there and it has a valid url and the image appears when I put the url in my browser. Am I missing something?
I had to do something similar and since there's no answer to this question, here is how I do it:
// take a cfs file and return a base64 string
var getBase64Data = function(file, callback) {
// callback has the form function (err, res) {}
var readStream = file.createReadStream();
var buffer = [];
readStream.on('data', function(chunk) {
buffer.push(chunk);
});
readStream.on('error', function(err) {
callback(err, null);
});
readStream.on('end', function() {
callback(null, buffer.concat()[0].toString('base64'));
});
};
// wrap it to make it sync
var getBase64DataSync = Meteor.wrapAsync(getBase64Data);
// get a cfs file
var file = Files.findOne();
// get the base64 string
var base64str = getBase64DataSync(file);
// get the buffer from the string
var buffer = new Buffer(base64str, 'base64')
Hope it'll help!