We Keep Coding

nodejs multer diskstorage to delete file after saving to disk

I am using multer diskstorage to save a file to disk.
I first save it to the disk and do some operations with the file and then i upload it to remote bucket using another function and lib.
Once the upload is finished, i would like to delete it from the disk.
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage }).single('file')
and here is how i use it:
app.post('/api/photo', function (req, res) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
});
how can i use the diskStorage remove function to remove the files in the temp folder?
https://github.com/expressjs/multer/blob/master/storage/disk.js#L54
update:
I have decided to make it modular and put it in another file:
const fileUpload = function(req, res, cb) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
}
module.exports = { fileUpload };

You don't need to use multer to delete the file and besides _removeFile is a private function that you should not use.
You'd delete the file as you normally would via fs.unlink. So wherever you have access to req.file, you can do the following:
const fs = require('fs')
const { promisify } = require('util')
const unlinkAsync = promisify(fs.unlink)
// ...
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}`)
}
})
const upload = multer({ storage: storage }).single('file')
app.post('/api/photo', upload, async (req, res) =>{
// You aren't doing anything with data so no need for the return value
await uploadToRemoteBucket(req.file.path)
// Delete the file like normal
await unlinkAsync(req.file.path)
res.end("UPLOAD COMPLETED!")
})

Multer isn't needed. Just use this code.
const fs = require('fs')
const path = './file.txt'
fs.unlink(path, (err) => {
if (err) {
console.error(err)
return
}
//file removed
})

You may also consider using MemoryStorage for this purpose, with this storage the file is never stored in the disk but in memory and is deleted from the memory automatically after execution comes out of controller block, i.e., after you serve the response in most of the cases.
When you will use this storage option, you won't get the fields file.destination, file.path and file.filename, instead you will get a field file.buffer which as name suggests is a buffer, you can convert this buffer to desired format to do operations on and then upload using a stream object.
Most of the popular libraries support streams so you should be able to use stream to upload your file directly, code for converting buffer to stream:
const Readable = require('stream').Readable;
var stream = new Readable();
stream._read = () => { }
stream.push(file.buffer);
stream.push(null);
// now you can pass this stream object to your upload function
This approach would be more efficient as files will be stored in memory which will result in faster access, but it does have a con as mentioned in multer documentation:
WARNING: Uploading very large files, or relatively small files in
large numbers very quickly, can cause your application to run out of
memory when memory storage is used.

To do it truly automatically across all routes I used this strategy :
when the request ends, we delete all the uploaded files (req.files). Before that, if you want to keep the files on the server, you need to save them in another path.
var express = require('express');
var app = express();
var http = require('http');
var server = http.Server(app);
// classic multer instantiation
var multer = require('multer');
var upload = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
cb(null, `${__dirname}/web/uploads/tmp/`);
},
filename: function (req, file, cb) {
cb(null, uniqid() + path.extname(file.originalname));
},
}),
});
app.use(upload.any());
// automatically deletes uploaded files when express finishes the request
app.use(function(req, res, next) {
var writeHead = res.writeHead;
var writeHeadbound = writeHead.bind(res);
res.writeHead = function (statusCode, statusMessage, headers) {
if (req.files) {
for (var file of req.files) {
fs.unlink(file.path, function (err) {
if (err) console.error(err);
});
}
}
writeHeadbound(statusCode, statusMessage, headers);
};
next();
});
// route to upload a file
router.post('/profile/edit', access.isLogged(), async function (req, res, next) {
try {
// we copy uploaded files to a custom folder or the middleware will delete them
for (let file of req.files)
if (file.fieldname == 'picture')
await fs.promises.copy(file.path, `${__dirname}/../uploads/user/photo.jpg`);
} catch (err) {
next(err);
}
});

I have removed directory after file uploaded using fs-extra
const fs = require('fs-extra');
// after you uploaded to bucket
await fs.remove('uploads/abc.png'); // remove upload dir when uploaded bucket

How to configure API endpoint to receive file from ember-uploader component

I'm trying to figure out how to use ember-uploader, I have the following component (like the one in the README)
export default EmberUploader.FileField.extend({
filesDidChange: function(files) {
const uploader = EmberUploader.Uploader.create({
url: (ENV.APP.API_HOST || '') + '/api/v1/images/',
});
console.log(uploader);
if (!Ember.isEmpty(files)) {
var photo = files[0];
console.log(photo);
uploader.upload(photo)
.then(data => {
// Handle success
console.log("Success uploading file");
console.log(data);
}, error => {
// Handle failure
console.log("ERROR uploading file");
console.log(error);
});
}
}
});
The express API endpoint is listening for a POST request.
var saveImage = (req, res, next) => {
let body = req.body;
res.json({
data: body
});
};
But the body is empty after the request is done. I really don't know how to implement the API endpoint in order to get the file, I tried to see the req object and it doesn't contains the file.
Debugging it, After select a file using the component I get the following info in the console.
Seems that the API endpoint works because I get the following output:
POST /api/v1/images/ 200 27.284 ms - 11
But I can't get the file.

SOLUTION
In Express 4, req.files is no longer available on the req object by
default. To access uploaded files on the req.files object, use a
multipart-handling middleware like busboy, multer, formidable,
multiparty, connect-multiparty, or pez.
Following this blog, the code below was added to the API and the ember-uploader code posted in the question worked as expected.
import formidable from 'formidable';
var saveImage = (req, res, next) => {
var form = new formidable.IncomingForm();
form.parse(req);
form.on('fileBegin', function (name, file){
file.path = __dirname + '/tmp/' + file.name;
});
form.on('file', function (name, file){
res.json({
data: file.name
});
});
};

Combining angular-file-upload and multer

I have big headache to combine angular file upload plugin with multer to make it fully SPA. I stucked on uploading multiple files through multer.
This is how my multer options looks like: (node route.js file)
var upload = multer({
storage: storage,
limits: {
//fileSize: 819200
}
}).array('myFile');
this is my POST: (node route.js file)
router.post('/add/file', function(req, res, next) {
upload(req,res,function(err) {
console.log(req.files);
if(err) {
console.log("Error uploading file.");
}
});
});
this is inside my angular controller:
var uploader = $scope.uploader = new FileUploader({
url: 'http://localhost:3000/add/file',
alias: 'myFile'
});
uploader.filters.push({
name: 'imageFilter',
fn: function(item /*{File|FileLikeObject}*/, options) {
var type = '|' + item.type.slice(item.type.lastIndexOf('/') + 1) + '|';
return '|jpg|png|jpeg|bmp|gif|'.indexOf(type) !== -1;
}
});
It adds only 1st file and stucks - I don't get any error it just stucks - whole page works and I can send files again, but again only 1st file will be uploaded. Console shows that req.files have only 1 file (that first one)
I couldn't find any tutorial or anything on the Internet with angular-file-upload plugin, that's why I ask you guys

Not sure if you figured this out yet or not, but with sending multiple files over, the 'uploadAll' function will not send the next file until it receives a response back from the server. So the route should look like this. I also saw somewhere in the documentation that the response needs to be json...haven't tested whether or not this is true though
router.post('/add/file', function(req, res, next) {
upload(req,res,function(err) {
console.log(req.files);
if(err) {
console.log("Error uploading file.");
} else {
res.status(200).json({response: 'some response...'})
}
});
});

How to send response to client when files is too large with Multer

I'm using NodeJs Multer to upload files. I need to send response back to a client when file user tries to upload is too large. The problem is that onFileSizeLimit only has file as argument and I dont know how to send response to client. What I need to do is basically soomething like below:
app.use('/users/gyms/upload-logo', multer({
// other settings here then:
onFileSizeLimit: function (file) {
// but res (response) object is not existing here
res.json({
message: "Upload failed",
status: MARankings.Enums.Status.FILE_TOO_LARGE
// status: -6
});
}
});
res object dosent exists in there however and I'm wondering what is the best way to send some sort of response to client.

try this:
app.use('/users/gyms/upload-logo', multer({
// other settings here then:
onFileSizeLimit: function (file) {
// but res (response) object is not existing here
file.error = {
message: "Upload failed",
status: MARankings.Enums.Status.FILE_TOO_LARGE
// status: -6
};
}, onFileUploadComplete: function (file, req, res) {
if (file.error){
res.send(file.error);
}
}
});

In this case, it's good to remember that Multer itself is just a (middleware) function that Express calls to get its response.
You could perhaps try with this:
app.use('/users/gyms/upload-logo', function(req, res, next) {
var handler = multer({
// other settings here then:
onFileSizeLimit: function (file) {
// res does exist here now :)
res.json({
message: "Upload failed",
status: MARankings.Enums.Status.FILE_TOO_LARGE
// status: -6
});
}
});
handler(req, res, next);
});
This basically aliases multer to handler, passes req, res, next from the Express callback, and means you get access to the req, res, next variables from within Multer's configuration.
I haven't tested this but I think the theory is sound!

This is an issue which has not been resolved by the author of multer yet. This github issue has quite a lot of discussion about it:

There is one work around which I have used in my current project
File: make-middleware.js
change "function done(err)" at end of this function
Replace
Line 52: onFinished(req, function () { next(err) })
With:
Line 52: onFinished(req, function () { if(err.code == 'LIMIT_FILE_SIZE') { req.fileSizeError = 1; next() } else next(err) })
And in app file you can change the code to
app.post('/upload', upload.single('upload'), function (req, res, next) {
if(typeof req.fileSizeError != "undefined") {
res.send({"error":"File too large"});// to display filesize error
} else {
res.send({"file":req.file}); // when file uploaded successfully
}
});

The multer file object actually contains a property indicating whether the file exceeded the size limit. See https://www.npmjs.com/package/multer#multer-file-object
To accept only one file with a maximum size of 2 MB and the name "data" I did something like this:
app.use(multer({
dest: "./uploads/",
putSingleFilesInArray: true, // see https://www.npmjs.com/package/multer#putsinglefilesinarray
limits: {
files: 1,
fileSize: 2097152 // 2 MB
}
}));
app.post("/foo", function(request, response) {
if (!request.files.hasOwnProperty("data")) {
// 400 Bad Request
return response.status(400).end();
}
var file = request.files.data[0];
if (file.truncated) {
// 413 Request Entity Too Large
console.log("Request aborted.");
return response.status(413).end();
}
// do stuff with file
});

How to serve (uploaded) images using Meteor

I have this Meteor application in which it is possible to upload images. The uploading parts seem to work. I store the images in .uploads. Now I would like to make these images accessable by the following URL
http://localhost:3000/uploads
After a bit of googling I was able to create the following server side code:
var fs = Meteor.require('fs');
if (Meteor.isServer) {
WebApp.connectHandlers.stack.splice(0, 0, {
route: '/uploads',
handle: function (req, res, next) {
var path = process.env.PWD + '/.' + req.originalUrl.substr(1);
fs.readFile(path, {encoding: 'binary'}, function (err,data) {
if (err) {
throw err;
}
res.writeHead(200, {
'Content-Type': 'image/png'
});
//res.setEncoding("binary"); // this method does not exist
res.write(data);
res.end();
});
}
});
}
This code works, the path constructed is correct and in the browser I receive the 200 code, except it cannot display the image. Something is wrong with the data the browser receives. I checked the image on disk which is fine. So the code above must do something wrong with the data. Any suggestions what that might be?

Here is the code I found after googling (and works for me) a few days ago when I wanted to do what you need to do
files are in .screenshots directory mapped to :
http://localhost:3000/screenshots
code :
//directly serve screenshot files from /.screenshots dir
var fs = Npm.require('fs');
WebApp.connectHandlers.use(function(req, res, next) {
var re = /^\/screenshots\/(.*)$/.exec(req.url);
if (re !== null) { // Only handle URLs that start with /screenshots/*
var filePath = process.env.PWD + '/.screenshots/' + re[1];
var data = fs.readFileSync(filePath, data);
res.writeHead(200, {
'Content-Type': 'image'
});
res.write(data);
res.end();
} else { // Other urls will have default behaviors
next();
}
});