We Keep Coding

nodejs multer diskstorage to delete file after saving to disk

I am using multer diskstorage to save a file to disk.
I first save it to the disk and do some operations with the file and then i upload it to remote bucket using another function and lib.
Once the upload is finished, i would like to delete it from the disk.
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage }).single('file')
and here is how i use it:
app.post('/api/photo', function (req, res) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
});
how can i use the diskStorage remove function to remove the files in the temp folder?
https://github.com/expressjs/multer/blob/master/storage/disk.js#L54
update:
I have decided to make it modular and put it in another file:
const fileUpload = function(req, res, cb) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
}
module.exports = { fileUpload };

You don't need to use multer to delete the file and besides _removeFile is a private function that you should not use.
You'd delete the file as you normally would via fs.unlink. So wherever you have access to req.file, you can do the following:
const fs = require('fs')
const { promisify } = require('util')
const unlinkAsync = promisify(fs.unlink)
// ...
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}`)
}
})
const upload = multer({ storage: storage }).single('file')
app.post('/api/photo', upload, async (req, res) =>{
// You aren't doing anything with data so no need for the return value
await uploadToRemoteBucket(req.file.path)
// Delete the file like normal
await unlinkAsync(req.file.path)
res.end("UPLOAD COMPLETED!")
})

Multer isn't needed. Just use this code.
const fs = require('fs')
const path = './file.txt'
fs.unlink(path, (err) => {
if (err) {
console.error(err)
return
}
//file removed
})

You may also consider using MemoryStorage for this purpose, with this storage the file is never stored in the disk but in memory and is deleted from the memory automatically after execution comes out of controller block, i.e., after you serve the response in most of the cases.
When you will use this storage option, you won't get the fields file.destination, file.path and file.filename, instead you will get a field file.buffer which as name suggests is a buffer, you can convert this buffer to desired format to do operations on and then upload using a stream object.
Most of the popular libraries support streams so you should be able to use stream to upload your file directly, code for converting buffer to stream:
const Readable = require('stream').Readable;
var stream = new Readable();
stream._read = () => { }
stream.push(file.buffer);
stream.push(null);
// now you can pass this stream object to your upload function
This approach would be more efficient as files will be stored in memory which will result in faster access, but it does have a con as mentioned in multer documentation:
WARNING: Uploading very large files, or relatively small files in
large numbers very quickly, can cause your application to run out of
memory when memory storage is used.

To do it truly automatically across all routes I used this strategy :
when the request ends, we delete all the uploaded files (req.files). Before that, if you want to keep the files on the server, you need to save them in another path.
var express = require('express');
var app = express();
var http = require('http');
var server = http.Server(app);
// classic multer instantiation
var multer = require('multer');
var upload = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
cb(null, `${__dirname}/web/uploads/tmp/`);
},
filename: function (req, file, cb) {
cb(null, uniqid() + path.extname(file.originalname));
},
}),
});
app.use(upload.any());
// automatically deletes uploaded files when express finishes the request
app.use(function(req, res, next) {
var writeHead = res.writeHead;
var writeHeadbound = writeHead.bind(res);
res.writeHead = function (statusCode, statusMessage, headers) {
if (req.files) {
for (var file of req.files) {
fs.unlink(file.path, function (err) {
if (err) console.error(err);
});
}
}
writeHeadbound(statusCode, statusMessage, headers);
};
next();
});
// route to upload a file
router.post('/profile/edit', access.isLogged(), async function (req, res, next) {
try {
// we copy uploaded files to a custom folder or the middleware will delete them
for (let file of req.files)
if (file.fieldname == 'picture')
await fs.promises.copy(file.path, `${__dirname}/../uploads/user/photo.jpg`);
} catch (err) {
next(err);
}
});

I have removed directory after file uploaded using fs-extra
const fs = require('fs-extra');
// after you uploaded to bucket
await fs.remove('uploads/abc.png'); // remove upload dir when uploaded bucket

How we can upload multiple blobs to azure using nodejs

I am trying to upload 6 images to azure blob from single endpoint that I get from a registration form. The code shows how to upload a single blob but I need to upload multiple blobs at the same time. How can I do it?
Here is my code:
app.post('/upload', function (req, res) {
//var dirname = require('path').dirname(__dirname);
//var dirname1 = require('path').dirname(dirname);
var filename = req.files[0].filename;
var path = req.files[0].path;
var type = req.files[0].mimetype;
var options = {
contentType: type,
metadata: { fileName: filename }
}
blobSvc.createBlockBlobFromLocalFile(containerName, filename, path, options, function (error, result, response) {
if (error != null) {
console.log('Azure Full Error: ', error)
} else {
console.log(result);
console.log(response);
var user = new User();
user.name = req.body.name;
user.picture = 'https://yourblob.blob.core.windows.net/profile/' + result.name;
user.save(function (err) {
if (err) {
return res.json(err.message);
}
else {
return res.json({ User: user });
}
});
}
});
});

As Azure Storage for node sdk is based on RESTful APIs, and we implement upload functionality via Put Blob.
There is no such RESTful API or function in SDK for us to directly upload multiple independent blobs to Azure at once time.
You can implement this functionality for yourself by uploading files in loop.

How to prompt save dialog to download file using angularjs?

I have a directory in server that contains file now i am sending file name from client and getting the file from server till that part is working good, below is the response from server now once i receive response i want to prompt for user to download that file for that i am trying to create blob using Angularjs but its not prompting for user to save the file. Any idea ?
ctrl.js
$scope.downloadFile = function(message){
DitFactory.getFile(message).then(function(response){
console.log('r',response);
var blob = new Blob([ response ], { type : 'text/plain' });
$scope.url = (window.URL || window.webkitURL).createObjectURL( blob );
console.log($scope.url);
});
};
serverResponse.json
{"level":"info","message":"another-2fdas message"}
server.js
app.get('/file', function (req, res) {
var dir = './ditLogs';
var root = path.resolve('./ditLogs');
var fileName = req.query.file_name;
var data;
fs.readdir(dir, function(err, items) {
items.forEach(function(file){
if(fileName === file){
data = file;
res.setHeader('Content-Disposition', 'attachment; filename=' + data);
res.sendFile(data, {root: root});
}
});
});
});

If your are using express, you can try below code in server.js file:
var file = 'path to your file';
res.download(file,function(err){
if(!err){
console.log('prompted successfully');
return;
}
});

How to upload file using multer or body-parser

I am a NodeJS beginner, following along a book "Web Development with MongoDB and NodeJS". I am stuck at its chapter 6 with 'multer'. When I use multer for file uploads the server throws the following error:
/Users/fk / Documents / imageuploader / node_modules / express / lib / application.js: 209
throw new TypeError('app.use() requires middleware functions'); ^
TypeError: app.use() requires middleware functions
but when I replace it with bodyParser the server fires up but when I click the upload button it gives me the following error on the browser.
500 TypeError: Cannot read property 'file' of undefined
However, it is supposed to redirect me towards another page, where the uploaded file is shown.
Here is my bodyParser code, please see if I am using it correctly because it gives me "body-parser deprecated" at the starting of the server. I've seen other questions like mine and I followed but none of them really work.
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
app.use(bodyParser({
uploadDir: path.join(__dirname, '../public/upload/temp')
}));
Following code shows how I use multer, just in case if there is something I shouldn't be doing please let me know. Which one would be better in case of uploading files, body-parser or multer?
app.use(multer({
dest: path.join(__dirname, '../public/upload/temp')
}));
var saveImage = function() {
var possible = 'abcdefghijklmnopqrstuvwxyz0123456789',
imgUrl = '';
for (var i = 0; i < 6; i += 1) {
imgUrl += possible.charAt(Math.floor(Math.random() * possible.length));
}
var tempPath = req.files.file.path,
ext = path.extname(req.files.file.name).toLowerCase(),
targetPath = path.resolve('./public/upload/' + imgUrl + ext);
if (ext === '.png' || ext === '.jpg' || ext === '.jpeg' || ext === '.gif') {
fs.rename(tempPath, targetPath, function(err) {
if (err) throw err;
res.redirect('/images/' + imgUrl);
});
} else {
fs.unlink(tempPath, function() {
if (err) throw err;
res.json(500, {
error: 'Only image files are allowed.'
});
});
}
};
saveImage();
Preceding block of code is the logic that I am using to upload the file. In the error it is referring to 'file' as undefined which is in the following line in the saveImage function. It is unable to get the path and therefore throws error 500 according to the else part of the saveImage function. Why is 'file' undefined here? I dont get it.
var tempPath = req.files.file.path,

multer() returns a middleware generator that uses the settings you specified, so you cannot pass its return value directly to app.use(). You can see all of the types of middleware it can generate in the documentation, but typically the generated middleware are added at the route level instead of globally like the other body parsers. This is because you will typically pass in the name of the file field(s) that you will be expecting.
For example, this will accept a single file (along with any non-file fields) whose form field name is foo:
var upload = multer({
dest: path.join(__dirname, '../public/upload/temp')
});
// ...
app.post('/upload', upload.single('foo'), function(req, res) {
if (req.file) {
console.dir(req.file);
return res.end('Thank you for the file');
}
res.end('Missing file');
});
Also, body-parser does not currently export a multipart/form-data-capable middleware, so you cannot use that module for handling uploaded files (well, short of passing a base64-encoded string in an application/x-www-form-urlencoded form or something, but that's much less efficient).

Here is the basic code for file upload in MEAN please check
HTML
<form id="frmDoc" name="frmDocument" ng-submit="upload()" class="form-horizontal form-bordered" enctype="multipart/form-data" >
<fieldset>
<div class="form-group">
<label class="col-md-4 control-label" for="val_email">Document<span class="text-danger">*</span></label>
<div class="col-md-4">
<div class="input-group">
<input type="file" name="file" id='file' required="required" />
</div>
</div>
</div>
</fieldset>
<div class="form-group form-actions">
<div class="col-md-8 col-md-offset-4">
<button type="submit" class="btn btn-sm btn-primary"><i class="fa fa-upload"></i> submit</button>
</div>
</div>
</form>
CLIENT SIDE CODE
app.controller ('myctrl',function($scope,$http){
$scope.upload = function () {
var file = angular.element(document.querySelector('#file')).prop("files")[0];
$scope.files = [];
$scope.files.push(file);
$http({
method: 'POST',
url: '/users/upload',
headers: { 'Content-Type': undefined },
transformRequest: function (data) {
var formData = new FormData();
formData.append('model', angular.toJson(data.model));
formData.append('file', data.files[0]);
return formData;
},
data: { model: { title: 'hello'}, files: $scope.files }
}).success(function (res) {
console.log(res)
});
}
});
SERVER SIDE CODE
var multer = require('multer');
var mkdirp = require('mkdirp');
var storage = multer.diskStorage({
destination: function (req, file, cb) {
//var code = JSON.parse(req.body.model).empCode;
var dest = 'public/uploads/';
mkdirp(dest, function (err) {
if (err) cb(err, dest);
else cb(null, dest);
});
},
filename: function (req, file, cb) {
cb(null, Date.now()+'-'+file.originalname);
}
});
var upload = multer({ storage: storage });
router.post('/upload', upload.any(), function(req , res){
console.log(req.body);
res.send(req.files);
});

I corrected the code of the book "Web Development with MongoDB and NodeJS" as follows:
app.use(multer({dest:path.join(__dirname,'../public/upload/temp')}).any());
.
.
.
.
const tempPath = req.files[0].path, // Temporary location of uploaded file
ext = path.extname(req.files[0].originalname).toLowerCase(), // Get file extension of the uploaded file
targetPath = path.resolve(`./public/upload/${imgUrl}${ ext}`); // The final path for the image file
The other parts of code remained intact. It worked and I could upload image files.
Best wishes,
Mehrdad Sheikhan

Code for upload file using Multer and save it to local folder
api- call fileUpload function
fileUpload(req)
.then(uploadRes => {
console.log('uploadRes', uploadRes)
})
.catch(err => {
console.log('err', err)
})
Create file upload service
const multer = require('multer') // import library
const moment = require('moment')
const q = require('q')
const _ = require('underscore')
const fs = require('fs')
let dir = './public'
/** Store file on local folder */
let storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, dir)
},
filename: function (req, file, cb) {
let date = moment(moment.now()).format('YYYYMMDDHHMMSS')
cb(null, date + '_' + file.originalname.replace(/-/g, '_').replace(/ /g, '_'))
}
})
/** Upload files */
let upload = multer({ storage: storage }).array('files')
/** Exports fileUpload function */
module.exports = {
fileUpload: function (req) {
let deferred = q.defer()
/** Create dir if not exist */
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
console.log(`\n\n ${dir} dose not exist, hence created \n\n`)
}
upload(req, {}, function (err) {
if (req && (_.isEmpty(req.files))) {
deferred.resolve({ status: 200, message: 'File not attached', data: [] })
} else {
if (err) {
deferred.reject({ status: 400, message: 'error', data: err })
} else {
deferred.resolve({
status: 200,
message: 'File attached',
filename: _.pluck(req.files,
'filename'),
data: req.files
})
}
}
})
return deferred.promise
}
}

Uploading Assets to s3 using Gulp

I am currently trying to upload the assets of a website to amazon s3 using aws-sdk and gulp, but for now I just achieved uploading single files using this code:
gulp.task('publish', function() {
var AWS = require('aws-sdk'),
fs = require('fs');
AWS.config.accessKeyId = 'access_id';
AWS.config.secretAccessKey = 'secret_key';
AWS.config.region = 'eu-central-1';
var fileStream = fs.createReadStream('folder/filename');
fileStream.on('error', function (err) {
if (err) { throw err; }
});
fileStream.on('open', function () {
var s3 = new AWS.S3();
s3.putObject({
Bucket: 'bucket_name',
Key: 'assets/filename',
Body: fileStream,
ACL:'public-read'
}, function (err) {
if (err) { throw err; }
else { console.log("Upload successfull"); }
});
});
});
Since I am neither a node.js nor a JS dev, I have no Idea on how to upload all my assets in the folder assets of the S3.
Idealy, applying the action I use to upload one file, but for each file would be neat. How would this be doable?

Found the solution to my problem. With this code, I finaly managed to upload all my assets to my bucket with the right ACL's. Hope this can help people not to spend as much time as I did on such a stupid problem.
/*
* Dependencies
*/
var gulp = require('gulp');
var AWS = require('aws-sdk');
var fs = require('fs');
var walk = require('walk');
/*
* Declaration of global variables
*/
var isPaused = false;
/*
* Bucket access informations
*/
AWS.config.accessKeyId = 'access_keyid'
AWS.config.secretAccessKey = 'secret_access_key'
AWS.config.region = 'region';
/*
* Publishing function: uses a stream to push the files on the AWS Bucket
*/
function publishit(filename) {
var file = filename.substring('./'.length);
var key = file.substring('src/'.length);
var fileStream = fs.createReadStream(file);
isPaused = true;
// Check if there is an error on the file
fileStream.on('error', function (err) {
if (err) { throw err; }
});
// Action to do on opening of the file
fileStream.on('open', function () {
var s3 = new AWS.S3();
// Uploading the stream to the bucket
s3.putObject({
Bucket: 'bucket_name',
Key: key,
Body: fileStream,
ACL:'public-read'
}, function (err) {
// Show the error if there is any
if (err) { throw err; }
// If everything went successfully, print which file is being uploaded
else { console.log("Uploading asset "+ file); }
// Closing the stream to avoid leaks and socket timeouts
fileStream.close();
// Changing the status of 'isPaused' to false to continue uploading the other assets
isPaused = false;
});
});
}
gulp.task('assets', function() {
var files = [];
// Walker options (first arg is the folder you want to upload)
var walker = walk.walk('./assets', { followLinks: false });
walker.on('file', function(root, stat, next) {
// Add this file to the list of files
files.push(root + '/' + stat.name);
next();
});
// Action after every file has been added to 'files'
walker.on('end', function() {
for (var filename in files){
// Publish every file added to 'files'
publishit(files[filename]);
// Wait for one push on the server to be done before calling the next one
function waitForIt(){
if (isPaused) {
setTimeout(function(){waitForIt()},100);
}
};
};
});
});