I am using multer diskstorage to save a file to disk.
I first save it to the disk and do some operations with the file and then i upload it to remote bucket using another function and lib.
Once the upload is finished, i would like to delete it from the disk.
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage }).single('file')
and here is how i use it:
app.post('/api/photo', function (req, res) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
});
how can i use the diskStorage remove function to remove the files in the temp folder?
https://github.com/expressjs/multer/blob/master/storage/disk.js#L54
update:
I have decided to make it modular and put it in another file:
const fileUpload = function(req, res, cb) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
}
module.exports = { fileUpload };
You don't need to use multer to delete the file and besides _removeFile is a private function that you should not use.
You'd delete the file as you normally would via fs.unlink. So wherever you have access to req.file, you can do the following:
const fs = require('fs')
const { promisify } = require('util')
const unlinkAsync = promisify(fs.unlink)
// ...
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}`)
}
})
const upload = multer({ storage: storage }).single('file')
app.post('/api/photo', upload, async (req, res) =>{
// You aren't doing anything with data so no need for the return value
await uploadToRemoteBucket(req.file.path)
// Delete the file like normal
await unlinkAsync(req.file.path)
res.end("UPLOAD COMPLETED!")
})
Multer isn't needed. Just use this code.
const fs = require('fs')
const path = './file.txt'
fs.unlink(path, (err) => {
if (err) {
console.error(err)
return
}
//file removed
})
You may also consider using MemoryStorage for this purpose, with this storage the file is never stored in the disk but in memory and is deleted from the memory automatically after execution comes out of controller block, i.e., after you serve the response in most of the cases.
When you will use this storage option, you won't get the fields file.destination, file.path and file.filename, instead you will get a field file.buffer which as name suggests is a buffer, you can convert this buffer to desired format to do operations on and then upload using a stream object.
Most of the popular libraries support streams so you should be able to use stream to upload your file directly, code for converting buffer to stream:
const Readable = require('stream').Readable;
var stream = new Readable();
stream._read = () => { }
stream.push(file.buffer);
stream.push(null);
// now you can pass this stream object to your upload function
This approach would be more efficient as files will be stored in memory which will result in faster access, but it does have a con as mentioned in multer documentation:
WARNING: Uploading very large files, or relatively small files in
large numbers very quickly, can cause your application to run out of
memory when memory storage is used.
To do it truly automatically across all routes I used this strategy :
when the request ends, we delete all the uploaded files (req.files). Before that, if you want to keep the files on the server, you need to save them in another path.
var express = require('express');
var app = express();
var http = require('http');
var server = http.Server(app);
// classic multer instantiation
var multer = require('multer');
var upload = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
cb(null, `${__dirname}/web/uploads/tmp/`);
},
filename: function (req, file, cb) {
cb(null, uniqid() + path.extname(file.originalname));
},
}),
});
app.use(upload.any());
// automatically deletes uploaded files when express finishes the request
app.use(function(req, res, next) {
var writeHead = res.writeHead;
var writeHeadbound = writeHead.bind(res);
res.writeHead = function (statusCode, statusMessage, headers) {
if (req.files) {
for (var file of req.files) {
fs.unlink(file.path, function (err) {
if (err) console.error(err);
});
}
}
writeHeadbound(statusCode, statusMessage, headers);
};
next();
});
// route to upload a file
router.post('/profile/edit', access.isLogged(), async function (req, res, next) {
try {
// we copy uploaded files to a custom folder or the middleware will delete them
for (let file of req.files)
if (file.fieldname == 'picture')
await fs.promises.copy(file.path, `${__dirname}/../uploads/user/photo.jpg`);
} catch (err) {
next(err);
}
});
I have removed directory after file uploaded using fs-extra
const fs = require('fs-extra');
// after you uploaded to bucket
await fs.remove('uploads/abc.png'); // remove upload dir when uploaded bucket
Related
I'm processing csv file data coming from request to create json object using multer,csv-parser. but can't process from long time. please help and thanks.following is example of csv file processing.
const multer = require('multer');
const fs = require('fs')
const csv = require('csv-parser')
const fileStorageEngine = multer.memoryStorage({
destination: (req, file, cb) => {
cb(null, './csv');
}
,
filename: (req, file, cb) => {
cb(null, file.originalname);
},
});
const upload = multer({storage:fileStorageEngine});
app.post('/uploadcsv',upload.single("upfile"),async(req,res)=>{
const file = req.file;
fs.createReadStream(`./csv/${file.originalname}`)
.pipe(csv())
.on("data",(data)=>console.log(data));
res.send("file uploaded")
})
here I'm using fs module that works for me but I don't want to store data in file instead I want to process buffer data coming from req.file.buffer in chunk
I'm stuck here please help. because storing file and reading same file getting process slow because csv file have thousands of data.
The idea is to create a readable stream from the buffer instead of writing it to the file as follows:
const multer = require('multer');
const { Readable } = require('stream');
const fs = require('fs')
const csv = require('csv-parser')
const fileStorageEngine = multer.memoryStorage({
destination: (req, file, cb) => {
cb(null, './csv');
}
,
filename: (req, file, cb) => {
cb(null, file.originalname);
},
});
const upload = multer({storage:fileStorageEngine});
app.post('/uploadcsv',upload.single("upfile"),async(req,res)=>{
const file = req.file;
const stream = Readable.from(file.buffer);
stream.pipe(csv()).on("data",(data)=>console.log(data));
res.send("file uploaded")
})
I am trying to implement File Upload functionality using multer and Express Router. I defined an endpoint /batch_upload using router.use like below
api.js
router.use(
"/batch_upload",
upload.single("emp_csv_data"),
userController.processBatchUserInformation
);
in userController.js
exports.processBatchUserInformation = async (req, res) => {
console.log(req.file);
if (req.method == "POST") {
try {
console.log("Upload route reached - POST");
console.log(req.file);
console.log(req.file.path);
return res.send(req.file);
} catch (err) {
console.log("Error Occurred");
return res.send(err);
}
}
};
In the FileUploader.js, I defined the upload variable and multer options like below
var multer = require("multer");
var storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, "uploads");
},
filename: (req, file, cb) => {
return cb(null, file.fieldname + "-" + Date.now());
}
});
exports.upload = multer({ storage: storage });
Finally, in the app.js I used the route using
app.use('/user',user_routes)
But when I send a file to http://localhost:5000/user/batch_upload, I get an undefined response for req.file
Irony is that I have the exact implementation in a sample test project and everything seems fine. I don't understand what am I missing. If you see something that seems off, please help me fix it.
So, the reason behind the file not being uploaded was that I did not add Content-type:multipart/form-data in the headers. Thank you guys for trying to help. I appreciate it.
I am creating API using express JS. Now, I have a router which will be used to upload image using multer.
Here is my router :
const multer = require('multer');
module.exports = (app) => {
const DIR = './public/';
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, DIR);
},
filename: (req, file, cb) => {
cb(null , file.originalname);
}
});
const upload = multer({ storage: storage });
// I have also tried this but not working
// const upload = multer({ dest: 'uploads/' });
app.post('/upload', upload.single('image'), (req, res, next) => {
res.status(201).json({
message: "File uploaded successfully"
});
});
}
Now, from my reactjs app I am calling this router using axios like this :
const headers = {
"Content-Type": "multipart/form-data"
}
const body = new FormData();
body.append('image', this.state.selectedCategoryImage);
axios.post('http://localhost:3000/upload', body, { headers }).then((res) => {
console.log(res);
}).catch((err) => {
console.log(err);
});
In above code this.state.selectedCategoryImage is a selected image from html <input> tag.
Now, When I call this api I am getting my response "file uploaded successfully", but I am not able to see my uploaded image anywhere in public directory. My image is not uploading.
Please anyone can help me what's the issue ?
Pass file Object not URL
URL.createObjectURL(file) // it return file url that you can use to show file preview
For upload file, send actual file in axios API as you got from file input
var file = event.target.files[0]; // return actual file
this way it actually send file object.
i am using a api in server.js file.
app.post('/api/uploadalumniresume', api.UploadAlumniResume);
my api.js file code is:
exports.UploadAlumniResume = (req, res) => {
const storage = multer.diskStorage({
destination(req, file, cb) {
// cb(null, `${__dirname}public/images/uploads`);
cb(null, `${__dirname}/../public/imports`);
},
filename(req, file, cb) {
cb(null, `${Date.now()}-${file.originalname}`);
},
});
const importupload = multer({
storage,
});
const importmulter = importupload.single('file');
};
The server.js as call the api but the exports api not saving the uploading file to the destination folder
And i need to store the upload file on destination follder.
And if the file as uploaded then function need to work on create fuction in the same api.
please give a any solution to me!
I tried to receive the file and store it in the multer storage
Node js code
enter code here
app.post('/createLicence', upload.single('photo'),function(req, res ,next) {
// any logic goes here
console.log("filename" ,req.body.name)
if (!req.file) {
console.log("No file received");
return res.send({
success: false
});
} else {
console.log('file received');
var function_name = 'createLicence'
var arguments_array = [req.file.path,'Raghav','Mumbai','Approved']
invoke = require('/Users/sanjeev.natarajan/fabric-samples/fabcar/invoke.js');
invoke.invokechaincode(function_name,arguments_array)
return res.send({
success: true
})
}
});
but i am receiving no file is receivedi have send the request through postman
-
From : https://www.npmjs.com/package/multer
In order to use the multer package, you have first to define a few parameters so that it can work on your fileDirectory.
In your server.js :
let multer = require('multer');
let storage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, '/path/to/storage/')
},
filename: function(req, file, callback) {
callback(null, file.originalname + '-' + Date.now());
}
});
let upload = multer({
storage: storage
});
Now, configure your route
router.route('/your/payload')
.post(authController.isAuthenticated, upload.any(), albumController.postFile)
Note that upload.any() will allow you to upload multiple different formatted files at once. Feel free to use any other kind of upload.method() depending on your needs.
From this point, multer already is doing its job, however you might want to keep track of the files uploaded on your server.
So, in your own module, the logic is pretty much straight forward :
(I'm assuming that you're using mongoose models since you're not giving much information, but that's not the relevant part anyway)
exports.postFile = async (req, res) => {
if (!req || !req.files || !req.files[0]) return res.status(400).send("Bad request.");
for (let i = 0; req.files[i]; i++) {
await File.create({
path: req.files[i],
originalName: req.files[i].originalName,
mimetype: req.files[i].mimetype,
owner: req.user.userId
}, (err, file) => {
if (err) console.log("Something went wrong: " + err); else {
// Do something with file
}
});
}
return res.status(418).send("I'm a teapot.");
}
This configuration and middleware use is ONLY for testing purpose, never ever let anyone upload something to your server without carefully handle that uploading process (file integrity, resource management, ...). An open uploading system can become a very wide backdoor getting straight to your server.
Hope this helps,
regards.