Multer validation with joi - javascript

I've created an endpoint to post and upload a csv file for processing in Node. It works fine but I'm trying to figure out how to validate a few things before uploading the file.
A sample request would be:
{
"test_doc": "/path/to/file/test.csv"
"offset": [0,1]
}
I want the form to require "test_doc" and only accept csv files and have "offset" be optional
The schema for the "offset" works but I'm unsure how to validate the file with multer as well, especially before uploading it.
Sample code below
const upload = multer({ dest: "/tmp" });
router.post("/", upload.single("test_doc"), async (req, res) => {
const schema = joi.object().keys({
offset: joi.array().items(joi.number().min(-60).max(60)).min(1).max(2)
});
});

You can use the fileFilter method to validate file types in Multer. The fileFilter method is a built-in method which comes with Multer middleware:
const upload = multer({
dest: "/tmp",
fileFilter: (req, file, cb) => {
if (file.mimetype == "text/csv" && file.fieldname === "test_doc") {
cb(null, true);
} else {
cb(null, false);
return cb(new Error('Invalid upload: fieldname should be test_doc and .csv format '));
}
}
});

Related

Error: ENOENT: no such file or directory Nodejs

I'm trying to upload a file and store it in an uploads folder, but I get this error: no such file or directory
I get the message success in console but I get this error anyway.
POST /auth/register 500 21.023 ms - 260
Error: ENOENT: no such file or directory, open E:\IMPORTANT\INFO-DEV\DEV\ANGULAR NODEJS\API AUTH\uploads\1671534381494.jpeg
Here is my configuration code for upload.
const path = require("path");
const multer = require("multer");
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "uploads/");
},
filename: function (req, file, cb) {
const extension = path.extname(file.originalname);
cb(null, Date.now() + extension);
},
});
const upload = multer({
storage: storage,
fileFilter: function (req, file, callback) {
if (
file.mimetype == "image/png" ||
file.mimetype == "image/jpg" ||
file.mimetype == "image/jpeg"
) {
callback(null, true);
console.log("Image téléchargé avec succès"); // success message
} else {
callback(null, false);
console.log("Seulement du fichier de type png, jpg ou jpeg"); // error message
}
},
limits: {
fileSize: 1024 * 1024 * 2,
},
});
module.exports = upload;
I got the same error, but I was able to solve it by getting my current path.
import multer from "multer";
// Set up multer storage options
const storage = multer.diskStorage({
destination: function (req, file, cb) {
console.log("🚀 ~ file: upload.ts:11 ~ file", process.cwd());
cb(null, `${process.cwd()}/src/Images`);
},
filename: function (req, file, cb) {
cb(null, file.fieldname + "-" + Date.now());
},
});
// Create a multer instance with the storage options
const upload = multer({ storage });
export default upload;
The issue is that your uploads folder doesn't exist or you set the path incorrectly.
I don't know where exactly you created uploads folder (and if you created it at all).
So in the destination param you should pass either:
path.join(__dirname, '/uploads') - in case that folder is in the same location where current js file is located.
Or path.join(process.cwd(), '/uploads') - in case if uploads folder is in the root of the project (where you run npm start etc.)
So, in short words you need to make sure folder exists and then make sure the path is correct.
P.S.
Using ../../ syntax should also work, you can try ../uploads or ../../uploads if, for example, that folder is on higher levels of your folders structure.
Change this line
cb(null, './uploads/');
this line
cb(null, "uploads/");
or try
cb(__dirname, "uploads/");
you can try your path name as like
img: {
data: fs.readFileSync(
path.join(__dirname + "../../uploads" + req.file.filename)
),
contentType: "image/png",
},

File not uploading in Express JS using Multer

I am creating API using express JS. Now, I have a router which will be used to upload image using multer.
Here is my router :
const multer = require('multer');
module.exports = (app) => {
const DIR = './public/';
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, DIR);
},
filename: (req, file, cb) => {
cb(null , file.originalname);
}
});
const upload = multer({ storage: storage });
// I have also tried this but not working
// const upload = multer({ dest: 'uploads/' });
app.post('/upload', upload.single('image'), (req, res, next) => {
res.status(201).json({
message: "File uploaded successfully"
});
});
}
Now, from my reactjs app I am calling this router using axios like this :
const headers = {
"Content-Type": "multipart/form-data"
}
const body = new FormData();
body.append('image', this.state.selectedCategoryImage);
axios.post('http://localhost:3000/upload', body, { headers }).then((res) => {
console.log(res);
}).catch((err) => {
console.log(err);
});
In above code this.state.selectedCategoryImage is a selected image from html <input> tag.
Now, When I call this api I am getting my response "file uploaded successfully", but I am not able to see my uploaded image anywhere in public directory. My image is not uploading.
Please anyone can help me what's the issue ?
Pass file Object not URL
URL.createObjectURL(file) // it return file url that you can use to show file preview
For upload file, send actual file in axios API as you got from file input
var file = event.target.files[0]; // return actual file
this way it actually send file object.

Custom file name from frontend in Multer

I'm uploading a file using FormData and receiving it server-side using Multer. Everything works as expected, except since I'm using FileSystem API on the front-end (https://developer.mozilla.org/en-US/docs/Web/API/DataTransferItem/webkitGetAsEntry), the files I'm uploading come from sub-directories. Multer seems to only see the filename, even if I explicitly set an alias for the file as I append it to form data (https://developer.mozilla.org/en-US/docs/Web/API/FormData/append). It also seems like Multer performs its logic prior to the rest of my request handler and does not see the parameters I set on the body. How do I get multer to see the full path?
Here is a simplified version of what I currently have setup:
Client (alias represents full name with path, file.name is the base name automatically set by FileSystem API):
function upload(file, alias) {
let url = window.location.origin + '/upload';
let xhr = new XMLHttpRequest();
let formData = new FormData();
xhr.open('POST', url, true);
return new Promise(function (resolve, reject) {
xhr.addEventListener('readystatechange', function(e) {
if (xhr.readyState == 4 && xhr.status == 200) {
resolve(file.name);
}
else if (xhr.readyState == 4 && xhr.status != 200) {
reject(file.name);
}
})
formData.append('file', file, alias || file.name); // this should in theory replace filename, but doesn't
formData.append('alias', alias || file.name); // an extra field that I can't see in multer function at all
xhr.send(formData);
});
}
Server:
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, 'uploads/');
},
filename: function (req, file, cb) {
// neither req nor file seems to contain any hint of the alias here
cb(null, file.originalname);
}
});
const upload = multer({storage: storage});
const bodyParser = require('body-parser');
app.use(bodyParser.json());
app.post('/upload', upload.single('file'), function (req, res, next) {
// by this time the file seems to already be on disk with whatever name multer picked
if (req.file) {
res.status(200).end();
} else {
res.status(500).end();
}
});
In order to get this to work, use the preservePath option when configuring multer. The following will work:
const upload = multer({storage: storage, preservePath: true});
However, it's important to note, multer will not create the directories or subdirectories. Those have to be created beforehand. (I tested this too. If directories are created and empty, upload succeeds, however, if directories do not exist, uploads fail).
In their readme, they say:
"Note: You are responsible for creating the directory when providing destination as a function. When passing a string, multer will make sure that the directory is created for you."
A follow-up to that note would be: "you are responsible for creating any sub-directories too".
The relative paths of files uploaded will be accessible in originalname property. So, backend would look like this: (as you had it, but with updated comments)
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, 'uploads/');
},
filename: function (req, file, cb) {
// If you uploaded for example, the directory: myDir/myFile.txt,
// file.originalname *would* be set to that (myDir/myFile.txt)
// and myFile.txt would get saved to uploads/myDir
// *provided that* uploads/myDir already exists.
// (if it doesn't upload will fail)
// /* if( [ uploads/myDir doesn't exist ] ) { mkdir } */
cb(null, file.originalname);
}
});
Helpful tip:
On the front end, I found it easier to test directory / subdirectory upload with: (tested on Chrome latest ok)
<form action="/uploads/multipleFiles" method="post" enctype="multipart/form-data">
<input type="file" name="multiple" webkitdirectory accept="text/*" onchange="console.log(this.files)" />
<input type="text" name="tester" value="uploadTester" />
<input type="submit"/>
</form>
If you want to upload Passport image as a front and back side then pass parameter from frontend like this user:"username" and type:"front" OR type:"back"
then Use it in node side like this
const upload = multer({
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, 'uploads/passport/');
},
filename: function (req, file, cb) {
cb(null, req.body.user+"-"+req.body.type+".jpg");
}
})
});

NodeJS - multer - change filename depending on request attributes

I know that I can change the filename with multer by means of the storage object like following:
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, process.env.UPLOAD_DIR);
},
filename: (req, file, cb) => {
cb(null, 'bla.png');
}
});
const upload = multer({ storage: storage } );
My request, besides having the file, also contains some text attributes such as name: myPic.png.
Is it possible to dynamically change the filename dependent on other request attributes or within the controller like following:
filename: (req, file, cb) => {
cb(null, `${req.body.name}.png`);
}
or
router.post('/upload', upload.single('pic'), myController.upload);
/* in controller */
upload = async (req: Request, res: Response) => {
try {
/* change the filename of multer here? */
} catch (err) {
winston.error(`Error while uploading: ${err.message}`);
winston.error(`Stack trace: ${err.stack}`);
sendJSONResponse(res, err, HttpStatus.INTERNAL_SERVER_ERROR);
}
}
Multer is the middleware which both populates req.body AND stores the file.
Also, when it reaches the filename() function, there is no guarantee that the text fields will be populated in req.body because it depends on which order the client sends them in (see last note).
From what I see, you have two options:
1) Rename the uploaded file after the multer upload middleware does its thing and populates req.body as well as req.file. So in your controller upload middleware, you'd do something like:
if (req.file) {
fs.renameSync(req.file.path, req.file.destination + req.body.name);
}
2) Change the request body text field into a query parameter. Then, inside filename() you can do a req.query.name.
Con: Not a very RESTful design, but maybe that is not so important to you.
According to the multer documentation it does not have access to req.body for other additional fields, if you test it it receives an undefined value, then a not so perfect but functional solution is the following, once the file is uploaded you can rename it as follows.
Add the native class fs for access to files option
const fs = require('fs');
In diskStorage configuration add the name you want, for example bla.png
var storage = multer.diskStorage({
destination: path.join('public/images/'),
filename: function ( req, file, cb ) {
cb(null, 'bla.png');
}
});
The form with the text field for the custom name
<form action="/upload" enctype="multipart/form-data" method="POST">
<input type="file" accept="image/*" name="photo" >
<br><!--here is the custom file name-->
<input type="text" name="file_name">
<br>
<button type="submit">Send</button>
</form>
Within the post path, once you have sent the file whose name will be bla.png, you can replace that name with the one in a field of the form by accessing req.body.field_name
router.post('/upload', upload.single('photo'), (req, res) => {
//Here change the file name bla.png for the new value in req.body.field_name + original ext of file
fs.renameSync(req.file.path, req.file.path.replace('bla.png',
req.body.field_name + path.extname(req.file.originalname)));
if(req.file) {
res.json(req.file);
}
else throw 'error';
});

nodejs multer diskstorage to delete file after saving to disk

I am using multer diskstorage to save a file to disk.
I first save it to the disk and do some operations with the file and then i upload it to remote bucket using another function and lib.
Once the upload is finished, i would like to delete it from the disk.
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage }).single('file')
and here is how i use it:
app.post('/api/photo', function (req, res) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
});
how can i use the diskStorage remove function to remove the files in the temp folder?
https://github.com/expressjs/multer/blob/master/storage/disk.js#L54
update:
I have decided to make it modular and put it in another file:
const fileUpload = function(req, res, cb) {
upload(req, res, function (err) {
uploadToRemoteBucket(req.file.path)
.then(data => {
// delete from disk first
res.end("UPLOAD COMPLETED!");
})
})
}
module.exports = { fileUpload };
You don't need to use multer to delete the file and besides _removeFile is a private function that you should not use.
You'd delete the file as you normally would via fs.unlink. So wherever you have access to req.file, you can do the following:
const fs = require('fs')
const { promisify } = require('util')
const unlinkAsync = promisify(fs.unlink)
// ...
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}`)
}
})
const upload = multer({ storage: storage }).single('file')
app.post('/api/photo', upload, async (req, res) =>{
// You aren't doing anything with data so no need for the return value
await uploadToRemoteBucket(req.file.path)
// Delete the file like normal
await unlinkAsync(req.file.path)
res.end("UPLOAD COMPLETED!")
})
Multer isn't needed. Just use this code.
const fs = require('fs')
const path = './file.txt'
fs.unlink(path, (err) => {
if (err) {
console.error(err)
return
}
//file removed
})
You may also consider using MemoryStorage for this purpose, with this storage the file is never stored in the disk but in memory and is deleted from the memory automatically after execution comes out of controller block, i.e., after you serve the response in most of the cases.
When you will use this storage option, you won't get the fields file.destination, file.path and file.filename, instead you will get a field file.buffer which as name suggests is a buffer, you can convert this buffer to desired format to do operations on and then upload using a stream object.
Most of the popular libraries support streams so you should be able to use stream to upload your file directly, code for converting buffer to stream:
const Readable = require('stream').Readable;
var stream = new Readable();
stream._read = () => { }
stream.push(file.buffer);
stream.push(null);
// now you can pass this stream object to your upload function
This approach would be more efficient as files will be stored in memory which will result in faster access, but it does have a con as mentioned in multer documentation:
WARNING: Uploading very large files, or relatively small files in
large numbers very quickly, can cause your application to run out of
memory when memory storage is used.
To do it truly automatically across all routes I used this strategy :
when the request ends, we delete all the uploaded files (req.files). Before that, if you want to keep the files on the server, you need to save them in another path.
var express = require('express');
var app = express();
var http = require('http');
var server = http.Server(app);
// classic multer instantiation
var multer = require('multer');
var upload = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
cb(null, `${__dirname}/web/uploads/tmp/`);
},
filename: function (req, file, cb) {
cb(null, uniqid() + path.extname(file.originalname));
},
}),
});
app.use(upload.any());
// automatically deletes uploaded files when express finishes the request
app.use(function(req, res, next) {
var writeHead = res.writeHead;
var writeHeadbound = writeHead.bind(res);
res.writeHead = function (statusCode, statusMessage, headers) {
if (req.files) {
for (var file of req.files) {
fs.unlink(file.path, function (err) {
if (err) console.error(err);
});
}
}
writeHeadbound(statusCode, statusMessage, headers);
};
next();
});
// route to upload a file
router.post('/profile/edit', access.isLogged(), async function (req, res, next) {
try {
// we copy uploaded files to a custom folder or the middleware will delete them
for (let file of req.files)
if (file.fieldname == 'picture')
await fs.promises.copy(file.path, `${__dirname}/../uploads/user/photo.jpg`);
} catch (err) {
next(err);
}
});
I have removed directory after file uploaded using fs-extra
const fs = require('fs-extra');
// after you uploaded to bucket
await fs.remove('uploads/abc.png'); // remove upload dir when uploaded bucket

Categories