Uploading multiple files to multiple locations with Multer & Node.JS - javascript

I am trying to write an app using NodeJS and Express which will upload both a user avatar which is a single image (jpeg, png or webp) and 'legalDocs' which is essentially also a single file that can either be an image (jpeg, png or webp) or a pdf file. The avatar needs to be uploaded to ./resources/(userID)/userAvatar and the legal docs to ./resources/(userID)/legalDocs. Along with some other text info that will be posted along with the request as a multipart/form-data.
The userID is only generated after processing the text data posted.
Here's my current non-working code
APP.JS
//ENV
require("dotenv").config();
require("express-async-errors");
//EXPRESS
const express = require("express");
const server = express();
global.__basedir = __dirname;
//routes
const authRouter = require("./routes/authRoutes.js");
server.use(express.json());
server.use(express.urlencoded({extended: true}));
server.use(express.static("./public"));
server.use("/api/v0/auth", authRouter);
const port = process.env.PORT || 3000;
const start = async () => {
try {
server.listen(port, () => {
console.log(`--> Listening on port ${port}...`);
});
} catch (error) {
console.log(error);
console.log("Fatal error!");
}
}
start();
authRoutes.js:
const express = require("express");
const router = express.Router();
const {signupWithEmail} = require("../controllers/auth.js");
router.route("/signup").post(signupWithEmail);
module.exports = router;
../controllers/auth.js:
const multer = require("multer");
const processReq = require("../middleware/upload.js");
const signupWithEmail = async (req, res, next) => {
//Processing to get the ID happens here.
//For simplicity we can pretend like the ID is directly
//posted in a field with the name id.
const id = req.body.id;
console.log("id:");
console.log(id);
await processReq(id)(req, res, next);
res.status(200).send("done!");
}
upload.js
const multer = require("multer");
const errors = require("../errors");
const maxSize = 2 * 1024 * 1024;
const acceptableFileMIMETypes = ["image/jpeg", "image/png", "image/webp", "application/pdf"];
const userAvatarFilter = function (req, file, cb) {
if ((file.fieldname === "userAvatar" || file.fieldname === "legalDocs")
&& acceptableFileMIMETypes.includes(file.mimetype)
&& !(file.fieldname === "userAvatar" && file.mimetype === "application/pdf")) {
if (file.fieldname === "userAvatar") {
cb(null, true);
} else {
cb(null, false);
}
} else {
cb(new errors.BadRequestError("Both files are required, and must be JPEG, PNG, WEBP. Legal documents can also be PDF."));
}
}
const userlegalDocsFilter = function (req, file, cb) {
if ((file.fieldname === "userAvatar" || file.fieldname === "legalDocs")
&& acceptableFileMIMETypes.includes(file.mimetype)
&& !(file.fieldname === "userAvatar" && file.mimetype === "application/pdf")) {
if (file.fieldname === "userAvatar") {
cb(null, false);
} else {
cb(null, true);
}
} else {
cb(new errors.BadRequestError("Both files are required, and must be JPEG, PNG, WEBP. Legal documents can also be PDF."));
}
}
const uploadFunc = (id) => {
return (req, res, next) => {
const userAvatarStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, __basedir + `/resources/${id}/userAvatar`);
},
filename: (req, file, cb) => {
console.log("Uploaded avatar!");
console.log(file.originalname);
console.log(file);
cb(null, file.originalname);
}
});
const legalDocsStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, __basedir + `/resources/${id}/legalDocs`);
},
filename: (req, file, cb) => {
console.log("Uploaded legalDocs!");
console.log(file.originalname);
console.log(file);
cb(null, file.originalname);
}
});
const uploadAvatar = multer({
storage: userAvatarStorage,
limits: {
fileSize: maxSize,
files: 2
},
fileFilter: userAvatarFilter
}).any();
const uploadLegalDocs = multer({
storage: legalDocsStorage,
limits: {
fileSize: maxSize,
files: 2
},
fileFilter: userlegalDocsFilter
}).any();
uploadAvatar(req, res, next);
uploadLegalDocs(req, res, next);
next();
}
}
module.exports = uploadFunc;
The Problem
The first problem with my code is quite obvious, I cannot parse text data from the request unless it had been processed by multer (with the current code, console.logging the id shows that it's undefined), but I need the ID to be passed to multer in order to save the files to the correct directory. It seems like I need some way to process text only and generate the ID, then process the files.
The second problem is that, even if the ID is static - meaning that in ./controllers/auth.js the signupWithEmail function became:
const multer = require("multer");
const processReq = require("../middleware/upload.js");
const signupWithEmail = async (req, res, next) => {
const id = 1337;
console.log("id:");
console.log(id);
await processReq(id)(req, res, next);
res.status(200).send("done!");
}
I this is the output I get on the console when I try to post:
id:
1337
Uploaded legalDocs!
cours-analyse-numerique-ch1-21_22.pdf
{
fieldname: 'legalDocs',
originalname: 'cours-analyse-numerique-ch1-21_22.pdf',
encoding: '7bit',
mimetype: 'application/pdf'
}
node:_http_outgoing:771
throw new ERR_HTTP_HEADERS_SENT('remove');
^
Error [ERR_HTTP_HEADERS_SENT]: Cannot remove headers after they are sent to the client
at new NodeError (node:internal/errors:400:5)
at ServerResponse.removeHeader (node:_http_outgoing:771:11)
at write (C:\Users\der_u\source\repos\ATEEK\node_modules\finalhandler\index.js:282:9)
at AsyncResource.runInAsyncScope (node:async_hooks:204:9)
at listener (C:\Users\der_u\source\repos\ATEEK\node_modules\on-finished\index.js:170:15)
at onFinish (C:\Users\der_u\source\repos\ATEEK\node_modules\on-finished\index.js:101:5)
at callback (C:\Users\der_u\source\repos\ATEEK\node_modules\ee-first\index.js:55:10)
at IncomingMessage.onevent (C:\Users\der_u\source\repos\ATEEK\node_modules\ee-first\index.js:93:5)
at IncomingMessage.emit (node:events:513:28)
at endReadableNT (node:internal/streams/readable:1359:12) {
code: 'ERR_HTTP_HEADERS_SENT'
}
and only 'legalDocs' get uploaded.
Is there any way to solve these problems without using a temporary directory to save all the files then move them once the ID had been generated? And what does the [ERR_HTTP_HEADERS_SENT] error mean?

Related

file upload multier MulterError: Unexpected field

Not able to upload image in MongoDB database using NodeJS
Middleware fieldName and key name is also same still not working. no solution on internet
server.js
import cookieParser from "cookie-parser";
import multer from "multer";
export const app = express();
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({extended: true}))
let upload = multer()
app.use(upload.array())
app.use("/",routLog,router)
app.listen(process.env.PORT,()=>{
db()
console.log(`Server is Runing on http://localhost:${process.env.PORT}`)
})
blogroute.js
import { fileUploder } from "../middelware/fileUploder.js";
export const blogRouter = Router()
blogRouter.post('/:user/create-post', fileUploder.single("file"), blogsController.createPos
This middleware file fileUplad.js which is use to store in upload folder and then from that will store in MongoDB in buffer ferment
import { randomBytes } from "crypto";
import multer from "multer";
import path from "path";
const storage = multer.diskStorage({
destination: './upload',
filename: async (req, file, callback) => {
return await randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
const filename = buf.toString("hex") + path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: "upload"
};
resolve(fileInfo);
})
}
})
export const fileUploder = multer({
storage: storage,
limits: { fileSize: 2000 },
fileFilter: (req, file, callback) => {
allowedFileTypes = /jpeg|jpg|png|gif/;
if (file.mimetype == 'jpg' || file.mimetype == 'jpeg' || file.mimetype == 'png', file.mimetype == 'gif') {
return cb(null, true);
} else {
cb("Invalid file type. Only JPEG, PNG and GIF file are allowed.")
}
}
})
MulterError: Unexpected field
at wrappedFileFilter (/mnt/ssd/BlogProject/backendapi/node_modules/multer/index.js:40:19)
at Multipart.<anonymous> (/mnt/ssd/BlogProject/backendapi/node_modules/multer/lib/make-middleware.js:107:7)
at Multipart.emit (node:events:527:28)
at HeaderParser.cb (/mnt/ssd/BlogProject/backendapi/node_modules/busboy/lib/types/multipart.js:358:14)
at HeaderParser.push (/mnt/ssd/BlogProject/backendapi/node_modules/busboy/lib/types/multipart.js:162:20)
at SBMH.ssCb [as _cb] (/mnt/ssd/BlogProject/backendapi/node_modules/busboy/lib/types/multipart.js:394:37)
at feed (/mnt/ssd/BlogProject/backendapi/node_modules/streamsearch/lib/sbmh.js:248:10)
at SBMH.push (/mnt/ssd/BlogProject/backendapi/node_modules/streamsearch/lib/sbmh.js:104:16)
at Multipart._write (/mnt/ssd/BlogProject/backendapi/node_modules/busboy/lib/types/multipart.js:567:19)
at writeOrBuffer (node:internal/streams/writable:389:12)```
This is simple example :
let multer = require('multer');
let multerImage = multer().single('file');
function uploadAvatar(req, res) {
let phone = data.phoneNumber;
multerImage(req, res, () => {
let file = req.file;
if (isUndefined(file))
return Json.builder(Response.HTTP_BAD_REQUEST);
let {
fileUrl
} = File.validationAndWriteFile(file.buffer, Util.getFileFormat(file.originalname));
Update.img(phone, fileUrl, result => {
if (!result)
return Json.builder(Response.HTTP_BAD_REQUEST);
Json.builder(Response.HTTP_OK);
});
});
}

uploading two files from two fields with multer as middleware

i'm trying to upload two files with different file extensions with multer from two fields but when i try it with postman the result always for the file is null, what is the solution for my problem? here is my code
middleware/uploadEpub
const multer = require('multer')
exports.uploadEpub = (epubFile, coverFile) => {
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "uploads")
},
filename: function (req, file, cb) {
cb(null, Date.now() + '-' + file.originalname.replace(/\s/g, ""))
}
})
const upload = multer({
storage
}).fields([{name: "bookFile", maxCount: 1},{name: "coverFile", maxCount: 1}])
}
controller/book
exports.addBook = async (req, res) => {
try {
const { ...data } = req.body
const newBook = await book.create({
...data,
bookFile: req.file,
coverFile: req.file
})
let bookData = await book.findOne({
where: {
id: newBook.id
},
attributes:{
exclude: ['createdAt','updatedAt']
}
})
bookData = JSON.parse(JSON.stringify(bookData))
res.send({
status: "Success",
Book: {
...bookData
}
})
} catch (error) {
console.log(error)
res.status(500).send({
status: "Failed",
message: "Server Error"
})
}
}
Multer set up
const multer = require('multer')
const path = require('path')
const { nanoid } = require('nanoid')
//Set Storage Engine
const storage = multer.diskStorage({
destination: './public/uploads/',
filename: (req, file, callback) => {
const id = nanoid(6)
const newFilename = `${file.fieldname}_${id}-${new Date().toISOString().replace(/:/g, '-')}${path.extname(file.originalname)}`
callback(null, newFilename)
}
})
const upload = multer({
storage: storage,
limits: { fileSize: 5572864 }, // up to 5.5 MB
fileFilter: (req, file, callback) => {
checkFileType(file, callback)
},
})
//Check File Type
const checkFileType = (file, cb) => {
//Allowed extensions
const fileType = /jpeg|jpg|png|gif|svg|pdf|epub/
//Check extension
const extname = fileType.test(path.extname(file.originalname).toLowerCase())
//Check mimetype
const mimetype = fileType.test(file.mimetype)
if (extname && mimetype) {
return cb(null, true)
} else {
return cb('Error: wrong file type!')
}
}
module.exports = upload
/***** middlewares.js *****/
module.exports.imageUploader = (req, res, next)=>{
const files = req.files
const uploadedFiles = []
for (var i = 0; i <images.length; i++){
uploadFiles.push('https://yourserver.com/public/uploads/' + files[i].filename)
}
req.uploadFiles = uploadFiles.toString() //appending files to req
next()
return
}
/**** index.js or app.js ****/ //where your routes are defined
router.post('/books/add', upload.array('images', 10), imageUploader, book.addBook) // allowing up to 10 files to be uploaded, calling imageUploader as a middleware
/*** Controllers/book ***/
exports.addBook = async (req, res) => {
const uploadedFiles = req.uploadFiles; //catching files from imageUploader middleware
// ... rest of your code
}
Try to upload the file as form-data in postman, and put the key with the same name that you have set in multer, the postman will look like this:
Try this:
const cpUpload = upload.fields([{ name: 'avatar', maxCount: 1 }, { name: 'gallery', maxCount: 8 }])
app.post('/cool-profile', cpUpload, function (req, res, next) {
})

Multer doesn't save the file

Multer does not save the file and I cannot see any logs... The folder is empty, no any files.
The image data is actually sent to the server as its console logged:
Screenshot of req.files from log
What am I doing wrong? Please help me find
const express = require('express');
const router = express.Router();
const path = require('path');
const multer = require('multer');
const storage = multer.diskStorage({
destination: (req, file, cb) => {
console.log('destination') // can't see this log
cb(null, './uploads/');
},
filename: (req, file, cb) => {
console.log('filename') // can't see this log
const ext = file.mimetype.split("/")[1];
cb(null, Date.now() + path.extname(file.originalname));
}
});
const uploadImage = multer({
storage,
limits: { fieldSize: 1024 * 1024 },
fileFilter: (req, file, cb) => {
console.log('log!') // can't see this log
const ext = path.extname(file.originalname);
if (ext !== '.jpg' && ext !== '.jpeg' && ext !== '.png') {
console.log('not log!') // can't see this log
const err = new Error('Extention');
err.code = 'EXTENTION';
return cb(err);
}
console.log('log!') // can't see this log
cb(null, true);
}
}).single('photo');
router.post('/image', async (req, res) => {
console.log(req.files) // can see this log
uploadImage(req, res, err => {
console.log('upload') // can see this log
let error = '';
if (err) {
console.log(err)
if (err.code === 'LIMIT_FILE_SIZE') {
error = 'Max 500kb!';
}
if (err.code === 'EXTENTION') {
error = 'Only jpg and png!';
}
}
res.json({
ok: !error,
error
});
})
});
Multer finally gave an error, so the problem was resolved... The problem was that the names of these fields are different:
Server:
.single('pic');
And in the request to the server:
formData.append('pic', file);
The code seems to work fine for me but I just had to manually create the uploads directory.
I used postman to test this and the picture appeared in my uploads directory.
# Console Output
SERVER RUNNING ON 3000
undefined
log!
log!
destination
filename
upload

How to set field name to upload file single/array Multer NodeJs

I have a code for upload images in nodejs using multer and sharp. But i want to ask how to set dynamic array name at line 19? I want to set that array name from the nodejs routes, because my field name in mongodb collections is different in every collection.
This is my middleware upload code:
const multer = require("multer");
const sharp = require("sharp");
const multerStorage = multer.memoryStorage();
const multerFilter = (req, file, cb) => {
if (file.mimetype.startsWith("image")) {
cb(null, true);
} else {
cb("Please upload only images.", false);
}
};
const upload = multer({
storage: multerStorage,
fileFilter: multerFilter
});
const uploadFiles = upload.array("images", 10); //Change this 'images' name from route
const uploadImages = (req, res, next) => {
uploadFiles(req, res, err => {
if (err instanceof multer.MulterError) {
if (err.code === "LIMIT_UNEXPECTED_FILE") {
return res.send("Too many files to upload.");
}
} else if (err) {
return res.send(err);
}
next();
});
};
const resizeImages = async (req, res, next) => {
if (!req.files) return next();
req.body.images = [];
await Promise.all(
req.files.map(async file => {
const filename = file.originalname.replace(/\..+$/, "");
const newFilename = `cariboss-${filename}-${Date.now()}.jpeg`;
await sharp(file.buffer)
.resize(640, 320)
.toFormat("jpeg")
.jpeg({ quality: 90 })
.toFile(`uploads/${newFilename}`);
req.body.images.push(newFilename);
})
);
next();
};
const getResult = async (req, res) => {
if (req.body.images.length <= 0) {
return res.send(`You must select at least 1 image.`);
}
const images = req.body.images
.map(image => "" + image + "")
.join("");
return res.send(`Images were uploaded:${images}`);
};
module.exports = {
uploadImages: uploadImages,
resizeImages: resizeImages,
getResult: getResult
};
My routes code:
module.exports = app => {
const articles = require("../controllers/articles.controller.js");
const upload = require("../middlewares/upload");
var router = require("express").Router();
router.post("/", upload.uploadImages, upload.resizeImages, upload.getResult, articles.create);
app.use("/api/article", verifyToken, router);
};
I have already search the solution on internet, but there is nothing. Thanks for your help sir.

Node.js: Multer upload with promise?

I have this express route with multer file-upload. When the upload is complete, I would like to encode the image to base64 and send with response.
However when I do it like this, the code tries to execute the base64 encoding before the file is created to the folder.
Edit: Added storage & upload functions
const storage = multer.diskStorage({
destination: (req, file, callback) => {
if (!fs.existsSync('./uploads')) {
fs.mkdirSync('./uploads');
}
let path = './uploads';
callback(null, path);
},
filename(req, file, cb) {
let fileExt = file.originalname.substring(file.originalname.lastIndexOf('.')).toLowerCase();
if (!imageFilter(fileExt)) {
return false;
} else {
cb(null, file.originalname);
}
},
onError: function (err, next) {
console.log('error', err);
next(err);
},
});
const upload = multer({
storage,
limits: {
fileSize: 1000 * 1000 * 2 // 2 MB
}
}).single('file');
router.post('/upload', function (req, res) {
var directory = 'uploads';
fs.readdir(directory, (err, files) => {
if (err) throw err;
for (var file of files) {
fs.unlink(path.join(directory, file), err => {
if (err) throw err;
});
}
});
upload(req, res, function (err) {
if (err) {
return res.status(404).json({
success: false,
message: 'File is too large. (Max 2MB)'
});
}
var file = req.file;
var base64str = base64_encode('./uploads/' + file.originalname);
return res.status(200).json({
success: true,
url: 'http://' + ip.address() + ':' + constants.PORT + '/api/uploads/' + file.originalname,
image: 'data:image/png;base64,' + base64str
});
});
});
What would be the smartest way to achieve the right order of operations. Possibly promises or async/await?
This solution worked for me :
Node v8.4.0 is required for this
//app.js
const fs = require('fs');
const express = require('express');
const cors = require('cors');
const bodyParser = require('body-parser');
const app = express();
app.use(cors({credentials: true, origin: 'http://localhost:4200'}));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
const Uploader = require('./Uploader.js');
const uploader = new Uploader();
app.post('/upload', uploader.startUpload);
//Uploader.js
const util = require("util");
const crypto = require("crypto");
const multer = require('multer');
class Uploader {
constructor() {
const storageOptions = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, __dirname + '/uploads/')
},
filename: function(req, file, cb) {
crypto.pseudoRandomBytes(16, function(err, raw) {
cb(null, raw.toString('hex') + Date.now() + '.' + file.originalname);
});
}
});
this.upload = multer({ storage: storageOptions });
}
async startUpload(req, res) {
let filename;
try {
const upload = util.promisify(this.upload.any());
await upload(req, res);
filename = req.files[0].filename;
} catch (e) {
//Handle your exception here
}
return res.json({fileUploaded: filename});
}
}
Edit :
The library "util" provide you a "promisify" method which will give you the possibility to avoid something called the "callback hell". It converts a callback-based function to a Promise-based one.
This is a small example to understand my code above:
const util = require('util');
function wait(seconds, callback) {
setTimeout(() => {
callback();
}, seconds);
}
function doSomething(callType) {
console.log('I have done something with ' + callType + ' !');
}
// Default use case
wait(2 * 1000, () => {
doSomething('callback');
});
const waitPromisified = util.promisify(wait);
// same with promises
waitPromisified(2000).then((response) => {
doSomething('promise');
}).catch((error) => {
console.log(error);
});
// same with async/await
(async () => {
await waitPromisified(2 * 1000);
doSomething('async/await');
})();
Will print :
I have done something with callback !
I have done something with promise !
I have done something with async/await !

Categories