We Keep Coding

Uploading multiple files to multiple locations with Multer & Node.JS

I am trying to write an app using NodeJS and Express which will upload both a user avatar which is a single image (jpeg, png or webp) and 'legalDocs' which is essentially also a single file that can either be an image (jpeg, png or webp) or a pdf file. The avatar needs to be uploaded to ./resources/(userID)/userAvatar and the legal docs to ./resources/(userID)/legalDocs. Along with some other text info that will be posted along with the request as a multipart/form-data.
The userID is only generated after processing the text data posted.
Here's my current non-working code
APP.JS
//ENV
require("dotenv").config();
require("express-async-errors");
//EXPRESS
const express = require("express");
const server = express();
global.__basedir = __dirname;
//routes
const authRouter = require("./routes/authRoutes.js");
server.use(express.json());
server.use(express.urlencoded({extended: true}));
server.use(express.static("./public"));
server.use("/api/v0/auth", authRouter);
const port = process.env.PORT || 3000;
const start = async () => {
try {
server.listen(port, () => {
console.log(`--> Listening on port ${port}...`);
});
} catch (error) {
console.log(error);
console.log("Fatal error!");
}
}
start();
authRoutes.js:
const express = require("express");
const router = express.Router();
const {signupWithEmail} = require("../controllers/auth.js");
router.route("/signup").post(signupWithEmail);
module.exports = router;
../controllers/auth.js:
const multer = require("multer");
const processReq = require("../middleware/upload.js");
const signupWithEmail = async (req, res, next) => {
//Processing to get the ID happens here.
//For simplicity we can pretend like the ID is directly
//posted in a field with the name id.
const id = req.body.id;
console.log("id:");
console.log(id);
await processReq(id)(req, res, next);
res.status(200).send("done!");
}
upload.js
const multer = require("multer");
const errors = require("../errors");
const maxSize = 2 * 1024 * 1024;
const acceptableFileMIMETypes = ["image/jpeg", "image/png", "image/webp", "application/pdf"];
const userAvatarFilter = function (req, file, cb) {
if ((file.fieldname === "userAvatar" || file.fieldname === "legalDocs")
&& acceptableFileMIMETypes.includes(file.mimetype)
&& !(file.fieldname === "userAvatar" && file.mimetype === "application/pdf")) {
if (file.fieldname === "userAvatar") {
cb(null, true);
} else {
cb(null, false);
}
} else {
cb(new errors.BadRequestError("Both files are required, and must be JPEG, PNG, WEBP. Legal documents can also be PDF."));
}
}
const userlegalDocsFilter = function (req, file, cb) {
if ((file.fieldname === "userAvatar" || file.fieldname === "legalDocs")
&& acceptableFileMIMETypes.includes(file.mimetype)
&& !(file.fieldname === "userAvatar" && file.mimetype === "application/pdf")) {
if (file.fieldname === "userAvatar") {
cb(null, false);
} else {
cb(null, true);
}
} else {
cb(new errors.BadRequestError("Both files are required, and must be JPEG, PNG, WEBP. Legal documents can also be PDF."));
}
}
const uploadFunc = (id) => {
return (req, res, next) => {
const userAvatarStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, __basedir + `/resources/${id}/userAvatar`);
},
filename: (req, file, cb) => {
console.log("Uploaded avatar!");
console.log(file.originalname);
console.log(file);
cb(null, file.originalname);
}
});
const legalDocsStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, __basedir + `/resources/${id}/legalDocs`);
},
filename: (req, file, cb) => {
console.log("Uploaded legalDocs!");
console.log(file.originalname);
console.log(file);
cb(null, file.originalname);
}
});
const uploadAvatar = multer({
storage: userAvatarStorage,
limits: {
fileSize: maxSize,
files: 2
},
fileFilter: userAvatarFilter
}).any();
const uploadLegalDocs = multer({
storage: legalDocsStorage,
limits: {
fileSize: maxSize,
files: 2
},
fileFilter: userlegalDocsFilter
}).any();
uploadAvatar(req, res, next);
uploadLegalDocs(req, res, next);
next();
}
}
module.exports = uploadFunc;
The Problem
The first problem with my code is quite obvious, I cannot parse text data from the request unless it had been processed by multer (with the current code, console.logging the id shows that it's undefined), but I need the ID to be passed to multer in order to save the files to the correct directory. It seems like I need some way to process text only and generate the ID, then process the files.
The second problem is that, even if the ID is static - meaning that in ./controllers/auth.js the signupWithEmail function became:
const multer = require("multer");
const processReq = require("../middleware/upload.js");
const signupWithEmail = async (req, res, next) => {
const id = 1337;
console.log("id:");
console.log(id);
await processReq(id)(req, res, next);
res.status(200).send("done!");
}
I this is the output I get on the console when I try to post:
id:
1337
Uploaded legalDocs!
cours-analyse-numerique-ch1-21_22.pdf
{
fieldname: 'legalDocs',
originalname: 'cours-analyse-numerique-ch1-21_22.pdf',
encoding: '7bit',
mimetype: 'application/pdf'
}
node:_http_outgoing:771
throw new ERR_HTTP_HEADERS_SENT('remove');
^
Error [ERR_HTTP_HEADERS_SENT]: Cannot remove headers after they are sent to the client
at new NodeError (node:internal/errors:400:5)
at ServerResponse.removeHeader (node:_http_outgoing:771:11)
at write (C:\Users\der_u\source\repos\ATEEK\node_modules\finalhandler\index.js:282:9)
at AsyncResource.runInAsyncScope (node:async_hooks:204:9)
at listener (C:\Users\der_u\source\repos\ATEEK\node_modules\on-finished\index.js:170:15)
at onFinish (C:\Users\der_u\source\repos\ATEEK\node_modules\on-finished\index.js:101:5)
at callback (C:\Users\der_u\source\repos\ATEEK\node_modules\ee-first\index.js:55:10)
at IncomingMessage.onevent (C:\Users\der_u\source\repos\ATEEK\node_modules\ee-first\index.js:93:5)
at IncomingMessage.emit (node:events:513:28)
at endReadableNT (node:internal/streams/readable:1359:12) {
code: 'ERR_HTTP_HEADERS_SENT'
}
and only 'legalDocs' get uploaded.
Is there any way to solve these problems without using a temporary directory to save all the files then move them once the ID had been generated? And what does the [ERR_HTTP_HEADERS_SENT] error mean?

uploading two files from two fields with multer as middleware

i'm trying to upload two files with different file extensions with multer from two fields but when i try it with postman the result always for the file is null, what is the solution for my problem? here is my code
middleware/uploadEpub
const multer = require('multer')
exports.uploadEpub = (epubFile, coverFile) => {
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "uploads")
},
filename: function (req, file, cb) {
cb(null, Date.now() + '-' + file.originalname.replace(/\s/g, ""))
}
})
const upload = multer({
storage
}).fields([{name: "bookFile", maxCount: 1},{name: "coverFile", maxCount: 1}])
}
controller/book
exports.addBook = async (req, res) => {
try {
const { ...data } = req.body
const newBook = await book.create({
...data,
bookFile: req.file,
coverFile: req.file
})
let bookData = await book.findOne({
where: {
id: newBook.id
},
attributes:{
exclude: ['createdAt','updatedAt']
}
})
bookData = JSON.parse(JSON.stringify(bookData))
res.send({
status: "Success",
Book: {
...bookData
}
})
} catch (error) {
console.log(error)
res.status(500).send({
status: "Failed",
message: "Server Error"
})
}
}

Multer set up
const multer = require('multer')
const path = require('path')
const { nanoid } = require('nanoid')
//Set Storage Engine
const storage = multer.diskStorage({
destination: './public/uploads/',
filename: (req, file, callback) => {
const id = nanoid(6)
const newFilename = `${file.fieldname}_${id}-${new Date().toISOString().replace(/:/g, '-')}${path.extname(file.originalname)}`
callback(null, newFilename)
}
})
const upload = multer({
storage: storage,
limits: { fileSize: 5572864 }, // up to 5.5 MB
fileFilter: (req, file, callback) => {
checkFileType(file, callback)
},
})
//Check File Type
const checkFileType = (file, cb) => {
//Allowed extensions
const fileType = /jpeg|jpg|png|gif|svg|pdf|epub/
//Check extension
const extname = fileType.test(path.extname(file.originalname).toLowerCase())
//Check mimetype
const mimetype = fileType.test(file.mimetype)
if (extname && mimetype) {
return cb(null, true)
} else {
return cb('Error: wrong file type!')
}
}
module.exports = upload
/***** middlewares.js *****/
module.exports.imageUploader = (req, res, next)=>{
const files = req.files
const uploadedFiles = []
for (var i = 0; i <images.length; i++){
uploadFiles.push('https://yourserver.com/public/uploads/' + files[i].filename)
}
req.uploadFiles = uploadFiles.toString() //appending files to req
next()
return
}
/**** index.js or app.js ****/ //where your routes are defined
router.post('/books/add', upload.array('images', 10), imageUploader, book.addBook) // allowing up to 10 files to be uploaded, calling imageUploader as a middleware
/*** Controllers/book ***/
exports.addBook = async (req, res) => {
const uploadedFiles = req.uploadFiles; //catching files from imageUploader middleware
// ... rest of your code
}

Try to upload the file as form-data in postman, and put the key with the same name that you have set in multer, the postman will look like this:

Try this:
const cpUpload = upload.fields([{ name: 'avatar', maxCount: 1 }, { name: 'gallery', maxCount: 8 }])
app.post('/cool-profile', cpUpload, function (req, res, next) {
})

Get image path with Multer Express Nodejs

I am using Multer to save images but I need to get the path of the image to save it to MongoDB. I am trying to get the path with req.file but it always tells me on the console that it is undefined.
this is my route:
import { Router } from 'express';
import { check, validationResult } from 'express-validator';
const multer = require('multer');
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, 'uploads/products')
},
filename: function (req, file, cb) {
cb(null, new Date().toISOString().replace(/:/g, '-') + file.originalname)
}
});
const fileFilter = (req, file, cb) => {
// reject a file
if (file.mimetype === 'image/jpeg' ||file.mimetype === 'image/png') {
cb(null, true);
} else {
cb(null, false);
//cb(new Error('I don\'t have a clue!'))
}
}
const upload = multer(
{ storage: storage,
limits:{
fileSize: 1024 * 1024
},
fileFilter: fileFilter
});
let router = Router();
router.post('/', upload.single('img'),
newProduct
);
And in the new Product controller I am trying to read the req.file but the console tells me that it is undefined:
Controller:
import { Product } from '../models'
let newProduct = async (req, res = response ) => {
console.log('file ' + req.file); //UNDEFINED
try {
let { status, user, ...body } = req.body;
let productDB = await Product.findOne ( { 'name': body.name } );
if (productDB) {
return res.status(400).json({
msg:`El producto ${ productDB.name } ya existe`
})
}
let data = {
...body,
name: body.name,
user: req.user._id
}
let product = new Product( data );
await product.save();
res.status(200).json( product );
} catch (error) {
return res.status(400).json({
error
});
}
}
Console:
Thanks for your help.

you can try to do this in filename instead:
filename: function (req, file, cb) {
req.imageName = new Date().toISOString().replace(/:/g, '-') + file.originalname
cb(null, req.imageName)
}
then there:
console.log('file ' + req.file); //UNDEFINED
//you can get imageName instead
console.log('imageName',req.imageName)
//if you want url to store in database you can do this
//supposing your have images directory in root of your node server
const url = `${req.protocol}://${req.get('host')}/images/${req.body.image}`

Refactored AWS S3 Uploader to reusable code doesn't invoke uploader

Consider the Controller :
authControoler:
const mongoose = require('mongoose');
const User = mongoose.model("User");
const crypto = require('crypto');
const bcrypt = require('bcryptjs');
const jwt = require('jsonwebtoken');
const { JWT_SECRET } = require('../config/keys');
const AWS = require('aws-sdk');
const multer = require("multer");
const multerS3 = require("multer-s3");
const uuid = require('uuid');
const s3 = new AWS.S3({
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECRET_KEY,
region: process.env.AWS_REGION,
});
const uuid_name = uuid.v4();
const signUpUploader = multer({
storage: multerS3({
s3: s3,
bucket: process.env.AWS_S3_BUCKET_NAME,
metadata: function (req, file, cb) {
console.log("Uploading file to S3...");
cb(null, {
OriginalFileName: file.originalname.toLowerCase(),
RandomName: uuid_name,
});
},
key: function (req, file, cb) {
cb(null, file.originalname);
}
})
}).single('singleFile');
exports.signUp = async (req, res, next) => {
// call the uploader to AWS S3
signUpUploader(req, res, function (err) {
if (err) {
// An error occurred when uploading
console.log('Something went wrong :', err);
return;
}
// Everything went fine
console.log('Everything went fine');
const image_s3_url = req.file.location;
const { name, email, password } = req.body;
if (!email || !password || !name) {
return res.status(422).json({ error: "please add all the fields" })
}
// update User in Mongo ...
});
};
I've refactored the code into reusable aws code , as follows :
aws.js:
const AWS = require('aws-sdk');
const multer = require("multer");
const multerS3 = require("multer-s3");
const uuid = require('uuid');
const uuid_name = uuid.v4();
const s3 = new AWS.S3({
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECRET_KEY,
region: process.env.AWS_REGION,
});
module.exports = class AWSUtils {
constructor(filePropertyName) {
this.filePropertyName = filePropertyName;
}
getFilePropertyName() {
return this.filePropertyName;
}
uploadImage() {
const signUpUploader = multer({
storage: multerS3({
s3: s3,
bucket: process.env.AWS_S3_BUCKET_NAME,
metadata: function (req, file, cb) {
console.log("Uploading file to S3...");
cb(null, {
OriginalFileName: file.originalname.toLowerCase(),
RandomName: uuid_name,
});
},
key: function (req, file, cb) {
cb(null, file.originalname);
}
})
}).single(this.filePropertyName);
return signUpUploader;
}
}
authControoler:
const mongoose = require('mongoose');
const User = mongoose.model("User");
const crypto = require('crypto');
const bcrypt = require('bcryptjs');
const jwt = require('jsonwebtoken');
const { JWT_SECRET } = require('../config/keys');
const AWSUtils = require('../utils/aws');
const _utils = new AWSUtils('singleFile');
exports.signUp = async (req, res, next) => {
console.log('In signUp');
// call the uploader to AWS S3
_utils.uploadImage(req, res, function (err) { // Refactored !!!
if (err) {
// An error occurred when uploading
console.log('Something went wrong :', err);
return;
}
// Everything went fine
console.log('Everything went fine');
const image_s3_url = req.file.location;
const { name, email, password } = req.body;
if (!email || !password || !name) {
return res.status(422).json({ error: "please add all the fields" })
}
// update User in Mongo ...
});
};
After refactored and extracted out the AWS code into its own file (aws.js) , I call uploadImage however nothing happens , no error , no upload , nothing.
What am I missing here ?

multer returns an Express middleware function. In the original code, that middleware function was assigned directly to signUpUploader
In your refactored code, the _util.uploadImage function is actually returning the middleware function, but in your code, you're not using the result as middleware, instead trying to use _util.uploadImage.
There are a few options, but a quick fix would be to do something like this:
Replace:
_utils.uploadImage(req, res, function (err) {
// The function body
})
With:
const signUpUploader = _utils.uploadImage();
signUpUploader(req, res, function (err) {
// The function body
})
That will use the mutler middleware the way you are expecting.

Node.js: Multer upload with promise?

I have this express route with multer file-upload. When the upload is complete, I would like to encode the image to base64 and send with response.
However when I do it like this, the code tries to execute the base64 encoding before the file is created to the folder.
Edit: Added storage & upload functions
const storage = multer.diskStorage({
destination: (req, file, callback) => {
if (!fs.existsSync('./uploads')) {
fs.mkdirSync('./uploads');
}
let path = './uploads';
callback(null, path);
},
filename(req, file, cb) {
let fileExt = file.originalname.substring(file.originalname.lastIndexOf('.')).toLowerCase();
if (!imageFilter(fileExt)) {
return false;
} else {
cb(null, file.originalname);
}
},
onError: function (err, next) {
console.log('error', err);
next(err);
},
});
const upload = multer({
storage,
limits: {
fileSize: 1000 * 1000 * 2 // 2 MB
}
}).single('file');
router.post('/upload', function (req, res) {
var directory = 'uploads';
fs.readdir(directory, (err, files) => {
if (err) throw err;
for (var file of files) {
fs.unlink(path.join(directory, file), err => {
if (err) throw err;
});
}
});
upload(req, res, function (err) {
if (err) {
return res.status(404).json({
success: false,
message: 'File is too large. (Max 2MB)'
});
}
var file = req.file;
var base64str = base64_encode('./uploads/' + file.originalname);
return res.status(200).json({
success: true,
url: 'http://' + ip.address() + ':' + constants.PORT + '/api/uploads/' + file.originalname,
image: 'data:image/png;base64,' + base64str
});
});
});
What would be the smartest way to achieve the right order of operations. Possibly promises or async/await?

This solution worked for me :
Node v8.4.0 is required for this
//app.js
const fs = require('fs');
const express = require('express');
const cors = require('cors');
const bodyParser = require('body-parser');
const app = express();
app.use(cors({credentials: true, origin: 'http://localhost:4200'}));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
const Uploader = require('./Uploader.js');
const uploader = new Uploader();
app.post('/upload', uploader.startUpload);
//Uploader.js
const util = require("util");
const crypto = require("crypto");
const multer = require('multer');
class Uploader {
constructor() {
const storageOptions = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, __dirname + '/uploads/')
},
filename: function(req, file, cb) {
crypto.pseudoRandomBytes(16, function(err, raw) {
cb(null, raw.toString('hex') + Date.now() + '.' + file.originalname);
});
}
});
this.upload = multer({ storage: storageOptions });
}
async startUpload(req, res) {
let filename;
try {
const upload = util.promisify(this.upload.any());
await upload(req, res);
filename = req.files[0].filename;
} catch (e) {
//Handle your exception here
}
return res.json({fileUploaded: filename});
}
}
Edit :
The library "util" provide you a "promisify" method which will give you the possibility to avoid something called the "callback hell". It converts a callback-based function to a Promise-based one.
This is a small example to understand my code above:
const util = require('util');
function wait(seconds, callback) {
setTimeout(() => {
callback();
}, seconds);
}
function doSomething(callType) {
console.log('I have done something with ' + callType + ' !');
}
// Default use case
wait(2 * 1000, () => {
doSomething('callback');
});
const waitPromisified = util.promisify(wait);
// same with promises
waitPromisified(2000).then((response) => {
doSomething('promise');
}).catch((error) => {
console.log(error);
});
// same with async/await
(async () => {
await waitPromisified(2 * 1000);
doSomething('async/await');
})();
Will print :
I have done something with callback !
I have done something with promise !
I have done something with async/await !