File upload to a third-party bucket architecture - javascript

I'm implementing an upload functionality for the Node.js-based system. I want to store the uploaded files on third-party cloud storage, e.g. AWS, Dropbox, etc.
To upload the data, I need to provide a storage API-token, but at the same time, I don't want to expose it on the client-side. As a workaround, I'm firstly uploading a file to my server and then forward the file to the cloud storage.
Client-side:
function sendFile(formData, callback) {
let ip = location.host;
$.ajax({
processData: false,
contentType: false,
type: "POST",
url: http() + ip + "/documentFileUpload",
data: formData,
success: callback,
error: function (err) {
return false;
}
});
}
Server-side:
app.post("/documentFileUpload", function (req, res) {
const storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, "./../uploads/");
},
filename: function (req, file, callback) {
callback(null, file.fieldname + '-' + Date.now());
}
});
const upload = multer({ storage : storage}).single('file');
upload(req, res, function(err) {
res.setHeader("Content-Type", "application/json");
if(err) {
res.send(JSON.stringify({
"result": false,
"message": "Cant save the local file: " + err.message
}));
} else {
// pseudo-code starts here
upladToExternalService(req.file.path, SECRET_KEY, function(err) {
if (err) {
res.send(JSON.stringify({
"result": false,
"message": "Cant forward the file: " + err.message
}));
} else {
deleteLocalFile(req.file.path, function (err) {
if (err) {
res.send(JSON.stringify({
"result": false,
"message": "Cant delete the local file: " + err.message
}));
} else {
res.send(JSON.stringify({
"result": true,
"message": "The file has been saved successfully"
}));
}
});
}
});
}
});
});
Question:
How to implement a file uploading functionality directly to cloud storage without exposing its API-token?

Related

I have to send file as well as some json data to server using axios and decode it into express server to store json in mongodb and file using multer

I want to create an endpoint which should store file on server side as well as the some json data which should be received from same point is to be stored on mongodb.
I am using axios to send request from React App. Here is my Code.
const [companyFile, setCompanyFile] = useState(null);
const [company, setCompany] = useState({
name: "",
websiteUrl: "",
email: "",
companyLocation: "",
});
const AddCompany = async (e) => {
if (companyFile) {
e.preventDefault();
let formData = new FormData();
formData.append("company-file", companyFile);
formData.append("company", JSON.stringify(company));
axios({
method: "post",
url: `http://localhost:8080/company/add`,
data: formData,
withCredentials: true,
header: {
Accept: "application/json",
"Content-Type": "multipart/form-data",
},
}).then((res) => console.log(res.data));
} else {
console.log("file not selected!!!!");
}
};
Now I don't know how to check if it is coming to backend express server or not? Or if coming then how to retrive application/json data from request for further process.
My question is that if data is sent to backend express then how to process that data in backend (i.e. get json data to create an document in mongodb).
Here is my code for company/add
let upload = multer({
storage: multer.diskStorage({
destination: async (req, file, cb) => {
if (!company) {
throw Error("Company cannot be found!");
}
let companyName = req.company.name;
let path = `./uploads/${companyName}/files`;
if (!fs.existsSync(path)) {
fs.mkdirSync(path, { recursive: true });
}
cb(null, path);
},
filename: async (req, file, cb) => {
// ** with student auth Code
req.filename = req.company.name;
cb(null, filename + path.extname(file.originalname));
},
}),
}).single("company-file");
// Add Company
module.exports.add_company = async (req, res) => {
try {
// Here I want to extract that company object to create new company
console.log(req.file);
try {
const newcompany = await Company.create(company);
req.company = newcompany;
upload(req, res, async () => {
try {
const companyFile = await CompanyFile.create({
companyId: req.company._id,
path: `./uploads/${req.company.name}/file/${req.filename}.pdf`,
});
req.companyFile = companyFile;
} catch (err) {
res.status(400).json({ success: false, message: err.message });
// ** code for resume-upload using student authentication middleware
if (
fs.existsSync(
`./uploads/${req.company.name}/file/${req.filename}.pdf`
)
) {
fs.unlink(`./uploads/${req.company.name}/file/${req.filename}.pdf`);
}
}
});
res.status(201).json({
success: true,
message: "Company Drive Added Successfully.",
company: req.company,
companyFile: req.companyFile,
});
} catch (err) {
res.status(400).json({
success: false,
errors: err,
message: "Error while applying company drive.",
});
}
} catch (err) {
console.log(err);
}
};
From the docs:
Multer adds a body object and a file or files object to the request
object. The body object contains the values of the text fields of the
form, the file or files object contains the files uploaded via the
form.
So in your case, you can access and parse the company field by doing:
const company = JSON.parse(req.body.company);
Also, you need to make sure to apply the multer middleware to your controller, e.g.
app.post('/company/add', upload.single('company-file'), function (req, res, next) {
// req.file is the `company-file` file
// req.body.company will hold the company text field
})

Express.js: TypeError: path argument is required to res.sendFile

I am uploading my files in a directory using Multer.
I want to download those same files which I have uploaded as I am displaying them in the form of a table on the template with each file having a download option. Here's my code:
Express.js
router.get('/downloadfile', (req, res, next) => {
var options = {
root: path.join(__dirname, './uploads'), //all my files are saved in uploads folder
dotfiles: 'deny',
headers: {
'x-timestamp': Date.now(),
'x-sent': true
}
}
var fileName = req.query.id3
res.sendFile(fileName, options, function (err) {
if (!err)
{
console.log('File sent successfully');
}
else
{
console.log('Error occurred while sending file.' + err.message)
}
})
});
Angular
onDownloadFiles(i: any)
{
this.fileToDownload = i;
console.log(this.fileToDownload);
const params = new HttpParams().set('id3', this.fileToDownload);
this.http.get('http://localhost:3000/downloadfile', {params})
.pipe(map(responseData => { return responseData; }))
.subscribe(response => {
console.log(response);
})
}
Here is the error while clicking on the download button.
TypeError: path argument is required to res.sendFile
Well, you can use modern try-catch methods when you're programming with asynchronous javascript rather than using conventional if-else statements to log errors.
Secondly, instead of using a callback function inside res.sendFile it is better to check whether the file exists and then send it.
Here's a sample code.
module.exports.getExe = async (req, res) => {
try {
const file = getExeFilePath(req.params.filename)
if (!fs.existsSync(file.path)) {
res.status(200).json({ 'status': false, 'result': 'File not found!' })
} else {
res.setHeader('Content-disposition', 'attachment; filename=' + file.name);
//filename is the name which client will see. Don't put full path here.
res.setHeader('Content-type', file.type);
var sendFile = fs.createReadStream(file.path);
sendFile.pipe(res);
}
} catch (error) {
console.log(error)
res.status(200).json({ 'status': false, 'result': 'Failed!' });
}
}

how to send image name in database using multer and express

When I'm trying to upload a photo with multer and express, everything is OK. But I'm not able to send the image name in mongoose database.
The images are uploaded successfully in the upload directory. I got the imageUrl in the body, but I'm not able to update the image name in mongoose database. However the other details are updated successfully.
Check the image below:
//file upload using multer
var storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, './uploads');
},
filename: function (req, file, callback) {
if (!file.originalname.match(/\.(jpg|png|JPEG)$/)) {
var err = new Error();
err.code = 'filetype';
return callback(err);
} else {
callback(null, Date.now() + file.originalname);
}
}
});
var upload = multer({
storage: storage,
limits: {
fileSize: 1000000
}
}).single('userImage');
app.patch('/updateProfile', authenticate, (req, res) => {
upload(req, res, function(err) {
var body = _.pick(req.body, ['name', 'email', 'mobile', 'imageUrl']);
User.findOneAndUpdate({
_id: req.user._id
}, {
$set: body
}, {
new: true
}).then((user) => {
if (!req.file) {
return res.send({
success: false,
msg: 'No file selected'
})
}
if (!user) {
res.status(404).send({
success: false,
msg: 'user not found'
})
} else {
body.imageUrl = req.file.filename;
console.log(body)
res.send({
sucess: true,
msg: 'update sucessfully',
user
})
}
}).catch((err) => {
res.send({
success: false,
msg: 'something wrong',
error: err
});
});
if (err) {
if (err.code === 'LIMIT_FILE_SIZE') {
return res.send({
success: false,
msg: 'limit file size 1MB '
})
} else if (err.code === 'filetype') {
return res.send({
success: false,
msg: 'Must be valid file extension only jpg or png'
})
} else {
return res.send({
success: false,
msg: 'something went wrong'
})
}
}
});
});
var storage = multer.diskStorage({
    destination: function (req, file, callback) {
        callback(null, './images/post');
    },
    filename: function (req, file, callback) {
        if (!file.originalname.match(/\.(jpg|png|JPEG)$/)) {
            var err = new Error();
            err.code = 'filetype';
            return callback(err);
        } else {
            callback(null, Date.now() + file.originalname);
        }
    }
});
var upload = multer({ storage: storage, limits: { fileSize: 1000000 } }).single('imageUrl');
router.post('/posts',authenticate, (req, res) => {
    upload(req, res, function (err) {
        if (err) {
            if (err.code === 'LIMIT_FILE_SIZE') {
                return res.send({ success: false, msg: 'limit file size 1MB ' })
            } else if (err.code === 'filetype') {
                return res.send({ success: false, msg: 'Must be valid file extension only jpg or png' })
            } else {
                return res.send({ success: false, msg: 'something went wrong' })
            }
        } else {
            if (!req.file) {
                return res.send({ success: false, msg: 'No file selected' })
            }
            var post = new Post({
                heading: req.body.heading,
                body: req.body.body,
                imageUrl: req.file.filename,
                creator:req.user.id,
            });
            post.save().then((result) => {
                res.send({ post: result, sucess: true, msg: 'Post created' });
            }).catch((err) => {
                res.send({ sucess: false, msg: 'post not created', error: err })
            });
        }
    });
});

How to get the body before uploading file in multer?

In my project the admins have the ability to upload MP3 files and submit parameters to it like song name.
I decided using multer middleware for handling multipart/form-data.
My problem is req.body.gender returns always undefined, because I have to use it inside the uploadSong listener. I want to upload the song when the gender is zero.
index.ejs
<form method="post" action="/acp" role="publish" enctype="multipart/form-data">
<div class="form-group">
<input type="file" name="song" id="song" accept="audio/mpeg">
</div>
<input type="checkbox" name="gender" checked data-toggle="toggle" data-on="Male" data-off="Female">
</form>
app.js
var uploadSong = upload.single('song');
app.post('/acp', isLoggedIn, function (req, res) {
console.log(req.body.gender); // returns "undefined"
if(req.body.gender == 0) { // returns "undefined"
uploadSong(req, res, function (err) {
if (err) {
res.send('uploaded');
return;
}
res.redirect('/');
});
}
});
(A) Not possible with multer.
(B) Use busboy. It uses streams to parse the form data and so you can get form elements values before the file upload and the fields are made available as events.
(C) Another solution (if you prefer using multer) is to use multer but add a header to send the value of the parameter to check before file upload. Headers are available as soon as the request reaches the server.
by using multer form-data parser you can parse form and access req.body before multer starts just register this app middle-ware:
import * as multer from "multer";
// parse form-data
app.use(multer().any());
This is my sample code, it is woking fine, if you need further explanation please let me know. hope helpful.
var Hotel = require('../models/hotel');
var path = require('path');
var multer = require('multer');
var uplodedImages = [];
var storageHotelGallery = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, './uploads/hotelGallery');
},
filename: function (req, file, cb) {
console.log(req.body);
var newFileName = Date.now() + path.extname(file.originalname);
req.newFileName = newFileName;
cb(null, newFileName);
uplodedImages.push(newFileName);
}
});
var uploadHotelGallery = multer({ storage: storageHotelGallery}).fields([{ name: 'imgArr', maxCount: 8 }]);
module.exports = function(router) {
// ADD HOTEL BASIC DATA TO CREATE HOTEL OBJECT
router.post('/createHotelStep1', function(req, res) {
if( req.body.name == null || req.body.name == '' ) {
res.json({ success: false, message: "Hotel name required" });
res.end();
}
else if( req.body.addressline1 == null || req.body.addressline1 == '' ) {
res.json({ success: false, message: "Address line 1 is required" });
res.end();
}
else if( req.body.city == null || req.body.city == '') {
res.json({ success: false, message: "City is required" });
res.end();
}
else {
var hotel = new Hotel();
hotel.name = req.body.name;
hotel.addressline1 = req.body.addressline1;
hotel.addressline2 = req.body.addressline2;
hotel.phone = req.body.phone;
hotel.city = req.body.city;
hotel.email = req.body.email;
hotel.save(function(err) {
if (err) {
res.json({ success: false, message: "Unable to Complete Hotel Step 1" });
} else {
res.json({ success: true, message: 'Create Hotel Step 1 Complete', _id : hotel._id });
}
});
}
});
router.post('/createHotelGallery', function (req, res, next) {
uplodedImages = [];
uploadHotelGallery(req, res, function(err) {
if(err) {
res.json({ success: false, message: 'Could not upload images'});
res.end();
}
else {
Hotel.findOne({ _id:req.body._id }).populate('users').exec(function (err, hotel) {
if (err) {
res.json({ success: false, message: 'Could not save uploaded images to database'});
res.end();
}
else {
for(var x=0; x<uplodedImages.length; x++)
hotel.images.push(uplodedImages[x]);
hotel.save();
res.json({ success: true, message: 'Gallery image uploaded' });
res.end();
}
});
}
});
});
return router;
}
This is my sample code, it is woking fine
const upload = multer({
storage,
fileFilter(req, file, cb) {
if(req.body.name===''){
return cb(new Error('Invalid name'), false)
}
const extname = path.extname(file.originalname).toLowerCase() === '.gz'
const mimetype = file.mimetype === 'application/x-gzip'
if (mimetype && extname) {
return cb(null, true)
} else {
return cb(new Error('Invalid mime type'), false)
}
},
})

MULTER-S3: TypeError: Cannot convert object to primitive valu…diate [as _immediateCallback]

I'm building a MEAN stack app using Multer-s3 to upload images to my database. However when I call the upload function I get this error:
POST http://localhost:3000/api/upload/single 500 (Internal Server Error)
TypeError: Cannot convert object to primitive valu…diate [as _immediateCallback](timers.js:354:15)↵", status: 500, config: Object, statusText: "Internal Server Error"
In my server side app.js
// MULTER
var s3config = require('./config/s3');
var upload = multer({
storage: s3({
dirname: s3config.dirname,
bucket: s3config.bucket,
secretAccessKey: process.env.AWS_ACCESS_SECRET,
accessKeyId: process.env.AWS_ACCESS_KEY,
region: s3config.region,
contentType: function(req, file, next) {
next(null, file.mimetype);
},
filename: function(req, file, next) {
var ext = '.' + file.originalname.split('.').splice(-1)[0];
var filename = uuid.v1() + ext;
next(null, filename);
}
})
});
// UPLOAD SINGLE FILE
app.post('/api/upload/single', upload.single('file'), function(req, res) {
var clothing_1 = {
type: req.body.type,
image: req.file.key
};
console.log('req.body =' + req.body);
console.log('req.body.type =' + req.body.type);
console.log('req.file.key =' + req.file.key);
// get the user model User.findOne({ id: req.user._id })
User.findOne({ _id: req.user._id }, {}, { new: true }, function(err, user){
user.clothing.push(clothing_1);
// save this user
user.save(function(err, user){
if(err) return res.status(401).send({ message: 'your error:' + err });
else return res.json({ user: user })
});
});
});
In my Front-end Controller:
self.uploadSingle = function() {
Upload.upload({
url: API + '/upload/single',
data: { file: self.file, type: self.clothingType }
})
.then(function(res) {
self.usersClothing = res.data.user.clothing.map(function(clothing) {
clothing.image = S3_BUCKET + clothing.image;
return clothing;
});
})
.catch(function(err) {
console.error(err);
});
}
This function is executed when I press upload button.
Was simply a matter of server not being able to find the user as req.body gave _id inside _doc.
User.findOne({ _id: req.user._doc._id } fixed the issue

Categories