I have been trying to upload files (mostly images) to firebase storage through firebase cloud function (onRequest method). I had to upload files from its base64 form. With the below code, i was able to achieve it, yet the file seems to be broken after upload.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const bucket = admin.storage().bucket();
const database = admin.database();
const express = require('express');
const cors = require('cors');
const safetyLogsAPI = express();
safetyLogsAPI.use(cors({ origin: true }));
safetyLogsAPI.post('/', async (req, res) => {
try {
const {
attachments,
projectId
} = req.body;
const safetyKey = database.ref('safetyLogs')
.child(projectId)
.push().key;
if(attachments && Array.isArray(attachments)) {
if (attachments.length > 0) {
for (let index = 0; index < attachments.length; index++) {
const base64Obj = attachments[index];
const { content, fileName, contentType } = base64Obj;
const stream = require('stream');
const bufferStream = new stream.PassThrough();
bufferStream.end(Buffer.from(content, 'base64'));
const fullPath = `SafetyIncidentLog/Images/${projectId}/${safetyKey}/${fileName}`;
const file = bucket.file(fullPath);
const metadata = {
projectId,
safetyLogId: safetyKey,
createdTimestamp: Date.now(),
systemGenerated: 'false',
fileType: 'Image',
fileName,
path: fullPath
};
bufferStream.pipe(file.createWriteStream({
metadata: {
contentType,
metadata
},
public: true,
validation: "md5"
}))
.on('error', (err) => {
console.log('Error Occured');
console.log(err);
})
.on('finish', () => {
console.log('File Upload Successfull');
});
}
}
}
return res.status(200).send({
code: 200,
message: 'Success'
});
} catch (error) {
console.log(error);
return res.status(500).send({
code:500,
message: 'Internal Server Error',
error: error.message
});
}
});
module.exports = functions.https.onRequest(safetyLogsAPI);
I have tried this approach with both the prefix part data:image/png;base64 present and eliminated. In both ways i see broken image. So where have I gone wrong. Is there a better way to make it?
Thanks in advance.
Also, is the approach i try to do so is a recommended way?. For use cases like, profile picture upload, and conversation image attachments, is this way recommended, or the a direct upload from client is recommended?
With Cloud Functions HTTP triggers, the request is terminated and the function is shut down as soon as you send a respond to the client. Any asynchronous work that isn't finished might never finish.
What I'm seeing in your code is that you send a response before the upload is complete. I can see that your call to res.status(200).send() happens immediately after you start the upload. Instead, your code should wait to send the response until after it completes, perhaps using on('finish') and on('error').
Related
What I'm trying to do:
I'm trying to make a api that saves images. I'd send a post request to the api with the image in body and it saves to the static folder 'public/images'.
Problem: I tried to do this on localhost and it works perfectly. I've got the cross-origin error's before but I've fixed them. After I hosted the api on plesk, it doesn't save the image, or send a error message back. I've checked the logs on plesk and it says that it received the post request. but it's not doing anything with it.
front-end code on client-side (hosted on plesk):
const formData = new FormData();
var fileField = document.querySelector('input[type="file"]');
formData.append('image', fileField.files[0]);
const result = await fetch('https://cdn.dhulun.com/upload25single', {
method: 'POST',
body: formData
}).then(res => res.json())
if (result.status === "ok") {
console.log("Success");
window.location.href = '/';
} else if (result.status === "error") {
console.log("error", result.error)
} else {
console.log("Something went wrong...")
}
back-end code on api (hosted on plesk):
const express = require('express');
const multer = require('multer');
const path = require('path');
const dotenv = require('dotenv');
const mongoose = require('mongoose');
const cors = require('cors')
dotenv.config();
// CONNECT TO DATABASE
async function connectDB() {
await mongoose.connect(process.env.DB_CONNECT,
{ useNewUrlParser: true, useUnifiedTopology: true },
() => {
console.log("Connected to Portal Base [Server]")
});
}
connectDB();
var Media = require('./models/media')
// INIT APP
const app = express();
app.use(cors())
// STATIC FILES
app.use(express.static(__dirname + '/public'));
app.use('/image', express.static('./public/images'))
// RULES
// INIT MULTER
// Storage Engine
const storage = multer.diskStorage({
destination: './public/images',
filename: (req, file, cb) => {
return cb(null, `${file.fieldname}_${Date.now()}${path.extname(file.originalname)}`)
}
})
const upload = multer({
storage: storage,
limits: { fileSize: 8000000 },
})
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html')
});
app.post('/upload25single', upload.single('image'), async(req, res) => {
var image_url = `http://localhost:2525/image/${req.file.filename}`;
console.log({
status: 'ok',
image_url: image_url,
})
res.json({
status: 'ok',
image_url: image_url,
});
/* var location = req.body.location;
var category = req.body.category;
var sourceby = req.body.sourceby;
var tags = req.body.tags;
const media = new Media({
url: image_url,
location: location,
category: category,
source_by: sourceby,
tags: tags
});
try {
const saved = await media.saved();
res.json({
status: 'ok',
image_url: image_url,
});
} catch (error) {
res.json({
status: 'error',
err: error,
});
}
*/
})
function errHandler(err, req, res, next) {
if (err instanceof multer.MulterError) {
res.json({
status: 'error',
err: err.message
})
}
}
// ERROR HANDLING
app.use(errHandler)
app.listen(process.env.PORT || 2525, () => console.log("Server Started..."));
I get this error on the browser console (hosted on plesk): Uncaught (in promise) SyntaxError: JSON.parse: unexpected character at line 1 column 1 of the JSON data
When i click the code line number in the log i got sent to this line:
console.log("Something went wrong...")
EDIT: I got suggested to turn res.json() to res.text() and i got this:
<p>iisnode encountered an error when processing the request.</p><pre style="background-color: eeeeee">HRESULT: 0x6d
HTTP status: 500
HTTP subStatus: 1013
HTTP reason: Internal Server Error</pre><p>You are receiving this HTTP 200 response because <a href=https://github.com/tjanczuk/iisnode/blob/master/src/samples/configuration/web.config>system.webServer/iisnode/#devErrorsEnabled</a> configuration setting is 'true'.</p><p>In addition to the log of stdout and stderr of the node.exe process, consider using <a href=http://tomasz.janczuk.org/2011/11/debug-nodejs-applications-on-windows.html>debugging</a> and <a href=http://tomasz.janczuk.org/2011/09/using-event-tracing-for-windows-to.html>ETW traces</a> to further diagnose the problem.</p><p>The last 64k of the output generated by the node.exe process to stderr is shown below:</p><pre style="background-color: eeeeee">(node:7696) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.
Using Node.Js: Express & Multer
I have uploaded image by using multer library in express.
in path Backend->Uploads/
and stored image path in mongodb.
I have project structure as
DirectoryName
Backend
Uploads
Frontend
I can get the image path in frontend component , but How to get actual images from backend folder.
Can I use file moving to store it in public in frontend , or retrieve stream from server.
Will moving File from backend to frontend works actually in deployment.
I think you have done most of the work. Just set the image source or uri to the path and it will serve the image.
A simple example of implementation from one of my projects.
Mayby not the best, but it works and, most important, you can get the idea.
In this case I keep files on server.
Created a route in my API:
router.get('/download/:photo_id', async (req, res) => {
try {
const photo = await Photo.findOne({
photoID: req.params.photo_id,
});
if (!photo) {
return res.status(404).json({ msg: 'Photo not found' });
}
const filename = photo.photoFileName;
const downloadPath = path.join(__dirname, '../../../imgs/', `${filename}`);
res.download(downloadPath);
} catch (err) {
console.error(err.message);
if (err.kind === 'ObjectId') {
return res.status(404).json({ msg: 'Photo not found' });
}
res.status(500).send('Server error');
}
});
And this route is called from front something like this:
const handleDownload = async () => {
const res = await fetch(`/api/photo/download/${photoID}`);
const blob = await res.blob();
download(
blob,
`${photoID}-${title}-${contributorName}.jpg`
);
};
I am learning how to upload images from my React website to my Mongo database through an express server. In every tutorial I have read, the author saves the file locally in the express server before sending it to the Mongo database. Is there a way to avoid having to store the file locally by keeping it in a local variable which is then uploaded to the database?
Here are the tutorials I am referring to:
https://www.positronx.io/react-file-upload-tutorial-with-node-express-and-multer/
https://medium.com/ecmastack/uploading-files-with-react-js-and-node-js-e7e6b707f4ef
Thank you for your help.
I guess The GridFS API would be helpful to you.It says :
you can .pipe() directly from file streams to MongoDB
Here is the sample example according to doc :
const assert = require('assert');
const fs = require('fs');
const mongodb = require('mongodb');
const uri = 'mongodb://localhost:27017';
const dbName = 'test';
mongodb.MongoClient.connect(uri, function(error, client) {
assert.ifError(error);
const db = client.db(dbName);
var bucket = new mongodb.GridFSBucket(db);
fs.createReadStream('./meistersinger.mp3').
pipe(bucket.openUploadStream('meistersinger.mp3')).
on('error', function(error) {
assert.ifError(error);
}).
on('finish', function() {
console.log('done!');
process.exit(0);
});
});
documentation link : https://mongodb.github.io/node-mongodb-native/3.0/tutorials/gridfs/streaming/
Hope this help !
yes you want to store you files locally. I used an NFS server (FreeNas) and mounted it to that local folder.
So when i saved a file to that location, it was stored on the other NFS server. Then i sent that image location as a response back to react, which then saved that location in Mongodb.
Example uploads.js
router.post('/', auth, async (req, res) => {
let CurrentDate = moment().unix();
if (req.files.file === null) {
return res.status(400).json({ msg: 'no file uploaded' });
}
let user = await User.findById(req.user.id).select('-password');
let file = req.files.file;
file.name = CurrentDate + user._id + '.jpg';
file.mv(`./client/public/uploads/${file.name}`, (err) => {
if (err) {
console.error(err);
return res.status(500).send(err);
}
res.json({ fileName: file.name, filePath: `/uploads/${file.name}`});
});
});
This is what the mongodb entry looks like
image:"/uploads/15951066675f1365239d46882312332d20.jpg"
I tried to receive the file and store it in the multer storage
Node js code
enter code here
app.post('/createLicence', upload.single('photo'),function(req, res ,next) {
// any logic goes here
console.log("filename" ,req.body.name)
if (!req.file) {
console.log("No file received");
return res.send({
success: false
});
} else {
console.log('file received');
var function_name = 'createLicence'
var arguments_array = [req.file.path,'Raghav','Mumbai','Approved']
invoke = require('/Users/sanjeev.natarajan/fabric-samples/fabcar/invoke.js');
invoke.invokechaincode(function_name,arguments_array)
return res.send({
success: true
})
}
});
but i am receiving no file is receivedi have send the request through postman
-
From : https://www.npmjs.com/package/multer
In order to use the multer package, you have first to define a few parameters so that it can work on your fileDirectory.
In your server.js :
let multer = require('multer');
let storage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, '/path/to/storage/')
},
filename: function(req, file, callback) {
callback(null, file.originalname + '-' + Date.now());
}
});
let upload = multer({
storage: storage
});
Now, configure your route
router.route('/your/payload')
.post(authController.isAuthenticated, upload.any(), albumController.postFile)
Note that upload.any() will allow you to upload multiple different formatted files at once. Feel free to use any other kind of upload.method() depending on your needs.
From this point, multer already is doing its job, however you might want to keep track of the files uploaded on your server.
So, in your own module, the logic is pretty much straight forward :
(I'm assuming that you're using mongoose models since you're not giving much information, but that's not the relevant part anyway)
exports.postFile = async (req, res) => {
if (!req || !req.files || !req.files[0]) return res.status(400).send("Bad request.");
for (let i = 0; req.files[i]; i++) {
await File.create({
path: req.files[i],
originalName: req.files[i].originalName,
mimetype: req.files[i].mimetype,
owner: req.user.userId
}, (err, file) => {
if (err) console.log("Something went wrong: " + err); else {
// Do something with file
}
});
}
return res.status(418).send("I'm a teapot.");
}
This configuration and middleware use is ONLY for testing purpose, never ever let anyone upload something to your server without carefully handle that uploading process (file integrity, resource management, ...). An open uploading system can become a very wide backdoor getting straight to your server.
Hope this helps,
regards.
I'm using Telegram bot API and AWS S3 to read data from a bucket. I need to use the data from the s3 method in the Telgraf method, but I don't know how:
'use strict'
const Telegraf = require('telegraf');
const bot = new Telegraf('TOKEN')
var AWS = require('aws-sdk')
var s3 = new AWS.S3({
accessKeyId: 'key',
secretAccessKey: 'secret'
})
var params = {Bucket: 'myBucket', Key:"ipsum.txt"};
var s3Promise = s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack);
else
var words= data.Body.toString(); //WHAT I WANT IN IN COMMAND METHOD
console.log('\n' + words+ '\n') //Returns ipsum.txt as string on console
})
bot.command('s', (ctx) => { //Bot Command
s3Promise; //Returns ipsum.txt as string on console
ctx.reply('Check console') //Meesage in Telegram
//ctx.reply(<I WANT data.Body.toSting() HERE>)
});
const { PORT = 3000 } = process.env
bot.startWebhook('/', null, PORT)
How do I use the data from the s3.getObject method in ctx.reply() ?
If you want to send the file as an attachment, you have to use: ctx.replyWithDocument. Aside from that your problem is: How do I return the response from an asynchronous call?
In this particular case you can use s3.getObject(params).promise() in order to avoid the callback API, and use it easily inside your bot.command listener.
Using async/await (Node >= 7.6) your code can be written like this
'use strict';
const Telegraf = require('telegraf');
const bot = new Telegraf('TOKEN');
const AWS = require('aws-sdk');
const s3 = new AWS.S3({
accessKeyId: 'key',
secretAccessKey: 'secret'
});
const params = {
Bucket: 'myBucket',
Key: 'ipsum.txt'
};
bot.command('s', async ctx => { // Bot Command
try {
// If you're sending always the same file and it won't change
// too much, you can cache it to avoid the external call everytime
const data = await s3.getObject(params).promise();
ctx.reply('Check console'); // Message in Telegram
// This will send the file as an attachment
ctx.replyWithDocument({
source: data.Body,
filename: params.Key
});
// or just as text
ctx.reply(data.Body.toString());
} catch(e) {
// S3 failed
ctx.reply('Oops');
console.log(e);
}
});
const {
PORT = 3000
} = process.env;
bot.startWebhook('/', null, PORT);
More info on how to work with files can be found on telegraf docs
PS: I tested the code and it it's fully working:
While I haven't used S3, I do know that AWS services added support for Promises to their implementations to avoid using callbacks. Personally, I much prefer the use of promises as I think they lead to more readable code.
I think the following should handle the issue you're having.
'use strict'
const Telegraf = require('telegraf');
const bot = new Telegraf('TOKEN')
var AWS = require('aws-sdk')
var s3 = new AWS.S3({
accessKeyId: 'key',
secretAccessKey: 'secret'
})
var params = {Bucket: 'myBucket', Key:"ipsum.txt"};
bot.command('s', (ctx) => {
s3.getObject(params).promise()
.then(data => {
ctx.reply('Check console');
ctx.reply(data.Body.toString());
}, err => console.log(err, err.stack));
})
const { PORT = 3000 } = process.env
bot.startWebhook('/', null, PORT)
As suggested by Luca, I called bot.command inside of s3.getObject and it works!
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else
bot.command('s', (ctx) => {
ctx.reply('Succesfully read from S3:\n\n' + data.Body.toString())
});
})