I have created a file upload API using multer and express which works fine through POSTMAN but when i try to call the same api using another file upload API, it's not working:
My code is as follows which runs on http://localhost:7022/nuxeo/upload:
module.exports.upload = function (req, res, next) {
var path = req.body.path
var uploadFile = req.file; //get uploaded file
var stream = streamifier.createReadStream(req.file.buffer) //create a stream from file buffer
var blob = new Nuxeo.Blob({ //create a blob from file stream
content: stream,
name: uploadFile.originalname,
mimeType: uploadFile.mimetype,
size: uploadFile.size
});
var batch = nuxeo.batchUpload();
In the above code when I call the API through postman, my req.file is populated.
But calling the above API using the code below doesn't populate the req.file of the first API, it is undefined. I have also tried using form-data npm module without any luck :
module.exports.attach = function(req,res,next){
var uploadfile = req.file //file is populated here
formData = { 'file' : uploadfile, 'path' : '/FCA/DT/clause32a'}
var opts = {
url: "http://localhost:7022/nuxeo/upload",
headers: { 'enctype': 'multipart/form-data;boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' },
json: true,
body: formData
};
request.post(opts,function(err,response,body){
console.log(body)
})
}
module.exports.attach = function(req,res,next){
var uploadfile = req.file //file is populated here
var fs = require('fs');
var request = require('request');
request.post({
url: <URL>,
formData: {
file: fs.createReadStream(<FILE_PATH>),
filetype: <FILE_TYPE>,
filename: <FILE_NAME>,
title: <FILE_TITLE>,
},
}, function(error, response, body) {
console.log(body);
});
}
I solved it with the help of this post : how to upload file saved in memory by multer to another API
var stream = require('stream')
const { Duplex } = stream;
function bufferToStream(buffer) {
const duplexStream = new Duplex();
duplexStream.push(buffer);
duplexStream.push(null);
return duplexStream;
}
request.post({
headers: { 'enctype': 'multipart/form-data;boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' },
url: "http://localhost:7022/nuxeo/upload",
formData: {
file: {
value: bufferToStream(req.file.buffer),
options: {
filename: req.file.originalname,
contentType: req.file.mimetype,
knownLength: req.file.size
}
},
path: '/FCA/DT/clause32a'
}
}, function (error, response, body) {
if (error) { return next(error)
res.send(JSON.parse(body))
})
Related
What I'm trying to do:
Uploading a file with AWS S3, and then taking the filename and filetype and placing it at the end of the url to save it in sql, so that every time the person logs in, it will pull the picture up by user image url.
Problem:
Not uploading and not recognizing the file in filename or filetype. Coming up with undefined for filetype and filename in URL and signedURL
The code for my fileUploadService.js used in Node.js is shown below.
The getSignedURL looks like this:
https://sabio-training.s3.us-west-2.amazonaws.com/C56/filename/?AWSAccessKeyId=AKIAJF53EJKW7SJUV55Q&Content-Type=filetype&Expires=1536877443&Signature=WxSvLSzfyZKDRN9LawVOwj1ayVY%3D&x-amz-acl=public-read
The URL looks like this:
https://sabio-training.s3.amazonaws.com/C56/filename/filetype
const aws = require('aws-sdk');
aws.config.region = 'us-west-2';
aws.config.update({ accessKeyId: '', secretAccessKey: '' });
const PROFILE_S3_LINK = "https://sabio-training.s3.amazonaws.com/";
module.exports = {
getUrl: getUrl
}
function getUrl(req, res) {
const s3 = new aws.S3();
const fileName = 'C56/'+"filename"+'/' ; //hardcoded filename and filetype for it to work.
const fileType = "filetype"; //How to take filename from uploaded file to insert into "fileName" along with the "filetype"?
const s3Params = {
Bucket: 'sabio-training',
Key: fileName,
Expires: 3000,
ContentType: fileType,
ACL: 'public-read'
};
s3.getSignedUrl('putObject', s3Params, (err, data) => {
if (err) {
console.log(err);
return res.end();
}
const returnData = {
signedRequest: data,
url: `${PROFILE_S3_LINK}${fileName}${fileType}` //unsigned URL
};
res.write(JSON.stringify(returnData));
res.end();
});
}
=========================================================================
fileUploadRoute.js
const router = require("express").Router();
const fileUploadController = require("../controllers/fileUploadController")
router.put("/", fileUploadController.getUrl);
module.exports = router;
==========================================================================
fileUploadController.js
const fileUploadService = require('../services/fileUploadService')
const responses = require("../models/responses");
module.exports = {
getUrl: getUrl
}
function getUrl(req, res) {
fileUploadService.getUrl(req, res)
.then(response => {
res.send(response)
})
.catch(error => {
res.send(error)
})
}
===========================================================================
index.js in node portion
const router = require("express").Router();
const pogsRoutes = require("./pogs.routes");
const userFromJWT = require("../filters/jwt.user");
const validateUser = require("../filters/validate.user");
const testRoutes = require("./test.routes");
const profileRoute = require("../profile/profileRoute");
const fileUploadRoute = require("../fileUpload/fileUploadRoute")
module.exports = router;
// router.use("/api/profilePage", profileRoute)
router.use("/api/pogs", pogsRoutes);
router.use("/api/upload", fileUploadRoute)
router.use("/api/profilePage", profileRoute)
// -----------------------------------
// Authenticated routes go below this:
// -----------------------------------
router.use(userFromJWT);
router.use(validateUser);
router.use("/api/test", testRoutes); // TODO: remove this before delivery to the client
============================================================================
USED IN REACT
Axios pulled from profile page
handleClickUpload = evt => {
evt.preventDefault()
console.log("RESULT : ", this.state);
// var file = evt.target.files[0]; <-- havent used this yet but I know its for upload
axios.put(`${NODE_API_URL}/api/upload`, {
// f:file
})
.then(response =>{
console.log(
response,"URL SIGNED REQUEST : ",response.data.signedRequest, " URL : ",response.data.url
)
})
.catch(error => {
console.log(error);
})
}
Upload button and file upload portion inside profile page
<div method="post" encType="multipart/form-data" action="/">
<input type="file" name="fileName" className="btn" />
<input type="submit" value="Upload" className="btn" onClick={this.handleClickUpload}/>
Can anyone let me know what I'm doing wrong or if anything is missing?
First of all, take a look at https://s3browser.com/features-content-mime-types-editor.aspx after you can figure out what kind of type you have to use (for ContentType). I think it should be 'text/plain' or 'text/html'.
Then you have to add one more property to s3 params named Body (the body have to contain your entity that you want to upload to s3 bucket)
Here is a snippet that you can use to upload your entity to s3 and then get a location:
let s3 = new AWS.S3({ region: process.env.AWS_REGION, apiVersion: '2006-03-01' });
let params =
{
Bucket: // a bucket's location,
Key: fileName // a path,
Body: // your entity to upload,
ACL: 'public-read',
ContentType: 'text/plain'
};
let s3Response = await s3.upload(params).promise();
console.log(`Entity uploaded to S3 at ${s3Response.Bucket} bucket. Location: ${s3Response.Location}`);
return s3Response.Location;
I created a .wav file from a recording from my microphone using this javascript library Unfortuntely the file is 7-bit encoded .wav file. I send this file to my node.js server and now I need to send this file to google's speech to text api to be processed but google only accepts base64 linear 16 .wav files. What can I do?
Here is how I create and send the audio file to my server:
navigator.mediaDevices.getUserMedia({
audio: true,video:false
})
.then((stream) => {
context = new AudioContext()
var source = context.createMediaStreamSource(stream)
config = {
numChannels:1
}
var rec = new Recorder(source,config)
rec.record()
$('#stop').click(()=>{
rec.stop()
blob = rec.exportWAV(export)
function export(blob)
{
fd = new FormData()
fd.append('file',blob)
$.ajax({
type: "POST",
url: "http:localhost:3000/send",
data: fd,
contentType:false,
processData: false,
encType:"multipart/form-data",
success: function(data){
console.log('success')
}
});
}
})
Here is how I handle it on my node.js server:
var upload = multer()
var app = express({ dest: '/' })
app.use(function(req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
next();
});
var upload = multer().any()
app.get('/',(req,res)=>{
res.send('Hello World!')
})
app.post('/send',(req,res)=>{
upload(req, res, function (err) {
if (err) {
console.log(err)
return
}
file = req.files[0]
buffer = file.buffer
var wstream = fs.createWriteStream('audiofile.wav');
wstream.write(buffer)
wstream.end();
path = "path/audiofile.wav"
So in my react front-end, I am using the 'react-drop-to-upload' module to allow the user to drag a file and upload. I followed the example on the npm module page and created a handler called handleDrop. The code looks like:
handleDrop(files) {
var data = new FormData();
alert((files[0]) instanceof File);
files.forEach((file, index) => {
data.append('file' + index, file);
});
fetch('/file_upload', {
method: 'POST',
body: data
});
}
At my express backend, I have the following code:
app.post('/file_upload', function(req , res){
var body = '';
req.on('data', function (data) {
body += data;
});
var post = "";
req.on('end', function () {
//post = qs.parse(body);
console.log(body);
// this won't create a buffer for me
//var fileBuffer = new Buffer(body);
//console.log(fileBuffer.toString('ascii'));
//pdfText(body, function(err, chunks) {
//console.log(chunks);
//});
});
//console.log(typeof post);
});
If I drop a txt file and do a console log on the body, it would give me:
------WebKitFormBoundaryqlp9eomS0BxhFJkQ
Content-Disposition: form-data; name="file0"; filename="lec16.txt"
Content-Type: text/plain
The content of my data!
------WebKitFormBoundaryqlp9eomS0BxhFJkQ--
I am trying to use the pdfText module which takes in a buffer or a pathname to the pdf file, extract text from it into an array of text 'chunks' . I want to convert the body object into a buffer using var fileBuffer = new Buffer(body); but that won't work. Can someone help me with this? Thanks!
You need a parser for multi-part data. You can look into multer regarding that.
Example code for you,
app.post('/file_upload', function(req , res){
var storage = multer.diskStorage({
destination: tmpUploadsPath
});
var upload = multer({
storage: storage
}).any();
upload(req, res, function(err) {
if (err) {
console.log(err);
return res.end('Error');
} else {
console.log(req.body);
req.files.forEach(function(item) {
// console.log(item);
// do something with the item,
const data = fs.readFileSync(item.path);
console.log(data);
});
res.end('File uploaded');
}
});
});
To understand the example code in depth, head here. Remember, you will get the file data as a buffer and not as actual data.
I am looking to use Digital Oceans spaces (which seems to have an identical API to S3), and would like to try it by uploading a sample file. I am having lots of difficulty. Here's what I've done so far
{'hi' : 'world'}
Is the contents of a file hiworld.json that I would like to upload. I understand that I need to create an aws v4 signature before I can make this request.
var aws4 = require('aws4')
var request = require('request')
var opts = {'json': true,'body': "{'hi':'world'}",host: '${myspace}.nyc3.digitaloceanspaces.com', path: '/hiworld.json'}
aws4.sign(opts, {accessKeyId: '${SECRET}', secretAccessKey: '${SECRET}'})
Then I send the request
request.put(opts,function(error, response) {
if(error) {
console.log(error);
}
console.log(response.body);
});
However, when I check my Digital Ocean space, I see that my file was not created. I have noticed that if I changed my PUT to GET and try to access an existing file, I have no issues.
Here's what my headers look like
headers:
{ Host: '${myspace}.nyc3.digitaloceanspaces.com',
'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
'Content-Length': 14,
'X-Amz-Date': '20171008T175325Z',
Authorization: 'AWS4-HMAC-SHA256 Credential=${mykey}/20171008/us-east-1//aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date, Signature=475e691d4ddb81cca28eb0dcdc7c926359797d5e383e7bef70989656822accc0' },
method: 'POST' }
As an alternative, using aws-sdk:
// 1. Importing the SDK
import AWS from 'aws-sdk';
// 2. Configuring the S3 instance for Digital Ocean Spaces
const spacesEndpoint = new AWS.Endpoint(
`${REGION}.digitaloceanspaces.com`
);
const url = `https://${BUCKET}.${REGION}.digitaloceanspaces.com/${file.path}`;
const S3 = new AWS.S3({
endpoint: spacesEndpoint,
accessKeyId: ACCESS_KEY_ID,
secretAccessKey: SECRET_ACCESS_KEY
});
// 3. Using .putObject() to make the PUT request, S3 signs the request
const params = { Body: file.stream, Bucket: BUCKET, Key: file.path };
S3.putObject(params)
.on('build', request => {
request.httpRequest.headers.Host = `https://${BUCKET}.${REGION}.digitaloceanspaces.com`;
// Note: I am assigning the size to the file Stream manually
request.httpRequest.headers['Content-Length'] = file.size;
request.httpRequest.headers['Content-Type'] = file.mimetype;
request.httpRequest.headers['x-amz-acl'] = 'public-read';
})
.send((err, data) => {
if (err) logger(err, err.stack);
else logger(JSON.stringify(data, '', 2));
});
var str = {
'hi': 'world'
}
var c = JSON.stringify(str);
request(aws4.sign({
'uri': 'https://${space}.nyc3.digitaloceanspaces.com/newworlds.json',
'method': 'PUT',
'path': '/newworlds.json',
'headers': {
"Cache-Control":"no-cache",
"Content-Type":"application/x-www-form-urlencoded",
"accept":"*/*",
"host":"${space}.nyc3.digitaloceanspaces.com",
"accept-encoding":"gzip, deflate",
"content-length": c.length
},
body: c
},{accessKeyId: '${secret}', secretAccessKey: '${secret}'}),function(err,res){
if(err) {
console.log(err);
} else {
console.log(res);
}
})
This gave me a successful PUT
It can be done using multer and aws sdk. It worked for me.
const aws = require('aws-sdk');
const multer = require('multer');
const express = require('express');
const multerS3 = require('multer-s3');
const app = express();
const spacesEndpoint = new aws.Endpoint('sgp1.digitaloceanspaces.com');
const spaces = new aws.S3({
endpoint: spacesEndpoint,
accessKeyId: 'your_access_key_from_API',
secretAccessKey: 'your_secret_key'
});
const upload = multer({
storage: multerS3({
s3: spaces,
bucket: 'bucket-name',
acl: 'public-read',
key: function (request, file, cb) {
console.log(file);
cb(null, file.originalname);
}
})
}).array('upload', 1);
Now you can also call this using an API like this
app.post('/upload', function (request, response, next) {
upload(request, response, function (error) {
if (error) {
console.log(error);
}
console.log('File uploaded successfully.');
});
});
HTML would look like this
<form method="post" enctype="multipart/form-data" action="/upload">
<label for="file">Upload a file</label>
<input type="file" name="upload">
<input type="submit" class="button">
</form>
I have two node servers and I am trying to send files between them using a rest api. However when I am sending the data I get a "Unexpected token -"on the receiving server. On the sender I get an [Error: write after end].
My router code:
var express = require('express');
var multer = require('multer');
var path = require('path');
var Router = express.Router;
const MODULES_PACKAGES_UPLOAD_DIR = path.resolve('/tmp');
module.exports = function() {
var router = new Router();
var storage = multer.diskStorage({
destination: function(req, file, cb){
cb(null, MODULES_PACKAGES_UPLOAD_DIR);
}
});
var upload = multer({storage: storage});
router.post('/fileUpload', upload.array(), function(req, res){
debug('We have a a file');
//Send the ok response
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain; charset=utf-8');
res.end('\n');
}
The sending code:
var Util = require('util');
var http = require('request-promise');
var request = require('request');
var fs = require('fs');
var Post = require('http');
var FormData = require('form-data');
//Generate the form data
var formdata = modules.map(function(fileName){
return fs.createReadStream('/opt/files/'+fileName);
});
var data = getData(); //Gets the body of the code as a promise
return Promise.all(data)
.then(function(dataResults){
var options = {
method: 'POST',
uri: 'https://' + name +'/file',
rejectUnauthorized: false,
timeout: 2000,
body: {
keys: keyResults,
modules: modules,
},
formData: { <====== If I remove this section everything works
'module-package': formdata,
},
json: true // Automatically stringifies the body to JSON
};
request.post(options, function(err, response){
if( err){
debug('Error: ',err);
}
else{
debug('We posted');
}
});
The weird thing is that if I remove the formData section then everything works but when it is there I get an exception that says:
SyntaxError: Unexpected token -
at parse (/home/.../projects/node_modules/body-parser/lib/types/json.js:83:15)
Does anyone have any idea what I could be doing wrong??
Just in case anyone in the future comes with the same problem. As #Bergi mentioned. You cant have both json data and form data. You need to choose either one. The solution is to just pass the json data as apart of the form like.
var options = {
method: 'POST',
uri: 'https://' + name +'/file',
rejectUnauthorized: false,
timeout: 2000,
body: {
},
formData: {
'module-package': formdata,
keys: keyResults,
modules: modules,
},
json: true // Automatically stringifies the body to JSON
};
request.post(options, function(err, response){
if( err){
debug('Error: ',err);
}
else{
debug('We posted');
}
});
In my case, the header of the HTTP Request contained "Content-Type" as "application/json".
So here are the things to check:
Send only either form-data or json body. NOT BOTH.
Check for Headers if the "Content-Type" is mentioned. Remove that.