Node request module - trouble sending formdata - javascript

I'm trying to send a post request from my node server to an api that expects a file and other form data as an multipart/form-data.
Here is what my code looks like
var importResponse = function(csv){
stringify(csv, function(err, output){
request.post({
headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
url: url,
formData: {
surveyId: surveyId,
file: {
value: output,
options: {
fileName: 'test.csv',
contentType:'text/csv'
}
}
}
}, function(error, response, body){
console.log(body);
});
});
}
Using request-debug here is the request:
request:
{ debugId: 1,
uri: 'https://co1.qualtrics.com/API/v3/responseimports',
method: 'POST',
headers:
{ 'X-API-TOKEN': 'removed',
'content-type':
'multipart/form-data; boundary=--------------------------010815605562947295265820',
host: 'co1.qualtrics.com',
'content-length': 575 } } }
and the response:
response:
{ debugId: 1,
headers:
{ 'content-type': 'application/json',
'content-length': '188',
'x-edgeconnect-midmile-rtt': '28',
'x-edgeconnect-origin-mex-latency': '56',
date: 'Wed, 18 Jul 2018 03:57:59 GMT',
connection: 'close',
'set-cookie': [Array],
'strict-transport-security': 'max-age=31536000; includeSubDomains; preload' },
statusCode: 400,
body:
'{"meta":{"httpStatus":"400 - Bad Request","error":{"errorMessage":"Missing Content-Type for file part. name=file","errorCode":"MFDP_3"},"requestId":"322a16db-97f4-49e5-bf10-2ecd7665972e"}}' } }
The error I'm getting is: Missing Content-Type for file part.
I've added this in the options:
options: {
fileName: 'test.csv',
contentType:'text/csv'
}
When I look at the request, it seems as though the form data isn't included. But perhaps that is just the request-debug not showing it.
I saw a similar SO question and the answer was to use JSON.stringify.
I tried changing my code to the following:
request.post({
headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
url: url,
body: JSON.stringify({
surveyId: surveyId,
file: {
value: output,
options: {
fileName: 'test.csv',
contentType:'text/csv'
}
}
})
However, I got the following error:
{"meta":{"httpStatus":"400 - Bad Request","error":{"errorMessage":"Missing boundary header"}}}
What am I doing wrong?
UPDATE
When I tried changing the file value to a csv on my computer fs.createReadStream('test.csv'), it worked fine
file: {
value: fs.createReadStream('test.csv'),
options: {
contentType: 'text/csv'
}
}
So I assume there is something wrong with the way I'm giving the file. The output variable that I'm using as the file just looks like "QID1,QID2\nQID1,QID2\n1,2". I assume this is causing the problems, even though the error is a bit misleading. I tried creating a Readable that I found as a StackOverFlow answer like so:
var s = new Readable
s.push(output)
s.push(null)
However, this lead to a Unexpected end of input
{"meta":{"httpStatus":"400 - Bad Request","error":{"errorMessage":"Unexpected end of input"}}}

I found the issue. My first solution was fine, but instead of fileName it should have been filename
var importResponse = function(csv){
stringify(csv, function(err, output){
request.post({
headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
url: url,
formData: {
surveyId: surveyId,
file: {
value: output,
options: {
filename: 'test.csv', //filename NOT fileName
contentType:'text/csv'
}
}
}
}, function(error, response, body){
console.log(body);
});
});
}

Is it possible that you are using the incorrect property name for your file?
A quick read of the forms info for the request Node module makes me think you should be using custom_file instead of file.
You can read more about it here: https://github.com/request/request#forms

Hey Eric check in which format they accept multipartRequest ,as I did for uploading the file on drive like this:
request(options, function (err, response) {
var boundary = '-------314159265358979323846';
var delimiter = "\r\n--" + boundary + "\r\n";
var close_delim = "\r\n--" + boundary + "--";
var fileContent = 'Sample upload :)';
var metadata = {
'name': 'myFile.txt',
'mimeType': 'text/plain\r\n\r\n'
};
var multipartRequestBody = delimiter + 'Content-Type: application/json\r\n\r\n' + JSON.stringify(metadata) + delimiter + 'Content-Type: ' + 'text/plain\r\n\r\n' + fileContent + close_delim;
request(options, function (err, response) {
var url = 'https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&access_token=' + JSON.parse(response.body).access_token;
var options = {
method: 'POST',
url: url,
headers: {
'Content-Type': 'multipart/related; boundary="' + boundary + '"'
},
body: multipartRequestBody
};
request(options, function (err, response) {
res.send({resultdata: response.body});
});
});
});
Set the multi-part as per your endpoint accepting.

Using frameworks like express(they automatically parse headers and response) and npm modules like multer for handling multipart form data helps cause they do all the heavy lifting for you

Related

nodejs - AxiosError: Request failed with status code 500

I am trying to convert the below code which is using request module to axios module to send the POST request.
request module code:
const imageFile = fs.createReadStream('image.jpeg');
const imgName = "image" + ".jpeg";
var options = {
'method': 'POST',
'url': url,
'headers': {
'Content-Type': 'application/json',
'Cache-Control': 'no-cache',
'Accept': 'application/json'
},
formData: {
'image': {
'value': imageFile,
'options': {
'filename': imgName,
'contentType': null
}
}
}
};
request(options, function (error, response) {
if (error) throw new Error(error);
console.log("SUCCESS");
});
The above code works fine and the image is posted successfully with the request module. But when I convert the same to axios, I am getting a 500 Error. (AxiosError: Request failed with status code 500)
axios module code:
const FormData = require('form-data')
const imageFile = fs.createReadStream('image.jpeg');
const imgName = "image" + ".jpeg";
const bodyFormData = new FormData();
bodyFormData.append("image['value']", imageFile);
bodyFormData.append("image['options']['filename']", imgName)
// bodyFormData.append("image['options']['contentType']", null)
console.log(bodyFormData)
const formHeaders = bodyFormData.getHeaders();
axios.post(url, bodyFormData, {
headers: {
...formHeaders,
'Cache-Control': 'no-cache',
'Accept': 'application/json',
}
}).then(function (response) {
console.log('SUCCESS');
}).catch(function (error) {
throw new Error(error);
});
Can anyone find out what I am doing wrong here?
Is there any other way to post the image using axios other than using form-data?
See the documentation for FormData#append(). You can provide extra data like the file name as the 3rd options parameter
const bodyFormData = new FormData();
// Pass filename as a string
bodyFormData.append("image", imageFile, imgName);
// or to specify more meta-data, pass an object
bodyFormData.append("image", imageFile, {
filename: imgName,
contentType: "image/jpeg",
});
axios.post(url, bodyFormData, {
headers: {
Accept: "application/json",
"Cache-Control": "no-cache",
...bodyFormData.getHeaders(),
},
});
Under the hood, request() does something very similar with the exact same form-data library. See request.js

Parsing multipart/mixed responses with Node.js and axios

I've trying to parse a multipart/mixed response with (1) JSON and (2) .zip file.
I've been using Axios to GET the response, no problem there. I've seen npm packages like meros and multipart-mixed-parser but both just returned an empty array.
the response looks like
--Boundary_137_1895017408_1627074248456
Content-Type: application/json
{
"createdDate" : "2021-07-22T01:46:05.149+0000",
more JSON...
}
--Boundary_137_1895017408_1627074248456
Content-Type: application/octet-stream
Content-Disposition: form-data; filename="347b3bd1-e6e3-49a0-928d-9956df5c5af11671844264911435718.tmp"; modification-date="Fri, 23 Jul 2021 21:04:08 GMT"; size=620991; name="attachments.zip"
binary data here ...
--Boundary_137_1895017408_1627074248456--
async function eManGet (mtn, zipPath) {
try {
mtn = mtn.toUpperCase()
if (zipPath) {
const res = eManAPI.get({
url: `/emanifest/manifest/${mtn}/attachments`,
headers: {
Accept: 'multipart/mixed'
}
})
return res
} else {
...
}
// console.log(res.data)
} catch (error) {
...
}
}
The package I want to use this for is here
If you've faced this problem before, what package(s) did you use?
The dicer npm package does the job pretty well. You'll have to extract the boudary from headers yourself tho.
How to send multipart/mixed with axios
There is example of source code with request
const messages = body[endpoint].map(function (m) {
return {
'Content-Type': 'application/http',
body: 'GET ' + api + '/gmail/v1/users/me/' + endpoint + '/' + m.id + query + '\n'
}
})
const r = request({
method: 'POST',
url: api + '/batch/gmail/v1',
multipart: messages,
timeout: opts.timeout,
headers: {
'Authorization': 'Bearer ' + key,
'content-type': 'multipart/mixed'
}
})
from this library
https://github.com/SpiderStrategies/node-gmail-api
To solve this problem I used request catcher and wrote the following tests:
it('request catcher of request', () => {
const catcher = 'https://pl.requestcatcher.com/';
const messages = [1,2,3].map(function (index:number) {
return {
'Content-Type': 'application/http',
body: 'GET ' + `/gmail/v1/users/me/${index}\n`
}
})
const r = request({
method: 'POST',
url: catcher,
multipart: messages,
headers: {
'Authorization': 'Bearer KEY',
'content-type': 'multipart/mixed'
}
})
expect(r).toBeNull();
})
and saw the following request on catcher:
For axios there is not ready answer in all internet. This is my research:
You can use a FormData object in the browser. In Node.js the feature isn't supported yet by axios (see #789).
https://github.com/axios/axios/issues/803
[RFC2388] suggested that multiple files for a single form field be
transmitted using a nested "multipart/mixed" part. This usage is
deprecated.
https://datatracker.ietf.org/doc/html/rfc7578
info how to do it in request ( that is also depreciated )
https://github.com/request/request#multipartrelated
Function for generation boundary
https://github.com/form-data/form-data/blob/53adbd81e9bde27007b28083068f2fc8272614dc/lib/form_data.js#L347
So we can send multipart/mixed by this code
it('request catcher of axios', () => {
const catcher = 'https://pl.requestcatcher.com/';
const r = axios.post(catcher, `--3e50fc2b-7a5c-4565-99f2-1628010018be
Content-Type: application/http
GET /gmail/v1/users/me/1
--3e50fc2b-7a5c-4565-99f2-1628010018be
Content-Type: application/http
GET /gmail/v1/users/me/2
--3e50fc2b-7a5c-4565-99f2-1628010018be
Content-Type: application/http
GET /gmail/v1/users/me/3
--3e50fc2b-7a5c-4565-99f2-1628010018be--`,{
headers: {
'Authorization': 'Bearer KEY',
'content-type': 'multipart/mixed; boundary=3e50fc2b-7a5c-4565-99f2-1628010018be'
}
})
expect(r).not.toBeNull();
})
and now oly generation of boundary is problem to solve. Lets check differences between multipart/mixed and and multipart/form-data bounduaries.
This is exmple of bounduary from form-data package:
--------------------------942880057551350107765038
and this is from mixed (from request):
--3e50fc2b-7a5c-4565-99f2-1628010018be
and this is proper RFC
https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html
so we can use this code
> `--${crypto.randomUUID()}`
'--7784084f-e129-44ea-aa64-8f480a26c75f'
And this is code that sends mutipart/mixed by axios
it('request catcher of axios', () => {
const catcher = 'https://pl.requestcatcher.com/';
interface SingleMultipartMixedMessage {
'Content-Type': string,
body: string
}
const messages = [1, 2, 3].map(function (index: number) {
return {
'Content-Type': 'application/http',
body: 'GET ' + `/gmail/v1/users/me/${index}\n`
}
})
const boundary = `--${crypto.randomUUID()}`
const body = messages.reduce((p: string, n: SingleMultipartMixedMessage) => {
return p + `\nContent-Type: ${n['Content-Type']}\n\n${n.body}\n${boundary}`
}, boundary)
const r = axios.post(catcher, body, {
headers: {
'Authorization': 'Bearer KEY',
'content-type': `multipart/mixed; boundary=${boundary.replace(/^--/, '')}`
}
})
expect(r).not.toBeNull();
})
and comparison of requests caught by caught between reference implementation [ request ] and our code [ axios ]
How to parse multipart mixed
Get boundary from content-type header
Add -- before it
Split body by this boundary
Any part split by double \n
Parse any pair of headers and data individually

"Failed: There was an error while trying to create this media item."

I'm trying to upload a photo on Google Photos, with google api, but i always get this error:
"status":{"code":3,"message":"Failed: There was an error while trying to create this media item."}}
I think that the problem is in the way i take the binary data of photo.
In this code i follow google photo api developers documentation, and it says, after take authorization, to:
1- Upload the raw bytes to a Google Server: to take upload-token
2- Use the upload token to create the media item.
This my node js code:
const photo = fs.readFileSync("fbimages/"+req.session.id_client+"/Prague/2020-05-30T17:29:14+0000_0.png", {
'encoding': 'binary',
'flag' : 'r'
});
var url= 'https://photoslibrary.googleapis.com/v1/uploads';
var headers= {
'Authorization': 'Bearer '+token,
'Content-type': 'application/octet-stream',
'X-Goog-Upload-Content-Type': 'image/png',
'X-Goog-Upload-Protocol': 'raw',
'X-Goog-Upload-File-Name': "2020-05-30T17:29:14+0000_0.png",
};
var body= photo
request({
url: url,
method:'POST',
headers: headers,
rejectUnauthorized: false,
body: JSON.stringify(body)
}, function(error, response, body1){
if(error)
{
console.log(error);
}
else
{
var upToken = body1.toString();
console.log(upToken);
var url= 'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate';
var headers= {
'Authorization' : 'Bearer '+token,
'Content-type': 'application/json',
};
var body= {
'newMediaItems': [
{
'description': 'Prague',
'simpleMediaItem': {
'fileName': 'prova',
'uploadToken': upToken,
}
}
]
};
request({
url: url,
method: 'POST',
headers: headers,
rejectUnauthorized: false,
body: JSON.stringify(body),
}, function(error, response, body){
if(error)
{
console.log(error);
}
else
{
res.send(JSON.parse(body));
}
});
}});
Please help me, if anyone has any idea!!
How about this modification?
Modification points:
I think that encoding: "binary" is not required.
In this case, ``const photo = fs.readFileSync("fig1.png", { flag: "r" });andconst photo = fs.readFileSync("fig1.png");` can be used.
Please modify body: JSON.stringify(body) to photo of const photo = fs.readFileSync("fig1.png", { flag: "r" });.
When above points are reflected to your script, it becomes as follows.
Modified script:
From:
const photo = fs.readFileSync("fbimages/"+req.session.id_client+"/Prague/2020-05-30T17:29:14+0000_0.png", {
'encoding': 'binary',
'flag' : 'r'
});
var url= 'https://photoslibrary.googleapis.com/v1/uploads';
var headers= {
'Authorization': 'Bearer '+token,
'Content-type': 'application/octet-stream',
'X-Goog-Upload-Content-Type': 'image/png',
'X-Goog-Upload-Protocol': 'raw',
'X-Goog-Upload-File-Name': "2020-05-30T17:29:14+0000_0.png",
};
var body= { 'media-binary-data': photo }
request({
url: url,
method:'POST',
headers: headers,
rejectUnauthorized: false,
body: JSON.stringify(body)
}, function(error, response, body1){
To:
const photo = fs.readFileSync("fbimages/"+req.session.id_client+"/Prague/2020-05-30T17:29:14+0000_0.png", { flag: "r" });
var url = "https://photoslibrary.googleapis.com/v1/uploads";
var headers = {
Authorization: "Bearer " + token,
"Content-type": "application/octet-stream",
"X-Goog-Upload-Content-Type": "image/png",
"X-Goog-Upload-Protocol": "raw",
"X-Goog-Upload-File-Name": "2020-05-30T17:29:14+0000_0.png",
};
request(
{
url: url,
method: "POST",
headers: headers,
rejectUnauthorized: false,
body: photo,
},
function (error, response, body1) {
Note:
In this modification, it supposes that your token can be used for this API. Please be careful this.
When you want to directly put the uploaded image to an alubmn, please include albumId for requesting to https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate.
Reference:
Upload media

Google Drive Rest API

I am running into a problem with google drive rest api. I have a button and upon the user click, I get a blob excel file from my backend and upload the file to google drive. The file is being uploaded to the google drive, but when I opened it, it says '[object blob]'. The actual content isn't in the file. Here is my function for creating the file. I found this solution from here: Create File with Google Drive Api v3 (javascript)
var UploadExcelFile = function(name, data, callback){
const boundary = '-------314159265358979323846';
const delimiter = "\r\n--" + boundary + "\r\n";
const close_delim = "\r\n--" + boundary + "--";
const contentType = "application/vnd.google-apps.spreadsheet";
var metadata = {
'name': name,
'mimeType': contentType
};
var multipartRequestBody =
delimiter +
'Content-Type: application/json\r\n\r\n' +
JSON.stringify(metadata) +
delimiter +
'Content-Type: ' + contentType + '\r\n\r\n' +
data +
close_delim;
var request = gapi.client.request({
'path': '/upload/drive/v3/files',
'method': 'POST',
'params': {'uploadType': 'multipart'},
'headers': {
'Content-Type': 'multipart/related; boundary="' + boundary + '"'
},
'body': multipartRequestBody});
if (!callback) {
callback = function(file) {
console.log(file)
};
}
request.execute(callback);
}```
```This is the response from the server:
Response {type: "basic", url:
"http://localhost:54878/home/generateexcel", redirected: false,
status:
200, ok: true, …}
body: ReadableStream
locked: true
__proto__: ReadableStream
bodyUsed: true
headers: Headers
__proto__: Headers
ok: true
redirected: false
status: 200
statusText: "OK"
type: "basic"
url: "http://localhost:54878/home/generateexcel"}
I am passing in the name of the file and the blob excel file from the backend like so:
fetch('/home/generateexcel', {
method: 'POST',
body: JSON.stringify(postData),
headers: {
"Content-Type": "application/json"
},
}).then(function (response) {
response.blob().then(function (result)
UploadExcelFile('newfile', result)
});
}).catch(function (err) {
// Error :(
});
You want to upload the downloaded xlsx file to Google Drive.
You have already confirmed that the xlsx file could be downloaded.
When a xlsx file is uploaded, you want to convert to Google Spreadsheet.
You can use Drive API and the access token for uploading files.
If my understanding is correct, how about this modification? In this modification, I used FormData() for creating the request body and used fetch() for requesting to Drive API. I think that there are several solutions for your situation. So please think of this as just one of them.
Modified script:
I modified UploadExcelFile(). Please modify as follows and try again.
function UploadExcelFile(name, data) {
var metadata = {
name: name,
mimeType: "application/vnd.google-apps.spreadsheet",
};
var form = new FormData();
form.append('metadata', new Blob([JSON.stringify(metadata)], {type: 'application/json'}));
form.append('file', data);
fetch('https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=id,name,kind', {
method: 'POST',
headers: new Headers({'Authorization': 'Bearer ' + gapi.auth.getToken().access_token}),
body: form
}).then((res) => {
return res.json();
}).then(function(val) {
console.log(val);
});
}
In my environment, I could confirm that this script worked. But if this didn't work in your environment, I apologize.

Header section has more than 10240 bytes (maybe it is not properly terminated)

I'm using NodeJs to try to upload an attachment to a Jira Issue via the Jira Rest API.
The api expects multipart/form-data so this is how I'm calling it in Node:
function uploadAttachments(supportFormData, callback) {
const url =
'https://somewhere.com/jira/rest/api/2/issue/' +
supportFormData.issueId +
'/attachments';
var options = {
url: url,
headers: {
Authorization: { user: username, password: password },
'X-Atlassian-Token': 'nocheck'
}
};
var r = request.post(options, function(err, res, body) {
if (err) {
console.error(err);
callback(false);
} else {
console.log('Upload successful! Server responded with:', body);
callback(false);
}
});
var form = r.form();
form.append('file', supportFormData.attachments[0].contents, {
filename: supportFormData.attachments[0].fileName,
contentType: supportFormData.attachments[0].contents
});
}
The error I'm receiving is:
org.apache.commons.fileupload.FileUploadException: Header section
has more than 10240 bytes (maybe it is not properly terminated)
The "supportFormData.attachments[0].contents" is ofType Buffer.
Any suggestions as to what could be causing this error?
I ran into this same issue and it turns out JIRA (or Java) requires \r\n as new line character. After I changed \n to \r\n my requests went through without problem.
If its a basic auth change options object to
let auth = new Buffer(`${username}:${password}`).toString('base64');
var options = {
url: url,
headers: {
Authorization: `Basic ${auth}`,
'X-Atlassian-Token': 'nocheck'
}
};

Categories