We Keep Coding

What is best practice for using fs.createWriteStream over an http.request in nodejs

I am streaming an xml file in, using fs.createWriteStream. For whatever reason, if the chunk coming in happens to break up xml closing tag, I am seeing a bunch of null characters being added to the data at the end of the chunk and it breaks the xml.
This may be caused since we have to replace all the xml reserved values(&lt,&gt, &#xD, etc) with the proper characters(<,>, etc.) after we stream the data file in. Any ideas on why this occurs and how to properly handle it?
Here's the basic stream code:
var writableStream = fs.createWriteStream(file_path + send_id + '.xml', { highWaterMark: 128 * 1024 });
var options = {
host: domain_name,
path: f_path,
method: 'POST',
'Content-Type': 'text/html',
'Content-Length':Buffer.byteLength(send_id)
};
var req = http.request(options, (res) =>{
res.setEncoding('utf8');
res.on('data', (chunk) => {
writableStream.write(chunk);
});
res.on('end', () => {
console.log('No more data in response');
});
});
req.write(send_id);
req.end();

nodejs https request throws error when using Content-Type

The following code is with the Dropbox API, it worked during testing but when I added the "Content-Type":"application/json" in the parameters it broke.
https.request(options, function(resp){
let creds = [];
let ts = "";
resp.on('data', function(body){
creds.push(body);
let jsonarray = Array.prototype.slice.call(creds, 0).toString().split(",");
let token = "Bearer " + jsonarray[0].substring(jsonarray[0].indexOf(':') + 3, jsonarray[0].length - 1);
ts = token;
});
resp.on('end', function(){
let connectfilelist = {
hostname: 'api.dropboxapi.com',
path: '/2/files/list_folder',
method: 'POST',
headers: {Authorization: ts,
"Content-Type", "application/json"}
};
https.request(connectfilelist, function(resp){
let flist = [];
resp.on('data', function(chunk){
flist.push(chunk.toString());
});
resp.on('end', function(){
console.log(flist);
res.render('mydropbox', flist);
});
}).end(bodyParams);
});
}).end(bodyParams);
This is the error I get when i omit that parameter:
[ 'Error in call to API function "files/list_folder": This function requires its argument in the HTTP request body, but your request body is empty.' ]
When I include the Content-Type, it throws this:
[ 'Error in call to API function "files/list_folder": request body: could not decode input as JSON' ]
I think that the way I am stating the parameters might be in a string and not JSON format, but I am not sure where I would change something.

Returning from an API request [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 3 years ago.
I have a function that does POST request to get access tokens from an API. The function works insofar as printing the token to the console but I couldn't find a way to parse and save the access-token to a variable for using in other methods.
I've searched a lot about these on Stack Overflow, and one thing I came across was the notion of callbacks, async/await... However there is no concrete answers that show me how to RETURN the value after the request is made.
I'm getting undefined values, probably because my function is executing too fast before the request ends but nonetheless I would like to know a specific code implementation (using another callback function or something different) that allows me to get the value accessible outside these asynchronous functions.
My code is below along with a screenshot. Keep in mind this is a Lambda code for an Alexa skill
function ParseToken(obj) {
var string = obj + "";
return string;
}
function httpRetrieveToken() {
var http = require("https");
var options = {
"method": "POST",
"host": "hostName",
"path": "pathVal",
"headers": {
"content-type": "contentType",
"Authorization": "Bearer token",
"cache-control": "no-cache",
}
};
var req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.write("stuff here");
req.end();
}

Parsing token
Once you have the text representing your token object you can use JSON.parse to convert it to an object:
var json = '{"token":"ABCDEFG123456789", "expires":36000}';
obj = JSON.parse(json);
console.log(obj.token);
// expected output: ABCDEFG123456789
Using Callback
You can send a callback function as parameter to the httpRetrieveToken function like:
var http = require("https");
function httpRetrieveToken(cb) {
var options = {
"method": "POST",
"host": "hostName",
"path": "pathVal",
"headers": {
"content-type": "contentType",
"Authorization": "Bearer token",
"cache-control": "no-cache",
}
};
var req = http.request(options, function (res) {
var _cb = cb;
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = Buffer.concat(chunks);
var token = JSON.parse(body.toString());
_cb(token); // Callback function
});
});
req.write("stuff here");
req.end();
}
Then you can use function like:
httpRetrieveToken(function(token) {
// Do something with the token
});
The reason to use a callback function is to execute it once the asynchronous process has been finished, then enclose in that callback the processes to follow with the required data.
Using Promise
Using promise object, you provide a function who executes either resolve when your process was successful or reject when you had an error. Then you set a function to be executed on sucess with then method, where you can use the token retrived by your http request.
var http = require("https");
function httpRetrieveToken() {
var options = {
"method": "POST",
"host": "hostName",
"path": "pathVal",
"headers": {
"content-type": "contentType",
"Authorization": "Bearer token",
"cache-control": "no-cache",
}
};
return new Promise(function(resolve, reject) {
let req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = Buffer.concat(chunks);
var token = JSON.parse(body.toString());
resolve(null, token); // Callback function
});
});
req.on("error", function(err) {
reject(err);
});
});
}
httpRetrieveToken().then((err, token) => {
if(err) console.log(err);
// Do something with token
});

Post Method works using https.request but not with Needle or Request libraries

I have nodeApp. It does stuff.
At a particular time I need to communicate with an API that out there in the wild. Using the API in rest tool like Postman is straight forward:
Postman
Url:
https://epicurl
Headers:
Content-Type : application/json
Accept : application/json
x-key : secret
Body:
{
"some":"kickass"
"data":"here"
}
Sending the above in Postman I get a nice quick response! Yay for rest tools.
So their API works, now I need to make that same response in my Node.js application.
This is where things get odd...
Request Module: FAILS
var request = require('request')
...lots_of_other_stuff...
var options = {
uri: 'https://epicURL',
method: 'POST',
json: true,
headers : {
"Content-Type":"application/json",
"Accept":"application/json",
"x-key":"secretbro"
},
body : JSON.stringify(bodyModel)
};
request(options, function(error, response, body) {
if (!error) {
console.log('Body is:');
console.log(body);
} else {
console.log('Error is:');
logger.info(error);
}
cb(body); //Callback sends request back...
});
The above fails.. It throws the good'ol ECONNRESET error that we all love! Why? Who knows?
https.request() - WORKS!
var https = require("https");
https.globalAgent.options.secureProtocol = 'SSLv3_method';
var headers = {
"Content-Type":"application/json",
"Accept":"application/json",
"x-key":"nicetrybro"
}
var options = {
host: 'www.l33turls.com',
port:443,
path: "/sweetpathsofjebus",
method: 'POST',
headers: headers
};
var req = https.request(options, function(res) {
res.setEncoding('utf-8');
var responseString = '';
res.on('data', function(data) {
responseString += data;
});
res.on('end', function() {
var resultObject = responseString;
//Call the callback function to get this response object back to the router.
cb(resultObject);
});
});
req.on('error', function(e) {
console.log(e);
});
req.write(bodyString);
req.end();
But then I notice...
If i leave this line of code in place when using the Request Module it then works...
var https = require("https");
https.globalAgent.options.secureProtocol = 'SSLv3_method';
Is this documented somewhere and I am missing it? Anyone explain this to me?

POST to PHP from Node.js

I am trying to POST some data from a Node.js application to a PHP script. For the time being I am just building a proof of concept but I am unable to get the actual data over to the PHP side. The request goes through and I get 200 back but PHP thinks the $_POST array is empty.
Here is my node code:
// simple end point just for testing
exports.testPost = function(request, response) {
data = request.body.data;
postToPHP(data);
response.end(data);
}
function postToPHP (data) {
var http = require('http');
var options = {
host : 'localhost',
port : 8050,
path : '/machines/test/index.php',
method : 'POST',
headers : {
'Content-Type' : 'application/json',
'Content-Length' : Buffer.byteLength(data)
}
};
var buffer = "";
var reqPost = http.request(options, function(res) {
console.log("statusCode: ", res.statusCode);
res.on('data', function(d) {
console.info('POST Result:\n');
//process.stdout.write(d);
buffer = buffer+data;
console.info('\n\nPOST completed');
});
res.on('end', function() {
console.log(buffer);
});
});
console.log("before write: "+data);
reqPost.write(data);
reqPost.end();
}
Again, the request makes it to localhost:8050/machines/test/index.php but when I do a var_dump of $_POST it is an empty array.
[29-Jan-2014 21:12:44] array(0) {
}
I suspect I am doing something wrong with the .write() method but I can't quite figure out what. Any input on what I am missing or doing incorrectly would be greatly appreciated.
* Update:
As some of the comments indicate using file_get_contents('php://input'); does work to get the data on the PHP side but I would still prefer to be able to access the $_POST array directly.

Since you are sending the data with Content-Type: application/json you would need to read the raw input as php does not know how to read json into their globals like _GET and _POST unless you have some php extension that does it.
You can use the querystring library to parse a object into a name-value pair query string that you could than transmit with Content-Type:application/x-www-form-urlencoded so that the data will be parsed into the globals
var data = {
var1:"something",
var2:"something else"
};
var querystring = require("querystring");
var qs = querystring.stringify(data);
var qslength = qs.length;
var options = {
hostname: "example.com",
port: 80,
path: "some.php",
method: 'POST',
headers:{
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': qslength
}
};
var buffer = "";
var req = http.request(options, function(res) {
res.on('data', function (chunk) {
buffer+=chunk;
});
res.on('end', function() {
console.log(buffer);
});
});
req.write(qs);
req.end();