Get body json specific data in node - javascript

I use the req res for node js from the following module and I want to send in post message body the following json
{
"Actions": [
{
"file1": {
"name": "file 1",
"content": "file 2 content"
},
"file2": {
"name": "file 2",
"content": "file 2 content"
}
}
]
}
How can I get from the req body the name and the content
I use the create server and there I've req and res
https://github.com/nodejitsu/node-http-proxy
UPDATE
this is my code
var http = require('http'),
httpProxy = require('http-proxy'),
url = require('url');
http.createServer(function (req, res) {
var hostname = req.headers.host.split(":")[0];
console.log(req.body);

The problem is that Node's http API is awful. To get the body, you need to listen for data events and build the body string yourself.
var http = require('http'),
url = require('url');
var server = http.createServer(function (req, res) {
var body = '';
req.on('data', function (chunk) {
body += chunk;
});
req.on('end', function () {
var json = JSON.parse(body);
console.log(json.Actions[0].file1.content);
res.writeHead(200);
res.end();
});
});
server.listen(8080);
I highly recommend using something like Express that hides all these details.

Related

NodeJS multi site web scrape

I am learning NodeJS and trying to scrape a fan wikia to get names of characters and store them in a json file. I have an array of character names and I want to loop through them and scrape each character name from each url in the array. The issue I am running into is:
throw new Error('Can\'t set headers after they are sent.');
Here is my source code at the moment:
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/', function(req, res){
var bosses = ["Boss1","Boss2"];
for (boss in bosses) {
url = 'http://wikiasiteexample.com/' + bosses[boss];
request(url, function (error, response, html) {
if (!error) {
var $ = cheerio.load(html);
var title;
var json = { title: "" };
$('.page-header__title').filter(function () {
var data = $(this);
title = data.text();
json.title = title;
})
}
fs.writeFile('output.json', JSON.stringify(json, null, 4), {'flag':'a'}, function(err) {
if (err) {
return console.error(err);
}
});
res.send('Check your console!')
})
}
})
app.listen('8081')
console.log('Running on port 8081');
exports = module.exports = app;
You're calling res.send() for every request you make.
Your HTTP request can only have one response, so that gives an error.
You must call res.send() exactly once.
Promises (and Promise.all()) will help you do that.

Redirect image request on NodeJS

Inside my application code, for a specific set of APIs, I'm making a NodeJS request like following, which should return a image as the body. This same request works fine on Postman (and I can see the image).
module.exports = {
getThumbnail: function (thumbnailUrn, env, token, onsuccess){
request({
url: config.baseURL(env) + config.thumbail(thumbnailUrn),
method: "GET",
headers: {
'Authorization': 'Bearer ' + token,
}
}, function (error, response, body) {
// error check removed for simplicity...
onsuccess(body);
});
}
}
The above code run under my own security checks and adds the token header. It works fine (request calls return 200/OK).
Now on my app router I want to respond this as an image, but the output is not being interpreted as an image. Here is what I have:
var dm = require(/*the above code*/);
// express router
var express = require('express');
var router = express.Router();
router.get('/getThumbnail', function (req, res) {
var urn = req.query.urn;
dm.getThumbnail(urn, req.session.env, req.session.oauthcode, function (thumb) {
res.writeHead(200,
{
'Content-Type': 'image/png'
}
);
// at this point, the 'thumb' variable is filled
// but I believe is not properly encoded...
// or maybe the res.end output is missing something...
res.end(thumb, 'binary');
});
});
module.exports = router;
EDIT: as commented by Nodari Lipartiya, this is kind of proxy behaviour ( server(responds with image) -> proxy (node.js/resends to client) -> end user)
I'm not sure what is coming back in thumb, but the following snippet seemed to work for me (bypassing Express for simplicity):
var http = require("http")
var fs = require("fs")
var server = http.createServer(listener)
server.listen(() => {
console.log(server.address().port)
})
var binary = fs.readFileSync("path to local image")
function listener(req, resp) {
resp.writeHead(200,
{
'Content-Type': 'image/png'
}
);
resp.end(new Buffer(binary), "binary")
}
What happens if you wrap it in a Buffer?
If I've understood everything correctly:
I did this
server.js
var fs = require('fs');
var express = require('express');
var app = express();
app.get('/img', function(req, res, next) {
var stream = fs.createReadStream('img.jpeg');
var filename = "img.jpeg";
filename = encodeURIComponent(filename);
res.setHeader('Content-disposition', 'inline; filename="' + filename + '"');
res.setHeader('Content-type', 'image/jpeg');
stream.pipe(res);
});
app.listen(9999, function () {
console.log('Example app listening on port 9999!');
});
proxy.js
var request = require('request');
var express = require('express');
var app = express();
app.get('/img', function(req, res, next) {
console.log('proxy/img');
request({
url: 'http://localhost:9999/img',
method: "GET",
}, function (error, response, body) {
res.end(body, 'binary');
});
});
app.listen(9998, function () {
console.log('Example app listening on port 9998!');
});
req.js
var request = require('request');
request({
url: 'http://localhost:9998/img',
method: "GET",
}, function (error, response, body) {
console.log('body', body);
});
works for me. Please, let me know if you'll need help.

Error: write after end - node post request (module) pipe post request in get

I have a REST API server which is running on one VM1. On other VM2 machine I have built a node js server which is running as proxy. On the same VM2 machine I have application (hosted with apache which serves only html, js and css files). My node js server only resends the api calls back to the API server. This is working fine, until new requirement arrive - to add a new API endpoint (on the node js server) to download files (csv). In order to make download happen, I need to use GET method. The thing is that the required data is available only on POST endpoint from the main API server, and I am calling the API endpoint to get the data and send it back. This is the code I am trying to work it out:
var express = require('express');
var cors = require('cors');
var request = require('request');
var http = require('http');
var csv = require("fast-csv");
var config = require("./config.js");
var corsOptions = {
origin: function(origin, callback){
var originIsWhitelisted = config.whitelist.indexOf(origin) !== -1;
callback(null, originIsWhitelisted);
}
};
var handler = function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('Hello World\n');
};
var app = express();
// Enable CORS for all requests
app.use(cors(corsOptions));
app.options('*', cors(corsOptions)); // specially for pre-flight requests
app.get('/download', function(req, res){
var limit = req.query.limit;
var offset = req.query.offset;
var options = {
method: 'POST',
url: config.apiServerHost + '/search',
useQuerystring: true,
qs: {'limit': limit, 'offset': offset},
rejectUnauthorized: false,
body: 'from=date&to=date'
};
var filename = 'data.csv';
res.setHeader('Content-disposition', 'attachment; filename=\"data.csv\"');
res.setHeader('content-type', 'text/csv');
var csvStream = csv.createWriteStream({
headers: true,
objectMode: true,
transform: function (row) {
return row;
}
});
console.log(options);
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var data = JSON.parse(body);
for(var i = 0; i < data.length; i++)
csvStream.write({
"col1": "value1-"+data[0][i],
"col2": "value2-"+data[1][i],
"col3": "value3-"+data[2][i],
"col4": "value4-"+data[3][i]
});
}
csvStream.end();
}
else {
console.log("Error:", error, body);
}
}
req.pipe(request(options, callback));//.pipe(res)
csvStream.pipe(res);
});
app.use('/api', function(req, res) {
var url = config.apiServerHost + req.url;
console.log(url);
req.pipe(request({
"rejectUnauthorized": false,
"url": url
}, function(error, response, body){
if(error) {
console.log(new Date().toLocaleString(), error);
}
})).pipe(res);
});
This all code works fine when request method is POST (the same as main API server). However I receive "[Error: write after end]" when I add the body in options object. Can someone help me figure out what is happening and how to solve this problem? Thanks.
The [Error: write after end] show pip data after .end(), for your codes
req.pipe(request(options, callback));//.pipe(res)
csvStream.pipe(res);
In the callback function, the csvStream.end(); is called, then invoke csvStream.pipe could cause this error.

Cannot save a remote image with node.js and request

I want to save an image with node.js and the request library. So far I have this simple code:
var request = require('request');
var fs = require('fs');
request('http://upload.wikimedia.org/wikipedia/commons/8/8c/JPEG_example_JPG_RIP_025.jpg', function(error, response, body)
{
// further logic that decides
// whether or not the image will be saved
fs.writeFile('downloaded.jpg', body, function(){});
});
But it doesn't work. The image always arrives corrupt. I assume it's an encoding error but I cannot figure out how to fix this.
var request = require('request'),
fs = require('fs'),
url = 'http://upload.wikimedia.org/wikipedia/commons/8/8c/JPEG_example_JPG_RIP_025.jpg';
request(url, {encoding: 'binary'}, function(error, response, body) {
fs.writeFile('downloaded.jpg', body, 'binary', function (err) {});
});
var fs = require('fs'),
request = require('request'),
url='http://upload.wikimedia.org/wikipedia/commons/8/8c/JPEG_example_JPG_RIP_025.jpg';
request(url).pipe(fs.createWriteStream('downloaded.jpg'));
Here's how I did it using stream and pipe, (I was using express but you may not need that)
var express = require('express');
var app = express();
var filesystem = require('fs');
var https = require('https');
var download = function(url, dest, cb) {
var file = filesystem.createWriteStream(dest);
var request = https.get(url, function(httpResponse) {
httpResponse.pipe(file);
file.on('finish', function() {
console.log("piping to file finished")
file.close(cb); // close() is async, call cb after close completes.
});
}).on('error', function(err) { // Handle errors
filesystem.unlink(dest); // Delete the file async. (But we don't check the result)
if (cb) cb(err.message);
});
};
app.get('/image', (req, res) => {
download('https://lastfm-img2.akamaized.net/i/u/64s/15cc734fb0e045e3baac02674d2092d6.png',
'porcupine.png',
() => {console.log("downloaded to porcupine.png")})
})
When I run using node server.js and hit the url localhost:3000/image, it will download and save the file to porcupine.png in the base directory.

Extracting POST parameters from request in Nodejs

I am trying to get parameters from a POST in the variable postData by using the request by -
( i used this because it was suggested here - How do I get the post request with express js? )
and here -
How to retrieve POST query parameters?
var express = require('express');
var app = express();
var fs = require('fs');
var json = require('json');
app.use(express.json()); // to support JSON-encoded bodies
app.use(express.urlencoded()); // to support URL-encoded bodies
app.post('/shrib/:file/:data',function(req,res){
var fileName = req.params.file;
var data = req.params.data;
req.on('data',function(data){ body+=data; } );
req.on('end' ,function(){
var postData = qs.parse(body);
var writeStream = fs.createWriteStream(fileName);
var postData = req.body.text;
if(postData)
console.log(postData);
else
console.log("failed miserably");
res.write(200);
res.end();
});
});
app.get('/shrib/:file',function(req,res){
var fileName = req.params.file;
if(fileName != ''){
var readStream = fs.createReadStream(fileName);
var content;
readStream.on('data',function(chunk){
content+=chunk.toString();
console.log(content);
});
readStream.on('end',function(){
res.writeHead(200,{"Content-Type":"text/html"});
res.write("<form id=\"submitForm\" method=\"POST\">");
res.write("<textarea id=\"text\"rows=50 cols=50 >");
console.log(content);
if(content)
res.write(content.toString());
res.write("</textarea>");
res.write("<input type=\"submit\" value=\"submit\" />");
res.write("</form>");
res.write("<script>");
res.write("var windowLocation = location.href;");
res.write("document.getElementById(\"submitForm\").action=windowLocation + \'/data\';");
res.write("</script>");
res.end();
});
}else{
res.writeHead(200);
res.write("invalid/empty path name");
}
});
app.listen(8080);
and got this error -
Error: Most middleware (like json) is no longer bundled with Express and must be installed separately. Please see https://github.com/senchalabs/connect#middleware.
at Function.Object.defineProperty.get (/home/unknown/public_html/node/node_modules/express/lib/express.js:89:13)
I was using body parser before which i read in some solutions here and it gave me the same error middleware missing, i installed it globally then also got the same error and after that i read about json , so i installed it globally using
npm install -g json
did not work, then too. then i tried adding the dependancies -
{
"name": "express_shrib.js",
"version": "0.0.1",
"description": "Creating Shrib Using Express",
"main": "express_shrib.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "https://github.com/iamdeadman/nodejs.git"
},
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/iamdeadman/nodejs/issues"
},
"homepage": "https://github.com/iamdeadman/nodejs",
"dependencies": {
"express": ">= 1.2.0",
"json": ">= 9.0.0"
}
}
and ran npm install
still the same error -
Error: Most middleware (like json) is no longer bundled with Express and must be installed separately. Please see https://github.com/senchalabs/connect#middleware.
at Function.Object.defineProperty.get (/home/unknown/public_html/node/node_modules/express/lib/express.js:89:13)
Edit** - Code with the new body-parser module
var express = require('express');
var app = express();
var fs = require('fs');
var bodyParser = require('body-parser');
app.use(bodyParser());
app.post('/shrib/:file/:data',function(req,res){
var fileName = req.params.file;
var data = req.params.data;
req.on('data',function(data){ body+=data; } );
req.on('end' ,function(){
var postData = req.body;
var writeStream = fs.createWriteStream(fileName);
if(postData)
console.log(postData);
else{
console.log("failed miserably");
console.log(postData);
}
res.writeHead(200);
res.end();
});
});
app.get('/shrib/:file',function(req,res){
var fileName = req.params.file;
if(fileName != ''){
var readStream = fs.createReadStream(fileName);
var content;
readStream.on('data',function(chunk){
content+=chunk.toString();
console.log(content);
});
readStream.on('end',function(){
res.writeHead(200,{"Content-Type":"text/html"});
res.write("<form id=\"submitForm\" method=\"POST\">");
res.write("<textarea id=\"text\"rows=50 cols=50 >");
console.log(content);
if(content)
res.write(content.toString());
res.write("</textarea>");
res.write("<input type=\"submit\" value=\"submit\" />");
res.write("</form>");
res.write("<script>");
res.write("var windowLocation = location.href;");
res.write("document.getElementById(\"submitForm\").action=windowLocation + \'/data\';");
res.write("</script>");
res.end();
});
}else{
res.writeHead(200);
res.write("invalid/empty path name");
}
});
app.listen(8080);
and here i get
{}
in the console which means that the body object is empty for some reason.
With Express 4, the body parsing middleware (like other previously built-in middleware) was extracted out into the 'body-parser' module. However, this new module only handles JSON and urlencoded form submissions, not multipart.
If you need multipart support, you'd need to use something like connect-busboy or multer or connect-multiparty (connect-multiparty is essentially the old Express bodyParser middleware).
EDIT: Also, the name attribute is missing for the textarea input field. This is required, otherwise the field will not be sent with the form.
When using express 4 use body-parser middleware to get parameters.
Multipart has issue that it creates loads of temp files. So its better to avoid it whenever possible and use upload services directly.
app.use(function (req, res, next) {
var urlParser = require('url');
var url = urlParser.parse(req.url, true);
if (url.pathname == "/rest/file/upload") {
next();
} else {
var contentType = req.header("content-type");
if (contentType && contentType.indexOf("application/json") != -1) {
bodyParser.json({limit: 1024 * 1024 * 10})(req, res, next);
} else {
bodyParser.urlencoded({ extended: true, limit: 1024 * 1024 * 10})(req, res, next);
}
}
});
then just get your request parameter as :
console.log(req.param("parameter-name"));

Categories