We Keep Coding

Error: write after end - node post request (module) pipe post request in get

I have a REST API server which is running on one VM1. On other VM2 machine I have built a node js server which is running as proxy. On the same VM2 machine I have application (hosted with apache which serves only html, js and css files). My node js server only resends the api calls back to the API server. This is working fine, until new requirement arrive - to add a new API endpoint (on the node js server) to download files (csv). In order to make download happen, I need to use GET method. The thing is that the required data is available only on POST endpoint from the main API server, and I am calling the API endpoint to get the data and send it back. This is the code I am trying to work it out:
var express = require('express');
var cors = require('cors');
var request = require('request');
var http = require('http');
var csv = require("fast-csv");
var config = require("./config.js");
var corsOptions = {
origin: function(origin, callback){
var originIsWhitelisted = config.whitelist.indexOf(origin) !== -1;
callback(null, originIsWhitelisted);
}
};
var handler = function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('Hello World\n');
};
var app = express();
// Enable CORS for all requests
app.use(cors(corsOptions));
app.options('*', cors(corsOptions)); // specially for pre-flight requests
app.get('/download', function(req, res){
var limit = req.query.limit;
var offset = req.query.offset;
var options = {
method: 'POST',
url: config.apiServerHost + '/search',
useQuerystring: true,
qs: {'limit': limit, 'offset': offset},
rejectUnauthorized: false,
body: 'from=date&to=date'
};
var filename = 'data.csv';
res.setHeader('Content-disposition', 'attachment; filename=\"data.csv\"');
res.setHeader('content-type', 'text/csv');
var csvStream = csv.createWriteStream({
headers: true,
objectMode: true,
transform: function (row) {
return row;
}
});
console.log(options);
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var data = JSON.parse(body);
for(var i = 0; i < data.length; i++)
csvStream.write({
"col1": "value1-"+data[0][i],
"col2": "value2-"+data[1][i],
"col3": "value3-"+data[2][i],
"col4": "value4-"+data[3][i]
});
}
csvStream.end();
}
else {
console.log("Error:", error, body);
}
}
req.pipe(request(options, callback));//.pipe(res)
csvStream.pipe(res);
});
app.use('/api', function(req, res) {
var url = config.apiServerHost + req.url;
console.log(url);
req.pipe(request({
"rejectUnauthorized": false,
"url": url
}, function(error, response, body){
if(error) {
console.log(new Date().toLocaleString(), error);
}
})).pipe(res);
});
This all code works fine when request method is POST (the same as main API server). However I receive "[Error: write after end]" when I add the body in options object. Can someone help me figure out what is happening and how to solve this problem? Thanks.

The [Error: write after end] show pip data after .end(), for your codes
req.pipe(request(options, callback));//.pipe(res)
csvStream.pipe(res);
In the callback function, the csvStream.end(); is called, then invoke csvStream.pipe could cause this error.

How do I write an image to buffer using native Node JS HTTP library?

I am really new to JavaScript and Node JS. I have various image URLs that I want to buffer. I have tried the request npm module but want a lower level library for what I want to achieve.
For example:
http://assets.loeildelaphotographie.com/uploads/article_photo/image/128456/_Santu_Mofokeng_-_TOWNSHIPS_Shebeen_Soweto_1987.jpg
I see lots of examples that suggest using the request module or examples that save files to disk. However, I cannot find an HTTP GET request example that simply buffers the image so I can pass to another function. It needs to have an "end" event so I upload the buffered image data with confidence in another step. Is there a sample pattern or "how to" on this someone could provide? Thanks!

This is the native way:
var http=require('http'), imageBuffer;
http.get(
'http://www.kame.net/img/kame-anime-small.gif',
function(res) {
var body=new Buffer(0);
if (res.statusCode!==200) {
return console.error('HTTP '+res.statusCode);
}
res.on('data', function(chunk) {
body=Buffer.concat([body, chunk]);
});
res.on('end', function() {
imageBuffer=body;
});
res.on('error', function(err) {
console.error(err);
});
}
);
// Small webserver serving the image at http://127.0.0.1:4567
http.createServer(function(req, res) {
res.write(imageBuffer || 'Please reload page');
res.end();
}).listen(4567, '127.0.0.1');
and using request (encoding:null for binary response):
var request=require('request'), imageBuffer;
request({
uri: 'http://www.kame.net/img/kame-anime-small.gif',
encoding: null
}, function(err, res, body) {
if (err) {
return console.error(err);
} else if (res.statusCode!==200) {
return console.error('HTTP '+res.statusCode);
}
imageBuffer=body;
});
// Small webserver serving the image at http://127.0.0.1:4567
require('http').createServer(function(req, res) {
res.write(imageBuffer || 'Please reload page');
res.end();
}).listen(4567, '127.0.0.1');

Here's a simple example using the built-in streaming that the http response has:
var http = require('http');
var fs = require('fs');
var file = fs.createWriteStream("test.png");
var request = http.get("some URL to an image", function(response) {
response.pipe(file);
});
I ran this myself and successfully downloaded an image from an external web site and saved it to a file and then loaded the file into the browser to see the same image.

Download image from url using request and save to variable

Is there someway I can download an image from request and save it to a variable?
request.head(url, function(err, res, body){
request(url).pipe(fs.createWriteStream(image_path));
});
right now I'm piping the result to a write stream. But instead I would like to save it to a variable so I can use it in my program. Is there any way to do this?

As what you request is an image, so you can get the response as Buffer .
var request = require('request'), fs = require('fs');
request({
url : 'http://www.google.com/images/srpr/logo11w.png',
//make the returned body a Buffer
encoding : null
}, function(error, response, body) {
//will be true, body is Buffer( http://nodejs.org/api/buffer.html )
console.log(body instanceof Buffer);
//do what you want with body
//like writing the buffer to a file
fs.writeFile('test.png', body, {
encoding : null
}, function(err) {
if (err)
throw err;
console.log('It\'s saved!');
});
});

Node: How to read file?

I want to read a file and return is as a response to GET request
This is what I am doing
app.get('/', function (request, response) {
fs.readFileSync('./index.html', 'utf8', function (err, data) {
if (err) {
return 'some issue on reading file';
}
var buffer = new Buffer(data, 'utf8');
console.log(buffer.toString());
response.send(buffer.toString());
});
});
index.html is
hello world!
When I load page localhost:5000, the page spins and nothing happens, what is I am doing incorrect here
I am newbie to Node.

You're using the synchronous version of the readFile method. If that's what you intended, don't pass it a callback. It returns a string (if you pass an encoding):
app.get('/', function (request, response) {
response.send(fs.readFileSync('./index.html', 'utf8'));
});
Alternatively (and generally more appropriately) you can use the asynchronous method (and get rid of the encoding, since you appear to be expecting a Buffer):
app.get('/', function (request, response) {
fs.readFile('./index.html', { encoding: 'utf8' }, function (err, data) {
// In here, `data` is a string containing the contents of the file
});
});

Node.js wait for callback of REST service that makes HTTP request

I am using the express module to build a RESTful API within Node.js. In my service I am making additional HTTP requests to external endpoints (server side). I need to return the data from those HTTP requests to the request body of my Web service.
On all the actions the Web service is conducting, by using console.log() I have confirmed that I get the data I need. However, when I try to return those values to the service they come back with value null. I know this is because a request is asynchronous and the callback is not waiting for the HTTP request to finish.
Is there a way to make this work?

A common practice is to use the async module.
npm install async
The async module has primitives to handle various forms of asynchronous events.
In your case, the async#parallel call will allow you to make requests to all external APIs at the same time and then combine the results for return to the requester.
Since you're making external http requests, you will probably find the request module helpful as well.
npm install request
Using request and async#parallel your route handler would look something like this...
var request = require('request');
var async = require('async');
exports.handler = function(req, res) {
async.parallel([
/*
* First external endpoint
*/
function(callback) {
var url = "http://external1.com/api/some_endpoint";
request(url, function(err, response, body) {
// JSON body
if(err) { console.log(err); callback(true); return; }
obj = JSON.parse(body);
callback(false, obj);
});
},
/*
* Second external endpoint
*/
function(callback) {
var url = "http://external2.com/api/some_endpoint";
request(url, function(err, response, body) {
// JSON body
if(err) { console.log(err); callback(true); return; }
obj = JSON.parse(body);
callback(false, obj);
});
},
],
/*
* Collate results
*/
function(err, results) {
if(err) { console.log(err); res.send(500,"Server Error"); return; }
res.send({api1:results[0], api2:results[1]});
}
);
};
You can also read about other callback sequencing methods here.

Node.js is all about callbacks. Unless the API call is synchronous (rare and shouldn't be done) you never return values from those calls, but callback with the result from within the callback method, or call the express method res.send
A great library for invoking web requests is request.js
Let's take the really simple example of calling google. Using res.send, your express.js code could look like:
var request = require('request');
app.get('/callGoogle', function(req, res){
request('http://www.google.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
// from within the callback, write data to response, essentially returning it.
res.send(body);
}
})
});
Alternatively, you can pass a callback to the method that invokes the web request, and invoke that callback from within that method:
app.get('/callGoogle', function(req, res){
invokeAndProcessGoogleResponse(function(err, result){
if(err){
res.send(500, { error: 'something blew up' });
} else {
res.send(result);
}
});
});
var invokeAndProcessGoogleResponse = function(callback){
request('http://www.google.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
status = "succeeded";
callback(null, {status : status});
} else {
callback(error);
}
})
}

Wait.for
https://github.com/luciotato/waitfor
Other answer's examples using wait.for:
Example from from Daniel's Answer (async), but using Wait.for
var request = require('request');
var wait = require('wait.for');
exports.handler = function(req, res) {
try {
//execute parallel, 2 endpoints, wait for results
var result = wait.parallel.map(["http://external1.com/api/some_endpoint"
,"http://external2.com/api/some_endpoint"]
, request.standardGetJSON);
//return result
res.send(result);
}
catch(err){
console.log(err);
res.end(500,"Server Error")
}
};
//wait.for requires standard callbacks(err,data)
//standardized request.get:
request.standardGetJSON = function ( options, callback) {
request.get(options,
function (error, response, body) {
//standardized callback
var data;
if (!error) data={ response: response, obj:JSON.parse(body)};
callback(error,data);
});
}