We Keep Coding

event-stream in node: works fine in console but doesn't write to stream

I want to stream events to localhost/czml - which works fine in the console or in the get request window. But I can't stream those variables to the page because req.query always ends up being undefined
I'm a bloody beginner in programming and most of the time I have no clue what I'm doing (that's why the code is so bad...). I got that code through trial and error and mostly through copying from somewhere
var express = require('express'),
fs = require('fs'),
morgan = require('morgan'),
path = require('path'),
os = require('os'),
http = require('http');
const app = express();
const EventEmitter = require('events');
const stream = new EventEmitter();
var czmlstream = fs.createWriteStream('czml.czml',{flags: 'a'});
app.get('/czml', function (req, res, next) {
//don't log favicon
if (req.url === '/favicon.ico'){
res.end();
return;
}
//only log GET and set to stream
if (req.method === 'GET' ) {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'
});
function createCzml() {
//get the query value from the request
var id = req.query.id;
var lon = parseInt(req.query.lon);
var lat = parseInt(req.query.lat);
var alt = parseInt(req.query.alt);
// custom json format for czml file
var entity = {
"id": id,
"position": {
"cartographicDegrees": [lat, lon, alt]
},
"point": {
"color" : {"rgba": [0,0,255,255]},
"pixelSize": 20
}
};
return entity;
}
//first 2 lines for the event stream
res.write('event: czml\n');
res.write('data:' + JSON.stringify({ "id":"document", "version":"1.0" })+
'\n\n');
//always tells me that 10 listeners are added .... ?
stream.setMaxListeners(0);
//stream.on(req) = emit event on get request?
stream.on('req', function() {
res.write('event: czml\n');
res.write('data:' +JSON.stringify(createCzml)+ '\n\n'); //this
doesn't work
});
//not sure why this is needed
stream.emit('req');
}else{
res.WriteHead(405, {'Content-Type': 'text/plain'});
res.end('No GET Request - not allowed');
}
//morgan(format, {stream: czmlstream})(req,res,next);
}).listen(8000);
console.log('Server running');
What I want to achieve:
someone sends a get request to localhost/czml/?id=1&lon=-40&lat=30&alt=5000 => those queries are parsed and sent to localhost/whatever as event-stream in the format of:
event: czml
data: {json}
I'm nearly there (even if the code is bad) - it's just the last part left where I have to write those pesky queries to localhost/whatever. Right now it loggs everything fine in the console, but undefined is written to localhost/whatever...
I would be very grateful if you can point me in the right direction - keep in mind though, that I need easy and good explanations ;)

ok I solved this on my own and just for reference for some other newcomers:
It's basically this Example, only with listeners (as I understood them) for get requests
// most basic dependencies
var express = require('express')
, http = require('http')
, os = require('os')
, path = require('path')
, url = require('url')
, fs = require('fs');
// create the app
var app = express();
// configure everything, just basic setup
//app.set('port', process.env.PORT || 8000);
app.use(function(req, resp, next) {
resp.header("Access-Control-Allow-Origin", "*");
resp.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
next();
});
// Serve the www directory statically
app.use(express.static('www'));
//---------------------------------------
// Handle Get request and event-stream every second
//---------------------------------------
var openConnections = [];
var id, lon, lat, alt;
app.get('/czml', function(req, res, next) {
//don't log favicon
if (req.url === '/favicon.ico'){
res.end();
return;
} else {
var queryData = url.parse(req.url, true).query;
id = queryData.id;
lon = queryData.lon;
lat = queryData.lat;
alt = queryData.alt;
req.socket.setTimeout(2 * 60 * 1000);
// send headers for event-stream connection
// see spec for more information
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'
});
res.write('\n');
// push this res object to our global variable
openConnections.push(res);
// send document packet
res.write('event: czml\ndata:' + JSON.stringify({ "id":"document", "version":"1.0" })+ '\n\n');
// When the request is closed, e.g. the browser window
// is closed. We search through the open connections
// array and remove this connection.
req.on("close", function() {
var toRemove;
for (var j =0 ; j < openConnections.length ; j++) {
if (openConnections[j] == res) {
toRemove =j;
break;
}
}
openConnections.splice(j,1);
});
next();
}
}).listen(8000);
function createMsg() {
var entity = {
"id" : id,
"position" : {
"cartographicDegrees": [lon,lat,alt]
},
"point" : {
"color" : {
"rgba" : [0,0,255,255]
},
"pixelSize" : 15
}
};
return JSON.stringify(entity);;
}
setInterval(function() {
// we walk through each connection
openConnections.forEach(function(res) {
// send doc
res.write('event: czml\n');
res.write('data:' + createMsg() + '\n\n');
});
}, 1000);
I don't know how this works here on SO - the above isn't really the answer to my question - more of a workaround. But it works, so I guess it's fine :)

Append objects to a json file

I have a task to implement a pseudo cart page and when I click on checkout i want to send a request to a json file "ordersTest.json" with a following structure:
{ "orders": [] }. So when a post request is sent i have to put the data in that orders array in the json. I am completely new to Nodejs and express. This is my first project on it and i came up with a very simple server.
const express = require('express')
const path = require('path')
const fs = require('fs')
const url = require('url')
const bodyParser = require('body-parser')
const app = express()
const ordersJson = require('./public/ordersTest.json');
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
app.post('/api/orders', (req, res) => {
let body = req.body;
console.log(body);
fs.appendFile('./public/ordersTest.json', JSON.stringify(body), err => {
if (err) console.log(err);
})
})
But this thing only appends it to the end of the file. I need to put it inside this orders array
This is my ajax passing an example object in the body of the post:
$(".btn-checkout").on('click', function() {
let date = new Date();
$.ajax({
method : "POST",
url: "/api/orders",
data : {a: "abc"},//{ order: "order",date: date.toDateString(), order: JSON.stringify(cart)},
success : function(success){
console.log(success,'success');
},
error : function(err) {
console.log(err);
}
});
clearCart();
displayClearedCart();
});

You need to parse the JSON file and then treat it like an object. Once you are done with it, convert it to JSON again and overwrite your file. like this
app.post('/api/orders', (req, res) => {
let body = req.body;
var ordersTest = require('./public/ordersTest.json');
ordersTest.orders.push(body);
fs.writeFile('./public/ordersTest.json', JSON.stringify(ordersTest), function(err) {
if (err) res.sendStatus(500)
res.sendStatus(200);
});
})
Not tested, please fix typo error if any.

Unexpected token - while parsing json request

I have two node servers and I am trying to send files between them using a rest api. However when I am sending the data I get a "Unexpected token -"on the receiving server. On the sender I get an [Error: write after end].
My router code:
var express = require('express');
var multer = require('multer');
var path = require('path');
var Router = express.Router;
const MODULES_PACKAGES_UPLOAD_DIR = path.resolve('/tmp');
module.exports = function() {
var router = new Router();
var storage = multer.diskStorage({
destination: function(req, file, cb){
cb(null, MODULES_PACKAGES_UPLOAD_DIR);
}
});
var upload = multer({storage: storage});
router.post('/fileUpload', upload.array(), function(req, res){
debug('We have a a file');
//Send the ok response
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain; charset=utf-8');
res.end('\n');
}
The sending code:
var Util = require('util');
var http = require('request-promise');
var request = require('request');
var fs = require('fs');
var Post = require('http');
var FormData = require('form-data');
//Generate the form data
var formdata = modules.map(function(fileName){
return fs.createReadStream('/opt/files/'+fileName);
});
var data = getData(); //Gets the body of the code as a promise
return Promise.all(data)
.then(function(dataResults){
var options = {
method: 'POST',
uri: 'https://' + name +'/file',
rejectUnauthorized: false,
timeout: 2000,
body: {
keys: keyResults,
modules: modules,
},
formData: { <====== If I remove this section everything works
'module-package': formdata,
},
json: true // Automatically stringifies the body to JSON
};
request.post(options, function(err, response){
if( err){
debug('Error: ',err);
}
else{
debug('We posted');
}
});
The weird thing is that if I remove the formData section then everything works but when it is there I get an exception that says:
SyntaxError: Unexpected token -
at parse (/home/.../projects/node_modules/body-parser/lib/types/json.js:83:15)
Does anyone have any idea what I could be doing wrong??

Just in case anyone in the future comes with the same problem. As #Bergi mentioned. You cant have both json data and form data. You need to choose either one. The solution is to just pass the json data as apart of the form like.
var options = {
method: 'POST',
uri: 'https://' + name +'/file',
rejectUnauthorized: false,
timeout: 2000,
body: {
},
formData: {
'module-package': formdata,
keys: keyResults,
modules: modules,
},
json: true // Automatically stringifies the body to JSON
};
request.post(options, function(err, response){
if( err){
debug('Error: ',err);
}
else{
debug('We posted');
}
});

In my case, the header of the HTTP Request contained "Content-Type" as "application/json".
So here are the things to check:
Send only either form-data or json body. NOT BOTH.
Check for Headers if the "Content-Type" is mentioned. Remove that.

Redirect image request on NodeJS

Inside my application code, for a specific set of APIs, I'm making a NodeJS request like following, which should return a image as the body. This same request works fine on Postman (and I can see the image).
module.exports = {
getThumbnail: function (thumbnailUrn, env, token, onsuccess){
request({
url: config.baseURL(env) + config.thumbail(thumbnailUrn),
method: "GET",
headers: {
'Authorization': 'Bearer ' + token,
}
}, function (error, response, body) {
// error check removed for simplicity...
onsuccess(body);
});
}
}
The above code run under my own security checks and adds the token header. It works fine (request calls return 200/OK).
Now on my app router I want to respond this as an image, but the output is not being interpreted as an image. Here is what I have:
var dm = require(/*the above code*/);
// express router
var express = require('express');
var router = express.Router();
router.get('/getThumbnail', function (req, res) {
var urn = req.query.urn;
dm.getThumbnail(urn, req.session.env, req.session.oauthcode, function (thumb) {
res.writeHead(200,
{
'Content-Type': 'image/png'
}
);
// at this point, the 'thumb' variable is filled
// but I believe is not properly encoded...
// or maybe the res.end output is missing something...
res.end(thumb, 'binary');
});
});
module.exports = router;
EDIT: as commented by Nodari Lipartiya, this is kind of proxy behaviour ( server(responds with image) -> proxy (node.js/resends to client) -> end user)

I'm not sure what is coming back in thumb, but the following snippet seemed to work for me (bypassing Express for simplicity):
var http = require("http")
var fs = require("fs")
var server = http.createServer(listener)
server.listen(() => {
console.log(server.address().port)
})
var binary = fs.readFileSync("path to local image")
function listener(req, resp) {
resp.writeHead(200,
{
'Content-Type': 'image/png'
}
);
resp.end(new Buffer(binary), "binary")
}
What happens if you wrap it in a Buffer?

If I've understood everything correctly:
I did this
server.js
var fs = require('fs');
var express = require('express');
var app = express();
app.get('/img', function(req, res, next) {
var stream = fs.createReadStream('img.jpeg');
var filename = "img.jpeg";
filename = encodeURIComponent(filename);
res.setHeader('Content-disposition', 'inline; filename="' + filename + '"');
res.setHeader('Content-type', 'image/jpeg');
stream.pipe(res);
});
app.listen(9999, function () {
console.log('Example app listening on port 9999!');
});
proxy.js
var request = require('request');
var express = require('express');
var app = express();
app.get('/img', function(req, res, next) {
console.log('proxy/img');
request({
url: 'http://localhost:9999/img',
method: "GET",
}, function (error, response, body) {
res.end(body, 'binary');
});
});
app.listen(9998, function () {
console.log('Example app listening on port 9998!');
});
req.js
var request = require('request');
request({
url: 'http://localhost:9998/img',
method: "GET",
}, function (error, response, body) {
console.log('body', body);
});
works for me. Please, let me know if you'll need help.

Error: write after end - node post request (module) pipe post request in get

I have a REST API server which is running on one VM1. On other VM2 machine I have built a node js server which is running as proxy. On the same VM2 machine I have application (hosted with apache which serves only html, js and css files). My node js server only resends the api calls back to the API server. This is working fine, until new requirement arrive - to add a new API endpoint (on the node js server) to download files (csv). In order to make download happen, I need to use GET method. The thing is that the required data is available only on POST endpoint from the main API server, and I am calling the API endpoint to get the data and send it back. This is the code I am trying to work it out:
var express = require('express');
var cors = require('cors');
var request = require('request');
var http = require('http');
var csv = require("fast-csv");
var config = require("./config.js");
var corsOptions = {
origin: function(origin, callback){
var originIsWhitelisted = config.whitelist.indexOf(origin) !== -1;
callback(null, originIsWhitelisted);
}
};
var handler = function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('Hello World\n');
};
var app = express();
// Enable CORS for all requests
app.use(cors(corsOptions));
app.options('*', cors(corsOptions)); // specially for pre-flight requests
app.get('/download', function(req, res){
var limit = req.query.limit;
var offset = req.query.offset;
var options = {
method: 'POST',
url: config.apiServerHost + '/search',
useQuerystring: true,
qs: {'limit': limit, 'offset': offset},
rejectUnauthorized: false,
body: 'from=date&to=date'
};
var filename = 'data.csv';
res.setHeader('Content-disposition', 'attachment; filename=\"data.csv\"');
res.setHeader('content-type', 'text/csv');
var csvStream = csv.createWriteStream({
headers: true,
objectMode: true,
transform: function (row) {
return row;
}
});
console.log(options);
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var data = JSON.parse(body);
for(var i = 0; i < data.length; i++)
csvStream.write({
"col1": "value1-"+data[0][i],
"col2": "value2-"+data[1][i],
"col3": "value3-"+data[2][i],
"col4": "value4-"+data[3][i]
});
}
csvStream.end();
}
else {
console.log("Error:", error, body);
}
}
req.pipe(request(options, callback));//.pipe(res)
csvStream.pipe(res);
});
app.use('/api', function(req, res) {
var url = config.apiServerHost + req.url;
console.log(url);
req.pipe(request({
"rejectUnauthorized": false,
"url": url
}, function(error, response, body){
if(error) {
console.log(new Date().toLocaleString(), error);
}
})).pipe(res);
});
This all code works fine when request method is POST (the same as main API server). However I receive "[Error: write after end]" when I add the body in options object. Can someone help me figure out what is happening and how to solve this problem? Thanks.

The [Error: write after end] show pip data after .end(), for your codes
req.pipe(request(options, callback));//.pipe(res)
csvStream.pipe(res);
In the callback function, the csvStream.end(); is called, then invoke csvStream.pipe could cause this error.