node express.js Can't set headers after they are sent.' - javascript

I am new to both node and express so I figure I am doing something stupid.
Complete source code can be found at:
https://github.com/wa1gon/aclogGate/tree/master/server
logRouter.get("/loggate/v1/listall", function(req, res) {
let countStr = req.param('count');
let count: number;
if (!countStr) {
count = null;
} else {
count = Number.parseInt(countStr);
if (count == NaN) count = null;
}
acConn.listAllDatabase(count, (err: string, result: Array<LogGateResp>) => {
console.log("got list all data resp")
return res.json(result).end();
});
}
);
app.use('/', logRouter);
It works the first time though, but blows up the second.
listallDatabase connects to a network socket which gets XML database back, parses it and calls back with an JS object. Which in turn calls res.json.
Suggestions?

Remove the end() after res.json().
res.josn() send the response to frontend and end() try to send the response again.
That why you are getting the error. Because node.js don't allow the API to send response twice. Either use res.end() or res.json().

Related

Node.js how to return an array from within a asynchronous callback for use in another file

File called testing.js
I can do whatever I like with the data in saveWeatherData but cannot call this function and return the data without getting 'undefined'
For example if i tried the below code in saveWeatherData it will print out the summary as expected...
console.log(The summary of the weather today is: ${dataArray[0]});
However I want to use these values within another file such as a server file that when connected to will display weather summary temperature etc.
So I need to return an array with these values in it so that I can call this function and get my data stored in an array for further use.
I know that the reason the array --dataArray is returning undefined is because asynchronous code.
The array is returned before we have gotten the data using the callback.
My question, is there anyway to do what I am trying to do?
I tried my best to explain the problem and what I want to do, hopefully its understandable.
Would I have to use a callback inside of a callback? To callback here to return the data when its been fetched?
I just cant get my head about it and have tried multiple things to try and get the result I am looking for.
My last idea and something i would prefer not to do is the use the 'fs' module to save the data to a text or json file for use in my other files through reading the data from the saved file...
I feel im close but cant get over the last hurdle, so ive decided to ask for a little help, even just point me on the right track and Ill continue to try and figure it out.
Phew...
Thank you for your time!
const request = require("request");
let dataArray = [];
let saveWeatherData = function(weatherData) {
dataArray = weatherData;
return dataArray;
};
let getWeatherData = function(callback) {
request({
url: `https://api.forecast.io/forecast/someexamplekey/1,-1`,
json: true
}, (error, response, body) => {
//Creating array to hold weather data until we can save it using callback...
let array = [];
if (error) {
console.log("Unable to connect with Dark Sky API servers.")
}
else {
console.log(`Successfully connected to Dark Sky API servers!\n`);
array.push(body.currently.summary, body.currently.temperature, body.currently.apparentTemperature, body.currently.windSpeed, body.currently.windBearing);
callback(array);
}
});
};
getWeatherData(saveWeatherData);
module.exports = {
saveWeatherData
};
My Other File...
File called server.js
const http = require("http");
const testing = require("./testing");
function onRequest(request, response){
let data = testing.saveWeatherData();
console.log(`A user made a request: ${request.url}`);
response.writeHead(200, {"context-type": "text/plain"});
response.write("<!DOCTYPE html>");
response.write("<html>");
response.write("<head>");
response.write("<title>Weather</title>");
response.write("</head>");
response.write("<body>");
response.write("Weather summary for today: " + data[0]);
response.write("</body>");
response.write("</html>");
response.end();
}
http.createServer(onRequest).listen(8888);
console.log("Server is now running on port 8888...");
I'm still not sure about what are you trying to do. However, I think you're not exporting what you suppose to be exporting. To avoid the use of so many callbacks you may use async/await.
Change this part of your server.js
async function onRequest(request, response) {
let data = await testing.getWeatherData();
console.log(`A user made a request: ${request.url}`);
response.writeHead(200, { 'context-type': 'text/plain' });
response.write('<!DOCTYPE html>');
response.write('<html>');
response.write('<head>');
response.write('<title>Weather</title>');
response.write('</head>');
response.write('<body>');
response.write('Weather summary for today: ' + data[0]);
response.write('</body>');
response.write('</html>');
response.end();
}
And this of your testing.
let getWeatherData = function() {
return new Promise(resolve =>
request(
{
url: `https://api.darksky.net/forecast/someexamplekey/1,-1`,
json: true
},
(error, response, body) => {
//Creating array to hold weather data until we can save it using callback...
let array = [];
if (error) {
console.log('Unable to connect with Dark Sky API servers.');
} else {
console.log(`Successfully connected to Dark Sky API servers!\n`);
array.push(
body.currently.summary,
body.currently.temperature,
body.currently.apparentTemperature,
body.currently.windSpeed,
body.currently.windBearing
);
resolve(array);
}
}
)
);
};
module.exports = {
getWeatherData
};
It will check for new Weather in each request. If you want to save the result to avoid checking every single time you might need to do something else. But I think for a weather app the important is to keep it updated.

Chaining routes in NodeJs with values after sending API response

I want to chain routes in NodeJs with values after sending API response to end-ser,
WHY: > The uploaded files would be somewhat large (5-50mb each) and require some processing, can not make my API user wait/timeout while my NodeJS code is working.. so need, 1: Upload files and send success immediately to user, Process files (few promises) and return/log success/failure for notification system.
My individual code blocks are done and working fine (i.e. upload service and file processing service both are good under tests and work nicely when tested individually.)
now with the API to upload in place, I've added following code:
router.post('/upload', upload.array('upload_data', multerMaxFiles), (req, res, next) => {
////some uploading and processing stuff - works nicely
res.json({ 'message': 'File uploaded successfully.' });// shown to API client nicely
console.log("what next? " + utilz.inspect(uploaded_file_paths)) //prints file names on console
next();
});
PROBLEM:
app.use('/api', uploadRoute); //The above code route
//want some processing to be done
app.use(function(req, res, next) {
**want those uploaded file names here**
tried with few response object options but stabs with error
});
OR
use something like ....
app.use(someFunction(uploaded_file_names)); **want those uploaded file names as params**
PS:
Any promise after the file upload success would result in 'Error: Can't set headers after they are sent.', so not helpful writing anything there.
Any suggestions folks.
--
N Baua
Once you've sent a response back to the browser (to keep it from timing out during your long processing time), that http request is done. You cannot send any more data on it and trying to do so will trigger a server-side error. You cannot "chain routes" the way you were asking as you seem to want to do because you simply can't send more data over that http request once you've sent the first response.
There are two common ways to deal with this issue.
As part of your initial response, send back a transaction ID and then have the client poll back every few seconds with an Ajax call asking what the final status is of that transaction. The server can return "in progress" until it is finally done and then it can return the final status.
You can connect a webSocket or socket.io connection from client to server. As part of your initial response to the upload, send back a transaction ID. Then, when the transaction is done server-side, it sends a notification on the webSocket or socket.io connection for that particular client with the transactionID with the final status. The client can then respond accordingly to that final status. You can either keep the webSocket/socket.io connection open for use with other requests or you can then close that connection.
Using either technique, you could also return/send a progress value (like percent complete) that the client could use to display completion progress. This is generally very helpful on the client-side to keep an impatient user from giving up or refreshing the page. If they can see that the processing is proceeding, they won't give up thinking that maybe it stopped working.
This should work with res.write(). But it does depend on your clients cache i think.
I tried this, but it does not work in my firefox.
app.get('/test', function(req, res) {
var count = 0;
var interval = setInterval(function() {
if (count++ === 100) {
clearInterval(interval);
res.end();
}
res.write('This is line #' + count + '\n');
}, 100);
});
After I increased frequency and number of writes it seems to work.
So try:
router.post('/upload', upload.array('upload_data', multerMaxFiles), (req, res, next) => {
////some uploading and processing stuff - works nicely
res.write(JSON.stringify({ 'message': 'File uploaded successfully.' }));// shown to API client nicely
console.log("what next? " + utilz.inspect(uploaded_file_paths)) //prints file names on console
next();
});
//STEP (1)
//Example simulate upload multiple files with chained api calls. In this example the parameters "idFile" and "arrayidFileExample" are helpful.
//You should create and send this data from view.
{
"idFile": "04fe640f6e4w", //id first file
"arrayidFileExample": ["04fe640f6e4w","03g5er4g65erg","g0er1g654er65g4er","0g4er4g654reg654re"] //simulate idFiles array from view
}
//STEP (2)
//your upload files api
app.post('/upload', function(req, res) {
//Express headers, no important in this code
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
let arrayIdFiles = req.body.arrayidFileExample; //create arrayIdFiles
let isBlock = false; //flag to block call loop to api
let countIndex = 0;
let currentIndex = 0;
//STEP (3)
//If "arrayIdFiles" index is not exist, set isBlock to true. Remeber flag to block call loop to api
for(let count in arrayIdFiles) {
if(req.body.idFile == arrayIdFiles[count]) {
console.log("current index --> ", countIndex)
countIndex++;
console.log("next index --> ", countIndex)
if(arrayIdFiles[countIndex] == undefined) {
isBlock = true;
}
break;
}
countIndex++;
currentIndex++;
}
//STEP (4)
//If isBlock is equal false, call get upload api with next idFile. this is simulate "recursive api loop"
if(isBlock == false) {
postUploadFile(
'http://localhost:3500/upload',
{
"idFile":arrayIdFiles[currentIndex + 1], //send next idFile from arrayIdFiles
"arrayidFileExample": arrayIdFiles //initial arrayIdFiles
});
}
//STEP (6)
//response json example
const json = JSON.stringify({
error:false,
statusCode: 200,
body:{
message:'current id file '+req.body.idFile,
}
});
res.write(json);
return res.end();
});
//STEP (5)
//call "/upload" api post
const postUploadFile = (url = '', body = {}) => {
return new Promise((resolve, reject)=>{
axios.post(url, body).then(response => {
return resolve(response.data);
}).catch(error => {});
});
};
//server listen instance
server.listen(3500,() => {
});

NodeJS Undefined JSON object

just posting a question as I have seen some other similar questions on here but none with a method that seemingly works for me.
I'm new to NodeJS and playing around with requesting data from an API. For my test here im just trying to pull ticker prices based on the input of a prompt from the user.
This works fine, however the object
This is the code I am using to try and make this work:
prompt.start();
prompt.get(['coin'], function (err, result) {
request({url: `https://min-api.cryptocompare.com/data/price?fsym=${result.coin}&tsyms=BTC,USD`, json:true}, function(err, res, json) {
if (err) {
throw err;
}
console.log(json);
var json = JSON.stringify(json);
var string2 = JSON.parse(json);
console.log(string2.btc_price);
console.log(json);
});
console.log('Retrieving: ' + result.coin);
});
The API request works, however it returns JSON that looks like this with my 3 console logs:
{ set_attributes: { btc_price: 1, usd_price: 15839.35 } }
undefined
{"set_attributes":{"btc_price":1,"usd_price":15839.35}} -- (Stringify'd response)
I want to be able to extract the btc_price & usd_price as variables, ive tried a few different methods and can't figure out where exactly im going wrong. Any help would be greatly appreciated!
Cheers,
J
When you attempt to extract the btc_price attribute, it's actually nested so your second console should read console.log(string2.set_attributes.btc_price);
axios has more stars on Github, more followers on Github and more forks.
Features
Make XMLHttpRequests from the browser
Make http requests from node.js
Supports the Promise API
Intercept request and response
Transform request and response data
Cancel requests
Automatic transforms for JSON data
Client side support for protecting against XSRF
Using async / await
// Make a request for a user with a given ID
var preload = null;
async function getPrice(symbol) {
preload = await axios.get('https://min-api.cryptocompare.com/data/price?fsym=${symbol}&tsyms=BTC,USD')
.then(function (response) {
preload = response.data;
})
.catch(function (error) {
console.log(error);
});
return `preload.BTC = ${preload.BTC}; preload.BTC = ${preload.BTC}`;
};
getPrice('ETH');
// return preload.BTC = 0.04689; preload.USD = 742.85

Using Node.js to find the value of Bitcoin on a webpage at real time

I'm trying to make a .js file that will constantly have the price of bitcoin updated (every five minutes or so). I've tried tons of different ways to web scrape but they always output with either null or nothing. Here is my latest code, any ideas?
var express = require('express');
var path = require('path');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var app = express();
var url = 'https://blockchain.info/charts/';
var port = 9945;
function BTC() {
request(url, function (err, res, body) {
var $ = cheerio.load(body);
var a = $(".market-price");
var b = a.text();
console.log(b);
})
setInterval(BTC, 300000)
}
BTC();
app.listen(port);
console.log('server is running on '+port);
It successfully says what port it's running on, that's not the problem. This example (when outputting) just makes a line break every time the function happens.
UPDATE:
I changed the new code I got from Wartoshika and it stopped working, but im not sure why. Here it is:
function BTCPrice() {
request('https://blockchain.info/de/ticker', (error, response, body) => {
const data = JSON.parse(body);
var value = (parseInt(data.USD.buy, 10) + parseInt(data.USD.sell, 10)) / 2;
return value;
});
};
console.log(BTCPrice());
If I have it console.log directly from inside the function it works, but when I have it console.log the output of the function it outputs undefined. Any ideas?
I would rather use a JSON api to get the current bitcoin value instead of an HTML parser. With the JSON api you get a strait forward result set that is parsable by your browser.
Checkout Exchange Rates API
Url will look like https://blockchain.info/de/ticker
Working script:
const request = require('request');
function BTC() {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
console.log(value);
});
}
BTC();
Using the value as callback:
const request = require('request');
function BTC() {
return new Promise((resolve) => {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
resolve(value);
});
});
}
BTC().then(val => console.log(val));
As the other answer stated, you should really use an API. You should also think about what type of price you want to request. If you just want a sort of index price that aggregates prices from multiple exchanges, use something like the CoinGecko API. Also if you need real-time data you need a websocket-based API, not a REST API.
If you need prices for a particular exchange, for example you're building a trading bot for one or more exchanges, you;ll need to communicate with each exchange's websoceket API directly. For that I would recommend something like the Coygo API, a node.js package that connects you directly to each exchange's real-time data feeds. You want something that doesn't add a middleman since that would add latency to your data.

Node.js respond with asynchronous data

Recently I started learning a little bit about Node.js and it's capabilities and tried to use it for some web services.
I wanted to create a web service which will serve as a proxy for web requests.
I wanted my service to work that way:
User will access my service -> http://myproxyservice.com/api/getuserinfo/tom
My service will perform request to -> http://targetsite.com/user?name=tom
Responded data would get reflected to the user.
To implement it I used the following code:
app.js:
var express = require('express');
var bodyParser = require('body-parser');
var app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
var proxy = require('./proxy_query.js')
function makeProxyApiRequest(name) {
return proxy.getUserData(name, parseProxyApiRequest);
}
function parseProxyApiRequest(data) {
returned_data = JSON.parse(data);
if (returned_data.error) {
console.log('An eror has occoured. details: ' + JSON.stringify(returned_data));
returned_data = '';
}
return JSON.stringify(returned_data);
}
app.post('/api/getuserinfo/tom', function(request, response) {
makeProxyApiRequest('tom', response);
//response.end(result);
});
var port = 7331;
proxy_query.js:
var https = require('https');
var callback = undefined;
var options = {
host: 'targetsite.com',
port: 443,
method: 'GET',
};
function resultHandlerCallback(result) {
var buffer = '';
result.setEncoding('utf8');
result.on('data', function(chunk){
buffer += chunk;
});
result.on('end', function(){
if (callback) {
callback(buffer);
}
});
}
exports.getUserData = function(name, user_callback) {
callback = user_callback
options['path'] = user + '?name=' + name;
var request = https.get(options, resultHandlerCallback);
request.on('error', function(e){
console.log('error from proxy_query:getUserData: ' + e.message)
});
request.end();
}
app.listen(port);
I wish I didn't screwed this code because I replaced some stuff to fit my example.
Anyway, the problem is that I want to post the response to the user when the HTTP request is done and I cant find how to do so because I use express and express uses asynchronous calls and so do the http request.
I know that if I want to do so, I should pass the makeProxyApiRequest the response object so he would be able to pass it to the callback but it is not possible because of asyn problems.
any suggestions?
help will be appreciated.
As you're using your functions to process requests inside your route handling, it's better to write them as express middleware functions, taking the specific request/response pair, and making use of express's next cascade model:
function makeProxyApiRequest(req, res, next) {
var name = parseProxyApiRequest(req.name);
res.locals.userdata = proxy.getUserData(name);
next();
}
function parseProxyApiRequest(req, res, next) {
try {
// remember that JSON.parse will throw if it fails!
data = JSON.parse(res.locals.userdata);
if (data .error) {
next('An eror has occoured. details: ' + JSON.stringify(data));
}
res.locals.proxyData = data;
next();
}
catch (e) { next("could not parse user data JSON."); }
}
app.post('/api/getuserinfo/tom',
makeProxyApiRequest,
parseProxyApiRequest,
function(req, res) {
// res.write or res.json or res.render or
// something, with this specific request's
// data that we stored in res.locals.proxyData
}
);
Even better would be to move those middleware functions into their own file now, so you can simply do:
var middleware = require("./lib/proxy_middleware");
app.post('/api/getuserinfo/tom',
middleware.makeProxyApiRequest,
middleware.parseProxyApiRequest,
function(req, res) {
// res.write or res.json or res.render or
// something, with this specific request's
// data that we stored in res.locals.proxyData
}
);
And keep your app.js as small as possible. Note that the client's browser will simply wait for a response by express, which happens once res.write, res.json or res.render etc is used. Until then the connection is simply kept open between the browser and the server, so if your middleware calls take a long time, that's fine - the browser will happily wait a long time for a response to get sent back, and will be doing other things in the mean time.
Now, in order to get the name, we can use express's parameter construct:
app.param("name", function(req, res, next, value) {
req.params.name = value;
// do something if we need to here, like verify it's a legal name, etc.
// for instance:
var isvalidname = validator.checkValidName(name);
if(!isvalidname) { return next("Username not valid"); }
next();
});
...
app.post("/api/getuserinfo/:name", ..., ..., ...);
Using this system, the :name part of any route will be treated based on the name parameter we defined using app.param. Note that we don't need to define this more than once: we can do the following and it'll all just work:
app.post("/api/getuserinfo/:name", ..., ..., ...);
app.post("/register/:name", ..., ..., ... );
app.get("/api/account/:name", ..., ..., ... );
and for every route with :name, the code for the "name" parameter handler will kick in.
As for the proxy_query.js file, rewriting this to a proper module is probably safer than using individual exports:
// let's not do more work than we need: http://npmjs.org/package/request
// is way easier than rolling our own URL fetcher. In Node.js the idea is
// to write as little as possible, relying on npmjs.org to find you all
// the components that you need to glue together. If you're writing more
// than just the glue, you're *probably* doing more than you need to.
var request = require("request");
module.exports = {
getURL: function(name, url, callback) {
request.get(url, function(err, result) {
if(err) return callback(err);
// do whatever processing you need to do to result:
var processedResult = ....
callback(false, processedResult);
});
}
};
and then we can use that as proxy = require("./lib/proxy_query"); in the middleware we need to actually do the URL data fetching.

Categories