We Keep Coding

How to make a HTTP request using Node.js upon user input?

So I started this simple project in node.js where the client sends a POST request containing in it's body a windows CMD command.
The server receives the POST request, extracts the CMD command, and after running it, responds with the output of that command.
This worked fine when I had one request being sent out, but then I set up a system to repeatedly ask the user for the command and then send the POST request with the inputted command as it's body.
Here is the client-side code for this: (server side not included as it's irrelevant)
var http = require("http");
var readline = require("readline");
var rl = readline.createInterface(process.stdin, process.stdout);
var options = {
hostname: "localhost",
port: 3001,
path: "/",
method: "POST",
headers: {
"Content-Type": "text/plain", // all command must be string
"Content-Length": 0 // changes based on what the command is
}
};
function requestCommand(hostname, port, command) {
// Some of the options of the request need to be changed based on
// what the command is
options.hostname = hostname;
options.port = port;
options.headers["Content-Length"] = command.length;
var req = http.request(options, function(res) {
console.log(`Got response. Status code: ${res.statusCode}`);
var resData = "";
res.setEncoding("utf-8");
res.on("data", function(chunk){
resData += chunk
});
res.on("end", function(){
return resData;
})
})
req.on("error", function (e){
return "\n\n\n ---------------\nERROR OCCURED IN THE REQUEST.\nREQUEST NOT SENT\n--------------" + e.stack;
})
req.write(command);
req.end();
}
rl.setPrompt("What command would you like to request from the server?: ");
rl.prompt();
rl.on("line", function(data){
if (data === "exit") {
console.log("\n\n\Exiting appplication...\n\n");
process.exit();
} else {
console.log("processing..");
var out = requestCommand("localhost", 3001, data);
console.log(`\n\n${out}\n\n`);
rl.prompt();
}
});
If I run this without creating the server first, instead of getting the error message, I get undefined.
Currently, I think this is because the requestCommand function ends before the error is handled (before the error event is emitted and the callback function that returns the error is called), since the callback function for http.request is obviously asynchronous, and before the server responds or an error is emitted, the function ends and hence return nothing (undefined)
So my question is: Can I keep that function running until the asynchronous command is done?
Or if this is not possible, is there a different approach to this? How would you send requests to this server upon a certain event fired by the user, such as data input?
EDIT: I am really not interested in using a 3rd party modules as I already can. This project is really pointless and only here for me to learn, so I am only using core modules for this. More specifically, I am only using HTTP for making the requests (not sync-request, e.t.c..)

requestCommand has to return a promise that gets resolved with resData and rejected on error:
function requestCommand(hostname, port, command) {
return new Promise((resolve, reject) => { // immeadiately return a Promise that can be resolved/rejected later
// Some of the options of the request need to be changed based on
// what the command is
options.hostname = hostname;
options.port = port;
options.headers["Content-Length"] = command.length;
var req = http.request(options, function(res) {
console.log(`Got response. Status code: ${res.statusCode}`);
var resData = "";
res.setEncoding("utf-8");
res.on("data", function(chunk){
resData += chunk
});
res.on("end", function(){
resolve(resData); // resolve, return won't work here
});
});
req.on("error", function (e){
reject(e); // reject here, don't handle it
});
req.write(command);
req.end();
});
}
That way, you can simply make the request handler async and then await the function call:
rl.on("line", async function(data){ // make the function async so that we can work with promises more easily
if (data === "exit") {
console.log("\n\n\Exiting appplication...\n\n");
process.exit();
} else {
console.log("processing..");
try {
var out = await requestCommand("localhost", 3001, data); // "await" lets async code look like sync code, but it is still async
console.log(`\n\n${out}\n\n`);
} catch(e) { // the reject() call will end up here
console.log("invalid command " + e.stack); // handle here
}
rl.prompt();
}
});

Returning parsed JSON API data as variable with Callbacks and https.get [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I am trying to build a simple web app that pulls API data on the price of Bitcoin. I am able to parse the data and extract the price. However, given the asynchronous nature of Node.js, when I try to return the data it returns before the http request is complete.
This is why I inserted the callback function. However, when it returns the price it is still undefined. Hence, I assume that the request is not complete when the return is sent even with the callback?
I have also heard of other solutions such as promises and async yet I cannot find a way to return the price variable from the function. I need to be able to do this so that I can do separate calculations on the returned variable.
Disclaimer: I am a node newbie so any help would be appreciated
var https = require('https');
function myCallback(result) {
return result;
}
var price = 0;
function getData(callback) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
callback(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
});
};
var y = getData(myCallback);
console.log(y);

The hard thing when you start in javascript is start thinking in asynchronous operations. When console.log(y) is executed the HTTPS call isn't completed yet so it can't be called that way once majorly IO functions in node are non-blocking. What you need to do is to call console.log inside the myCallback function.
var https = require('https');
function myCallback(result) {
console.log(result);
}
var price = 0;
function getData(callback) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
callback(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
});
}
getData(myCallback);
EDIT:
The thing is, you can set the value of a variable outside the callback by you still have to ensure that you'll be using this variable only after the attribution. You still have to use callbacks one way or another.
There's another way of pass callbacks around and it has a nicer way for doing that, promises. I believe that is what you're looking for.
var https = require('https');
var price = 0;
function getData() {
var promise = new Promise(function(resolve, reject) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
resolve(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
reject(e);
});
})
return promise;
}
var y = getData()
y.then(function (result) {
console.log('price: ' + result);
});

Node.js Generators

I am unsure how to structure Javascript generator code such that it will execute correctly.
...
var http = require('http');
emails.send = function *(next) {
// Pull the HTML for email to be sent from another local server
mailOptions['html'] = yield* emailHtml();
...
};
function* emailHtml() {
// Get the data from the database -- ends up using node-sqlite
var data = yield mea.getMeasurements();
// Callback function to deal with http request below
function callback(response) {
var str = '';
response.on('data', function(chunk) {
str += chunk;
});
response.on('end', function(chunk) {
return str;
});
}
// Make a request to the other server to get it's HTML
var req = http.request(options, callback);
// Post the data from this server's database connection to the other server to be processed and sent back
req.write(JSON.stringify(data));
req.end();
return ??????;
}
...
I already have the emailHtml() function yielding data from the local sqlite database and passing that data via POST with the http.request but can't figure out how to structure my code to have the emailHtml() function return the callback's final string.
Do I need to make callback a generator function also? I've tried var req = yield http.request(options, callback); but since that stops the request, the POST data is never written and the request is never completed in the following two lines.
What other options do I have if a generator is not the right way to go about this?

You need to turn the HTTP call into something you can yield on. It's messy as currently written, so time to pull in some other tools - in particular, promises. Since you're using Koa, which under the hood uses a library called co, promises are probably the easiest way to to this. I tend to use a library called Bluebird for my promise implementation, there are other options.
So basically you want something like this:
var http = require('http');
var Promise = require('bluebird');
emails.send = function *(next) {
// Pull the HTML for email to be sent from another local server
mailOptions['html'] = yield* emailHtml();
...
};
function makeHttpRequest(options, data) {
// Note we're returning a promise here
return new Promise(function (resolve, reject) {
var req = http.request(options, callback);
req.write(JSON.stringify(data));
req.end();
function callback(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function (chunk) {
// -- Resolve promise to complete the request
resolve(str);
});
}
});
}
function* emailHtml() {
// Get the data from the database -- ends up using node-sqlite
var data = yield mea.getMeasurements();
// Callback function to deal with http request below
function callback(response) {
var str = '';
response.on('data', function(chunk) {
str += chunk;
});
response.on('end', function(chunk) {
return str;
});
}
// Make a request to the other server to get it's HTML
var str = yield makeHttpRequest(options, data);
// do whatever you want with the result
return ??????;
}
This wraps up the http stuff inside a promise object, which you generator runner at the outer layer knows how to wait for completion of.
There's other ways to do this, and libraries (like co-request) that wrap this stuff natively, but this is the basic idea.

Just to add on Chris' answer, here's the cleaned up version of the code I'm now using:
var http = require('http');
emails.send = function *(next) {
// Pull the HTML for email to be sent from another local server
mailOptions['html'] = yield* emailHtml();
};
function makeHttpRequest(options, data) {
// Note we're returning a promise here
return new Promise(function (resolve, reject) {
var req = http.request(options, callback);
req.write(JSON.stringify(data));
req.end();
function callback(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function (chunk) {
// -- Resolve promise to complete the request
resolve(str);
});
}
});
}
function* emailHtml() {
// Get the data from the database -- ends up using node-sqlite
var data = yield mea.getMeasurements()
// Make a request to the other server to get it's HTML
return yield makeHttpRequest(options, data);
}
Promises are already built in to Node v0.12.7 when you use the --harmony flag so an additional library is not necessary

Make a blocking call to a function in Node.js required in this case?

I am starting to learn node.js. I am stuck with a problem here. I am calling a weather service which returns a JSON(url below).
http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json
I want to display the results from the api on an HTML page.
I have written the following code(getInfo module) to retrieve and return the JSON.
var fbResponse;
module.exports = function (url) {
var http=require('http');
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
fbResponse = JSON.parse(body);
console.log("Got response: ", fbResponse.response);
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
return fbResponse;
};
So to use this module I created a test.js file as follows:
var relay = require('./getInfo');
var url = 'http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json';
var response;
var x=relay(url);
console.log (x);
Output is as follows:
undefined // console.log(x) from test.js
Got response: { version: '0.1',
termsofService: 'http://www.wunderground.com/weather/api/d/terms.html',
features: { conditions: 1 } }
The console output in the test code runs first with no data in it. The HTTP get competes later and displays the actual output I need.
How can I modify the test code to make a blocking call such that var x in the test code actually have the JSON output instead of undefined?
Can I achieve the desired result without a blocking call to the getInfo module?

As you know, node is asynchronous, so the callback of http.get and res.on('end', .. will fire after relay function is executed and it is returned. So normally you can't return the result from it.
You have a couple of choices:
Pass a callback to relay and use that:
module.exports = function (url, cb) {
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
cb(null, JSON.parse(body));
});
}).on('error', cb);
};
Then use it like this:
var relay = require('./getInfo');
relay(url, function (err, x) {
if (err) {
console.error('oohh i got a error: ', err)
}
console.log('oohh i got a response: ', x)
});
Use promises. This is almost same as passing callbacks. A little less lightweight, but when combining different asynchronous operations, you will understand how awesome they are. For just one asynchronous call there might not be any difference. Here I use q. You can also use bluebird which is way more performant but lacks some of the sugar of q. You can read this article to understand why promises are cleaner than callbacks in some cases.
module.exports = function (url) {
var deferred = Q.defer();
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
deferred.resolve(JSON.parse(body));
});
}).on('error', function(e) {
deferred.reject(e);
});
return deferred.promise;
};
var relay = require('./getInfo');
relay(url).then(function responseHandler(x) {
console.log('my awesome response')
}, function errorHandler(err) {
console.error('got an error', err);
});
Use generators. It is part of Ecmascript 6 specification it only exists in node v0.11.x and later. But it would be almost what you want.
With that past promise example we can do this:
Q.async(function *() {
var relay = require('./getInfo');
var x = yield relay(url);
console.log('my awesome response', x)
});
This is almost what you want. You can also achieve it using the callback solution with co library:
co(function *() {
var relay = require('./getInfo');
var x = yield relay.bind(null, url);
console.log('my awesome response', x);
});
You can also use node-fibers in above example which is almost a similar tool like generators.
If you want to use bleeding edge Javascript, you can use Async/Await instead of generators with promises.

You need to pass in a callback instead:
var http = require('http');
module.exports = function(url, cb) {
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var resp, err;
try {
resp = JSON.parse(body);
} catch (ex) {
err = ex;
}
cb(err, resp);
});
}).on('error', function(e) {
console.log("Got error: ", e);
cb(e);
});
};
Then use it like:
var relay = require('./getInfo');
var url = 'http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json';
var response;
relay(url, function(err, resp) {
if (err) throw err; // TODO: handle better
console.dir(resp);
});

you can take a callback in your module function to return the result.
module.exports = function (url, onsuccess) {
...
res.on('end', function() {
fbResponse = JSON.parse(body);
if(onsuccess){
onsuccess(null, fbResponse);
}
Then in your caller code:
relay(url, function(err, result){
console.log(result);
});
Another option is to use httpsync module which provides synchronous apis for the same functionality that 'http' module provides. But in node js programming, you should always avoid synchronous calls.

Callback on node.js http REST call issue

Have a following node.js call to http get REST API. When I run it, it returns the JSON result. However, I am trying to get the result to console.log on var r after assigning the result. Why do I always get undefined? Looks like I am messing up the javascript scope. Any help is appreciated.
var http = require('http'),
oex_appId = 'myappId',
_hostname = 'openexchangerates.org',
_path = '/api/latest.json?app_id=' + oex_appId;
var r;
function getRates(cb) {
var options = {
hostname: _hostname,
path: _path,
method: 'GET'
};
var responseCallback = function(response) {
var latestRates = '';
var parseCurrencies = function(rates) {
var currencies = JSON.parse(rates);
currencies.rates['USD'] = 1;
return currencies.rates;
};
response.setEncoding('utf8');
response.on('data', function(chunk) {
latestRates += chunk;
}).on('error', function(err) {
throw new Error(err);
}).on('end', function() {
cb(parseCurrencies(latestRates));
});
};
var req = http.request(options, responseCallback);
req.on('error', function(e) {
throw e;
});
req.end();
};
var d = function(data) {
console.log(data["INR"]);
currencies["INR"].amount = data["INR"];
r = data;
};
getRates(d);
console.log(r);

Why do I always get undefined?
Your problem is not an scope issue but a misunderstanding of async execution. In this code you are printing the value of r before it is assigned.
The execution of your code is as follow:
getRates() is called
console.log(r) is called (hence you get undefined)
When the request executed in getRates finishes your callback is THEN executed. This is when you are assigning a value to r but by then is too late since you already printed it.
In general, you cannot expect the next line to have a value that will be assigned via an async callback. Instead you should reference the value inside the callback. In your case you should move the console.log(r) inside your callback function d.