We Keep Coding

Passing data from Nodejs http get request

I am trying to pass data out of my get request from resp.on function. I want to use 'var url' to make a separate get request from which I will parse data again. I am able to console.log variables from inside the function but not return (or access from outside). This seems to be a scoping or async issue.
const https = require('https');
https.get('https://collectionapi.metmuseum.org/public/collection/v1/objects', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
var json_data = JSON.parse(data);
var total = json_data.total
var random_objectID = Math.floor(Math.random()*total)
var url = 'https://collectionapi.metmuseum.org/public/collection/v1/objects/' + random_objectID
console.log(url);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
})
//'url' becomes unknown here. I want to pass it to another get request.

It's both an async and a scope issue!
If you declare var url; in the outermost scope you'll be able to set it inside that callback as intended, but since that's happening asynchronously you won't be able to use the value outside the scope unless you are checking after the callback completes.
Alternatively, you could wrap the whole thing in a promise, e.g.
const https = require('https');
new Promise((resolve,reject)=>{
let targetUrl = 'https://collectionapi.metmuseum.org/public/collection/v1/objects';
https.get(targetUrl,resp=>{
// ...
resp.on('end', () => {
// ...
resolve(url);
});
});
}).then(url=>{
// do stuff with that URL
});
If your goal is to automate fetching data from web resources, I'd recommend checking out the request module, which also has a promisified variant.

Returning parsed JSON API data as variable with Callbacks and https.get [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I am trying to build a simple web app that pulls API data on the price of Bitcoin. I am able to parse the data and extract the price. However, given the asynchronous nature of Node.js, when I try to return the data it returns before the http request is complete.
This is why I inserted the callback function. However, when it returns the price it is still undefined. Hence, I assume that the request is not complete when the return is sent even with the callback?
I have also heard of other solutions such as promises and async yet I cannot find a way to return the price variable from the function. I need to be able to do this so that I can do separate calculations on the returned variable.
Disclaimer: I am a node newbie so any help would be appreciated
var https = require('https');
function myCallback(result) {
return result;
}
var price = 0;
function getData(callback) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
callback(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
});
};
var y = getData(myCallback);
console.log(y);

The hard thing when you start in javascript is start thinking in asynchronous operations. When console.log(y) is executed the HTTPS call isn't completed yet so it can't be called that way once majorly IO functions in node are non-blocking. What you need to do is to call console.log inside the myCallback function.
var https = require('https');
function myCallback(result) {
console.log(result);
}
var price = 0;
function getData(callback) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
callback(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
});
}
getData(myCallback);
EDIT:
The thing is, you can set the value of a variable outside the callback by you still have to ensure that you'll be using this variable only after the attribution. You still have to use callbacks one way or another.
There's another way of pass callbacks around and it has a nicer way for doing that, promises. I believe that is what you're looking for.
var https = require('https');
var price = 0;
function getData() {
var promise = new Promise(function(resolve, reject) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
resolve(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
reject(e);
});
})
return promise;
}
var y = getData()
y.then(function (result) {
console.log('price: ' + result);
});

Using imported asynchronous method with promises in main entry point

I'm creating a Node.js module with an asynchronous method - a simple HTTP GET request. Here is the code:
//mymodule.js
var https = require('https');
function getSomething(url_str)
{
var callback_fn = function(response){
var body = '';
response.on('data', function (data) {
body += data;
});
response.on('end', function () {
//console.log(body);
return body;
});
};
return https.request(url_str, callback_fn).end();
}
var module_obj = {
getSome: getSomething
};
module.exports = module_obj;
This module is called by my app.js - a web server - like so:
//app.js
var myModule = require('./mymodule');
var http = require('http');
var qs = require('querystring');
var server_fn = function(request, response){
response.setHeader('Access-Control-Allow-Origin', '*');
response.setHeader('Access-Control-Request-Method', '*');
response.setHeader('Access-Control-Allow-Methods', 'OPTIONS, GET');
response.setHeader('Access-Control-Allow-Headers', '*');
if ( request.method === 'OPTIONS' ) {
response.writeHead(200);
response.end();
return;
}
if (request.method == 'POST') {
var body = '';
request.on('data', function (data) {
body += data;
// Too much POST data, kill the connection!
// 1e6 === 1 * Math.pow(10, 6) === 1 * 1000000 ~~~ 1MB
if (body.length > 1e6)
request.connection.destroy();
});
request.on('end', function () {
var post = qs.parse(body),
post_url = post.url,
post_method = post.method;
var promise_flow = new Promise(function(resolve, reject){
if(post_method === 'get_me_something')
{
response_str = myModule.getSome(post_url);
resolve(response_str);
}
else
{
resolve('nothing');
}
});
promise_flow
.then(function(response){
response.write(response);
response.end();
return;
}).catch(function(error){
response.write(error);
response.end();
return;
})
});
}
};
var http_server = http.createServer(server_fn);
http_server.listen(2270);
console.log("server listening on 2270");
So basically, I start things up via node app.js, and then I post the URL, and then the module should fetch the Web page and then return the content.
Unfortunately, I'm getting the following error:
UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 1): TypeError: First argument must be a string or Buffer
I believe this is because the response I'm getting from my modules getSomething method is false, as opposed to the content of the requested Web page.
I know I can fix this by moving the https.get operation from mymodule.js and putting it inline with app.js, and then calling resolve on end, but I'd like to keep the current module setup.
Is there a workaround to get the asynchronous method in the imported module to work with the existing promise chain as setup?
UPDATE
After further review, I noticed that I wasn't quite running things the right way. I updated the code as follows:
//...
var promise_flow = new Promise(function(resolve, reject){
if(post_method === 'get_me_something')
{
myModule.getSome(post_url)
.then(function(data){
resolve(data);
})
.catch(function(err){
reject(err);
});
}
else
{
resolve('nothing');
}
});
//...
This way, I think it fits with the true spirit of Promises.

Your getSomething function doesn't return a promise. Make it returns a promise, and fulfill the promise in response.on('end').
function getSomething(url_str)
{
return new Promise(function(resolve, reject) {
var callback_fn = function(response){
var body = '';
response.on('data', function (data) {
body += data;
});
response.on('end', function () {
//console.log(body);
resolve(body);
});
};
https.request(url_str, callback_fn).end();
});
}
Then in your main file, call it like this : myModule.getSomething(post_url).then(resolve);.

return data with JavaScript

I have a problem when I call my function JavaScript I can not return my Buffer because it's in other function
index.js:
var client = require('./sender');
//envoyer le paquet au seveur blizzard.com
var received =client.sendAndreceive(bufenc);
console.log('received: '+received);
sender.js:
var Http = require('http');
var Url = require('url');
function sendAndreceive(data) {
console.log('##########################################################################');
// connexion au serveur
var options = {
//proxy
host: "proxya.u-pec.fr",
port: 3128,
//hostname : 'http://m.eu.mobileservice.blizzard.com',
//port : 80,
path : 'http://m.eu.mobileservice.blizzard.com/enrollment/enroll.htm',
method : 'POST',
headers: {
Host: "http://m.eu.mobileservice.blizzard.com"
}
};
var req = Http.request(options, callback );
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.write(data);
req.end();
}
exports.sendAndreceive = sendAndreceive;
function callback (res) {
res.on('data', function(chunk) {
buf = new Buffer(chunk);
console.log('data length: ' + buf.length);
return buf; // my problem is her !!!!!!!!!!!!!!!!!!!!!!
});
}
output:
##########################################################################
received: undefined
data length: 45

Using the JavaScript callback pattern
You won't get a result as you returning the result from the callback function, which is nothing and wouldn't work either as your request to the server is asynchronously. Take a look at the example below, and see how it can work:
function callback (res, callback) {
res.on('data', function(chunk) {
buf = new Buffer(chunk);
console.log('data length: ' + buf.length);
// Make sure the argument is indeed a callable function
if (typeof callback === 'function') {
callback(buf);
}
});
}
Now what you do is simply implement a callback, that is executed when the result is finished.
var client = require('./sender');
//envoyer le paquet au seveur blizzard.com
client.sendAndreceive(bufenc, function(received) {
console.log('Received', received);
});
This JavaScript pattern is called the callback pattern, and can be very useful in these kinds of situations.
Using a promise
Another more sexy way of doing stuff like this, is through promises. I don't want to fully explain what a promise is or does (you can use search, right?), so I'll just show you an example:
// This is part of the node-promise module: https://github.com/kriszyp/node-promise
var deferred = require("promise").defer;
var client = require('./sender');
function callback (res) {
// Create a deffered object that will be returned
var deferred = defer();
res.on('data', function(chunk) {
buf = new Buffer(chunk);
console.log('data length: ' + buf.length);
// Now let the listeners know that this promise
// has been completed successfully, and return the buf
deffered.resolve(buf);
// Or if something went wrong, you could do the following:
// deffered.reject('Everything is wrong!');
});
return deferred;
}
//envoyer le paquet au seveur blizzard.com
client.sendAndreceive(bufenc)
.then(
// Function ended successfully
function(received) {
console.log('Received', received);
},
// Function returned an error
function(err) {
console.log('Oh noes!', err);
}
)
;

You are calling sendAndreceive(), and sendAndreceive() is starting a HTTP request that asynchronously calls callback(). So the return value of callback() is a return value given to the invoker inside the HTTP request object, while sendAndreceive() returns nothing.
There is no way to force asynchronous calls behave like one step-by-step-flow. Therefore I support the recommendation of Boy, using a callback.
OK, I see the promise pattern by Boy :-) cool. I will take a deeper look. It seems you can make asynchronous steps at least looking like synchronously written with constructs like .then(). Thx!

Make a blocking call to a function in Node.js required in this case?

I am starting to learn node.js. I am stuck with a problem here. I am calling a weather service which returns a JSON(url below).
http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json
I want to display the results from the api on an HTML page.
I have written the following code(getInfo module) to retrieve and return the JSON.
var fbResponse;
module.exports = function (url) {
var http=require('http');
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
fbResponse = JSON.parse(body);
console.log("Got response: ", fbResponse.response);
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
return fbResponse;
};
So to use this module I created a test.js file as follows:
var relay = require('./getInfo');
var url = 'http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json';
var response;
var x=relay(url);
console.log (x);
Output is as follows:
undefined // console.log(x) from test.js
Got response: { version: '0.1',
termsofService: 'http://www.wunderground.com/weather/api/d/terms.html',
features: { conditions: 1 } }
The console output in the test code runs first with no data in it. The HTTP get competes later and displays the actual output I need.
How can I modify the test code to make a blocking call such that var x in the test code actually have the JSON output instead of undefined?
Can I achieve the desired result without a blocking call to the getInfo module?

As you know, node is asynchronous, so the callback of http.get and res.on('end', .. will fire after relay function is executed and it is returned. So normally you can't return the result from it.
You have a couple of choices:
Pass a callback to relay and use that:
module.exports = function (url, cb) {
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
cb(null, JSON.parse(body));
});
}).on('error', cb);
};
Then use it like this:
var relay = require('./getInfo');
relay(url, function (err, x) {
if (err) {
console.error('oohh i got a error: ', err)
}
console.log('oohh i got a response: ', x)
});
Use promises. This is almost same as passing callbacks. A little less lightweight, but when combining different asynchronous operations, you will understand how awesome they are. For just one asynchronous call there might not be any difference. Here I use q. You can also use bluebird which is way more performant but lacks some of the sugar of q. You can read this article to understand why promises are cleaner than callbacks in some cases.
module.exports = function (url) {
var deferred = Q.defer();
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
deferred.resolve(JSON.parse(body));
});
}).on('error', function(e) {
deferred.reject(e);
});
return deferred.promise;
};
var relay = require('./getInfo');
relay(url).then(function responseHandler(x) {
console.log('my awesome response')
}, function errorHandler(err) {
console.error('got an error', err);
});
Use generators. It is part of Ecmascript 6 specification it only exists in node v0.11.x and later. But it would be almost what you want.
With that past promise example we can do this:
Q.async(function *() {
var relay = require('./getInfo');
var x = yield relay(url);
console.log('my awesome response', x)
});
This is almost what you want. You can also achieve it using the callback solution with co library:
co(function *() {
var relay = require('./getInfo');
var x = yield relay.bind(null, url);
console.log('my awesome response', x);
});
You can also use node-fibers in above example which is almost a similar tool like generators.
If you want to use bleeding edge Javascript, you can use Async/Await instead of generators with promises.

You need to pass in a callback instead:
var http = require('http');
module.exports = function(url, cb) {
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var resp, err;
try {
resp = JSON.parse(body);
} catch (ex) {
err = ex;
}
cb(err, resp);
});
}).on('error', function(e) {
console.log("Got error: ", e);
cb(e);
});
};
Then use it like:
var relay = require('./getInfo');
var url = 'http://api.wunderground.com/api/Your_key/conditions/q/CA/San_Francisco.json';
var response;
relay(url, function(err, resp) {
if (err) throw err; // TODO: handle better
console.dir(resp);
});

you can take a callback in your module function to return the result.
module.exports = function (url, onsuccess) {
...
res.on('end', function() {
fbResponse = JSON.parse(body);
if(onsuccess){
onsuccess(null, fbResponse);
}
Then in your caller code:
relay(url, function(err, result){
console.log(result);
});
Another option is to use httpsync module which provides synchronous apis for the same functionality that 'http' module provides. But in node js programming, you should always avoid synchronous calls.