Passing data from Nodejs http get request - javascript

I am trying to pass data out of my get request from resp.on function. I want to use 'var url' to make a separate get request from which I will parse data again. I am able to console.log variables from inside the function but not return (or access from outside). This seems to be a scoping or async issue.
const https = require('https');
https.get('https://collectionapi.metmuseum.org/public/collection/v1/objects', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
var json_data = JSON.parse(data);
var total = json_data.total
var random_objectID = Math.floor(Math.random()*total)
var url = 'https://collectionapi.metmuseum.org/public/collection/v1/objects/' + random_objectID
console.log(url);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
})
//'url' becomes unknown here. I want to pass it to another get request.

It's both an async and a scope issue!
If you declare var url; in the outermost scope you'll be able to set it inside that callback as intended, but since that's happening asynchronously you won't be able to use the value outside the scope unless you are checking after the callback completes.
Alternatively, you could wrap the whole thing in a promise, e.g.
const https = require('https');
new Promise((resolve,reject)=>{
let targetUrl = 'https://collectionapi.metmuseum.org/public/collection/v1/objects';
https.get(targetUrl,resp=>{
// ...
resp.on('end', () => {
// ...
resolve(url);
});
});
}).then(url=>{
// do stuff with that URL
});
If your goal is to automate fetching data from web resources, I'd recommend checking out the request module, which also has a promisified variant.

Related

Unable to get the variable value outside the http call

I have written below code to get the response data chunk in the variable called outBody, but it is showing undefined. Any suggestions please?
var options = {
host: 'jsonplaceholder.typicode.com',
port: 80,
path: '/todos/1'
};
var outBody;
http.get(options, function(res) {
console.log("Got response: " + res.statusCode);
res.on("data", function(chunk) {
outBody = chunk;
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
console.log(outBody);
I actually want to use the variable outBody outside the http request. How can I use it?
Your code is showing undefined, because your code is a asyncronous code and you are logging data synchronously.
A simple fix is that put your console.log inside the data event listener like this
res.on("data", function(chunk) {
outBody = chunk;
console.log(outBody);
});
and if you want to use the outBody variable outside the http.get then you either has to use promises or a callback function inside another event listener called end , which fires when the response is completed.
Like this
using CallBack :-
//Your code
res.on("data", chunk => {
outBody += chuck
});
res.on("end", () => {
// this event fires when the response body is completely loaded ...
myFunction(outBody); // pass the respone body to your callback function
}
// remaing code
Wanna learn how to deal with this problem again, then learn about async / await and promises on internet
OR
Using promises / async await :-
const options = {...}
var outBody = "";
const req = new Promise( (resolve, reject) => {
http.get(options, res => {
res.on("data", chunk => { outBody += chunk });
res.on("end", () => resolve(outBody));
res.on("error", console.error);
});
});
req.then( data => {
myFunction(data);
})
.catch(err => console.error(err));
// or u can try async / await
(async function (){
const data = await req;
})();
Note : You had done a mistake i.e., inside the data event listener you overwrite the outBody with the new chunk. It is a mistake because the http class loads the response in chuck which means parts. It means you are losing data , I also fixed it by adding += in place of outBody = chunk, it simply appends the new part the older part which gives you the complete response body...
Still confused about the promises and async / await , then them here
The http request is asynchronous, the reason why console.log is logging undefined is because you log the result of the request before the request is done, only after the request is done outbody gets the value of chunk. In order to access the outbody value you should write the console.log inside the callback block which is called right after the data is returned from the request like this:
http.get(options, function(res) {
console.log("Got response: " + res.statusCode);
res.on("data", function(chunk) {
outBody = chunk;
console.log(outbody)
});
res.on("error",e=>{
console.log("Got error: " + e.message);
})
})

Cannot perform http request with http module nodejs azure function

I am trying to make a http request with node module http on an azure function with javascript and for some reason http.request is not receiving data(no error printed its like the requested is blocked). There is anything wrong with azure configuration or the code?Am i missing something really obvious?
Code is running fine on local js file
context.log('JavaScript timer trigger function ran!', timeStamp);
printed as expected but context.log("SUCCESS", JSON.parse(data)); not printed at all.
Also tryed different libraries (request, axios) with no success
module.exports = async function (context, myTimer) {
var timeStamp = new Date().toISOString();
if (myTimer.IsPastDue)
{
context.log('JavaScript is running late!');
}
context.log('JavaScript timer trigger function ran!', timeStamp);
const http = require('http');
http.get('http://myAPIurl', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
context.log("SUCCESS", JSON.parse(data));
});
}).on("error", (err) => {
context.log("ERROR: " + err.message);
});
}
I rewrote your code using Axios. It has support for async/await out of the box, and simplifies a lot of the code to function as your would expect since it makes asynchronous code perform like synchronous code.
I think the main issue you may have been running into it that everything in JavaScript is async. As a result, the Azure Function runtime was exiting before your async function finished since it didn't block like synchronous code would when making the HTTP request. If you want to use callbacks, you need to call context.done() inside your callback function so the Azure Function doesn't exit before the callback is completed. By using async/await, you can guarantee that your code will block on the HTTP request until it receives a response or timeout. In the example below, Axios will return a response object that includes data as an element. I'm extracting data with a deconstruction operation, which allows me to log data.
const axios = require('axios');
const url = 'https://google.com';
module.exports = async function (context, myTimer) {
try {
const { data } = await axios.get(url);
context.log(data);
// do something with the data
return data;
} catch (err) {
context.log(err);
// do something with the error
}
context.done();
}
To Install a Package on an Azure Function
Go to the Azure Portal; select your Azure Function App
Select Platform Features. Then Advanced Tools (Kudu) under Development Tools
Using the file explorer, navigate to site/wwwroot/. You should see a package.json file in there, and several folders each with the name of your functions.
From that directory, run npm install --save axios
To confirm it worked, edit your package.json with the pencil icon. axios should be listed under dependencies json element
I've kept your code callback based.
I removed the async moniker from the definition and added a call to context.done (this signals the functions host when your function has ended) in your resp.end handler
module.exports = function (context, myTimer) {
var timeStamp = new Date().toISOString();
if (myTimer.IsPastDue)
{
context.log('JavaScript is running late!');
}
context.log('JavaScript timer trigger function ran!', timeStamp);
const https = require('https');
https.get('https://raw.githubusercontent.com/LearnWebCode/json example/master/animals-1.json', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
context.log("SUCCESS", JSON.parse(data));
context.done(null, data);
});
}).on("error", (err) => {
context.log("ERROR: " + err.message);
});
}
Another option would be to keep the function as async but you'd need to replace the callbacks with promise based calls. In some scenarios this can be achieved by wrapping them using util.promisify and then calling them with the await keyword

working with Node js Await/Async functions

This is my axios Request to call the API.
export function axiosGet (url) {
return opsutils.get(url)
.then(function (response) {
return response.data.data;
})
.catch(function (error) {
return 'An error occured..' + error;
})
}
From here i'm calling it asynchrously
async getDirList(data){
this.domainDir=data.domain_name
var apiurl="some URL"
var stat = await axiosGet(apiurl)
if (status){
this.domainlog= stat
}
From here i'm calling the async func defined above
Notify(data){
var filelist = this.getDirList(data)
if(filelist){
var status = createNotification(data.domain_name,message_stripped,'Degrading web server performance')
}
The ideal should be like this that it should go forward only after the promise is resolved ,right now the var filelist got empty.
How do i get to solve this problem ?Thanks in advance
The problem is this.getDirList(data) is not being accessed asynchronously as well. Remember, because that is async now, it's returning a promise, so you either need to chain it with .then():
Notify(data){
var filelist = this.getDirList(data)
.then(data => {
var status = createNotification(data.domain_name,message_stripped,'Degrading web server performance')
});
}
Or turn Notify() into an async function as well:
async Notify(data){
var filelist = await this.getDirList(data);
if(filelist){
var status = createNotification(data.domain_name,message_stripped,'Degrading web server performance')
}
Additionally, make sure you're actually returning data from getDirList so you can utilize the return value when you await it.

Returning parsed JSON API data as variable with Callbacks and https.get [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I am trying to build a simple web app that pulls API data on the price of Bitcoin. I am able to parse the data and extract the price. However, given the asynchronous nature of Node.js, when I try to return the data it returns before the http request is complete.
This is why I inserted the callback function. However, when it returns the price it is still undefined. Hence, I assume that the request is not complete when the return is sent even with the callback?
I have also heard of other solutions such as promises and async yet I cannot find a way to return the price variable from the function. I need to be able to do this so that I can do separate calculations on the returned variable.
Disclaimer: I am a node newbie so any help would be appreciated
var https = require('https');
function myCallback(result) {
return result;
}
var price = 0;
function getData(callback) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
callback(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
});
};
var y = getData(myCallback);
console.log(y);
The hard thing when you start in javascript is start thinking in asynchronous operations. When console.log(y) is executed the HTTPS call isn't completed yet so it can't be called that way once majorly IO functions in node are non-blocking. What you need to do is to call console.log inside the myCallback function.
var https = require('https');
function myCallback(result) {
console.log(result);
}
var price = 0;
function getData(callback) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
callback(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
});
}
getData(myCallback);
EDIT:
The thing is, you can set the value of a variable outside the callback by you still have to ensure that you'll be using this variable only after the attribution. You still have to use callbacks one way or another.
There's another way of pass callbacks around and it has a nicer way for doing that, promises. I believe that is what you're looking for.
var https = require('https');
var price = 0;
function getData() {
var promise = new Promise(function(resolve, reject) {
var url = "https://api.coinbase.com/v2/prices/spot?currency=USD";
https.get(url, function(res){
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var exData = JSON.parse(body);
price = Number((exData.data.amount))*14.5;
resolve(price);
});
}).on('error', function(e){
console.log("Got an Error: ", e);
reject(e);
});
})
return promise;
}
var y = getData()
y.then(function (result) {
console.log('price: ' + result);
});

Node.js Generators

I am unsure how to structure Javascript generator code such that it will execute correctly.
...
var http = require('http');
emails.send = function *(next) {
// Pull the HTML for email to be sent from another local server
mailOptions['html'] = yield* emailHtml();
...
};
function* emailHtml() {
// Get the data from the database -- ends up using node-sqlite
var data = yield mea.getMeasurements();
// Callback function to deal with http request below
function callback(response) {
var str = '';
response.on('data', function(chunk) {
str += chunk;
});
response.on('end', function(chunk) {
return str;
});
}
// Make a request to the other server to get it's HTML
var req = http.request(options, callback);
// Post the data from this server's database connection to the other server to be processed and sent back
req.write(JSON.stringify(data));
req.end();
return ??????;
}
...
I already have the emailHtml() function yielding data from the local sqlite database and passing that data via POST with the http.request but can't figure out how to structure my code to have the emailHtml() function return the callback's final string.
Do I need to make callback a generator function also? I've tried var req = yield http.request(options, callback); but since that stops the request, the POST data is never written and the request is never completed in the following two lines.
What other options do I have if a generator is not the right way to go about this?
You need to turn the HTTP call into something you can yield on. It's messy as currently written, so time to pull in some other tools - in particular, promises. Since you're using Koa, which under the hood uses a library called co, promises are probably the easiest way to to this. I tend to use a library called Bluebird for my promise implementation, there are other options.
So basically you want something like this:
var http = require('http');
var Promise = require('bluebird');
emails.send = function *(next) {
// Pull the HTML for email to be sent from another local server
mailOptions['html'] = yield* emailHtml();
...
};
function makeHttpRequest(options, data) {
// Note we're returning a promise here
return new Promise(function (resolve, reject) {
var req = http.request(options, callback);
req.write(JSON.stringify(data));
req.end();
function callback(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function (chunk) {
// -- Resolve promise to complete the request
resolve(str);
});
}
});
}
function* emailHtml() {
// Get the data from the database -- ends up using node-sqlite
var data = yield mea.getMeasurements();
// Callback function to deal with http request below
function callback(response) {
var str = '';
response.on('data', function(chunk) {
str += chunk;
});
response.on('end', function(chunk) {
return str;
});
}
// Make a request to the other server to get it's HTML
var str = yield makeHttpRequest(options, data);
// do whatever you want with the result
return ??????;
}
This wraps up the http stuff inside a promise object, which you generator runner at the outer layer knows how to wait for completion of.
There's other ways to do this, and libraries (like co-request) that wrap this stuff natively, but this is the basic idea.
Just to add on Chris' answer, here's the cleaned up version of the code I'm now using:
var http = require('http');
emails.send = function *(next) {
// Pull the HTML for email to be sent from another local server
mailOptions['html'] = yield* emailHtml();
};
function makeHttpRequest(options, data) {
// Note we're returning a promise here
return new Promise(function (resolve, reject) {
var req = http.request(options, callback);
req.write(JSON.stringify(data));
req.end();
function callback(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function (chunk) {
// -- Resolve promise to complete the request
resolve(str);
});
}
});
}
function* emailHtml() {
// Get the data from the database -- ends up using node-sqlite
var data = yield mea.getMeasurements()
// Make a request to the other server to get it's HTML
return yield makeHttpRequest(options, data);
}
Promises are already built in to Node v0.12.7 when you use the --harmony flag so an additional library is not necessary

Categories