I am using request nodejs module to get html for a website as below:
var request = require('request');
request("http://www.thenewschool.org/", function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("body>>>>>>>>>>");
} else {
console.log("error>>>>>>>>>"+error);
console.log("response statusCode>>>>>>>>>"+response.statusCode);
console.log("response body>>>>>>>>>"+response.body);
}
})
and this gives me this output
error>>>>>>>>>null
response statusCode>>>>>>>>>403
response body>>>>>>>>>Sorry, this request has been blocked due to an
invalid user agent.
This is passing for most of the cases, but fails in this case, can someone help me to solve this.
You just have to pass the user-agent in the headers (because the URL requires it), like:
var options = {
headers: {'user-agent': 'node.js'}
}
request("http://www.thenewschool.org/", options, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("body>>>>>>>>>>" + body);
} else {
console.log("error>>>>>>>>>"+error);
console.log("response statusCode>>>>>>>>>"+response.statusCode);
console.log("response body>>>>>>>>>"+response.body);
}
})
well you are receiving an HTTP 403 error code : Access Is Forbidden.
this probably means that your request has been "profiled" as "we don't want you here" :
this can be because your IP has been flagged
or because your are missing a header that would make your request look like the one of a real browser. Most probably the user-agent header given the body of the answer
Related
I am trying to use openweathermap.org weather api, I am getting the data but having trouble parsing it.
Here's my code:
request("https://api.openweathermap.org/data/2.5/weather?lat=30.4831&lon=76.595&appid=7beb6de85d3f3a28dabda1015684562f&units=metric&callback=test", function(error, response, body) {
if (!error && response.statusCode == 200) {
var parsedData = JSON.parse(body);
console.log(typeof body);
}
});
console.log(typeof body); returns string so I just can't figure out what 's the problem.
You are looking for this - the URL is JSONP so it expects a function called test and no need to parse
<script>
const test = data => console.log(data);
</script>
<script src="https://api.openweathermap.org/data/2.5/weather?lat=30.4831&lon=76.595&appid=7beb6de85d3f3a28dabda1015684562f&units=metric&callback=test"></script>
Alternatively remove the callback - here using fetch:
fetch("https://api.openweathermap.org/data/2.5/weather?lat=30.4831&lon=76.595&appid=7beb6de85d3f3a28dabda1015684562f&units=metric")
.then(response => response.json())
.then(data => console.log(data));
Or Axios
axios.get('https://api.openweathermap.org/data/2.5/weather?lat=30.4831&lon=76.595&appid=7beb6de85d3f3a28dabda1015684562f&units=metric')
.then(response => console.log(response.data));
<script src="https://unpkg.com/axios/dist/axios.min.js"></script>
This worked for me. Just set object for the first argument with url and json property. set json:true. Setting to true will handle the parsing for you :)
const request = require('request');
const url = "https://api.openweathermap.org/data/2.5/weather?lat=30.4831&lon=76.595&appid=7beb6de85d3f3a28dabda1015684562f&units=metric";
request({ url: url, json: true }, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
});
As #mplungjan pointed out. That's a JSONP url.
Remove &callback=test at the end of your URL then you don't have to deal with JSONP and you can work with it like normal.
Since you are using an API which has a callback function return as response. It is not JSON but JSONP (JSON with Padding).
Your code will work just fine if you remove the callback parameter from URL.
request("https://api.openweathermap.org/data/2.5/weather?lat=30.4831&lon=76.595&appid=7beb6de85d3f3a28dabda1015684562f&units=metric", function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(response);
}
});
For further reading about JSONP, You can refer this.
const request = require('request');
exports.helloWorld = function helloWorld(req, res) {
var headers = {
'scheme': 'https',
'authorization': 'Token 123',
'user-agent': 'mobile'
};
var options = {
url: 'https://url',
headers: headers
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
res.status(200).send(response.body);
}
}
request(options, callback);
};
I can submit a request without issue, but once i start doing a request with headers and options , i cant get it to work properly on Google Cloud Function.
Please advice on my mistake
Trigger type
: HTTP trigger
It's likely that an error is being returned from the call you're making using request, and since in the error case you don't end the function using res.end() or res.status(500).end(), Cloud Functions doesn't know if your function is done or not, so just runs it until a timeout is hit.
Try:
if (!error && response.statusCode == 200) {
console.log(body);
res.status(200).send(response.body);
} else {
console.log(error);
res.status(500).send(error);
}
You should get a response back without the Cloud Function hanging.
Using the request node library to make GET requests and and wondering how to grab the URL from the response returned.
Something like
var request = require('request');
request('http://www.somewebsite.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // grab the url here
}
})
assuming the response returned is not www.somewebsite.com but a redirect.
Start at the documentation. Search it for redirect and you'll find:
followRedirect - follow HTTP 3xx responses as redirects (default: true). This property can also be implemented as function which gets response object as a single argument and should return true if redirects should continue or false otherwise.
Note the status code. You only get the 200 after the redirect has been followed.
Therefore:
var request = require('request');
request({
uri: 'http://google.com',
followRedirect: function(response) {
console.log("Redirecting to " + response.headers.location);
return true;
}
},
function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body)
}
}
);
I want to read a header from a rest response and store it in a variable.
Till now I am able to fetch the header from the response. But not able to store it in a variable.
This is how I am posting the request and successfully fetching the header:
var options = {
uri: 'http://xxxyyyxxx.com',
method: 'POST',
json: {
"username": "someUserName",
"password": "somePassword"
},
};
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("AUTHORIZATION=",response.headers['authorization']);
}
});
Now I want to assign the header to a variable.
I tried this way but its not working:
var header = response.headers['authorization']; //Code not working
console.log(header); //Code not working
Could someone please help me?
Thanks in advance.
So, i'm trying to get the http status code of an URL that i pass to the funcion. so far i've used the request module but the problem is that request fetches all the content of the given URL and if the content is a big amount of data it's slow, but i just need to get the status code to make an if statement.
request(audioURL, function (error, response) {
if (!error && response.statusCode == 200) {
queue.push({
title: videoTitle,
user: message.author.username,
mention: message.author.mention(),
url: audioURL
});
bot.reply(message, "\"" + videoTitle + "\" has been added to the queue.")
}
else {
bot.reply(message, "Sorry, I couldn't get the audio track for that video. HTTP status code: " + response.statusCode);
}
});
This is what i got so far.audioURL is a basic string with a link to a media file
http.request can take an options Object as its first parameter. Use something like this:
const options = {
method: 'HEAD',
host: 'example.com',
};
const req = http.request(options, (res) => {
console.log(JSON.stringify(res.headers));
});
req.end();
The problem is that you'll want to turn your audioURL into its components, to pass as parameters of options. I'm using the HEAD method, that fetches only the headers of your request.