const request = require('request');
exports.helloWorld = function helloWorld(req, res) {
var headers = {
'scheme': 'https',
'authorization': 'Token 123',
'user-agent': 'mobile'
};
var options = {
url: 'https://url',
headers: headers
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
res.status(200).send(response.body);
}
}
request(options, callback);
};
I can submit a request without issue, but once i start doing a request with headers and options , i cant get it to work properly on Google Cloud Function.
Please advice on my mistake
Trigger type
: HTTP trigger
It's likely that an error is being returned from the call you're making using request, and since in the error case you don't end the function using res.end() or res.status(500).end(), Cloud Functions doesn't know if your function is done or not, so just runs it until a timeout is hit.
Try:
if (!error && response.statusCode == 200) {
console.log(body);
res.status(200).send(response.body);
} else {
console.log(error);
res.status(500).send(error);
}
You should get a response back without the Cloud Function hanging.
Related
I'm trying to access all repositories that have more than 5000 stars on Github. I've written this scraper to work with Node.js (it's running on a Cloud9 environment):
var request = require('request');
var fs = require('fs');
var options = {
url: 'https://api.github.com/repositories',
headers: {
'User-Agent': 'myusernamehere'
},
qs: {
stargazers: 5000
}
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(response.headers);
fs.writeFile('output_teste.json', body, function (err) {
if (err) throw err;
console.log('It\'s saved!');
console.log(response.statusCode);
});
} else {
console.log(response.statusCode);
}
}
request(options, callback);
But the result is not all of the repositories, just the first page of all of them. How can I use pagination with the Request module? I've tried to find examples within the documentation, but they aren't that clear. Or do I need to do this with another library or maybe another language?
Thanks!
you should modify your querystring to include the value of "since". You can read more on the github documentation.
https://developer.github.com/v3/repos/#list-all-public-repositories
Sample URL with query string of since
https://api.github.com/repositories?since=364
You could use the pagination data provided in response.headers.link that's received when making calls to the GitHub API to find out if there are any more pages left for your call.
One approach is to loop through the pages until there are no more new pages left, at which point you can write to file and return from function.
On each loop you can add to the data that you already have by using concat (I assume that the response body is delivered as an array) and then passing on the data to the next function call.
I rewrote your code to include a basic implementation of such a technique:
var request = require('request');
var fs = require('fs');
var requestOptions = function(page) {
var url = 'https://api.github.com/repositories?page=' + page;
return {
url: url,
headers: {
'User-Agent': 'myusernamehere'
},
qs: {
stargazers: 5000
}
};
};
function doRequest(page, incomingRepos) {
request(requestOptions(page), function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(response.headers);
var currentPageRepos = JSON.parse(body);
var joinedRepos = incomingRepos.concat(currentPageRepos);
var linkData = response.headers.link;
// if response does not include reference to next page
// then we have reached the last page and can save content and return
if (!(linkData.includes('rel="next"'))) {
fs.writeFile('output_teste.json', JSON.stringify(joinedRepos), function(err) {
if (err) throw err;
console.log('It\'s saved!');
});
return;
}
page++;
doRequest(page, joinedRepos);
} else {
console.log(response.statusCode);
}
});
}
doRequest(1, []);
Spent a few hours trying to solve this but keep on getting 400 errors. I think my payload is incorrect but not sure what I need to do.
I've tried the same request in POSTMAN with great success, and also have tried different combinations on formatting my payload (body).
I've also tried the following: var body = {"document":{"type":"PLAIN_TEXT", "content":"This is great!!"}};
function main(){
// Set the headers
var headers = {
'Authorization': 'Bearer ya29.GlyvBPhT4Y502Yo4NZ6sKI001AYlVU6om5ytXrzf2vjfEYERrsdMSBu-2ZkKir83jjj06-ewqYuBvvrx8mKfuTW1YjjlmtPmRdlK0I0Gjx',
'Content-Type': 'application/json',
'User-Agent': 'Untitled browser'
}
var uri = 'https://language.googleapis.com/v1/documents'+encodeURI(':analyzeSentiment');
var form = JSON.stringify({"document":{"type":"PLAIN_TEXT", "content":"This is great!!"}});
// Configure the request
var options = {
url: uri,
method: 'POST',
headers: headers,
form: form
}
// Start the request
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
// Print out the response body
console.log(body)
}
else{
console.log('Error code: '+response.statusCode)
}
})
}
main();
Update - this is how it works in POSTMAN
Header is the same too:
Solved!
Need to post as body rather than form
Using the request node library to make GET requests and and wondering how to grab the URL from the response returned.
Something like
var request = require('request');
request('http://www.somewebsite.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // grab the url here
}
})
assuming the response returned is not www.somewebsite.com but a redirect.
Start at the documentation. Search it for redirect and you'll find:
followRedirect - follow HTTP 3xx responses as redirects (default: true). This property can also be implemented as function which gets response object as a single argument and should return true if redirects should continue or false otherwise.
Note the status code. You only get the 200 after the redirect has been followed.
Therefore:
var request = require('request');
request({
uri: 'http://google.com',
followRedirect: function(response) {
console.log("Redirecting to " + response.headers.location);
return true;
}
},
function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body)
}
}
);
I am using request nodejs module to get html for a website as below:
var request = require('request');
request("http://www.thenewschool.org/", function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("body>>>>>>>>>>");
} else {
console.log("error>>>>>>>>>"+error);
console.log("response statusCode>>>>>>>>>"+response.statusCode);
console.log("response body>>>>>>>>>"+response.body);
}
})
and this gives me this output
error>>>>>>>>>null
response statusCode>>>>>>>>>403
response body>>>>>>>>>Sorry, this request has been blocked due to an
invalid user agent.
This is passing for most of the cases, but fails in this case, can someone help me to solve this.
You just have to pass the user-agent in the headers (because the URL requires it), like:
var options = {
headers: {'user-agent': 'node.js'}
}
request("http://www.thenewschool.org/", options, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("body>>>>>>>>>>" + body);
} else {
console.log("error>>>>>>>>>"+error);
console.log("response statusCode>>>>>>>>>"+response.statusCode);
console.log("response body>>>>>>>>>"+response.body);
}
})
well you are receiving an HTTP 403 error code : Access Is Forbidden.
this probably means that your request has been "profiled" as "we don't want you here" :
this can be because your IP has been flagged
or because your are missing a header that would make your request look like the one of a real browser. Most probably the user-agent header given the body of the answer
I created a small api to generate test data on the fly. Each call creates a fresh user and returns the created data.
To load the data I use the package request:
var flow = protractor.promise.controlFlow();
var result = flow.execute(function() {
var defer = protractor.promise.defer();
request('http://localhost/test/recipe/person', function (error, response, body) {
if (!error && response.statusCode === 200) {
defer.fulfill(JSON.parse(body));
}
});
return defer.promise;
});
To make any use of the retrieved data I have to resolve the promise and proceed the test script inside a callback:
result.then(function(data) {
element(by.model('username')).sendKeys(data.person.email);
element(by.model('password')).sendKeys('test');
$('button[type="submit"]').click();
});
I don't like this callback handling and the possible hell it will lead to. Besides that, Protractor is so good in hiding this messy callback handling thingy. So, the question is:
How to use the result of an async call?
At the end I would like to perform code like the following:
var flow = protractor.promise.controlFlow();
var result = flow.execute(function() {...});
element(by.model('username')).sendKeys(result.person.email);
//...
Any ideas?
You can either make the http request synchronous - In most cases that's a bad thing to do.
Or, you can insert the callback into the execute function:
var flow = protractor.promise.controlFlow();
var result = flow.execute(function() {
var defer = protractor.promise.defer();
request('http://localhost/test/recipe/person', function (error, response, body) {
if (!error && response.statusCode === 200) {
defer.fulfill(JSON.parse(body));
}
});
defer.promise.then(function(data) {
element(by.model('username')).sendKeys(data.person.email);
element(by.model('password')).sendKeys('test');
$('button[type="submit"]').click();
});
return defer.promise;
});
but result will stay a promise.
It worked for me:
var request = require('request');
var options = {
method: 'POST',
url: 'http://testurl.com/endpoint/test/',
headers: {'id': 'ABCD',
'sessionid': 'dummyId',
'Accept': 'application/json',
'Accept-Language': 'en-us'
},
body: '{ "pay_load": [] }'
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var info = JSON.parse(body);
console.log(body);
console.log(info);
}
}
request(options, callback);