We Keep Coding

Using Google Script, how to connect to an API using Macaroon

I was able to connect to this API using this javascript code.
function get_info(){
const fs = require('fs');
const request = require('request');
var macaroonFile = '/Path_to_Folder/access.macaroon'
var abc = fs.readFileSync (macaroonFile);
var macaroon = Buffer.from(abc).toString("base64");
console.log(macaroon)
let options = {
url: 'https://localhost:2000/v1/getinfo',
// Work-around for self-signed certificates.
rejectUnauthorized: false,
json: true,
headers: {
'macaroon': macaroon,
},
}
request.get(options, function(error, response, body) {
console.log(body);
});
}
However, when modifying this code to google script like bellow, I keep getting this error.
Exception: Bad request: https://localhost:2000/v1/getinfo.
function getinfo(){
var url = 'https://localhost:2000/v1/getinfo'
// ↓ access.macaroon encoded to base 64.
var macaroon = 'AgELYy1saWdodG5pbmcCN1RodSBOb3YgMTIgMjByMCAyMToyNjozOCBHTVQAMDkwMCAoSmBwYW4gU3RhbmRhcmQgVGltZSkAAAYg0lCjv2MeZJQ20NeB+L92W0mGHER92YGxEpTgYPaIct0=';
var options = {};
options.headers = {"Authorization": "Basic " + macaroon};
var response = UrlFetchApp.fetch(url, options)
var json = response.getContentText();
return json
}
Can someone help me to figure out what could be the problem. Thank you in advance.

How to send the form data to the JS program

user will give the url in the input type field in the Html page that url need to get in the JS program and then the JS program need to execute to fetch the data from webpage.
this is what have done so far.
var request = require('request');
var cheerio = require('cheerio');
var fs = require("fs");
var url = ""
request(url, function(err, response, html){
if(!err) {
var $ =cheerio.load(html);
var allItems = $('.clearfix').parent().children();
var items = [];
allItems.each(function(index) {
var result = $('.clearfix').eq(index).parent().children().eq(1).find("a").text();
if(result !== ""){
items.push(result);
}
});
fs.writeFile("output1.xls",JSON.stringify(items, null, 1),)
console.log(items);
}
});

Is this the solution to your problem?
var url = document.getElementById('myURL').value

How do I read Ajax parameter value in Node without framework?

I am trying to pass a variable to my Ajax send and read it in Node without using a framework. I am not trying to return the value back to the DOM, I just need to read the value passed to Node.js. Here is what I have:
Ajax:
const XHR = new XMLHttpRequest();
XHR.open('POST', document.url, true);
XHR.setRequestHeader('X-Requested-load', 'XMLHttpRequest2');
XHR.send(`password=${password}`);
Nodejs:
const QS = require('querystring');
let password = QS.parse(req.body);
req.on('data', (data) => {
password = QS.parse(data);
});
console.log(password);

Hope this will help you:
http.createServer(function (request, response) {
if (request.method == 'POST') {
// save all data received
var postdata = '';
// receiving data
request.on('data', function(chunk) {
postdata += chunk;
// Avoid too much POST data
if (postdata.length > 1e6)
request.connection.destroy();
});
// received all data
request.on('end', function() {
var post = qs.parse(postdata);
// handle post by accessing
// post['password']
// response.send(process(post['password']));
});
} else {
console.log("Non POST request received at " + request.url);
}
}).listen();
In your example you are trying to access password out of the callback where this data is provided.

const QS = require('querystring');
let password = QS.parse(req.body);
req.on('data', (data) => {
password = QS.parse(data);
//async. access data here
console.log(password);
});

iterate node.js request function

This question is about a crawler in node.js.
A start_url is given where he crawls for URLs, and "pushes" them to a .json-file (output.json).
At the moment, he runs the request function only with the start_url, and saves the collected URLs in output.json. I want that he uses the saved URLs by replacing the start_url with the first collected URL and collect links again ... and so on ...
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var start_url = ["http://stackoverflow.com/"]
var req = function(url){
request(url, function(error, response, html){
var $ = cheerio.load(html);
var data = [];
$("a").each(function() {
var link = $(this);
var exurls = {exurl: new Array(link.attr("href"))}
data.push(exurls);
// Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
// save to "output.json" from time to time, so you can stop it anytime
});
fs.writeFile("output.json", JSON.stringify(data, null, 4), function(err){
if(err){
console.log(err);
} else {
console.log("File successfully written!");
}
});
});
}
for (var i = 0; i < start_url.length; i++){
req(start_url[i]);
}

So what you can do is make the function call recursively. The below example should work:
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var start_url = ["http://stackoverflow.com/"]
var req = function(url){
var count = 0;
request(url, function(error, response, html){
var $ = cheerio.load(html);
$("a").each(function() {
var link = $(this);
var exurls = {exurl: new Array(link.attr("href"))}
start_url.push(exurls);
// Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
// save to "output.json" from time to time, so you can stop it anytime
});
try {
fs.writeFileSync("output.json");
console.log("File successfully written!");
}catch(err){
console.log(err);
}
++count;
if(start_url.length > count) {
req(start_url[count]);
}
});
}
return req(start_url[0]);
The problem with this is that you are completely rewriting the file each time. If this goes on for awhile you are going to run out of memory. Another option is to create a write stream
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var start_url = ["http://stackoverflow.com/"]
var wstream = fs.createWriteStream("output.json");
var req = function(url){
request(url, function(error, response, html){
var $ = cheerio.load(html);
$("a").each(function() {
var link = $(this);
var exurls = {exurl: new Array(link.attr("href"))}
start_url.push(exurls);
// Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
// save to "output.json" from time to time, so you can stop it anytime
wstream.write('"'+ exurls + '",');
});
start_url.shift();
if(start_url.length > 0) {
return req(start_url[0]);
}
wstream.end();
});
}
req(start_url[0]);
Edit: switched to a basic queue so combat memory problems

Handle on success event with npm package request

I have a node.js app that scrapes informations from a website. I'm using npm packages request and cheerio and the scraping works fine but I want to do something else when the request function is done. Here's some code:
app.js
var express = require('express');
var extractor = require("./extractor");
console.log(extractor('http://www.example.com'));
var app = express();
app.get('/', function (req, res) {
res.send('Hello world\n');
});
app.listen(3000);
extractor.js (all the fun)
var request = require('request');
var cheerio = require('cheerio');
var Extractor = function(url) {
var games = [];
request(url, function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('tr.game').each(function(i, v){
var game = { /* many attributes */ };
games.push(game);
});
}
});
this.extractedGames = games;
};
module.exports = function(url) {
return new Extractor(url);
};
Eventually when I run this it shows { extractedGames: [] } that is because the output was printed before the request treatment was over. So I want to add an on success event to extracedGames attribute when the request job is over.
Thanks

Solved it myself ! I hope this could help people in the future (though I felt like a complete noob)
The trick was to emit an event and handle it later.
var express = require('express');
var extractor = require("./extractor");
extractor('http://www.example.com');
var app = express();
app.get('/', function (req, res) {
res.send('Hello world\n');
});
app.listen(3000);
I removed console.log(extractor('http://www.example.com')) because this would run before the request job is done. So I moved it to the event handling function.
var request = require('request');
var cheerio = require('cheerio');
var Emitter = require('events').EventEmitter;
var extractEmitter = new Emitter();
extractEmitter.on('extracted', function(extractedGames){
console.log(extractedGames);
});
var Extractor = function(url) {
var games = [];
request(url, function (error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('tr.game').each(function(i, v){
var game = { /* many attributes */ };
games.push(game);
});
extractEmitter.emit('extracted', games);
}
});
this.extractedGames = games;
};
module.exports = function(url) {
return new Extractor(url);
};