I'm trying to scrape data from google map using the /search/ from maps.
When I search it myself, I tape this:
'https://www.google.com/maps/search/new york'
And then I'm redirected to this url:
'https://www.google.com/maps/place/New+York,+%C3%89tat+de+New+York,+%C3%89tats-Unis/#40.6974881,-73.979681,10z/data=!3m1!4b1!4m5!3m4!1s0x89c24fa5d33f083b:0xc80b8f06e177fe62!8m2!3d40.7127753!4d-74.0059728'
I can't manage to reproduce this behavior using axios. I guess there's maybe something to make with async / await but i didn't find any solution since then.
here's my code :
const axios = require('axios');
const cheerio = require('cheerio');
var map = 'www.google.com/maps/search/';
axios.get(map + 'New York')
.then(response => {
let getData = html => {
coor = [];
v= -1;
const $ = cheerio.load(html);
$('.widget-pane-content scrollable-y').each((i, elem) => {
coor.push({
adress : $(elem).find('span.widget-pane-link').text(),
});
});
console.log(coor);
console.log(coor.length);
}
getData(response.coor);
})
.catch(error => {console.log(error);})
When I execute the file, I get this error:
'Error: Request failed with status code 400'
If you have any clue to solve my problem, thanks for sharing it!
Look into a tool like Selenium, or Cypress.js (a wrapper around selenium)
(search "end to end testing" or "automated browser")
This unfortunately cannot be done with a tool like Axios. Google Maps doesn't return a redirect response, but rather uses JavaScript to reload the page.
example with cypress:
cy.visit("https://www.google.com/maps/search/new york");
cy.wait(2000); // sit for 2 seconds
cy.get('#thing-im-looking-for')
Try using cookieJar with axios
https://github.com/axios/axios/issues/943#issuecomment-599174929
Or set jar to true with request package
request(url, { jar: true })
https://stackoverflow.com/a/48912841/11686526
You can't get places info from Google Maps using axios, because results builds on the page via JavaScript, so you need to use some browser automation, e.g. Puppeteer. In the code below I show you how you can do this (also check it on the online IDE):
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());
const requestParams = {
baseURL: `http://google.com`,
query: "starbucks", // what we want to search
coordinates: "#47.6040174,-122.1854488,11z", // parameter defines GPS coordinates of location where you want your query to be applied
hl: "en", // parameter defines the language to use for the Google maps search
};
async function scrollPage(page, scrollContainer) {
let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
while (true) {
await page.evaluate(`document.querySelector("${scrollContainer}").scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
await page.waitForTimeout(5000);
let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
if (newHeight === lastHeight) {
break;
}
lastHeight = newHeight;
}
}
async function fillDataFromPage(page) {
const dataFromPage = await page.evaluate(() => {
return Array.from(document.querySelectorAll(".bfdHYd")).map((el) => {
const placeUrl = el.parentElement.querySelector(".hfpxzc")?.getAttribute("href");
const urlPattern = /!1s(?<id>[^!]+).+!3d(?<latitude>[^!]+)!4d(?<longitude>[^!]+)/gm; // https://regex101.com/r/KFE09c/1
const dataId = [...placeUrl.matchAll(urlPattern)].map(({ groups }) => groups.id)[0];
const latitude = [...placeUrl.matchAll(urlPattern)].map(({ groups }) => groups.latitude)[0];
const longitude = [...placeUrl.matchAll(urlPattern)].map(({ groups }) => groups.longitude)[0];
return {
title: el.querySelector(".qBF1Pd")?.textContent.trim(),
rating: el.querySelector(".MW4etd")?.textContent.trim(),
reviews: el.querySelector(".UY7F9")?.textContent.replace("(", "").replace(")", "").trim(),
type: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(1) > span:first-child")?.textContent.replaceAll("·", "").trim(),
address: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(1) > span:last-child")?.textContent.replaceAll("·", "").trim(),
openState: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(3) > span:first-child")?.textContent.replaceAll("·", "").trim(),
phone: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(3) > span:last-child")?.textContent.replaceAll("·", "").trim(),
website: el.querySelector("a[data-value]")?.getAttribute("href"),
description: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(2)")?.textContent.replace("·", "").trim(),
serviceOptions: el.querySelector(".qty3Ue")?.textContent.replaceAll("·", "").replaceAll(" ", " ").trim(),
gpsCoordinates: {
latitude,
longitude,
},
placeUrl,
dataId,
};
});
});
return dataFromPage;
}
async function getLocalPlacesInfo() {
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
const URL = `${requestParams.baseURL}/maps/search/${requestParams.query}/${requestParams.coordinates}?hl=${requestParams.hl}`;
await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForNavigation();
const scrollContainer = ".m6QErb[aria-label]";
const localPlacesInfo = [];
await page.waitForTimeout(2000);
await scrollPage(page, scrollContainer);
localPlacesInfo.push(...(await fillDataFromPage(page)));
await browser.close();
return localPlacesInfo;
}
getLocalPlacesInfo().then((result) => console.dir(result, { depth: null }));
Output
[
{
"title":"Starbucks",
"rating":"4.4",
"reviews":"210",
"type":"Coffee shop",
"address":"3300 W McGraw St",
"openState":"Closed ⋅ Opens 6AM",
"phone":"(206) 298-3390",
"description":"Iconic Seattle-based coffeehouse chain",
"serviceOptions":"Dine-in Takeout Delivery",
"gpsCoordinates":{
"latitude":"47.639704",
"longitude":"-122.399869"
},
"placeUrl":"https://www.google.com/maps/place/Starbucks/data=!4m7!3m6!1s0x54901580f2d8ba8b:0xcc4a61a86f6d87ec!8m2!3d47.639704!4d-122.399869!16s%2Fg%2F1td6mc1x!19sChIJi7rY8oAVkFQR7Idtb6hhSsw?authuser=0&hl=en&rclk=1",
"dataId":"0x54901580f2d8ba8b:0xcc4a61a86f6d87ec"
},
{
"title":"Starbucks",
"rating":"4.3",
"reviews":"201",
"type":"Coffee shop",
"address":"701 5th Ave",
"openState":"Closed ⋅ Opens 5:30AM Mon",
"phone":"(206) 447-9934",
"description":"Iconic Seattle-based coffeehouse chain",
"serviceOptions":"Dine-in Takeout No delivery",
"gpsCoordinates":{
"latitude":"47.604155",
"longitude":"-122.330827"
},
"placeUrl":"https://www.google.com/maps/place/Starbucks/data=!4m7!3m6!1s0x54906ab0bab91e09:0xd1284ac9106e9c7e!8m2!3d47.604155!4d-122.330827!16s%2Fg%2F1tdmk5c9!19sChIJCR65urBqkFQRfpxuEMlKKNE?authuser=0&hl=en&rclk=1",
"dataId":"0x54906ab0bab91e09:0xd1284ac9106e9c7e"
},
... and other places
]
You can read more about scraping Google Maps Places from my blog post Web Scraping Google Maps Places with Nodejs.
Related
I'm writing an API with express, puppeteer-cluster and cheerio that returns all anchor elements containing one or more words that can be added as query parameters. I want to use puppeteer in order to get elements that are javascript generated too. But for some reason it's not working, I get an empty array as an output printed on the browser.
I'm still trying to understand this library but has been 2 days and I made no progress. Any help is deeply appreciated.
Update: I added async to all my functions and they run now, but the result is still empty :(
Update 2: I started logging everything, every step and found that data.name is being passed to the cheerio function as a Promise. '-' I think that is the problem, but don't know how to fix it yet.
Update 3: One of the issues was that the page content (html code) was not being handled properly to the cheerio function. In the browser, however, the response is empty and the console shows an error:
Error handling response: TypeError: Cannot read properties of
undefined (reading 'innerText').
So, I think the response is not json formatted. Is res.json() not the right way to do it?
My code:
app.js
const PORT = process.env.PORT || 8000;
var path = require("path");
const express = require("express");
// Routes
const indexRouter = require("./routes/index");
const allNews = require("./routes/news");
const clusterRouter = require("./routes/cluster");
const app = express();
app.use(cors());
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(express.static(path.join(__dirname, "public")));
app.use("/", indexRouter);
app.use("/news", allNews);
app.use("/cluster", clusterRouter);
app.listen(PORT, () => console.log(`server running on PORT ${PORT}`));
cluster.js
const express = require("express");
const { Cluster } = require("puppeteer-cluster");
const puppeteer = require("puppeteer-extra");
const cheerio = require("cheerio");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
var router = express.Router();
const newspapers = [
{
"name": "CNN",
"address": "https://edition.cnn.com/specials/world/cnn-climate",
"base": "https://edition.cnn.com"
},
{
"name": "The Guardian",
"address": "https://www.theguardian.com/environment/climate-crisis",
"base": "https://www.theguardian.com"
}]
const app = express();
puppeteer.use(StealthPlugin());
const result = [];
router.get("/", async (req, res) => {
(async () => {
// Query String
const query = checkForQuery(req);
const wordsToSearch = query ? verifyQuery(query) : "";
console.log("Running tests.."); // This is printed on console
//Functions
function checkForQuery(request) {
if (request.originalUrl.indexOf("?") !== -1) {
console.log(request.query);
return request.query;
} else {
return false;
}
}
// // Validates query and remove invalid values
function verifyQuery(queryString) {
const queryParams = {
only: queryString.only ? queryString.only : "",
also: queryString.also ? queryString.also : "",
};
// Creates new list containing valid terms for search
var newList = {
only: [],
also: [],
};
for (const [key, value] of Object.entries(queryParams)) {
const tempId = key.toString();
const tempVal =
queryParams[tempId].length >= 2
? queryParams[tempId].split(",")
: queryParams[tempId];
console.log(queryParams[tempId], " and ", tempVal);
if (tempVal.length > 1) {
console.log("helloooooo");
tempVal.forEach((term) => {
if (topics.indexOf(term) != -1) {
newList[tempId].push(term);
}
});
} else {
if (topics.indexOf(queryParams[tempId]) != -1) {
newList[tempId].push(queryParams[tempId]);
}
}
}
console.log(newList);
return newList;
}
function storeData(element, base, name) {
const results = [];
element.find("style").remove();
const title = element.text();
const urlRaw = element.attr("href");
const url =
urlRaw.includes("www") || urlRaw.includes("http")
? urlRaw
: base + urlRaw;
// Check for duplicated url
if (tempUrls.indexOf(url) === -1) {
// Check for social media links and skip
if (!exceptions.some((el) => url.toLowerCase().includes(el))) {
tempUrls.push(url);
// Get img if child of anchor tag
const imageElement = element.find("img");
if (imageElement.length > 0) {
// Get the src attribute of the image element
results.push({
title: title.replace(/(\r\n|\n|\r)/gm, ""),
url,
source: name,
imgUrl: getImageFromElement(imageElement),
});
} else {
results.push({
title: title.replace(/(\r\n|\n|\r)/gm, ""),
url: url,
source: name,
});
}
}
}
return results;
}
function getElementsCheerio(html, base, name, searchterms) {
console.log(html, base, name);
const $ = cheerio.load(html);
console.log(searchterms);
const concatInfo = [];
if (searchterms) {
const termsAlso = searchterms.also;
const termsOnly = searchterms.only;
termsAlso.forEach((term) => {
$(`a:has(:contains("climate"):contains(${term}))`).each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
});
termsOnly.forEach((term) => {
// $(`a:has(:contains(${term}))`).each(function () {
$(`a:contains(${term})`).each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
});
} else {
$('a:contains("climate")').each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
}
return concatInfo;
}
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 2,
puppeteerOptions: {
headless: true,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
userDataDir: "./tmp",
defaultViewport: false,
},
});
await cluster.task(async ({ page, data }) => {
await page.goto(data.address);
await page.waitForSelector("body");
// console.log here prints that data.name is a Promise :(
const elements = await getElementsCheerio(
document.body.innerHTML,
data.base,
data.name,
wordsToSearch
);
result.push(elements);
});
newspapers.map((newspaper) => {
console.log("queue" + newspaper); // This logs correctly: queue[object Object]
cluster.queue(newspaper);
});
await cluster.idle();
await cluster.close();
// Display final object
res.json(result);
})();
});
module.exports = router;
I don't get any errors, but on screen I get an empty [ ]. Anyone can see what I am doing wrong here? :(
In general, it's an antipattern to mix Puppeteer with another selection library like Cheerio. In addition to being redundant, the extra HTML parser doesn't work on the live document as Puppeteer does, so you have to snapshot the HTML at a particular moment with Puppeteer to capture it as a string and plug that string into Cheerio, where it's re-parsed back to a traversible tree structure.
Introducing this extra step creates opportunity for bugs and confusion to creep in, and that's what happened here.
The code
const elements = await getElementsCheerio(
document.body.innerHTML,
data.base,
data.name,
wordsToSearch
);
is problematic. document.body.innerHTML doesn't refer to anything related to Puppeteer. Instead, use Puppeteer's await page.content() to snapshot the HTML.
As a minor point, there's no need for Cheerio functions to be async, because they never use await. It's a fully synchronous API.
Here's a minimal set up for using Cheerio with Puppeteer, assuming you accept the terms and conditions and are sure that intoducing this usually unnecessary layer of indirection is appropriate for your use case:
const cheerio = require("cheerio"); // 1.0.0-rc.12
const puppeteer = require("puppeteer"); // ^19.0.0
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const url = "https://www.example.com";
await page.goto(url, {waitUntil: "domcontentloaded"});
const html = await page.content();
const $ = cheerio.load(html);
// do cheerio stuff synchronously
console.log($("h1").text()); // => Example Domain
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
It's basically the same for puppeteer-cluster: just drop the lines starting with const html = await page.content(); into the cluster.task callback that operates on page.
I'm creating a generator that uses your google catchall domain to generate a list of email accounts and how I'm going about it, is that a function will generate a random first & last name from an array and merges it together with the catchall domain. Essentially the result would be fname + lname + domain = johndoe#domain.com, but for some reason I'm getting an error. The terminal says "Fetch is not defined" but when I define it by either the node-fetch package (const fetch = require('node-fetch');, it then says "fetch is not a function". I was attempting to use the built in Fetch API to fetch the data because the script I'm basing it off of instructed to do so, after the terminal said it wasn't defined, I tried using the node-fetch package to define the variable fetch in hopes of it fixing it, but no luck either. Does anyone have a solution on why I'm getting both fetch is not a function and fetch is not defined?
const prompt = require("prompt-sync") ({sigint: true });
const fs = require("fs").promises;
const request = require('request');
// const fetch = require('node-fetch');
const random_useragent = require('random-useragent');
const { Webhook, MessageBuilder } = require('discord-webhook-node');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());
( async () => {
const browser = await puppeteer.launch({
headless: false,
executablePath: `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`,
userDataDir: `/Users/bran_d0_n/Library/Application Support/Google/Chrome/Default`,
ignoreHTTPSErrors: true,
ignoreDefaultArgs: ['--enable-automation'],
args: [
`--disable-blink-features=AutomationControlled`,
`--enable-blink-feautres=IdleDetection`,
`--window-size=1920,1080`,
`--disable-features=IsolateOrigins,site-per-process`,
`--blink-settings=imagesEnabled=true`
]
});
//------------------ Random Password Generator Function ------------------//
function generatePassword() {
let pass = '';
let str = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' +
'abcdefghijklmnopqrstuvwxyz0123456789##$';
for ( let i = 1; i <= 8; i++) {
var char = Math.floor(Math.random() * str.length + 1);
pass += str.charAt(char)
}
return pass;
}
//------------------ First & Last Name Generator Function ------------------//
async function fetchData(url) {
const response = await fetch(url);
return response.json();
}
async function fetchData(url) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error('Network Response Invalid');
}
return response.json();
} catch (error) {
console.error('Unable To Fetch Data:', error)
}
}
function fetchNames(nameType) {
return fetchData(`https://www.randomlists.com/data/names-${nameType}.json`);
}
function pickRandom(list) {
return list[Math.floor(Math.random() * list.length)];
}
async function generateName(gender) {
try {
const response = await Promise.all ([
fetchNames(gender || pickRandom(['male', 'female'])),
fetchNames('surnames')
]);
const [ firstNames, lastNames] = response;
const firstName = pickRandom(firstNames.data);
const lastName = pickRandom(lastNames.data);
return `${firstName} ${lastName}`;
} catch (error) {
console.error('Unable To Generate Name:', error);
}
}
console.log('Loading Browser...');
// Account Values
var bDayval = '01/05/22' + (Math.floor((Math.random() * ( 99-55 )) + 55 )).toString();
var passwordVal = generatePassword();
var fnameVal = generateName();
var lnameVal = generateName();
var info;
var themessage;
var phoneNum;
var userpass;
Loading and configuring the module
node-fetch from v3 is an ESM-only module - you are not able to import it with require().
If you cannot switch to ESM, please use v2 which remains compatible with CommonJS. Critical bug fixes will continue to be published for v2.
You should either use
import fetch from 'node-fetch';
(Remember to add "type": "module" to the package.json)
Or install the older version
npm install node-fetch#2
I am somewhat new to coding and recently created this script in order to pull data from Zoom and push it to Google Drive via API. I am trying to push this to a Google Cloud Function, but when running it in a Cloud Function and console logging each step in the process, it seems like the uploadFile function, specifically the drive.files.create method, is being skipped. Every other step is being console logged, but neither the err or res is being logged after the drive.files.create method. Google Cloud Functions does not show errors, instead it shows OK and that the function took 1500ms to execute. It works fine on my local machine, I am only having issues in Cloud Functions. Any suggestions on how to get this to act right would be super helpful. Thank you!
const axios = require("axios");
require("dotenv").config();
const stream = require("stream");
const request = require("request");
const { google } = require("googleapis");
const KEYFILEPATH = "./credentials.json";
const SCOPES = ["https://www.googleapis.com/auth/drive"];
const auth = new google.auth.GoogleAuth({
keyFile: KEYFILEPATH,
scopes: SCOPES,
});
let today = new Date().toISOString();
let zoomAccessToken;
let zoomDownloadUrl;
///////////////////////////////////////////////////////////////// Searching for latest Town Hall recording in Google.
const searchFile = async (auth) => {
const service = google.drive({ version: "v3", auth });
const files = [];
try {
const res = await service.files.list({
corpora: "drive",
includeItemsFromAllDrives: true,
supportsAllDrives: true,
driveId: "XXXXXXXXXXXXXXXX",
q: '"XXXXXXXXXXXXXXX" in parents',
fields: "nextPageToken, files(id, name)",
spaces: "drive",
});
Array.prototype.push.apply(files, res.files);
const filesArray = res.data.files;
const filesName = filesArray.map((x) => x.name).sort().reverse()[0];
console.log(filesName);
return filesName;
} catch (err) {
throw err;
}
};
///////////////////////////////////////////////////////////////// Get Zoom OAuth access token.
const getAccessToken = async () => {
return axios({
method: "post",
url: `https://zoom.us/oauth/token?grant_type=account_credentials&account_id=${process.env.ZOOM_ACCOUNT_ID}`,
headers: {
Authorization: "Basic" +new Buffer.from(process.env.ZOOM_CLIENT_ID + ":" + process.env.ZOOM_CLIENT_SECRET).toString("base64"),
},
});
};
///////////////////////////////////////////////////////////////// Get the latest Town Hall recording's data.
const getRecordingData = async () => {
const token = await getAccessToken();
zoomAccessToken = await token.data.access_token;
return axios({
method: "get",
url: "https://api.zoom.us/v2/meetings/XXXXXXXXX/recordings",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${zoomAccessToken}`,
},
});
};
///////////////////////////////////////////////////////////////// Get the latest Town Hall recording's date.
const getRecordingDate = async () => {
const recording = await getRecordingData();
const lastRecordingDate = await recording.data.start_time;
const recordingDateFormatted = `${lastRecordingDate.substring(0,4)}.${lastRecordingDate.substring(5, 7)}.${lastRecordingDate.substring(8,10)} - Town Hall.mp4`;
return recordingDateFormatted;
};
///////////////////////////////////////////////////////////////// Get the latest Town Hall recording's download link.
const zoomDownloadLink = async () => {
const recording = await getRecordingData();
zoomDownloadUrl = `${recording.data.recording_files[0].download_url}?access_token=${zoomAccessToken}`;
return zoomDownloadUrl;
};
///////////////////////////////////////////////////////////////// Upload data from latest Town Hall recording's download link to Google Drive.
const uploadFile = async (auth) => {
const buffer = await zoomDownloadLink();
const bs = new stream.PassThrough();
request(buffer).pipe(bs);
const drive = google.drive({ version: "v3", auth });
var fileMetadata = {
name: `${today.substring(0, 4)}.${today.substring(5, 7)}.${today.substring(8,10)} - Town Hall.mp4`,
parents: ["XXXXXXXXXXXXXXXXX"],
};
var media = {
mimeType: "video/mp4",
body: bs,
};
drive.files.create(
{
resource: fileMetadata,
media: media,
fields: "id",
uploadType: "resumable",
supportsAllDrives: true,
},
function (err, res) {
if (err) {
console.log(err);
} else {
console.log("File Id: ", res.data.id);
}
}
);
};
///////////////////////////////////////////////////////////////// Compares Town Hall files in Google Drive and Zoom. If different, run uploadFile function.
exports.townHall = async () => {
const townHallFile = await searchFile(auth);
const lastRecordingDate = await getRecordingDate();
if (townHallFile != lastRecordingDate) {
await uploadFile(auth);
} else {
console.log("No Recording Today");
}
};
As you are calling an API inside a cloud function which is an async function but does not have a return statement, it will only execute the function but doesn't wait for the response, because the drive.files.create call is running.
So to fix that just need to await the result of the API. Just add
return await statement on the API call
like:
const uploadFile = async (auth) => {
const buffer = await zoomDownloadLink();
const bs = new stream.PassThrough();
request(buffer).pipe(bs);
const drive = google.drive({ version: "v3", auth });
var fileMetadata = {
name: `${today.substring(0, 4)}.${today.substring(5, 7)}.${today.substring(8,10)} - Town Hall.mp4`,
parents: ["XXXXXXXXXXXXXXXXX"],
};
var media = {
mimeType: "video/mp4",
body: bs,
};
return await drive.files.create(
{
resource: fileMetadata,
media: media,
fields: "id",
uploadType: "resumable",
supportsAllDrives: true,
},
function (err, res) {
if (err) {
console.log(err);
} else {
console.log("File Id: ", res.data.id);
}
}
);
};
Also, something important when you are calling APIs inside cloud functions is the time out. Check on your CF time out is enough to wait for the API call response.
Also, you can use the Promise function to force wait for the response:
const result = uploadFile(aut);
const _response = await Promise.all(result);
So i tried following the https://discordjs.guide/additional-info/rest-api.html guide before making my own. But I can't get either to work.
Firstly with /cat it crashes and the console returns with:
SyntaxError: Unexpected end of JSON input
at JSON.parse (<anonymous>)
at getJSONResponse (BOTLOCATION\index.js:77:14)
at processTicksAndRejections (node:internal/process/task_queues:96:5)
at async Client.<anonymous> (BOTLOCATION\index.js:90:20)
And /urban works but no matter what term I enter it returns with NULL.
Here is the code, its nearly identical from the guides apart from the added SlashCommandBuilder and REST.
const { request } = require('undici');
const clientId = 'CLIENTID_HERE';
const guildId = 'GUILDID_HERE';
const { SlashCommandBuilder } = require('#discordjs/builders');
const { REST } = require('#discordjs/rest');
const { Routes } = require('discord-api-types/v9');
const commands = [
new SlashCommandBuilder().setName('cat').setDescription('Cat thing idk'),
new SlashCommandBuilder().setName('urban').setDescription('Urban Dictionary Thing'),
]
.map(command => command.toJSON());
const rest = new REST({ version: '9' }).setToken("TOKEN_HERE");
rest.put(Routes.applicationGuildCommands(clientId, guildId), { body: commands })
//rest.put(Routes.applicationGuildCommands(clientId), { body: commands })
.then(() => console.log('Successfully registered application commands.'))
.catch(console.error);
const trim = (str, max) => (str.length > max ? `${str.slice(0, max - 3)}...` : str);
async function getJSONResponse(body) {
let fullBody = '';
for await (const data of body) {
fullBody += data.toString();
}
return JSON.parse(fullBody);
}
client.on('interactionCreate', async interaction => {
if (!interaction.isCommand()) return;
const { commandName } = interaction;
await interaction.deferReply();
if (commandName === 'cat') {
const catResult = await request('https://aws.random.cat/meow');
const { file } = await getJSONResponse(catResult.body);
interaction.reply({ files: [{ attachment: file, name: 'cat.png' }] });
} else if (commandName === 'urban') {
const term = interaction.options.getString('term');
const query = new URLSearchParams({ term });
const dictResult = await request(`https://api.urbandictionary.com/v0/define?${query}`);
const { list } = await getJSONResponse(dictResult.body);
if (!list.length) {
return interaction.editReply(`No results found for **${term}**.`);
}
const [answer] = list;
const embed = new MessageEmbed()
.setColor('#EFFF00')
.setTitle(answer.word)
.setURL(answer.permalink)
.addFields(
{ name: 'Definition', value: trim(answer.definition, 1024) },
{ name: 'Example', value: trim(answer.example, 1024) },
{
name: 'Rating',
value: `${answer.thumbs_up} thumbs up. ${answer.thumbs_down} thumbs down.`,
},
);
interaction.editReply({ embeds: [embed] });
}
});
So for the cat command since there is a deferReply first we need to use editReply since deferReply counts as the first/initial reply.
await interaction.deferReply();
const catResult = await request('https://aws.random.cat/meow').catch((err) => { console.log(err); });;
const { file } = await getJSONResponse(catResult.body).catch((err) => { console.log(err); });
return await interaction.editReply({ files: [{ attachment: file, name: 'cat.png' }] });
I also added a .catch to the end of each await, this was just for testing however I recommend it.
Now with the urban command, the reason it is using null is since you don't have the string option's text. We can check for it by adding an if statement.
await interaction.deferReply();
const term = interaction.options.getString('term');
if (!term) return await interaction.editReply('Please provide a term.'); // We need to add this check to see if the user provided the term option or not.
const query = new URLSearchParams({ term });
const dictResult = await request(`https://api.urbandictionary.com/v0/define?${query}`);
const { list } = await getJSONResponse(dictResult.body);
if (!list.length) {
return interaction.editReply(`No results found for **${term}**.`);
}
const [answer] = list;
const embed = new MessageEmbed()
.setColor('#EFFF00')
.setTitle(answer.word)
.setURL(answer.permalink)
.addFields(
{ name: 'Definition', value: trim(answer.definition, 1024) },
{ name: 'Example', value: trim(answer.example, 1024) },
{
name: 'Rating',
value: `${answer.thumbs_up} thumbs up. ${answer.thumbs_down} thumbs down.`,
},
);
return await interaction.editReply({ embeds: [embed] });
IMPORTANT: When you are building your slash command you are not setting a string option. In the commands array, when creating the second slash command called urban we will add the support for the string option there. (An example using the string option, discord.js guide all command options)
This is how we can do this:
const commands = [
new SlashCommandBuilder().setName('cat')
.setDescription('Cat thing idk'),
new SlashCommandBuilder()
.setName('urban')
.setDescription('Urban Dictionary Thing')
.addStringOption((option) => option.setName('term').setDescription('term')) // We first add the string option then set the name to 'term' which is what the code calls for and then the description.
].map((command) => command.toJSON());
If you would like to make the term input required, add .setRequired(true) which will not allow the command to be ran without entering the term to search.
Once you do that you should be all good! Tested the code and it's working once that's fixed
I am trying to run a simple javascript code straight from the documentation, but for some reason it's resulting in an error-
"SyntaxError: await is only valid in async functions and the top level bodies of modules"
const { MnemonicKey,Coin,MsgSwap,LCDClient,MsgExecuteContract } = require('#terra-money/terra.js');
const lcd = new LCDClient({
URL: 'https://lcd.terra.dev',
// URL: 'https://falling-empty-waterfall.terra-mainnet.quiknode.pro/94818b47fef9194193d08f7ffdc9e90ffdcdee17/',
chainId: 'columbus-5'
});
const mk = new MnemonicKey({
mnemonic: 'mnemo',
})
// UST <> SCRT
const pool = "terra1tq4mammgkqrxrmcfhwdz59mwvwf4qgy6rdrt46";
// Fetch the number of each asset in the pool.
const { assets } = await lcd.wasm.contractQuery(pool, { pool: {} });
// Calculate belief price using pool balances.
const beliefPrice = (assets[0].amount / assets[1].amount).toFixed(18);
// Swap 1 UST to SCRT with 1% slippage tolerance.
const terraSwap = new MsgExecuteContract(
wallet.key.accAddress,
pool,
{
swap: {
max_spread: "0.01",
offer_asset: {
info: {
native_token: {
denom: "uusd",
},
},
amount: "1000000",
},
belief_price: beliefPrice,
},
},
new Coins({ uusd: '1000000' }),
);
const tx = await wallet.createAndSignTx({ msgs: [terraSwap] });
const result = await lcd.tx.broadcast(tx);
console.log(result);
What could I be missing?
Thank you.
P.s this code is taken straight from the documentation - https://docs.terra.money/docs/develop/sdks/terra-js/common-examples.html
Wrap your code in async function or in async IIFE