We Keep Coding

how to download a generated pdf file using playwright?

I'm trying to download a file using playwright, this file is generated which means the url doesn't ends with a '.pdf' and forcing this link in the url doesn't start the download. I was abble to navigate until the pdf is generated but how can I download it now ?
here is my code:
const {chromium} = require('playwright');
const path = require('path');
(async () => {
const browser = await chromium.launch({
headless: true,
});
const context = await browser.newContext({
acceptDownloads: true
});
const page = await context.newPage();
const downloadPath = path.join(__dirname, 'edt');
await page.goto('https://planif.eppe.ui/jsp/custom/eppee/easyMyPlanning.jsp');
await page.getByLabel('Identifiant :').click();
await page.getByLabel('Identifiant :').fill('myusername');
await page.getByLabel('Identifiant :').press('Tab');
await page.getByLabel('Mot de passe :').fill('mysecretpassword');
await page.getByLabel('Mot de passe :').press('Enter');
await page.getByRole('combobox').selectOption('10');
await page.getByRole('button', { name: 'Submit' }).click();
await page.locator('.footer > a').click();
const page1Promise = page.waitForEvent('popup');
await page.frameLocator('frame[name="link"]').getByRole('link', { name: 'Export PDF...' }).click();
const page1 = await page1Promise;
const downloadPromise = page1.waitForEvent('download');
await page1.getByRole('button', { name: 'Submit' }).click();
const download = await downloadPromise;
await download.saveAs(path.join(downloadPath, 'edt.pdf'));
await browser.close();
})();
with this code I end up having the browser waiting for me to try to download the file how can I automate this ?
ps: with this code and with my browser in non headless mode I end up on this page end if I manualy press the download button it works

Puppeteer-cluster with cheerio in express router API returns empty response

I'm writing an API with express, puppeteer-cluster and cheerio that returns all anchor elements containing one or more words that can be added as query parameters. I want to use puppeteer in order to get elements that are javascript generated too. But for some reason it's not working, I get an empty array as an output printed on the browser.
I'm still trying to understand this library but has been 2 days and I made no progress. Any help is deeply appreciated.
Update: I added async to all my functions and they run now, but the result is still empty :(
Update 2: I started logging everything, every step and found that data.name is being passed to the cheerio function as a Promise. '-' I think that is the problem, but don't know how to fix it yet.
Update 3: One of the issues was that the page content (html code) was not being handled properly to the cheerio function. In the browser, however, the response is empty and the console shows an error:
Error handling response: TypeError: Cannot read properties of
undefined (reading 'innerText').
So, I think the response is not json formatted. Is res.json() not the right way to do it?
My code:
app.js
const PORT = process.env.PORT || 8000;
var path = require("path");
const express = require("express");
// Routes
const indexRouter = require("./routes/index");
const allNews = require("./routes/news");
const clusterRouter = require("./routes/cluster");
const app = express();
app.use(cors());
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(express.static(path.join(__dirname, "public")));
app.use("/", indexRouter);
app.use("/news", allNews);
app.use("/cluster", clusterRouter);
app.listen(PORT, () => console.log(`server running on PORT ${PORT}`));
cluster.js
const express = require("express");
const { Cluster } = require("puppeteer-cluster");
const puppeteer = require("puppeteer-extra");
const cheerio = require("cheerio");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
var router = express.Router();
const newspapers = [
{
"name": "CNN",
"address": "https://edition.cnn.com/specials/world/cnn-climate",
"base": "https://edition.cnn.com"
},
{
"name": "The Guardian",
"address": "https://www.theguardian.com/environment/climate-crisis",
"base": "https://www.theguardian.com"
}]
const app = express();
puppeteer.use(StealthPlugin());
const result = [];
router.get("/", async (req, res) => {
(async () => {
// Query String
const query = checkForQuery(req);
const wordsToSearch = query ? verifyQuery(query) : "";
console.log("Running tests.."); // This is printed on console
//Functions
function checkForQuery(request) {
if (request.originalUrl.indexOf("?") !== -1) {
console.log(request.query);
return request.query;
} else {
return false;
}
}
// // Validates query and remove invalid values
function verifyQuery(queryString) {
const queryParams = {
only: queryString.only ? queryString.only : "",
also: queryString.also ? queryString.also : "",
};
// Creates new list containing valid terms for search
var newList = {
only: [],
also: [],
};
for (const [key, value] of Object.entries(queryParams)) {
const tempId = key.toString();
const tempVal =
queryParams[tempId].length >= 2
? queryParams[tempId].split(",")
: queryParams[tempId];
console.log(queryParams[tempId], " and ", tempVal);
if (tempVal.length > 1) {
console.log("helloooooo");
tempVal.forEach((term) => {
if (topics.indexOf(term) != -1) {
newList[tempId].push(term);
}
});
} else {
if (topics.indexOf(queryParams[tempId]) != -1) {
newList[tempId].push(queryParams[tempId]);
}
}
}
console.log(newList);
return newList;
}
function storeData(element, base, name) {
const results = [];
element.find("style").remove();
const title = element.text();
const urlRaw = element.attr("href");
const url =
urlRaw.includes("www") || urlRaw.includes("http")
? urlRaw
: base + urlRaw;
// Check for duplicated url
if (tempUrls.indexOf(url) === -1) {
// Check for social media links and skip
if (!exceptions.some((el) => url.toLowerCase().includes(el))) {
tempUrls.push(url);
// Get img if child of anchor tag
const imageElement = element.find("img");
if (imageElement.length > 0) {
// Get the src attribute of the image element
results.push({
title: title.replace(/(\r\n|\n|\r)/gm, ""),
url,
source: name,
imgUrl: getImageFromElement(imageElement),
});
} else {
results.push({
title: title.replace(/(\r\n|\n|\r)/gm, ""),
url: url,
source: name,
});
}
}
}
return results;
}
function getElementsCheerio(html, base, name, searchterms) {
console.log(html, base, name);
const $ = cheerio.load(html);
console.log(searchterms);
const concatInfo = [];
if (searchterms) {
const termsAlso = searchterms.also;
const termsOnly = searchterms.only;
termsAlso.forEach((term) => {
$(`a:has(:contains("climate"):contains(${term}))`).each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
});
termsOnly.forEach((term) => {
// $(`a:has(:contains(${term}))`).each(function () {
$(`a:contains(${term})`).each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
});
} else {
$('a:contains("climate")').each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
}
return concatInfo;
}
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 2,
puppeteerOptions: {
headless: true,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
userDataDir: "./tmp",
defaultViewport: false,
},
});
await cluster.task(async ({ page, data }) => {
await page.goto(data.address);
await page.waitForSelector("body");
// console.log here prints that data.name is a Promise :(
const elements = await getElementsCheerio(
document.body.innerHTML,
data.base,
data.name,
wordsToSearch
);
result.push(elements);
});
newspapers.map((newspaper) => {
console.log("queue" + newspaper); // This logs correctly: queue[object Object]
cluster.queue(newspaper);
});
await cluster.idle();
await cluster.close();
// Display final object
res.json(result);
})();
});
module.exports = router;
I don't get any errors, but on screen I get an empty [ ]. Anyone can see what I am doing wrong here? :(

In general, it's an antipattern to mix Puppeteer with another selection library like Cheerio. In addition to being redundant, the extra HTML parser doesn't work on the live document as Puppeteer does, so you have to snapshot the HTML at a particular moment with Puppeteer to capture it as a string and plug that string into Cheerio, where it's re-parsed back to a traversible tree structure.
Introducing this extra step creates opportunity for bugs and confusion to creep in, and that's what happened here.
The code
const elements = await getElementsCheerio(
document.body.innerHTML,
data.base,
data.name,
wordsToSearch
);
is problematic. document.body.innerHTML doesn't refer to anything related to Puppeteer. Instead, use Puppeteer's await page.content() to snapshot the HTML.
As a minor point, there's no need for Cheerio functions to be async, because they never use await. It's a fully synchronous API.
Here's a minimal set up for using Cheerio with Puppeteer, assuming you accept the terms and conditions and are sure that intoducing this usually unnecessary layer of indirection is appropriate for your use case:
const cheerio = require("cheerio"); // 1.0.0-rc.12
const puppeteer = require("puppeteer"); // ^19.0.0
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const url = "https://www.example.com";
await page.goto(url, {waitUntil: "domcontentloaded"});
const html = await page.content();
const $ = cheerio.load(html);
// do cheerio stuff synchronously
console.log($("h1").text()); // => Example Domain
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
It's basically the same for puppeteer-cluster: just drop the lines starting with const html = await page.content(); into the cluster.task callback that operates on page.

Electron Spotify not opening in desktop mode

Electron is not opening spotify in desktop mode, as you can see in the screenshot below.
Here is the code:
const {BrowserWindow, app} = require("electron");
const pie = require("puppeteer-in-electron")
const puppeteer = require("puppeteer-core");
const fs = require("fs");
const path = require("path");
const main = async () => {
const cookiesPath = path.join(__dirname, "cookies/open.spotify.com.cookies.json");
const cookies = JSON.parse(await fs.readFileSync(cookiesPath, 'utf8'));
await pie.initialize(app);
const browser = await pie.connect(app, puppeteer);
const window = new BrowserWindow();
const url = "https://example.com/";
await window.loadURL(url);
const page = await pie.getPage(browser, window);
await page.goto("https://open.spotify.com");
for (const cookie of cookies) {
if (cookie.name !== 'ig_lang') {
await page.setCookie(cookie);
}
}
await page.reload();
};
main();
Note I'm using puppeteer-in-electron so that I can automate web process even in electron.
But, this is not an issue because even if I use electron normally without puppeteer the issue persists.
This is how it should've been: https://cdn.discordapp.com/attachments/1026704902925324410/1026710664611377202/unknown.png
This is how it is: https://cdn.discordapp.com/attachments/1026704902925324410/1026704903055343626/Screenshot_42.png
Hope I've explained it well.
Thanks

install package "https://github.com/castlabs/electron-releases#v20.0.0+wvcus"
like this:
npm install "https://github.com/castlabs/electron-releases#v20.0.0+wvcus" --save-dev
import also components
const { BrowserWindow, app, components } = require("electron");
to open in desktop mode just add userAgent:
window.loadURL(url, {
userAgent: "Chrome/105.0.0.0",
});
and create BrowserWindow after app and components are ready
app.whenReady().then(async () => {
await components.whenReady();
main();
});
full code:
const { BrowserWindow, app, components } = require("electron");
const pie = require("puppeteer-in-electron");
const puppeteer = require("puppeteer-core");
const fs = require("fs");
const path = require("path");
pie.initialize(app);
const main = async () => {
const cookiesPath = path.join(
__dirname,
"cookies/open.spotify.com.cookies.json",
);
const cookies = JSON.parse(await fs.readFileSync(cookiesPath, "utf8"));
const browser = await pie.connect(app, puppeteer);
const window = new BrowserWindow();
const url = "https://example.com/";
await window.loadURL(url, {
userAgent: "Chrome/105.0.0.0",
});
const page = await pie.getPage(browser, window);
await page.goto("https://open.spotify.com");
for (const cookie of cookies) {
if (cookie.name !== "ig_lang") {
await page.setCookie(cookie);
}
}
await page.reload();
};
app.whenReady().then(async () => {
await components.whenReady()
main();
});
short version of code:
const { BrowserWindow, app, components } = require("electron");
const main = () => {
const window = new BrowserWindow();
const url = "https://open.spotify.com";
window.loadURL(url, {
userAgent: "Chrome/105.0.0.0",
});
};
app.whenReady().then(async () => {
await components.whenReady();
main();
});

How get the selector of an element from a web page with more than one document html?

I try get information from a web page using puppeteer, but in I don't to find the selector tha I need, I suppose that's because the page contain more than one documents html and I can't to find the way for to get the data that I need.
the inpection of the page
that´s the code:
const puppeteer = require('puppeteer');
(async ()=>{
const browser = await puppeteer.launch({headless:false});
const page = await browser.newPage();
await page.goto('https://www.arrivia.com/careers/job-openings/');
await page.waitForSelector('.job-search-result');
const data = await page.evaluate(()=>{
const elements = document.querySelectorAll('.job-search-result .job-btn-container a');
vacancies = [];
for(element of elements){
vacancies.push(element.href);
}
return vacancies;
});
console.log(data.length);
const vacancies = [];
for (let i = 0; i <=2; i++){
var urljob = data[i];
await page.goto(data[i]);
await page.waitForSelector(".app-title"); //that´s one of the selectors that I can´t to find
from here I get an error`enter code here`
const jobs = await page.evaluate((urljob)=> {
const job = {};
job.title = document.querySelector(".app-title").innerText;
job.location = document.querySelector(".location").innerText;
job.url = urljob;
return job;close
});
vacancies.push(jobs);
}
console.log(vacancies);
//await page.screenshot({ path: 'xx1.jpg'});
await browser.close()
})();

Iframes are not always the easiest things to deal with, in Puppeteer. But a way to bypass this could be to access directly the URL of the iframe, instead of accessing the page which hosts the iframe. It's also faster:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({ headless: false, defaultViewport: null });
const page = await browser.newPage();
await page.goto("https://www.arrivia.com/careers/job-openings/", {
waitUntil: "domcontentloaded",
});
const jobUrls = await page.$$eval(".job-search-result .job-btn-container a",
els => els.map(el => el.href));
const vacancies = [];
for (let i = 0; i < 10; i++) { // don't forget to replace 10 with jobUrls.length later
const url = jobUrls[i];
const jobId = /job_id=(\d+)/.exec(url)[1]; // Extract the ID from the link
await page.goto(
`https://boards.greenhouse.io/embed/job_app?token=${jobId}`, // Go to iframe URL
{ waitUntil: "domcontentloaded" }
);
vacancies.push({
title: await page.$eval(".app-title", el => el.innerText),
location: await page.$eval(".location", el => el.innerText),
url,
});
}
console.log(vacancies);
await browser.close();
})();
Output:
[
{
title: 'Director of Account Management',
location: 'Scottsdale, AZ',
url: 'https://www.arrivia.com/careers/job/?job_id=2529695'
},
{
title: "Site Admin and Director's Assistant",
location: 'Albufeira, Portugal',
url: 'https://www.arrivia.com/careers/job/?job_id=2540303'
},
...
]

Make axios wait for redirection

I'm trying to scrape data from google map using the /search/ from maps.
When I search it myself, I tape this:
'https://www.google.com/maps/search/new york'
And then I'm redirected to this url:
'https://www.google.com/maps/place/New+York,+%C3%89tat+de+New+York,+%C3%89tats-Unis/#40.6974881,-73.979681,10z/data=!3m1!4b1!4m5!3m4!1s0x89c24fa5d33f083b:0xc80b8f06e177fe62!8m2!3d40.7127753!4d-74.0059728'
I can't manage to reproduce this behavior using axios. I guess there's maybe something to make with async / await but i didn't find any solution since then.
here's my code :
const axios = require('axios');
const cheerio = require('cheerio');
var map = 'www.google.com/maps/search/';
axios.get(map + 'New York')
.then(response => {
let getData = html => {
coor = [];
v= -1;
const $ = cheerio.load(html);
$('.widget-pane-content scrollable-y').each((i, elem) => {
coor.push({
adress : $(elem).find('span.widget-pane-link').text(),
});
});
console.log(coor);
console.log(coor.length);
}
getData(response.coor);
})
.catch(error => {console.log(error);})
When I execute the file, I get this error:
'Error: Request failed with status code 400'
If you have any clue to solve my problem, thanks for sharing it!

Look into a tool like Selenium, or Cypress.js (a wrapper around selenium)
(search "end to end testing" or "automated browser")
This unfortunately cannot be done with a tool like Axios. Google Maps doesn't return a redirect response, but rather uses JavaScript to reload the page.
example with cypress:
cy.visit("https://www.google.com/maps/search/new york");
cy.wait(2000); // sit for 2 seconds
cy.get('#thing-im-looking-for')

Try using cookieJar with axios
https://github.com/axios/axios/issues/943#issuecomment-599174929
Or set jar to true with request package
request(url, { jar: true })
https://stackoverflow.com/a/48912841/11686526

You can't get places info from Google Maps using axios, because results builds on the page via JavaScript, so you need to use some browser automation, e.g. Puppeteer. In the code below I show you how you can do this (also check it on the online IDE):
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());
const requestParams = {
baseURL: `http://google.com`,
query: "starbucks", // what we want to search
coordinates: "#47.6040174,-122.1854488,11z", // parameter defines GPS coordinates of location where you want your query to be applied
hl: "en", // parameter defines the language to use for the Google maps search
};
async function scrollPage(page, scrollContainer) {
let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
while (true) {
await page.evaluate(`document.querySelector("${scrollContainer}").scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
await page.waitForTimeout(5000);
let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
if (newHeight === lastHeight) {
break;
}
lastHeight = newHeight;
}
}
async function fillDataFromPage(page) {
const dataFromPage = await page.evaluate(() => {
return Array.from(document.querySelectorAll(".bfdHYd")).map((el) => {
const placeUrl = el.parentElement.querySelector(".hfpxzc")?.getAttribute("href");
const urlPattern = /!1s(?<id>[^!]+).+!3d(?<latitude>[^!]+)!4d(?<longitude>[^!]+)/gm; // https://regex101.com/r/KFE09c/1
const dataId = [...placeUrl.matchAll(urlPattern)].map(({ groups }) => groups.id)[0];
const latitude = [...placeUrl.matchAll(urlPattern)].map(({ groups }) => groups.latitude)[0];
const longitude = [...placeUrl.matchAll(urlPattern)].map(({ groups }) => groups.longitude)[0];
return {
title: el.querySelector(".qBF1Pd")?.textContent.trim(),
rating: el.querySelector(".MW4etd")?.textContent.trim(),
reviews: el.querySelector(".UY7F9")?.textContent.replace("(", "").replace(")", "").trim(),
type: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(1) > span:first-child")?.textContent.replaceAll("·", "").trim(),
address: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(1) > span:last-child")?.textContent.replaceAll("·", "").trim(),
openState: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(3) > span:first-child")?.textContent.replaceAll("·", "").trim(),
phone: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(3) > span:last-child")?.textContent.replaceAll("·", "").trim(),
website: el.querySelector("a[data-value]")?.getAttribute("href"),
description: el.querySelector(".W4Efsd:last-child > .W4Efsd:nth-of-type(2)")?.textContent.replace("·", "").trim(),
serviceOptions: el.querySelector(".qty3Ue")?.textContent.replaceAll("·", "").replaceAll(" ", " ").trim(),
gpsCoordinates: {
latitude,
longitude,
},
placeUrl,
dataId,
};
});
});
return dataFromPage;
}
async function getLocalPlacesInfo() {
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
const URL = `${requestParams.baseURL}/maps/search/${requestParams.query}/${requestParams.coordinates}?hl=${requestParams.hl}`;
await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForNavigation();
const scrollContainer = ".m6QErb[aria-label]";
const localPlacesInfo = [];
await page.waitForTimeout(2000);
await scrollPage(page, scrollContainer);
localPlacesInfo.push(...(await fillDataFromPage(page)));
await browser.close();
return localPlacesInfo;
}
getLocalPlacesInfo().then((result) => console.dir(result, { depth: null }));
Output
[
{
"title":"Starbucks",
"rating":"4.4",
"reviews":"210",
"type":"Coffee shop",
"address":"3300 W McGraw St",
"openState":"Closed ⋅ Opens 6AM",
"phone":"(206) 298-3390",
"description":"Iconic Seattle-based coffeehouse chain",
"serviceOptions":"Dine-in Takeout Delivery",
"gpsCoordinates":{
"latitude":"47.639704",
"longitude":"-122.399869"
},
"placeUrl":"https://www.google.com/maps/place/Starbucks/data=!4m7!3m6!1s0x54901580f2d8ba8b:0xcc4a61a86f6d87ec!8m2!3d47.639704!4d-122.399869!16s%2Fg%2F1td6mc1x!19sChIJi7rY8oAVkFQR7Idtb6hhSsw?authuser=0&hl=en&rclk=1",
"dataId":"0x54901580f2d8ba8b:0xcc4a61a86f6d87ec"
},
{
"title":"Starbucks",
"rating":"4.3",
"reviews":"201",
"type":"Coffee shop",
"address":"701 5th Ave",
"openState":"Closed ⋅ Opens 5:30AM Mon",
"phone":"(206) 447-9934",
"description":"Iconic Seattle-based coffeehouse chain",
"serviceOptions":"Dine-in Takeout No delivery",
"gpsCoordinates":{
"latitude":"47.604155",
"longitude":"-122.330827"
},
"placeUrl":"https://www.google.com/maps/place/Starbucks/data=!4m7!3m6!1s0x54906ab0bab91e09:0xd1284ac9106e9c7e!8m2!3d47.604155!4d-122.330827!16s%2Fg%2F1tdmk5c9!19sChIJCR65urBqkFQRfpxuEMlKKNE?authuser=0&hl=en&rclk=1",
"dataId":"0x54906ab0bab91e09:0xd1284ac9106e9c7e"
},
... and other places
]
You can read more about scraping Google Maps Places from my blog post Web Scraping Google Maps Places with Nodejs.