My code:
async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://example.com' , {
waitUntil: 'networkidle2'
});
await page.screenshot({path: 'home.png', fullPage: true});
let frames = await page.frames();
await page.screenshot({path: 'home.png', fullPage: true});
var iFrame = frames.find(f => f.url().indexOf("https://accounts.") > -1);
const usernameInput = await iFrame.waitForSelector("[name=username]" , {visible : true});
await usernameInput.type(email);
const passwordInput = await iFrame.waitForSelector("[name=password]" , {visible : true});
await passwordInput.type(password);
const navigationPromisePortal = page.waitForNavigation();
await iFrame.click(".primary");
await navigationPromisePortal;
var cookies = await page.cookies();
await browser.close();
return cookies;
}
I'm trying to get cookies from example.com. But every time it accounts iFrame is not loaded. This page is built by react. How do I solve this problem?
Related
Im trying to scrap a web page, and after following some tutorials, i found how to scrap different products, and how to change the page, but not at the same time. I tryed some ways to do it, but couln't find out.
This is my scraping code:
const puppeteer = require('puppeteer');
const xlsx = require("xlsx");
async function getPageData(url,page){
await page.goto(url);
const h1 = await page.$eval(".product_main h1", h1 => h1.textContent);
const price = await page.$eval(".price_color", price => price.textContent);
const instock = await page.$eval(".instock.availability", instock => instock.innerText);
return {
title: h1,
price: price,
instock: instock
}
//await browser.close();
};
async function getLinks(){
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://books.toscrape.com/');
const links = await page.$$eval('.product_pod .image_container a', allAs =>
allAs.map(a => a.href));
}
async function main(){
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
const data = await getPageData("https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",page);
console.log(data);
}
main();
And here is my code to change the page:
const puppeteer = require('puppeteer');
const xlsx = require("xlsx");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://books.toscrape.com/');
while(await page.$(".pager .next a")){
await page.click(".pager .next a");
await page.waitForTimeout(3000);
}
})();
I open the 'deliver to' popup but am not able to click on the input field and enter information.
(async () => {
const browser = await puppeteer.launch({headless: false});
const page = await browser.newPage();
const url = 'https://www.tntsupermarket.com/eng/store-flyer';
await page.goto(url, {waitUntil: 'networkidle0'});
const newPagePromise = new Promise(x => browser.once('targetcreated', target => x(target.page())));
await page.evaluate(()=> {
document.querySelector('span[class="deliverCss-city-FJJ"]').click();
});
const popup = await newPagePromise;
await popup.waitForSelector('input[aria-label="Enter your Postal Code"]');
await popup.focus('input[aria-label="Enter your Postal Code"]');
await popup.click('input[aria-label="Enter your Postal Code"]');
await popup.keyboard.type('a2b');
})();
The pop-up isn't a new page, just a modal element that's shown with JS and without navigation. Removing the navigation promise gives a pretty clear result:
const puppeteer = require("puppeteer"); // ^13.5.1
let browser;
(async () => {
browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
const url = "https://www.tntsupermarket.com/eng/store-flyer";
await page.goto(url, {waitUntil: "networkidle0", timeout: 90000});
const cityEl = await page.waitForSelector('span[class="deliverCss-city-FJJ"]');
await cityEl.evaluate(el => el.click());
const postalSel = 'input[aria-label="Enter your Postal Code"]';
const postalEl = await page.waitForSelector(postalSel);
await postalEl.type("a2b");
await page.waitForTimeout(30000); // just to show that the state is as we wish
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
This is a bit slow; there's an annoying pop-up you might wish to click off instead of using "networkidle0":
// ... same code
await page.goto(url, {waitUntil: "domcontentloaded", timeout: 90000});
const closeEl = await page.waitForSelector("#closeActivityPop");
await closeEl.click();
const cityEl = await page.waitForSelector('span[class="deliverCss-city-FJJ"]');
// same code ...
On quick glance, if the page is cached, the pop-up might not show, so you might want to abort page.waitForSelector("#closeActivityPop"); after 30 seconds or so and continue with the code without clicking on it, depending on how flexible you want the script to be.
I have this code:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto("https://www.sisal.it/scommesse-matchpoint/quote/calcio/serie-a");
const [button1] = await
page.$x('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
button1.click();
const [button2] = await page.$x('//div[#class="listItem_container__2IdVR white
marketList_listItemHeight__1aiAJ marketList_bgColorGrey__VdrVK"]/p[text()="1X2
ESITO FINALE"]');
button2.click();
})();
The proble is that after clicking button1 the page change and puppeteer executes immediately the following line of code, instead I want it to wait for the new page to be loaded becuase otherwise It will throw an error since It can't find button2.
I found this solution on stackoverflow:
const puppeteer = require("puppeteer");
function delay(time) {
return new Promise(function (resolve) {
setTimeout(resolve, time);
});
}
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto("https://www.sisal.it/scommesse-matchpoint/quote/calcio/serie-a");
const [button1] = await
page.$x('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
button1.click();
await delay(4000);
const [button2] = await page.$x('//div[#class="listItem_container__2IdVR white
marketList_listItemHeight__1aiAJ
marketList_bgColorGrey__VdrVK"]/p[text()="1X2
ESITO FINALE"]');
button2.click();
})();
But of course this in't the best solution.
I think you have to modify a bit in your code:
await button1.click();
await page.waitForNavigation({waitUntil: 'networkidle2'});
For reference, see the documentation.
I found a solution, here's the code:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto("https://www.sisal.it/scommesse
matchpoint/quote/calcio/serie-a");
await page.waitForXPath('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
const [button1] = await page.$x('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
await button1.click();
await page.waitForXPath('//div[#class="listItem_container__2IdVR white marketList_listItemHeight__1aiAJ marketList_bgColorGrey__VdrVK"]/p[text()="1X2 ESITO FINALE"]');
const [button2] = await page.$x('//div[#class="listItem_container__2IdVR white marketList_listItemHeight__1aiAJ marketList_bgColorGrey__VdrVK"]/p[text()="1X2 ESITO FINALE"]');
button2.click();
})();
I'm doing experiments for a bot im making but for some reason things i cant get it to type into the input box in youtube.
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://www.youtube.com/?hl=hr&gl=HR');
await page.waitForNavigation({
waitUntil: 'networkidle0',
});
await page.type('#search', `text`)
// await browser.waitForTarget(() => false)
// await browser.close();
})();
The #search is the id for the youtube search bar but it isn't working for some reason
Your problem is waitUntil: 'networkidle0'
async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://www.youtube.com/?hl=hr&gl=HR');
await page.waitForSelector('input#search')
await page.type('input#search', `text`)
})();
Here is my code:
I am trying to login to my linkedln account which is done successfully but i want to go to page that stores my connections and fetch their details but onclick event not working for going to that page. Can anyone help how to proceed?
const puppeteer = require('puppeteer');
const C = require('./constants');
const USERNAME_SELECTOR = '#username';
const PASSWORD_SELECTOR = '#password';
const CTA_SELECTOR = '#app__container > main > div:nth-child(2) > form >
div.login__form_action_container > button';
let selector= '#ember23';
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-
basic_nav-header-signin');
await page.click(USERNAME_SELECTOR);
await page.keyboard.type(C.username);
await page.click(PASSWORD_SELECTOR);
await page.keyboard.type(C.password);
await page.click(CTA_SELECTOR);
await page.waitForNavigation();
await page.evaluate((selector) => document.querySelector(selector).click(), selector);
// await page.click(MY_NETWORK);
await page.screenshot({path: 'linkedln.png'});
await browser.close();
})();