Here is my code:
I am trying to login to my linkedln account which is done successfully but i want to go to page that stores my connections and fetch their details but onclick event not working for going to that page. Can anyone help how to proceed?
const puppeteer = require('puppeteer');
const C = require('./constants');
const USERNAME_SELECTOR = '#username';
const PASSWORD_SELECTOR = '#password';
const CTA_SELECTOR = '#app__container > main > div:nth-child(2) > form >
div.login__form_action_container > button';
let selector= '#ember23';
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-
basic_nav-header-signin');
await page.click(USERNAME_SELECTOR);
await page.keyboard.type(C.username);
await page.click(PASSWORD_SELECTOR);
await page.keyboard.type(C.password);
await page.click(CTA_SELECTOR);
await page.waitForNavigation();
await page.evaluate((selector) => document.querySelector(selector).click(), selector);
// await page.click(MY_NETWORK);
await page.screenshot({path: 'linkedln.png'});
await browser.close();
})();
Related
I open the 'deliver to' popup but am not able to click on the input field and enter information.
(async () => {
const browser = await puppeteer.launch({headless: false});
const page = await browser.newPage();
const url = 'https://www.tntsupermarket.com/eng/store-flyer';
await page.goto(url, {waitUntil: 'networkidle0'});
const newPagePromise = new Promise(x => browser.once('targetcreated', target => x(target.page())));
await page.evaluate(()=> {
document.querySelector('span[class="deliverCss-city-FJJ"]').click();
});
const popup = await newPagePromise;
await popup.waitForSelector('input[aria-label="Enter your Postal Code"]');
await popup.focus('input[aria-label="Enter your Postal Code"]');
await popup.click('input[aria-label="Enter your Postal Code"]');
await popup.keyboard.type('a2b');
})();
The pop-up isn't a new page, just a modal element that's shown with JS and without navigation. Removing the navigation promise gives a pretty clear result:
const puppeteer = require("puppeteer"); // ^13.5.1
let browser;
(async () => {
browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
const url = "https://www.tntsupermarket.com/eng/store-flyer";
await page.goto(url, {waitUntil: "networkidle0", timeout: 90000});
const cityEl = await page.waitForSelector('span[class="deliverCss-city-FJJ"]');
await cityEl.evaluate(el => el.click());
const postalSel = 'input[aria-label="Enter your Postal Code"]';
const postalEl = await page.waitForSelector(postalSel);
await postalEl.type("a2b");
await page.waitForTimeout(30000); // just to show that the state is as we wish
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
This is a bit slow; there's an annoying pop-up you might wish to click off instead of using "networkidle0":
// ... same code
await page.goto(url, {waitUntil: "domcontentloaded", timeout: 90000});
const closeEl = await page.waitForSelector("#closeActivityPop");
await closeEl.click();
const cityEl = await page.waitForSelector('span[class="deliverCss-city-FJJ"]');
// same code ...
On quick glance, if the page is cached, the pop-up might not show, so you might want to abort page.waitForSelector("#closeActivityPop"); after 30 seconds or so and continue with the code without clicking on it, depending on how flexible you want the script to be.
Here is my script - it scrapes amazon to check for product availability using puppeteer
const puppeteer = require('puppeteer');
//URL
scrape('https://www.amazon.co.uk/PlayStation-9395003-5-Console/dp/B08H95Y452/ref=sr_1_1?dchild=1&keywords=ps5&qid=1613570801&sr=8-1');
/**
*
* Scrape PS5
*
*/
async function scrape(url)
{
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(url);
//get availability
const [el2] = await page.$x('//*[#id="availability"]/span');
const txt1 = el2.getProperty('textContent');
const rawTxt1 = (await txt1)._remoteObject.value;
//log data
console.log({rawTxt1});
//close browser
browser.close();
}
It returns this to the console after i log it
i want it to return only 'currently unavailable'
Try this
async function scrape(url)
{
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(url);
await page.waitForSelector("div#availability")
const result = await page.evaluate(()=>{
return Array.from(document.querySelectorAll("div#availability span.a-size-medium")).map((el)=> el.innerText)
});
//log data
console.log(result)
//close browser
browser.close();
}
Output:
["Currently unavailable."]
I need to wait until the new page is loaded, which was opened in a new tab after clicking on the button. That is, I click on a button, a new page opens (which should load) and on it I click another button. I have some example code, but it doesn't work for some reason:
const page = await browser.newPage();
await page.goto('https://twitter.com/amazon/');
await page.click('.css-1dbjc4n:nth-child(1) > .css-1dbjc4n > .css-1dbjc4n > .css-901oao > .css-4rbku5',{waitUntil: ['load', 'domcontentloaded', 'networkidle0', 'networkidle2']});
const page2 = (await browser.pages())[2];
await page2.click('#nav-main > .nav-fill > #nav-xshop-container > #nav-xshop > .nav-a:nth-child(2)');
If I understand correctly, the problem is detecting when a new tab ("page") has opened and getting the new page object associated with the tab.
There are at least a couple techniques available. One method is promisifying the browser's "targetcreated" event as described here:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
await page.goto("https://twitter.com/amazon");
const amzSel = `.css-1dbjc4n:nth-child(1) > .css-1dbjc4n >
.css-1dbjc4n > .css-901oao > .css-4rbku5`;
await page.waitForSelector(amzSel, {visible: true});
console.log((await browser.pages()).length); // => 1
// method 1
const newPagePromise = new Promise(resolve =>
browser.once("targetcreated", target => resolve(target.page()))
);
await page.click(amzSel);
const newPage = await newPagePromise;
// --------
console.log((await browser.pages()).length); // => 2
await newPage.waitForSelector("#nav-link-prime", {visible: true});
await newPage.click("#nav-link-prime");
const sel = "#prime-header-CTA-announce";
await newPage.waitForSelector(sel, {visible: true});
console.log(await newPage.$eval(sel, el => el.innerText.trim())); // => TRY PRIME
//await browser.close();
})();
Another approach is to use browser.waitForTarget to check when the target's opener() is the previous page target, as described here:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
await page.goto("https://twitter.com/amazon");
const amzSel = `.css-1dbjc4n:nth-child(1) > .css-1dbjc4n >
.css-1dbjc4n > .css-901oao > .css-4rbku5`;
await page.waitForSelector(amzSel, {visible: true});
console.log((await browser.pages()).length); // => 1
// method 2
const pageTarget = page.target();
await page.click(amzSel);
const newTarget = await browser.waitForTarget(target =>
target.opener() === pageTarget
);
const newPage = await newTarget.page();
// --------
console.log((await browser.pages()).length); // => 2
await newPage.waitForSelector("#nav-link-prime", {visible: true});
await newPage.click("#nav-link-prime");
const sel = "#prime-header-CTA-announce";
await newPage.waitForSelector(sel, {visible: true});
console.log(await newPage.$eval(sel, el => el.innerText.trim())); // => TRY PRIME
//await browser.close();
})();
As an aside, I'm not sure how important/significant this particular Twitter/Amazon example is, but #nav-xshop > .nav-a:nth-child(2) doesn't seem like a reliable selector (it appears to have a race condition between "Best Sellers" and "Prime")--I'd use #nav-link-prime since it's a direct id, if that's what you're looking for.
I'm trying to web scrape a live scores every score change. Can puppeteer do this? If it can what should I add in this code so it returns live data.
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('site to go');
await page.waitForSelector('input[name="username"]');
await page.type('input[name="username"]', 'username');
await page.type('input[name="password"]', 'password');
await page.click('button[type="submit"]');
let score = await page.evaluate(() => document.getElementById("scores").innerHTML);
})();
You could use exposeFunction to register a callback function:
await page.exposeFunction('newScore', s => console.log(s));
Then you can call that function on the DOMSubtreeModified event:
page.evaluate(() => document.getElementById('scores')
.addEventListener('DOMSubtreeModified', () => newScore(element.innerHTML)));
My code:
async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://example.com' , {
waitUntil: 'networkidle2'
});
await page.screenshot({path: 'home.png', fullPage: true});
let frames = await page.frames();
await page.screenshot({path: 'home.png', fullPage: true});
var iFrame = frames.find(f => f.url().indexOf("https://accounts.") > -1);
const usernameInput = await iFrame.waitForSelector("[name=username]" , {visible : true});
await usernameInput.type(email);
const passwordInput = await iFrame.waitForSelector("[name=password]" , {visible : true});
await passwordInput.type(password);
const navigationPromisePortal = page.waitForNavigation();
await iFrame.click(".primary");
await navigationPromisePortal;
var cookies = await page.cookies();
await browser.close();
return cookies;
}
I'm trying to get cookies from example.com. But every time it accounts iFrame is not loaded. This page is built by react. How do I solve this problem?