Been trying out puppeteer to login to a certain site but nothing I threw at it worked ... it's doing something really strange. Tried all kinds of clicks on element , moving mouse to the position of the element and doing mouse down / up , click , doing enter event from keyboard. Waiting for everything to load. I think there's something missing or the javascript is interpreted differently somehow inside the headless browser ?
Here's how it looks like after performing the login , screenshot took using puppeteer
const puppeteer = require('puppeteer');
//const {installMouseHelper} = require('./install-mouse-helper');
async function run () {
const browser = await puppeteer.launch({
headless: true,
ignoreHTTPSErrors: true,
args: ['--no-sandbox', '--disable-setuid-sandbox'],
});
const page = await browser.newPage();
//await installMouseHelper(page);
await page.goto("https://dash.bdroppy.com/login?lang=EN", {waitUntil: 'networkidle2'});
await page.waitFor('input[name=email]');
await page.$eval('input[name=email]', el => el.value = 'test#test.com');
await page.$eval('input[name=password]', el => el.value = 'abcdefg');
// const loginButton = await page.$('.AuthSubmit');
// const rect = await page.evaluate((loginButton) => {
// const {top, left, bottom, right} = loginButton.getBoundingClientRect();
// return {top, left, bottom, right};
// }, loginButton);
// const x = (rect.top + rect.right) / 2;
// const y = (rect.left + rect.bottom) / 2;
// console.log(x,y);
// console.log(rect);
await page.mouse.move(290, 539);
await page.mouse.click(290, 539, { button: 'left' })
// console.log(await page.evaluate( async() => {
// const loginbutton = document.elementFromPoint(290, 533);
// await page.waitFor(3000);
// await page.mouse.down();
// await page.mouse.up();
// }));
//await page.mouse.down();
//await page.mouse.up();
// await page.$$eval(selectors[0].click())
await page.waitFor(3000);
//await page.click('.AuthSubmit');
// await page.click('.AuthSubmit');
// await page.$eval('.AuthSubmit CC', el => {
// console.log(el);
// el.click()
// });
//await page.keyboard.press('Enter');
// await page.waitForNavigation();
await page.screenshot({path: 'screenshot.png'});
// page.on('dialog', async (dialog) => {
// console.log(dialog.defaultValue());
// // dialog.accept("Stack Overflow!");
// });
await browser.close();
}
run();
Here I used input type. This will get input by name.
Tested and working fine.
await page.goto("https://dash.bdroppy.com/login?lang=EN", {waitUntil: 'networkidle2'});
await page.waitFor('input[name=email]');
await page.type('input[name=email]', 'test comment', {delay: 20})
//await page.$eval('input[name=email]', el => el.value = 'test#test.com');
await page.type('input[name=password]', 'test comment', {delay: 20})
//add some delay here. This should work
await page.click('div[class="AuthSubmit CC "]');
Maybe parent element is changing but there is only one email/password input. We can use this.
Related
I open the 'deliver to' popup but am not able to click on the input field and enter information.
(async () => {
const browser = await puppeteer.launch({headless: false});
const page = await browser.newPage();
const url = 'https://www.tntsupermarket.com/eng/store-flyer';
await page.goto(url, {waitUntil: 'networkidle0'});
const newPagePromise = new Promise(x => browser.once('targetcreated', target => x(target.page())));
await page.evaluate(()=> {
document.querySelector('span[class="deliverCss-city-FJJ"]').click();
});
const popup = await newPagePromise;
await popup.waitForSelector('input[aria-label="Enter your Postal Code"]');
await popup.focus('input[aria-label="Enter your Postal Code"]');
await popup.click('input[aria-label="Enter your Postal Code"]');
await popup.keyboard.type('a2b');
})();
The pop-up isn't a new page, just a modal element that's shown with JS and without navigation. Removing the navigation promise gives a pretty clear result:
const puppeteer = require("puppeteer"); // ^13.5.1
let browser;
(async () => {
browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
const url = "https://www.tntsupermarket.com/eng/store-flyer";
await page.goto(url, {waitUntil: "networkidle0", timeout: 90000});
const cityEl = await page.waitForSelector('span[class="deliverCss-city-FJJ"]');
await cityEl.evaluate(el => el.click());
const postalSel = 'input[aria-label="Enter your Postal Code"]';
const postalEl = await page.waitForSelector(postalSel);
await postalEl.type("a2b");
await page.waitForTimeout(30000); // just to show that the state is as we wish
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
This is a bit slow; there's an annoying pop-up you might wish to click off instead of using "networkidle0":
// ... same code
await page.goto(url, {waitUntil: "domcontentloaded", timeout: 90000});
const closeEl = await page.waitForSelector("#closeActivityPop");
await closeEl.click();
const cityEl = await page.waitForSelector('span[class="deliverCss-city-FJJ"]');
// same code ...
On quick glance, if the page is cached, the pop-up might not show, so you might want to abort page.waitForSelector("#closeActivityPop"); after 30 seconds or so and continue with the code without clicking on it, depending on how flexible you want the script to be.
I try to use the code from this question, but it doesn't work out.
The waitForFunction seems to get skipped and is not evaluated.
chrome://downloads/ is shown, the file still downloads and the script ends.
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
headless: false, slowMo: 100, // Uncomment to visualize test
});
const page = await browser.newPage();
await page.goto('https://speed.hetzner.de/');
// Resize window to 1588 x 901 and await navigation
await page.setViewport({ width: 1588, height: 901 });
// Click on <a> "10MB.bin"
await page.waitForSelector('[href="10GB.bin"]');
await page.click('[href="10GB.bin"]');
dmPage = await browser.newPage()
await dmPage.goto('chrome://downloads/')
await dmPage.bringToFront()
await dmPage.waitForFunction(() => {
// monitoring the state of the first download item
// if finish than return true; if fail click
const dm = document.querySelector('downloads-manager').shadowRoot
const firstItem = dm.querySelector('#frb0')
if (firstItem) {
const thatArea = firstItem.shadowRoot.querySelector('.controls')
const atag = thatArea.querySelector('a')
if (atag && atag.textContent === 'Show in folder') {
return true
}
const btn = thatArea.querySelector('cr-button')
if (btn && btn.textContent === 'Retry') {
btn.click()
}
}
},
{ polling: 'raf', timeout: 0 }, // polling? yes. there is a 'polling: "mutation"' which kind of async
)
console.log('finish')
// await browser.close();
})();
I need to wait until the new page is loaded, which was opened in a new tab after clicking on the button. That is, I click on a button, a new page opens (which should load) and on it I click another button. I have some example code, but it doesn't work for some reason:
const page = await browser.newPage();
await page.goto('https://twitter.com/amazon/');
await page.click('.css-1dbjc4n:nth-child(1) > .css-1dbjc4n > .css-1dbjc4n > .css-901oao > .css-4rbku5',{waitUntil: ['load', 'domcontentloaded', 'networkidle0', 'networkidle2']});
const page2 = (await browser.pages())[2];
await page2.click('#nav-main > .nav-fill > #nav-xshop-container > #nav-xshop > .nav-a:nth-child(2)');
If I understand correctly, the problem is detecting when a new tab ("page") has opened and getting the new page object associated with the tab.
There are at least a couple techniques available. One method is promisifying the browser's "targetcreated" event as described here:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
await page.goto("https://twitter.com/amazon");
const amzSel = `.css-1dbjc4n:nth-child(1) > .css-1dbjc4n >
.css-1dbjc4n > .css-901oao > .css-4rbku5`;
await page.waitForSelector(amzSel, {visible: true});
console.log((await browser.pages()).length); // => 1
// method 1
const newPagePromise = new Promise(resolve =>
browser.once("targetcreated", target => resolve(target.page()))
);
await page.click(amzSel);
const newPage = await newPagePromise;
// --------
console.log((await browser.pages()).length); // => 2
await newPage.waitForSelector("#nav-link-prime", {visible: true});
await newPage.click("#nav-link-prime");
const sel = "#prime-header-CTA-announce";
await newPage.waitForSelector(sel, {visible: true});
console.log(await newPage.$eval(sel, el => el.innerText.trim())); // => TRY PRIME
//await browser.close();
})();
Another approach is to use browser.waitForTarget to check when the target's opener() is the previous page target, as described here:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
await page.goto("https://twitter.com/amazon");
const amzSel = `.css-1dbjc4n:nth-child(1) > .css-1dbjc4n >
.css-1dbjc4n > .css-901oao > .css-4rbku5`;
await page.waitForSelector(amzSel, {visible: true});
console.log((await browser.pages()).length); // => 1
// method 2
const pageTarget = page.target();
await page.click(amzSel);
const newTarget = await browser.waitForTarget(target =>
target.opener() === pageTarget
);
const newPage = await newTarget.page();
// --------
console.log((await browser.pages()).length); // => 2
await newPage.waitForSelector("#nav-link-prime", {visible: true});
await newPage.click("#nav-link-prime");
const sel = "#prime-header-CTA-announce";
await newPage.waitForSelector(sel, {visible: true});
console.log(await newPage.$eval(sel, el => el.innerText.trim())); // => TRY PRIME
//await browser.close();
})();
As an aside, I'm not sure how important/significant this particular Twitter/Amazon example is, but #nav-xshop > .nav-a:nth-child(2) doesn't seem like a reliable selector (it appears to have a race condition between "Best Sellers" and "Prime")--I'd use #nav-link-prime since it's a direct id, if that's what you're looking for.
My problem is that I need to set the comment selector to "all comments" whit puppeteer but the comments don't render after that puppeteer clicks on the correct button, "all the comments", the comment section just disappears, I will provide the code and a video of the browser in action.
const $ = require('cheerio');
const puppeteer = require('puppeteer');
const url = 'https://www.facebook.com/pg/SamsungGlobal/posts/';
const main = async () => {
const browser = await puppeteer.launch({
headless: false,
args: ['--no-sandbox', '--disable-setuid-sandbox']
});
const page = await browser.newPage();
await page.setViewport({
width: 1920,
height: 1080
});
await page.goto(url, {
waitUntil: 'networkidle2',
timeout: 0
});
page.mouse.click(50, 540, {});
for (var a = 0; a < 18; a++) {
setTimeout(() => {}, 16);
await page.keyboard.press('ArrowDown');
}
let bodyHTML = await page.evaluate(() => document.body.innerHTML);
var id = "#" + $("._427x ._4-u2.mbm._4mrt", bodyHTML).attr('id'); // selects id of first post
try {
var exp = await page.$(`${id} a._21q1`); // clicks on "most relevant" from the first post
await exp.evaluate(exp => exp.click());
await page.click('div[data-ordering="RANKED_UNFILTERED"]'); // selects "all the comments"
var exp = await page.$(`${id} a._42ft`); // should click on "more comments" but it doesn't load
await exp.evaluate(exp => exp.click());
await page.waitForSelector(`${id} a._5v47.fss`); // wait for the "others" in facebook comments
var exp = await page.$$(`${id} a._5v47.fss`);
await exp.evaluate(exp => exp.click());
await page.screenshot({
path: "./srn4.png"
});
// var post = await page.$eval(id + " .userContentWrapper", el => el.innerHTML);
// console.log("that's the post " + post);
} catch (e) {
console.log(e);
}
setTimeout(async function() {
await browser.close(); //close after some time
}, 1500);
};
main();
That's the video of the full execution process: https://youtu.be/jXpSOBfVskg
That's a slow motion of the moment it click on the menu: https://youtu.be/1OgfFNokxsA
You can try a variant with selectors:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch({ headless: false });
const [page] = await browser.pages();
await page.goto('https://www.facebook.com/pg/SamsungGlobal/posts/');
await page.waitForSelector('[data-ordering="RANKED_THREADED"]');
await page.click('[data-ordering="RANKED_THREADED"]');
await page.waitForSelector('[data-ordering="RANKED_UNFILTERED"]');
await page.click('[data-ordering="RANKED_UNFILTERED"]');
} catch (err) {
console.error(err);
}
})();
page.mouse.click(50, 540, {});
This is not going to work necessarily. What are you trying to click? You need to use CSS selectors to find elements that you want to click.
Also, dynamic elements might not appear in the page right away. You should use waitForSelector as needed.
I want to take a screenshot with puppeteer and it's working for one post. But I want to make it iterate.
If it's normal function I can just wrote the function name in the last side of the code so that it can iterate. But this is async function so I don't know how to iterate it.
const puppeteer = require('puppeteer');
let postNumber = 1;
let by;
(async () => {
const browser = await puppeteer.launch({
executablePath: 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
userDataDir: 'C:\\Users\\{computerName}\\AppData\\Local\\Google\\Chrome\\User Data',
headless: false
}); // default is true
const page = await browser.newPage();
await page.goto(`https://band.us/band/{someNumbers}/post/${postNumber}`, {
waitUntil: 'networkidle2'
});
let element = await page.$('.boardList');
by = await page.evaluate(() => document.getElementsByClassName('text')[0].textContent);
console.log(by);
await element.screenshot({
path: `./image/${postNumber}-${by}.png`
});
console.log(`SAVED : ${postNumber}-${by}.png`)
postNumber++;
await browser.close();
})();
After the function is finished, the postNumber variable should be increase by one. And then run the code again by new URLs.
As you want to run the code one iteration after another, a normal for (or while) loop can be used. async/await code works fine with these.
You can use a for in your case like this:
(async () => {
const browser = await puppeteer.launch(/* ... */);
const page = await browser.newPage();
for (let postNumber = 1; postNumber < 10; postNumber++) {
await page.goto(/* ... */);
let element = await page.$('.boardList');
// ...
}
await browser.close();
})();
You can use any appropriate loop, like while-loop:
'use strict';
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
executablePath: 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
userDataDir: 'C:\\Users\\{computerName}\\AppData\\Local\\Google\\Chrome\\User Data',
headless: false
}); // default is true
const page = await browser.newPage();
let postNumber = 1;
while (postNumber <= 10) {
await page.goto(`https://band.us/band/{someNumbers}/post/${postNumber}`, {
waitUntil: 'networkidle2'
});
const element = await page.$('.boardList');
const by = await page.evaluate(() => document.getElementsByClassName('text')[0].textContent);
console.log(by);
await element.screenshot({
path: `./image/${postNumber}-${by}.png`
});
console.log(`SAVED : ${postNumber}-${by}.png`)
postNumber++;
}
await browser.close();
})();