Im trying to scrap a web page, and after following some tutorials, i found how to scrap different products, and how to change the page, but not at the same time. I tryed some ways to do it, but couln't find out.
This is my scraping code:
const puppeteer = require('puppeteer');
const xlsx = require("xlsx");
async function getPageData(url,page){
await page.goto(url);
const h1 = await page.$eval(".product_main h1", h1 => h1.textContent);
const price = await page.$eval(".price_color", price => price.textContent);
const instock = await page.$eval(".instock.availability", instock => instock.innerText);
return {
title: h1,
price: price,
instock: instock
}
//await browser.close();
};
async function getLinks(){
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://books.toscrape.com/');
const links = await page.$$eval('.product_pod .image_container a', allAs =>
allAs.map(a => a.href));
}
async function main(){
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
const data = await getPageData("https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",page);
console.log(data);
}
main();
And here is my code to change the page:
const puppeteer = require('puppeteer');
const xlsx = require("xlsx");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://books.toscrape.com/');
while(await page.$(".pager .next a")){
await page.click(".pager .next a");
await page.waitForTimeout(3000);
}
})();
Related
I have this code:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto("https://www.sisal.it/scommesse-matchpoint/quote/calcio/serie-a");
const [button1] = await
page.$x('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
button1.click();
const [button2] = await page.$x('//div[#class="listItem_container__2IdVR white
marketList_listItemHeight__1aiAJ marketList_bgColorGrey__VdrVK"]/p[text()="1X2
ESITO FINALE"]');
button2.click();
})();
The proble is that after clicking button1 the page change and puppeteer executes immediately the following line of code, instead I want it to wait for the new page to be loaded becuase otherwise It will throw an error since It can't find button2.
I found this solution on stackoverflow:
const puppeteer = require("puppeteer");
function delay(time) {
return new Promise(function (resolve) {
setTimeout(resolve, time);
});
}
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto("https://www.sisal.it/scommesse-matchpoint/quote/calcio/serie-a");
const [button1] = await
page.$x('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
button1.click();
await delay(4000);
const [button2] = await page.$x('//div[#class="listItem_container__2IdVR white
marketList_listItemHeight__1aiAJ
marketList_bgColorGrey__VdrVK"]/p[text()="1X2
ESITO FINALE"]');
button2.click();
})();
But of course this in't the best solution.
I think you have to modify a bit in your code:
await button1.click();
await page.waitForNavigation({waitUntil: 'networkidle2'});
For reference, see the documentation.
I found a solution, here's the code:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto("https://www.sisal.it/scommesse
matchpoint/quote/calcio/serie-a");
await page.waitForXPath('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
const [button1] = await page.$x('//div[#class="marketBar_changeMarketLabel__l0vzl"]/p');
await button1.click();
await page.waitForXPath('//div[#class="listItem_container__2IdVR white marketList_listItemHeight__1aiAJ marketList_bgColorGrey__VdrVK"]/p[text()="1X2 ESITO FINALE"]');
const [button2] = await page.$x('//div[#class="listItem_container__2IdVR white marketList_listItemHeight__1aiAJ marketList_bgColorGrey__VdrVK"]/p[text()="1X2 ESITO FINALE"]');
button2.click();
})();
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
headless: false,
defaultViewport: null
})
const page = await browser.newPage()
await page.goto('https://www.supremenewyork.com/shop/sweatshirts/ftq968f24/lhrblx1z5')
var productName = await page.evaluate(() => {
document.querySelector('div[id="details"] > p[itemprop="model"]').innerText;
})
console.log(productName);
})()
When I run my code that is supposed to grab the name of the supreme item, it says undefined when it's supposed to log it in the console.
You are neither returning anything from the page.evaluate nor are you setting the value of productName. Try something like this instead that uses $eval to return the innerText of the matching element:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({
headless: false,
defaultViewport: null,
});
const page = await browser.newPage();
await page.goto(
"https://www.supremenewyork.com/shop/sweatshirts/ftq968f24/lhrblx1z5"
);
const productName = await page.$eval(
'div[id="details"] > p[itemprop="model"]',
(el) => el.innerText
);
console.log(productName);
})();
If you prefer to use evaluate it would look like:
const puppeteer = require("puppeteer");
(async () => {
const browser = await puppeteer.launch({
headless: false,
defaultViewport: null,
});
const page = await browser.newPage();
await page.goto(
"https://www.supremenewyork.com/shop/sweatshirts/ftq968f24/lhrblx1z5"
);
const productName = await page.evaluate(() => {
// notice the return
return document.querySelector('div[id="details"] > p[itemprop="model"]').innerText;
});
console.log(productName);
})();
If innerText doesn't return anything you may instead need to use something like textContent.
Hopefully that helps!
I want to take a screenshot with puppeteer and it's working for one post. But I want to make it iterate.
If it's normal function I can just wrote the function name in the last side of the code so that it can iterate. But this is async function so I don't know how to iterate it.
const puppeteer = require('puppeteer');
let postNumber = 1;
let by;
(async () => {
const browser = await puppeteer.launch({
executablePath: 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
userDataDir: 'C:\\Users\\{computerName}\\AppData\\Local\\Google\\Chrome\\User Data',
headless: false
}); // default is true
const page = await browser.newPage();
await page.goto(`https://band.us/band/{someNumbers}/post/${postNumber}`, {
waitUntil: 'networkidle2'
});
let element = await page.$('.boardList');
by = await page.evaluate(() => document.getElementsByClassName('text')[0].textContent);
console.log(by);
await element.screenshot({
path: `./image/${postNumber}-${by}.png`
});
console.log(`SAVED : ${postNumber}-${by}.png`)
postNumber++;
await browser.close();
})();
After the function is finished, the postNumber variable should be increase by one. And then run the code again by new URLs.
As you want to run the code one iteration after another, a normal for (or while) loop can be used. async/await code works fine with these.
You can use a for in your case like this:
(async () => {
const browser = await puppeteer.launch(/* ... */);
const page = await browser.newPage();
for (let postNumber = 1; postNumber < 10; postNumber++) {
await page.goto(/* ... */);
let element = await page.$('.boardList');
// ...
}
await browser.close();
})();
You can use any appropriate loop, like while-loop:
'use strict';
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
executablePath: 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
userDataDir: 'C:\\Users\\{computerName}\\AppData\\Local\\Google\\Chrome\\User Data',
headless: false
}); // default is true
const page = await browser.newPage();
let postNumber = 1;
while (postNumber <= 10) {
await page.goto(`https://band.us/band/{someNumbers}/post/${postNumber}`, {
waitUntil: 'networkidle2'
});
const element = await page.$('.boardList');
const by = await page.evaluate(() => document.getElementsByClassName('text')[0].textContent);
console.log(by);
await element.screenshot({
path: `./image/${postNumber}-${by}.png`
});
console.log(`SAVED : ${postNumber}-${by}.png`)
postNumber++;
}
await browser.close();
})();
How do I scroll through just the side bar of a page and not the whole page?
I saw this solution but I don't know exactly how to get x and y. Using this doc as an example, how do I scroll through the scroll view on the side bar.
There are some ways you can scroll this part to its bottom:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch({
headless: false,
args: ['--start-maximized'],
});
const [page] = await browser.pages();
await page.setViewport({ width: 1280, height: 600 });
await page.goto('https://nodejs.org/api/');
await page.evaluate(() => {
const container = document.querySelector('#column2');
container.scrollTop = container.scrollHeight;
});
await page.waitFor(3000);
await page.goto('https://nodejs.org/api/');
await page.evaluate(() => {
const container = document.querySelector('#column2');
container.scroll(0, container.scrollHeight);
});
await page.waitFor(3000);
await page.goto('https://nodejs.org/api/');
await page.evaluate(() => {
const container = document.querySelector('#column2');
container.scrollBy(0, container.scrollHeight);
});
await page.waitFor(3000);
await page.goto('https://nodejs.org/api/');
await page.evaluate(() => {
const container = document.querySelector('#column2');
container.scrollTo(0, container.scrollHeight);
});
await page.waitFor(3000);
await page.goto('https://nodejs.org/api/');
await page.evaluate(() => {
const lastElementInContainer = [...document.querySelectorAll('#column2 li')].pop();
lastElementInContainer.scrollIntoView();
});
} catch (err) {
console.error(err);
}
})();
Recently I started to crawl the web using Puppeteer. Below is a code for extracting a specific product name from the shopping mall.
const puppeteer = require('puppeteer');
(async () => {
const width = 1600, height = 1040;
const option = { headless: false, slowMo: true, args: [`--window-size=${width},${height}`] };
const browser = await puppeteer.launch(option);
const page = await browser.newPage();
const vp = {width: width, height: height};
await page.setViewport(vp);
const navigationPromise = page.waitForNavigation();
await page.goto('https://shopping.naver.com/home/p/index.nhn');
await navigationPromise;
await page.waitFor(2000);
const textBoxId = 'co_srh_input';
await page.type('.' + textBoxId, '양말', {delay: 100});
await page.keyboard.press('Enter');
await page.waitFor(5000);
await page.waitForSelector('div.info > a.tit');
const stores = await page.evaluate(() => {
const links = Array.from(document.querySelectorAll('div.info > a.tit'));
return links.map(link => link.innerText).slice(0, 10) // 10개 제품만 가져오기
});
console.log(stores);
await browser.close();
})();
I have a question. How can I output the crawled results to an HTML document (without using the database)? Please use sample code to explain it.
I used what was seen on blog.kowalczyk.info
const puppeteer = require("puppeteer");
const fs = require("fs");
async function run() {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto("https://www.google.com/", { waitUntil: "networkidle2" });
// hacky defensive move but I don't know a better way:
// wait a bit so that the browser finishes executing JavaScript
await page.waitFor(1 * 1000);
const html = await page.content();
fs.writeFileSync("index.html", html);
await browser.close();
}
run();
fs.writeFile()
You can use the following write_file function that returns a Promise that resolves or rejects when fs.writeFile() succeeds or fails.
Then, you can await the Promise from within your anonymous, asynchronous function and check whether or not the data was written to the file:
'use strict';
const fs = require('fs');
const puppeteer = require('puppeteer');
const write_file = (file, data) => new Promise((resolve, reject) => {
fs.writeFile(file, data, 'utf8', error => {
if (error) {
console.error(error);
reject(false);
} else {
resolve(true);
}
});
});
(async () => {
// ...
const stores = await page.evaluate(() => {
return Array.from(document.querySelectorAll('div.info > a.tit'), link => link.innerText).slice(0, 10); // 10개 제품만 가져오기
});
if (await write_file('example.html', stores.toString()) === false) {
console.error('Error: Unable to write stores to example.html.');
}
// ...
});