We Keep Coding

How to download historical-data (csv format) from investing.com with Puppeteer Js?

I tried this piece of code to download historical data in csv format from investing.com.
//collector.mjs
import puppeteer from "puppeteer";
import path from "path";
(async ()=>{
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.setUserAgent('Chrome/105.0.0.0');
await page.goto("https://www.investing.com/equities/tesla-motors-historical-data", {
waitUntil: "networkidle2",
});
const client = await page.target().createCDPSession();
await client .send('Page.setDownloadBehavior', {
behavior: 'allow',
downloadPath: path.resolve("./csvData"),
});
await page.evaluate(()=>{
document.querySelector("span[class='download-data_text__Myrn3']").click();
});
await browser.close();
})();
What i get all the time is "TSLA Historical Data.csv.crdownload" file (instead).
So, how to get this (picture under) instead in my folder csvData?

Use the page.setViewport().
//collector.mjs
import puppeteer from "puppeteer";
import path from "path";
(async ()=>{
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.setUserAgent('Chrome/105.0.0.0');
await page.setViewport({
width: 1920,
height: 1080
});
await page.goto("https://www.investing.com/equities/tesla-motors-historical-data", {
waitUntil: "networkidle2",
});
const client = await page.target().createCDPSession();
await client .send('Browser.setDownloadBehavior', {
behavior: 'allow',
downloadPath: path.resolve("./csvData"),
});
await page.click('.download-data_text__Myrn3');
await browser.close();
})();
Done,

Page loads in regular chrome but not in puppeteer

I am trying to load a page, http://www.nhc.gov.cn/wjw/index.shtml, on puppeteer as part of a covid-tracking program. The page loads very quickly in the regular chrome browser, but when I load it in puppeteer, the page load fails with a 412. What can I do to get the page to load and fully simulate a regular browser going to the page?
The code for reproduction of this phenomenon is below:
const puppeteer = require('puppeteer-core');
(async () => {
const browser = await puppeteer.launch({ executablePath: '..\\executables\\chrome.exe', headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu'] });
const page = await browser.newPage();
Object.assign(global, { browser, page });
page.on('console', msg => console.log(`chrome[${msg.text()}]`));
await page.goto('http://www.nhc.gov.cn/wjw/index.shtml', { waitUntil: 'networkidle0' });
await page.waitFor(15000);
await page.screenshot({path: 'nhc_scrape.png'});
await browser.close();
})();
Thank you in advance for your help!

you can use puppeteer-extra with the StealthPlugin.
https://www.npmjs.com/package/puppeteer-extra-plugin-stealth
Here is my code :
const puppeteer = require('puppeteer-extra')
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
(async () => {
puppeteer.use(StealthPlugin())
const browser = await puppeteer.launch({headless: false, ignoreHTTPSErrors: true})
const page = await browser.newPage();
await page.goto('http://www.nhc.gov.cn/wjw/index.shtml');
await page.waitForSelector('.inLists')
await page.screenshot({path: 'nhc_scrape.png'});
await browser.close();
})();

How do i type something into an input box with the puppeteer?

I'm doing experiments for a bot im making but for some reason things i cant get it to type into the input box in youtube.
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://www.youtube.com/?hl=hr&gl=HR');
await page.waitForNavigation({
waitUntil: 'networkidle0',
});
await page.type('#search', `text`)
// await browser.waitForTarget(() => false)
// await browser.close();
})();
The #search is the id for the youtube search bar but it isn't working for some reason

Your problem is waitUntil: 'networkidle0'
async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://www.youtube.com/?hl=hr&gl=HR');
await page.waitForSelector('input#search')
await page.type('input#search', `text`)
})();

How do I point the mouse to a specific part of the page and then scroll? (Puppeteer)

The URL I'm talking about is https://www.vudu.com/content/movies/movieslist. I'm trying to scroll through the part where the movies are. When I use the following code, it doesn't work.
await page.evaluate( () => {
window.scrollBy(0, window.innerHeight);
});
I think this is because you naturally have to hover over the movies before that part can scroll. How can I solve this problem? Thanks!
My full code:
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.setViewport({ width: 1280, height: 800 });
url = "https://www.vudu.com/content/movies/movieslist"
await page.goto(url, {waitUntil: 'load'});
await page.evaluate( () => {
window.scrollBy(0, window.innerHeight);
});
await page.waitFor(2000);
})()

You can use mouse.move() for this, but it seems this would not fix the issue as the document body is not scrollable. You need to find the scrollable element to scroll just it:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch({ headless: false, defaultViewport: null });
const [page] = await browser.pages();
await page.goto('https://www.vudu.com/content/movies/movieslist');
await page.waitForSelector('.nr-pt-40');
const data = await page.evaluate(() => {
const container = document.querySelector('.nr-pt-40');
container.scrollBy(0, container.clientHeight);
});
// await browser.close();
} catch (err) {
console.error(err);
}
})();

It's not the window that you need to scroll, but the div that wraps the thunbnails.
Try getting a handle to the wrapper using:
var first = document.querySelector(".contentPosterWrapper");
var wrapper = first.parentNode.parentNode;
wrapper.scrollBy(0, 500);

Puppeteer not working as expected when clicking button

My problem is that I need to set the comment selector to "all comments" whit puppeteer but the comments don't render after that puppeteer clicks on the correct button, "all the comments", the comment section just disappears, I will provide the code and a video of the browser in action.
const $ = require('cheerio');
const puppeteer = require('puppeteer');
const url = 'https://www.facebook.com/pg/SamsungGlobal/posts/';
const main = async () => {
const browser = await puppeteer.launch({
headless: false,
args: ['--no-sandbox', '--disable-setuid-sandbox']
});
const page = await browser.newPage();
await page.setViewport({
width: 1920,
height: 1080
});
await page.goto(url, {
waitUntil: 'networkidle2',
timeout: 0
});
page.mouse.click(50, 540, {});
for (var a = 0; a < 18; a++) {
setTimeout(() => {}, 16);
await page.keyboard.press('ArrowDown');
}
let bodyHTML = await page.evaluate(() => document.body.innerHTML);
var id = "#" + $("._427x ._4-u2.mbm._4mrt", bodyHTML).attr('id'); // selects id of first post
try {
var exp = await page.$(`${id} a._21q1`); // clicks on "most relevant" from the first post
await exp.evaluate(exp => exp.click());
await page.click('div[data-ordering="RANKED_UNFILTERED"]'); // selects "all the comments"
var exp = await page.$(`${id} a._42ft`); // should click on "more comments" but it doesn't load
await exp.evaluate(exp => exp.click());
await page.waitForSelector(`${id} a._5v47.fss`); // wait for the "others" in facebook comments
var exp = await page.$$(`${id} a._5v47.fss`);
await exp.evaluate(exp => exp.click());
await page.screenshot({
path: "./srn4.png"
});
// var post = await page.$eval(id + " .userContentWrapper", el => el.innerHTML);
// console.log("that's the post " + post);
} catch (e) {
console.log(e);
}
setTimeout(async function() {
await browser.close(); //close after some time
}, 1500);
};
main();
That's the video of the full execution process: https://youtu.be/jXpSOBfVskg
That's a slow motion of the moment it click on the menu: https://youtu.be/1OgfFNokxsA

You can try a variant with selectors:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch({ headless: false });
const [page] = await browser.pages();
await page.goto('https://www.facebook.com/pg/SamsungGlobal/posts/');
await page.waitForSelector('[data-ordering="RANKED_THREADED"]');
await page.click('[data-ordering="RANKED_THREADED"]');
await page.waitForSelector('[data-ordering="RANKED_UNFILTERED"]');
await page.click('[data-ordering="RANKED_UNFILTERED"]');
} catch (err) {
console.error(err);
}
})();

page.mouse.click(50, 540, {});
This is not going to work necessarily. What are you trying to click? You need to use CSS selectors to find elements that you want to click.
Also, dynamic elements might not appear in the page right away. You should use waitForSelector as needed.