I want to get the text of this webpage and the following when clicking on the Oui or Non button, which will be at the same place, and store them as a json file :
I am open to solutions in javascript and python. I tried teh following one:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto('http://www.leparisien.fr/elections/municipales/municipales-a-paris-notre-simulateur-pour-savoir-quel-candidat-vous-correspond-le-mieux-05-03-2020-8273238.php');
const pollFrame = page.frames().find(
frame => frame.url() === 'https://livemixr-assets.s3-eu-west-1.amazonaws.com/quel-candidat/index.html'
);
// getting first question
const data = await pollFrame.evaluate(
() => document.querySelector('html > body > div > div > div > div > div:nth-child(5) > h4').innerText
);
console.log(data);
// clicking on an answer
await page.$x('/html/body/div/div/div[1]/div/div[5]/div/div/label[1]')
const elements = await page.$x('/html/body/div/div/div[1]/div/div[5]/div/div/label[1]')
await elements[0].click()
// getting second question
const data2 = await pollFrame.evaluate(
() => document.querySelector('html > body > div > div > div > div > div:nth-child(5) > h4').innerText
);
console.log(data2);
await browser.close();
} catch (err) {
console.error(err);
}
})();
Which get the first text, click on one button, get the second text.
But got the following error:
C:\Users\antoi\Documents\Programming\Scraping>node scraper.js
Faut-il accélérer l’automatisation du métro ?
TypeError: Cannot read property 'click' of undefined
at main (C:\Users\antoi\Documents\Programming\Scraping\scraper.js:24:23)
at processTicksAndRejections (internal/process/task_queues.js:94:5)
So how to extract text from a webpage with puppeteer?
This selector is located inside an iframe, so you need to find this frame first.
While you are not so experienced in puppeteer, it would be easier to just use page.evaluate() (or frame.evaluate()) to get document data by executing web API code in the browser context.
For example:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto('http://www.leparisien.fr/elections/municipales/municipales-a-paris-notre-simulateur-pour-savoir-quel-candidat-vous-correspond-le-mieux-05-03-2020-8273238.php');
await page.waitForSelector('iframe[src="https://livemixr-assets.s3-eu-west-1.amazonaws.com/quel-candidat/index.html"]');
const pollFrame = page.frames().find(
frame => frame.url() === 'https://livemixr-assets.s3-eu-west-1.amazonaws.com/quel-candidat/index.html'
);
const data = await pollFrame.evaluate(() => {
const texts = [];
while (document.querySelector('div.ant-modal-content') === null) {
texts.push(document.querySelector('div:nth-child(5) > h4').innerText);
document.querySelectorAll('input[type="radio"]')[1].click();
// or: document.querySelector('input[type="radio"][value="Non"]').click();
}
return texts;
});
console.log(data);
await browser.close();
} catch (err) {
console.error(err);
}
})();
Output:
[
'Faut-il accélérer l’automatisation du métro ?',
'Faut-il faire payer le stationnement des deux-roues ?',
'Faut-il interdire les bus de tourisme dans la capitale ?',
'Faut-il piétonniser les quatre arrondissements centraux de la capitale ?',
'Faut-il réduire le nombre de places de stationnement en surface ?',
"Faut-il rendre le Vélib' gratuit ?",
'Faut-il renforcer la vidéoverbalisation ?',
'Faut-il rouvrir à la circulation les voies sur berges ?',
'Faut-il interdire les trottinettes électriques ? (free floating)',
'Le périphérique doit-il...',
'Les transports doivent-ils être rendus gratuits...',
"Faut-il demander l'annulation des Jeux olympiques ?",
'Faut-il mettre fin à la pratique du tirage au sort pour le conservatoire ?',
'Faut-il augmenter le nombre de caméras de vidéo-surveillance dans la rue ?',
'Faut-il créér une police municipale ?',
'Le future police municipale doit-elle être armée ?',
"Faut-il augmenter le montant des amendes (jets de mégots, urine, dépôt d'ordures...) ?",
'Faut-il privatiser le ramassage des déchets ?',
'À Paris, la société Airbnb doit-elle être ...',
'Faut-il baisser le nombre de nuitées autorisées à la location sur Airbnb (120 jours actuellement) ?',
"Faut-il maintenir l'encadrement des loyers?",
"En priorité, l'accès au logement social doit-il être attribué ...",
'Faut-il accroître le nombre de logements sociaux ?',
'En matière de finances, faut-il ...',
'Faut-il interdire les animaux sauvages dans les cirques ?',
'Faut-il proposer une alternative végétarienne tous les jours dans les cantines ?',
'Faut-il piétonniser les abords des écoles ?',
"Quelle est la priorité pour améliorer l'environnement ?",
'En cas de grèves, faut-il un service minimum pour les crèches et les écoles ?',
'Faut-il revenir à la semaine de 4 jours dans les écoles ?',
'Les tarifs des cantines scolaires doivent-ils ... ?',
'Faut-il étendre le travail du dimanche ?'
]
Related
When I try to use the .pipe() function in my controller I get an error and I don't know why. Attached my code below, thank you very much for the help in advance
const fs =require('fs');
module.exports={
inputs:{
name_file:{
type:"string",
required:true
}
},
exits: {
success: {
description: "Return matching data.",
},
defaultError: {
description: "It catches any error.",
responseType: "defaultError",
},
},
fn:function (inputs,exits){
//Comprobamos si el fichero viene en la peticion
if (!inputs.name_file) return exits.defaultError("No se ha adjuntado el nombre del archivo")
var nameFile=inputs.name_file
//Creamos ruta completa del fichero
var URL_FILE=process.cwd()
URL_FILE= URL_FILE.concat("\\.tmp\\uploads\\",nameFile,'.csv')
//Comprobamos si el fichero existe
fs.access(URL_FILE,fs.constants.F_OK,(err)=>{
return "El fichero no existe"
})
var response={}
var SkipperDisk = require('skipper-disk');
var fileAdapter = SkipperDisk(/* optional opts */);
// Stream the file down
var filedown=fileAdapter.read(URL_FILE)
filedown.on('error', function (err){
return exits.defaultError(err);
})
filedown.on('line',(linea)=>{
console.log(linea)
})
filedown.pipe(exits.success(response))
}
}
TypeError: Cannot read property 'on' of undefined```
so i have this schema validation :
const validationSchema = Yup.object().shape({
username: Yup.string()
.min(2, 'votre surnom doit contenir minimum 2 caractères')
.max(30, '30 caractères maximum autorisés')
.required('Veuillez renseigner un pseudo'),
email: Yup.string()
.email('Veuillez renseigner une adresse mail valide')
.required('Ce champs est obligatoire.')
.test(
'emailCheck',
'cet est email déjà utiliser, veuillez en choisir un autre',
async (value) => {
const res = await AuthenticationService.checkEmail(value);
if (res) {
return false;
} else {
return true;
}
}
),
password: Yup.string()
.required('Ce champs est obligatoire.')
.min(
8,
'Le mot de passe doit contenir 8 caractères ou plus, dont au moins un de chaque type : majuscule, minuscule, chiffre et spécial.'
)
.minLowercase(1, 'le mot de passe doit contenir au moins 1 lettre minuscule')
.minUppercase(1, 'le mot de passe doit contenir au moins 1 lettre majuscule')
.minNumbers(1, 'le mot de passe doit contenir au moins 1 chiffre')
.minSymbols(1, 'le mot de passe doit contenir au moins 1 caractère spécial'),
passwordConfirmation: Yup.string().oneOf(
[Yup.ref('password'), null],
'Les mots de passe doivent correspondre'
)
});
i'm testing if the email already exist it's work well but the test affecting all other input when i quit the username input and go on the password input for example the test for the email input is call. if i try to pu a test like a checkUsername function on the username input like this:
username: Yup.string()
.min(2, 'votre surnom doit contenir minimum 2 caractères')
.max(30, '30 caractères maximum autorisés')
.required('Veuillez renseigner un pseudo')
.test(
'usernameCheck',
'cet est identifiant déjà utiliser, veuillez en choisir un autre',
async (value) => {
const res = await AuthenticationService.checkUsername(value);
if (res) {
return false;
} else {
return true;
}
}
),
email: Yup.string()
.email('Veuillez renseigner une adresse mail valide')
.required('Ce champs est obligatoire.')
.test(
'emailCheck',
'cet est email déjà utiliser, veuillez en choisir un autre',
async (value) => {
const res = await AuthenticationService.checkEmail(value);
if (res) {
return false;
} else {
return true;
}
}
),
the checkEmail function email still be called for the username input.
My ionic app fires a modal twice when hitting ion-button.
I cannot figured why it is happening.
<ion-button (click)="editProduct(p.id)" fill="clear">
<ion-icon name="cloud-upload"></ion-icon>
</ion-button>
editProduct(id) {
// obter dados do produto pelo seu id
this.afs.collection("products").doc(id)
.valueChanges()
.subscribe(data => {
this.product = data
// chamar o modal e enviar o id do produto
this.modalProduct(id);
});
}
async modalProduct(id) {
const modal = await this.modalCtrl.create({
component: AdminProductPage,
componentProps: {
'id': id,
'title': this.product.title,
'description': this.product.description,
'price': this.product.price,
'image': this.product.image
}
});
await modal.present();
}
I figured out by myself.
I need to use a pipe from rxjs to prevent double execution of editProduct() subscrive method.
editProduct(id) {
// obter dados do produto pelo seu id
this.afs.collection("products").doc(id)
.valueChanges()
.pipe(
first()
)
.subscribe(data => {
this.product = data
// chamar o modal e enviar o id do produto
this.modalProduct(id);
});
}
I would like to know how to mention a member who has just arrived in some server and my bot sends to the welcome lounge and mentions the person who has just arrived.
Like:
Welcome {new member}! Blablablabla ...
client.on("guildMemberAdd", async member => {
let welcomeMsg = member.guild.channels.cache.find(channel => channel.name === "welcome");
let welcomeEmbed = new Discord.MessageEmbed()
.setColor("#e23922")
.setDescription("*```\n \n Nous te souhaitons la bienvenue.\n ```*\n\n:love_letter: ↬ 𝗖'𝗲𝘀𝘁 𝘂𝗻 𝗵𝗼𝗻𝗻𝗲𝘂𝗿 𝗱𝗲 𝘁'𝗮𝗰𝗰𝘂𝗲𝗶𝗹𝗹𝗶𝗿 𝗱𝗮𝗻𝘀 𝗻𝗼𝘁𝗿𝗲 𝗲́𝘁𝗮𝗯𝗹𝗶𝘀𝘀𝗲𝗺𝗲𝗻𝘁. 𝗧𝘂 𝗮𝘂𝗿𝗮𝘀 𝗱𝗿𝗼𝗶𝘁 𝗮̀ 𝗹𝗮 𝗺𝗲𝗶𝗹𝗹𝗲𝘂𝗿𝗲 𝗲́𝗱𝘂𝗰𝗮𝘁𝗶𝗼𝗻 𝗾𝘂'𝗶𝗹 𝘀𝗼𝗶𝘁 𝘀𝘂𝗿 𝗻𝗼𝘁𝗿𝗲 𝗺𝗲𝗿𝘃𝗲𝗶𝗹𝗹𝗲𝘂𝘀𝗲 𝗶̂𝗹𝗲 𝗾𝘂'𝗲𝘀𝘁 `🌹 ᙏᥙɾᥲ𝘀ᥲ Ꙇ𝘀Ꙇᥱᴿᴾ` ! 𝗡𝗼𝘂𝘀 𝘁𝗲 𝗽𝗿𝗶𝗶𝗼𝗻𝘀 𝗱'𝗮𝗹𝗹𝗲𝗿 𝗹𝗶𝗿𝗲 𝗹𝗲 𝗿𝗲̀𝗴𝗹𝗲𝗺𝗲𝗻𝘁 𝗲𝗻𝗰𝗼𝗿𝗲 𝘂𝗻𝗲 𝗳𝗼𝗶𝘀 𝗮𝘃𝗮𝗻𝘁 𝗱𝗲 𝘁𝗲 𝗹𝗮𝗻𝗰𝗲𝗿 𝗱𝗮𝗻𝘀 𝗹𝗮 𝗳𝗮𝗺𝗲𝘂𝘀𝗲 𝘀𝘂𝗶𝘁𝗲.\n\n**Grâce à toi nous sommes à member.guild.memberCount membres !**\n```diff\n- Chikin School -\n```")
.setThumbnail("https://i.pinimg.com/originals/1d/f4/ec/1df4ece417ea8174ac2a3c635cf871b3.gif")
.setImage("https://i.gifer.com/g27a.gif");
await client.channels.cache.get("737621052003450992").send({ embed: welcomeEmbed })
},
In this code, where could I put it?
Like this !
client.on("guildMemberAdd", async member => {
let welcomeMsg = member.guild.channels.cache.find(channel => channel.name === "welcome");
let welcomeEmbed = new Discord.MessageEmbed()
.setColor("#e23922")
.setDescription("*```\n \n Nous te souhaitons la bienvenue.\n ```*\n\n:love_letter: ↬ 𝗖'𝗲𝘀𝘁 𝘂𝗻 𝗵𝗼𝗻𝗻𝗲𝘂𝗿 𝗱𝗲 𝘁'𝗮𝗰𝗰𝘂𝗲𝗶𝗹𝗹𝗶𝗿 𝗱𝗮𝗻𝘀 𝗻𝗼𝘁𝗿𝗲 𝗲́𝘁𝗮𝗯𝗹𝗶𝘀𝘀𝗲𝗺𝗲𝗻𝘁. 𝗧𝘂 𝗮𝘂𝗿𝗮𝘀 𝗱𝗿𝗼𝗶𝘁 𝗮̀ 𝗹𝗮 𝗺𝗲𝗶𝗹𝗹𝗲𝘂𝗿𝗲 𝗲́𝗱𝘂𝗰𝗮𝘁𝗶𝗼𝗻 𝗾𝘂'𝗶𝗹 𝘀𝗼𝗶𝘁 𝘀𝘂𝗿 𝗻𝗼𝘁𝗿𝗲 𝗺𝗲𝗿𝘃𝗲𝗶𝗹𝗹𝗲𝘂𝘀𝗲 𝗶̂𝗹𝗲 𝗾𝘂'𝗲𝘀𝘁 `🌹 ᙏᥙɾᥲ𝘀ᥲ Ꙇ𝘀Ꙇᥱᴿᴾ` ! 𝗡𝗼𝘂𝘀 𝘁𝗲 𝗽𝗿𝗶𝗶𝗼𝗻𝘀 𝗱'𝗮𝗹𝗹𝗲𝗿 𝗹𝗶𝗿𝗲 𝗹𝗲 𝗿𝗲̀𝗴𝗹𝗲𝗺𝗲𝗻𝘁 𝗲𝗻𝗰𝗼𝗿𝗲 𝘂𝗻𝗲 𝗳𝗼𝗶𝘀 𝗮𝘃𝗮𝗻𝘁 𝗱𝗲 𝘁𝗲 𝗹𝗮𝗻𝗰𝗲𝗿 𝗱𝗮𝗻𝘀 𝗹𝗮 𝗳𝗮𝗺𝗲𝘂𝘀𝗲 𝘀𝘂𝗶𝘁𝗲.\n\n**Grâce à toi nous sommes à member.guild.memberCount membres !**\n```diff\n- Chikin School -\n```")
.setThumbnail("https://i.pinimg.com/originals/1d/f4/ec/1df4ece417ea8174ac2a3c635cf871b3.gif")
.setImage("https://i.gifer.com/g27a.gif");
await client.channels.cache.get("737621052003450992").send(`Bienvenue, ${member.toString()}`, { embed: welcomeEmbed })
},
ReferenceError: user is not defined = corriged thanks.
You can't mention a member in an embed, or they won't get a notification. You can use the following to mention the user just above the embed:
client.on("guildMemberAdd", async member => {
let welcomeMsg = member.guild.channels.cache.find(channel => channel.name === "welcome");
let welcomeEmbed = new Discord.MessageEmbed()
.setColor("#e23922")
.setDescription("Welcome " + member.toString() + "*```\n \n Nous te souhaitons la bienvenue.\n ```*\n\n:love_letter: ↬ 𝗖'𝗲𝘀𝘁 𝘂𝗻 𝗵𝗼𝗻𝗻𝗲𝘂𝗿 𝗱𝗲 𝘁'𝗮𝗰𝗰𝘂𝗲𝗶𝗹𝗹𝗶𝗿 𝗱𝗮𝗻𝘀 𝗻𝗼𝘁𝗿𝗲 𝗲́𝘁𝗮𝗯𝗹𝗶𝘀𝘀𝗲𝗺𝗲𝗻𝘁. 𝗧𝘂 𝗮𝘂𝗿𝗮𝘀 𝗱𝗿𝗼𝗶𝘁 𝗮̀ 𝗹𝗮 𝗺𝗲𝗶𝗹𝗹𝗲𝘂𝗿𝗲 𝗲́𝗱𝘂𝗰𝗮𝘁𝗶𝗼𝗻 𝗾𝘂'𝗶𝗹 𝘀𝗼𝗶𝘁 𝘀𝘂𝗿 𝗻𝗼𝘁𝗿𝗲 𝗺𝗲𝗿𝘃𝗲𝗶𝗹𝗹𝗲𝘂𝘀𝗲 𝗶̂𝗹𝗲 𝗾𝘂'𝗲𝘀𝘁 `🌹 ᙏᥙɾᥲ𝘀ᥲ Ꙇ𝘀Ꙇᥱᴿᴾ` ! 𝗡𝗼𝘂𝘀 𝘁𝗲 𝗽𝗿𝗶𝗶𝗼𝗻𝘀 𝗱'𝗮𝗹𝗹𝗲𝗿 𝗹𝗶𝗿𝗲 𝗹𝗲 𝗿𝗲̀𝗴𝗹𝗲𝗺𝗲𝗻𝘁 𝗲𝗻𝗰𝗼𝗿𝗲 𝘂𝗻𝗲 𝗳𝗼𝗶𝘀 𝗮𝘃𝗮𝗻𝘁 𝗱𝗲 𝘁𝗲 𝗹𝗮𝗻𝗰𝗲𝗿 𝗱𝗮𝗻𝘀 𝗹𝗮 𝗳𝗮𝗺𝗲𝘂𝘀𝗲 𝘀𝘂𝗶𝘁𝗲.\n\n**Grâce à toi nous sommes à member.guild.memberCount membres !**\n```diff\n- Chikin School -\n```")
.setThumbnail("https://i.pinimg.com/originals/1d/f4/ec/1df4ece417ea8174ac2a3c635cf871b3.gif")
.setImage("https://i.gifer.com/g27a.gif");
await client.channels.cache.get("737621052003450992").send({ embed: welcomeEmbed })
},
I'm new to symfony, I render forms throughout a JSON so whenever I click on an icon it shows different forms (to change firstname, lastname...).
I return thoses forms as a JSON :
public function profileSettings()
{
$user = $this->getUser();
// Formulaire d'informations concernant le compte
$formAccountSettings = $this->createForm(AccountSettingsType::class, $user, [
'userEmail' => $user->getEmail(),
'userUsername' => $user->getUsername(),
]);
// Formulaire d'informations personnel
$formPersonnalSettings = $this->createForm(PersonnalSettingsType::class, $user, [
'userFirstname' => $user->getFirstname(),
'userLastname' => $user->getLastname(),
]);
// Retour en format JSON des 3 requêtes sous tableau
return $this->json(
[
'formAccountSettingsView' =>
$this->render('user/_accountSettings.html.twig', [
'form' => $formAccountSettings->createView(),
]),
'currentUser' => $user,
'formPersonnalSettingsView' => $this->render('user/_accountSettings.html.twig', [
'form' => $formPersonnalSettings->createView(),
]),
]
);
}
Here is how I display this :
$('#settings-user li').on('click', function (e) {
$.ajax({
type: 'GET',
url: "/website-skeleton/public/index.php/json/profile",
success: function (response) {
if ($(e.target).hasClass('profile')) {
$('.display').empty().append(
`
<p id="welcome">Bonjour, <em><strong>${response['currentUser']['username']}</strong></em> vous êtes enregistré sous le nom de <strong>${response['currentUser']['firstname']} ${response['currentUser']['lastname']}</strong>.
<br>
Votre adresse email est : <strong>${response['currentUser']['email']}</strong>.
<br>
Pour modifiez vos informations veuillez cliquez sur le menu de navigation.
</p>`);
} else if ($(e.target).hasClass('security')) {
$('.display').empty().append(response['formAccountSettingsView']['content']);
} else if ($(e.target).hasClass('informations')) {
$('.display').empty().append(response['formPersonnalSettingsView']['content'])
}
}
})
});
The problem now is that I don't know how to handle thoses forms from another controller and validate it with the constraints I set on my entity User this is how I validate :
public function editCredentials(Request $request, UserPasswordEncoderInterface $encoder)
{
$user = $this->getUser();
$formPersonnalSettings = $this->createForm(PersonnalSettingsType::class, $user);
if ($request->isMethod('POST')) {
if ($request->request->has('personnal_settings')) {
if ($formPersonnalSettings->isSubmitted() && $formPersonnalSettings->isValid()) {
$user->setFirstname($request->request->get('personnal_settings')['firstname']);
$user->setLastname($request->request->get('personnal_settings')['lastname']);
$em = $this->getDoctrine()->getManager();
$em->persist($user);
$em->flush();
$this->addFlash('personnal_success', 'Vos informations personnels ont bien été enregistrées !');
return $this->redirectToRoute('user_profile');
}
}
Is that a good method? should I handle everything with ajax ? Thanks for your time !
You should use the handlerequest which automatically sets the values in the form to the entity added to it once the form has been submitted.
$form = $this->createCreateForm($entity); // private function to create the form
$form->handleRequest($request);
if ($form->isSubmitted()) {
if ($form->isValid()) {
$em->persist($entity);
$em->flush();
}
}
return $response;
Once handled you just need to return an ok response, or return the form back to the view if something has gone wrong.
If I remember well, the asserts set on the entity are also automatically processed with the handleRequest(), it depends on how you declared them.
Yo can also add other errors after the submit, just add them between the isSumbmitted() and isValid(). Documentation
I leave you here some documentation that may help.
Handle request
Validation
Validation in the form itself