This is the code to fetch all the results from the website.
const puppeteer = require('puppeteer');
let students = [];
let rollPrefix = '387EA';
let regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
let i = 0;
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
while(i < 4){
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
}catch{
currReg++;
continue;
}
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Here the variable value of currReg stays the same pls help
The code tries each roll no and reg no combinations but there are some reg no that doesnt match with roll no so in the code the roll no should stay the same but the reg no should increase by one..
Not really sure what should happen with each combination, but here's an implementation which inputs all combinations. Below a short explanation:
const puppeteer = require('puppeteer');
let students = [];
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let i = 0;
let j = 0;
const rollPrefix = '387EA';
const regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
while(i < 4){
while(j < 4) {
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
console.log("rolltemp = ", rolltemp, " regtemp = ", regTemp);
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
} catch {
continue;
}
currReg++;
j++;
}
currReg = 80;
j = 0;
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Explanation
so, assuming you want all combinations of pairs {currentRol, currentReg}, you'll definitely need two loops. There are gonna be 4x4=16 combinations in total (I assume, basing on i < 4 condition). First mistake which you made was assigning regTemp before while loop, effectively not changing the strings entered to the inputs, only some unused, temporary values (currentRoll, currentReg). So, first and foremost is to move rollTemp and regTemp definitions into the while loop. Now, as I said, you're gonna need two nested loops, as you need to generate all possible combinations (for each currentRol, all currentRegs). One more thing to remember is that you'll have to reset currentReg with each outer loop iteration, as you want to test each reg for given roll.
Note about variables' scopes
This is a great example why variables scopes are critical when programming. Not only it increases readability and comprehensibility of the given code - it prevents other functions/scopes from using symbols which do not really belong to them. Please notice where the variables definitions are within my snippet. Probably it's not perfect, but why would you pollute global namespace as in your example?
Related
I've tried all sorts of ways to dispose of tensors (tf.dispose(), start/endscope).
The closest I've got was through this code, where 1 unused tensor remains after each execution. It takes about 2 hours for this program to run enough to use up 64 GB of RAM (big memory leak).
I also suspect that other factors besides TFJS-based operations are contributing to the memory leak, though (in theory) garbage collection should clean this up.
The piece of code below is one event that gets processed by an Event Listener handler. Any help with this issue would be greatly appreciated!
'use strict';
global.fetch = require("node-fetch");
const { MessageActionRow, MessageButton, Permissions } = require('discord.js');
const { mod, eco, m, n } = require(`../../index.js`);
const { Readable } = require('stream');
const PImage = require('pureimage');
const tf = require('#tensorflow/tfjs');
const tfnode = require('#tensorflow/tfjs-node');
const wait = require('util').promisify(setTimeout);
let bufferToStream = (binary) => {
let readableInstanceStream = new Readable({
read() {
this.push(binary);
this.push(null);
}
});
return readableInstanceStream;
}
const predict = async (imageUrl, modelFile) => {
let model = await tf.loadLayersModel(modelFile);
let modelClasses = [ "NSFW", "SFW" ];
let data = await fetch(imageUrl);
let fileType = data.headers.get("Content-Type");
let buffer = await data.buffer();
let stream = bufferToStream(buffer);
let image;
if ((/png/).test(fileType)) {
image = await PImage.decodePNGFromStream(stream);
}
else if ((/jpe?g/).test(fileType)) {
image = await PImage.decodeJPEGFromStream(stream);
}
else {
return;
}
let rawArray;
rawArray = tf.tidy(() => {
let tensorImage;
tensorImage = tf.browser.fromPixels(image).toFloat();
tensorImage = tf.image.resizeNearestNeighbor(tensorImage, [model.inputs[0].shape[1], model.inputs[0].shape[2]]);
tensorImage = tensorImage.reshape([1, model.inputs[0].shape[1], model.inputs[0].shape[2], model.inputs[0].shape[3]]);
return model.predict(tensorImage);
});
rawArray = await rawArray.data();
rawArray = Array.from(rawArray);
tf.disposeVariables();
model.layers.forEach(l => {
l.dispose();
});
if (rawArray[1] > rawArray[0]) {
return [`SFW`, rawArray[1]];
}
else {
return [`NSFW`, rawArray[0]];
}
};
const getResults = async (imageLink, imageNumber) => {
let image = `${imageLink}`;
let prediction = await predict(image, `file://D:/retake7/sfwmodel/model.json`);
let className = `SFW`;
if (prediction[0] == `NSFW`) {
className = `**NSFW**`;
}
return [`[Image ${imageNumber+1}](${imageLink}): ${className} (${(prediction[1]*100).toFixed(2)}% Certainty)`, ((prediction[1]*100).toFixed(2))*1];
}
const main = async (message, client, Discord) => {
if (message.attachments.size == 0 || message.author.bot || message.channel.nsfw) return;
await client.shard.broadcastEval(c => {
console.log(`Scanning...`);
}).catch(e => {
return;
});
let inChannel = await eco.seid.get(`${message.guild.id}.${message.channel.id}.active`);
let sfwImage = await eco.seid.get(`${message.guild.id}.sfwAlerts`);
if (inChannel == `no`) return;
let atmentArr = Array.from(message.attachments);
let msgArr = [];
if (message.attachments.size > 1) {
msgArr.push(`**Images Scanned**`);
} else {
msgArr.push(`**Image Scanned**`);
}
let hasNSFW = false;
let uncertain = false;
for (i = 0; i < message.attachments.size; i++) {
let msg = await getResults(atmentArr[i][1][`proxyURL`], i);
if (msg[1] < 80) {
uncertain = true;
}
if (msg[0].includes(`NSFW`)) {
hasNSFW = true;
}
msgArr.push(msg[0]);
}
if (uncertain == false && hasNSFW == false) {
let cont = `${msgArr.join(`\n`)}`;
msgArr = null;
client.seid.set(`${message.channel.id}.previousScan`, cont);
return;
}
let embed = new Discord.MessageEmbed()
.setColor(`GREEN`)
.setDescription(msgArr.join(`\n`));
let cont2 = `${msgArr.join(`\n`)}`;
client.seid.set(`${message.channel.id}.previousScan`, cont2);
msgArr = null;
if (sfwImage != `no` || hasNSFW || msg[1] <= 80) {
embed.setColor(`RED`);
await message.delete();
let msgSent = await message.channel.send({embeds: [embed], components: [row]});
};
};
module.exports = {
event: 'messageCreate',
run: async (message, client, Discord) => {
await main(message, client, Discord);
},
};
first, separate model loading and inference - in your current code, you'd reload a model each time you need to run prediction on a new image.
and then look at any possible leaks in prediction function - so once model is loaded.
you're loading a model and disposing each layer, but that doesn't mean model itself gets unloaded so there more than a chance that part of model remains in memory.
but leak itself is this line:
rawArray = await rawArray.data();
that variable is already used and its a tensor.
now you're overwriting the same variable with a data array and tensor never gets disposed.
This question already has an answer here:
Getting Promise pending ..- ES6
(1 answer)
Closed 12 months ago.
When running the below code and dumping out the results variable it returns Promise {<pending>}, I have added the await key word to the being of the function call so const results = await getAllResults() however this returns the error of Unexpected reserved word 'await'.
Anyone have any ideas?
useEffect(() => {
async function getPageOfResults(page) {
const response = await axios.get('https://swapi.dev/api/starships/?page=' + page);
return response.data.results;
}
async function getAllResults() {
let starships = [];
let lastResultsLength = 10;
let page = 1;
while (lastResultsLength === 10) {
const newResults = await getPageOfResults(page);
page++;
lastResultsLength = newResults.length;
starships = starships.concat(newResults);
}
return starships;
}
const results = getAllResults();
}, []);
You need to add async in the useEffect like so:
useEffect(async () => {
async function getPageOfResults(page) {
const response = await axios.get('https://swapi.dev/api/starships/?page=' + page);
return response.data.results;
}
async function getAllResults() {
let starships = [];
let lastResultsLength = 10;
let page = 1;
while (lastResultsLength === 10) {
const newResults = await getPageOfResults(page);
page++;
lastResultsLength = newResults.length;
starships = starships.concat(newResults);
}
return starships;
}
const results = await getAllResults();
}, []);
async function getChampionID(randomChampionID) {
let response = await fetch('http://ddragon.leagueoflegends.com/cdn/11.10.1/data/en_US/champion.json');
let body = await response.json();
var championData = Object.keys(body.data)
var randomChampionKey = Math.floor(Math.random() * (154 - 0 + 1) + 0)
return randomChampionID = championData[randomChampionKey]
}
async function getChampionName(randomChampionName) {
let result = await getChampionID();
let response = await fetch(`http://ddragon.leagueoflegends.com/cdn/11.10.1/data/en_US/champion/${result}.json`)
let body = await response.json();
return randomChampionName = body.data[result].name
}
var randomChampion = document.getElementById('random-champion')
var bgimg = document.getElementById('background-image')
var championSquare = document.getElementById('square')
randomChampion.addEventListener("click", async () =>
{
let championID = await getChampionID()
let championName = await getChampionName()
bgimg.style.backgroundImage = `url('http://ddragon.leagueoflegends.com/cdn/img/champion/splash/${championID}_0.jpg')`
championSquare.src=`http://ddragon.leagueoflegends.com/cdn/11.11.1/img/champion/${championID}.png`
console.log(championID)
console.log(championName)
})
The getChampionName() function takes a random value from getChampionID, so whenever I call both of them through a button event listener, the getChampionID() generates a random ID (#1), the getChampionName() once again takes another value from getChampionID(), result in a different ID (#2), but I need the getChampionName() to take the #1 ID
Firstly, the functions are not being passed a parameter and therefore do not need anything within the parentheses when being defined.
Secondly, within the event listener, I would simply call a function called getChampion that gets a random ID, then gets the details, and returns both the champion detail and the ID for further use as an object.
My code would look like this.
async function getChampionID() {
let response = await fetch('http://ddragon.leagueoflegends.com/cdn/11.10.1/data/en_US/champion.json');
let body = await response.json();
var championData = Object.keys(body.data)
var randomChampionKey = Math.floor(Math.random() * (154 - 0 + 1) + 0)
return championData[randomChampionKey]
}
async function getChampion() {
let result = await getChampionID();
let response = await fetch(`http://ddragon.leagueoflegends.com/cdn/11.10.1/data/en_US/champion/${result}.json`)
let body = await response.json();
return { name: body.data[result].name, id: result }
}
var randomChampion = document.getElementById('random-champion')
var bgimg = document.getElementById('background-image')
var championSquare = document.getElementById('square')
randomChampion.addEventListener("click", async () =>
{
let champion = await getChampion()
bgimg.style.backgroundImage = `url('http://ddragon.leagueoflegends.com/cdn/img/champion/splash/${champion.id}_0.jpg')`
championSquare.src=`http://ddragon.leagueoflegends.com/cdn/11.11.1/img/champion/${champion.id}.png`
console.log(champion.id)
console.log(champion.name)
})
So I am trying to pull out information using data scraping from this real estate website (https://www.zillow.com/vancouver-bc/)
I am able to get all the information about the listing on the page but with images (image links/src), after a few of them, the result is some garbage. I tried researching and found it was because of lazy loading. For which is tried almost all the methods available and answered by others but none seem to work - this includes scrolling to the bottom, scrolling with delays (https://www.npmjs.com/package/puppeteer-autoscroll-down), zooming out the browser as much as I can to get the images to render. But it still doesn't work. I have been looking everywhere for hours now before I decided to post my question and code here itself for anyone else to figure it out.
let cheerio = require('cheerio')
let puppeteer = require('puppeteer-extra')
const pluginStealth = require("puppeteer-extra-plugin-stealth")
puppeteer.use(pluginStealth())
let userAgent = require('random-useragent')
const baseURL = "https://www.zillow.com/vancouver-bc"
let estateData = []
let urlLinks = []
let scrollPageToBottom = require('puppeteer-autoscroll-down')
let getEstateData = async () => {
estateData = []
urlLinks = []
let url
for (let pgNum = 1; pgNum <= 1; pgNum++) {
if (pgNum === 1) {
url = baseURL + "/"
} else {
url = baseURL + ("/" + pgNum + "_p")
}
urlLinks.push(url)
}
await searchWebsite()
console.log("search over")
return estateData
//module.exports = estateData
}
let searchWebsite = async () => {
await puppeteer
.launch({headless : false})
.then(async function (browser) {
let page = await browser.newPage();
// await page.setRequestInterception(true)
//
// page.on('request', (req) => {
// if( req.resourceType() === 'image' || req.resourceType() === 'stylesheet' || req.resourceType() === 'font'){
// req.abort()
// }
// else {
// req.continue()
// }
//
// })
let html
await page.setUserAgent(userAgent.getRandom())
for(let url of urlLinks){
console.log(url)
await page.goto(url).then(async function () {
html = await page.content();
let obj = await cheerio('.list-card-link.list-card-info', html)
let imgObj = await cheerio(".list-card-top", html)
let geoLocation = await cheerio(".photo-cards.photo-cards_wow", html)
// await page.waitForSelector('img',{
// visible: true,
// })
// await page.evaluate(() => { window.scrollTo(0, document.body.scrollHeight)})
const scrollStep = 250 // default
const scrollDelay = 100 // default
const lastPosition = await scrollPageToBottom(page, scrollStep, scrollDelay)
await page.waitFor(2000)
let num = 0
console.log(obj.length)
for (let key in obj) {
if (obj[key].attribs) {
try {
let geoStr = await geoLocation[0].children[0].children[0].children[0].data
let geoObj = await (JSON.parse(geoStr)["geo"])
let extractedInfo = {
estateName : await obj[key].children[0].children[0].data,
estatePrice : await obj[key].children[2].children[0].children[0].data,
saleType : await obj[key].children[1].children[0].next.data,
estateConfig : {
beds : await obj[key].children[2].children[1].children[0].children[0].data,
bath : await obj[key].children[2].children[1].children[1].children[0].data,
area : await obj[key].children[2].children[1].children[2].children[0].data
},
estateLocation : {
longitude : await geoObj.longitude,
latitude : await geoObj.latitude
},
estateLink : await obj[key].attribs.href,
estateCoverImgLink : await imgObj[num++].children[2].children[0].attribs.src
}
console.log(extractedInfo.estateName, imgObj[num].children[2].children[0].attribs.src)
await estateData.push(extractedInfo)
}
catch (e) {
console.log("Estate Skipped - ", obj[key].children[0].children[0].data, obj[key].attribs.href)
console.log(e)
}
}
}
console.log(estateData.length)
});
}
//Now read the page
console.log("total - ", estateData.length)
await page.close()
await browser.close()
})
.catch(function (err) {
console.log(err)
});
}
module.exports.getEstateData = getEstateData
I had a similar issue and found a working answer here. Hopefully this works for you too. The interval was a little slow so I changed it from 100 to 30.
I was able to solve this with a pretty simple implementation using the puppeteer-autoscroll-down library as you mentioned. I'm not sure which images you were specifically attempting to grab, but this worked for me.
// Set the initial viewport and navigate to the page
await page.setViewport({ width: 1300, height: 1000 });
await page.goto('https://www.zillow.com/vancouver-bc/', { waitUntil: 'load' });
// Scroll to the very top of the page
await page.evaluate(_ => {
window.scrollTo(0, 0);
});
// Scroll to the bottom of the page with puppeteer-autoscroll-down
await scrollPageToBottom(page);
// Get your image links
let imageLinks = await page.$$eval('.list-card img', imgLinks => {
return imgLinks.map((i) => i.src);
});
imageLinks was an array with 40 fully formed links, https://photos.zillowstatic.com/p_e/ISz7wlfm278p501000000000.jpg is one example.
Hope that helps you, this was a pretty brutal one for me to solve as well.
I have an async generator that yields a the updated value when next() is invoked.
I want to invoke next() on every clicks but it seems thatthe generaor is not invoked and it always return the first value
How can i handle this ?
const baseApi = `https://hacker-news.firebaseio.com/v0`;
const topStories = [
19571054,
19570735,
19570986,
19571139,
19571399,
19569865,
19561411,
19571027,
19560994
];
const fetchAsyncA = async url => await (await fetch(url)).json();
const sliceBy = (list, start, end) => list.slice(start, end);
async function* hackerNewsStreamer(ids, slice) {
let start = 0;
while (start <= ids.length) {
let urls = sliceBy(ids, start, start + slice).map(
id => `${baseApi}/item/${id}.json?print=pretty`
);
yield await Promise.all(urls.map(fetchAsyncA));
start += slice;
}
}
document.querySelector("button").addEventListener("click", async () => {
const articlesStream = hackerNewsStreamer(topStories, 5);
const { value: articles } = await articlesStream.next();
console.log(articles);
//console.log(await articlesStream.next());
//console.log(await articlesStream.next());
//console.log(await articlesStream.next());
});
https://codepen.io/daniele-bolla/pen/MRKMBX