I'm still trying to learn about global and local identifiers on nodejs, but still not clear for me.
I'm trying to use those variables from some query, then use them again with the results from database in page.evaluate().
Someone can help me? I've tried many ways.
let initial_time = null;
let end_time = null;
(async () => {
const query_time = await query('select time_i, time_f from time_config where config_id = 1', function(err, results) {
if (err) throw err;
for (var i in results) {
var time = results[0];
initial_time = time['time_i'];
end_time = time['time_f'];
}
})
})
()
const result_time = await page.evaluate(() => {
return [].concat.apply([], document.querySelectorAll('.ipo-Fixture.ipo-Fixture_Time'))
.filter(e => {
const timeString = e.innerText;
const minute = +timeString.split(':')[0];
return (minute > initial_time && minute < end_time);
})
.map(e => e.innerText);
})
One of the solutions is to pass the variables to your function something like this
(async () => {
const query_time = await query('select time_i, time_f from time_config where config_id = 1', function(err, results){
if(err) throw err;
for (var i in results){
var time = results[0];
var initial_time = time['time_i'];
var end_time = time['time_f'];
}
page.evaluate(initial_time,end_time)
})
})
()
const result_time = await page.evaluate(initial_time,end_time,() => {
return [].concat.apply([], document.querySelectorAll('.ipo-Fixture.ipo-Fixture_Time'))
.filter(e => {
const timeString = e.innerText;
const minute = +timeString.split(':')[0];
return (minute > initial_time && minute < end_time);
})
.map(e => e.innerText);
})
Related
I'm trying to query the token balance history of significant user with web3's batchRequest.
But the function can't guarantee the order of functions, so I get the balance datas not in order.
So I want to know other multicall functions which can get response with blockNumber to organaize the order after querying datas.
This is my current code
// this succeeded to query, and I'm using this
const Web3 = require('web3');
const { getTokens } = require('./utils/getTokens.js');
const { convertToNumber } = require('./utils/convertToNumber.js')
const { abi, rpcEndpoint, walletAddress, wsEndpoint } = require('./constant.js');
const { formatUnits } = require('ethers/lib/utils');
const web3 = new Web3(new Web3.providers.HttpProvider(rpcEndpoint));
const queryHistoryBalance = async (userAddress, tokenAddress, startBlock, endBlock, decimal) => {
const batch = new web3.BatchRequest(); // https://stackoverflow.com/questions/69788660/web3-batchrequest-always-returning-undefined-what-am-i-doing-wrong
const contract = new web3.eth.Contract(abi, tokenAddress);
/*
let step = Math.floor((endBlock - startBlock) / eachNumber);
if (step < 1) {
step = 1;
}
for (let n = 0; n < step; n++) {
}*/
let data = { [userAddress]: [] }
for (let n = startBlock; n <= endBlock; n++) {
batch.add(
contract.methods.balanceOf(userAddress).call.request({}, n, (err, res) => {
if (res) {
data[userAddress].push({ 'blockNumber': n, 'balance': res })
}
})
)
/*
batch.add(
contract.methods.balanceOf(userAddress).call.request({}, n, async (err, res) => {
const timestamp = await web3.eth.getBlock(n).timestamp;
data[userAddress].push({ 'blockNumber': n, 'blockTime': timestamp, 'balance': res })
//data[userAddress].push({ 'blockNumber': n,'balance': res })
})
)*/
}
await batch.execute();
//console.log(data);
let data2 = { [userAddress]: [] }
let formarNumber
await Promise.all(data[userAddress].map((element, i) => {
//console.log(element['balance'] / 1)
//data[userAddress][i]['balance'] = convertToNumber(element['balance'], 6)
let formatBalance = element['balance'] / (10 ** decimal);
data[userAddress][i]['balance'] = formatBalance;
if (formarNumber !== element['balance']) {
data2[userAddress].push({ 'blockNumber': element['blockNumber'], 'balance': formatBalance });
formarNumber = element['balance'];
}
if (i === 0) formarNumber = element['balance'];
}))
//console.log("data", data)
return data2
}
const main = async () => {
const data = await queryHistoryBalance('0x8d9e0f65569b3cc2f602a7158d02ab52976a4535', '0xdAC17F958D2ee523a2206206994597C13D831ec7', 14954947, 14958046, 6)
console.log(data)
}
main()
module.exports.queryHistoryBalance = queryHistoryBalance;
I appreciate any help and advice
I am doing some practice in node.js. In this exercise I been asked to find a country name through a GET Http Request to an endpoint passing a page integer as a parameter.
Where the important response structs are these {page, total_pages, data}.
page is the current page,
total_pages is the last page,
data is an array of 10 country object.
In getCountryName func I am able to retrieve the right answer only if the answer is on the 1st page, the 1 iteration of the loop. So, why the loop only happens once?
Aditional, I wanted to retrieve the total_pages to replace the hardcode '25' value but I do not figure it out how to return it along with the search.
Any hint you wanna give me? The whole problem is in getCountryCode func.
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
var res = '';
var pages = 25;
var i = 1;
while(i <= pages && res == ''){
console.log(i);
res = makeRequest(i)
.then(data => {
let f = ''
let p = data['total_pages'];
let search = data['data'].find(o => o.alpha3Code === code);
f = search != null ? search['name'] : f;
return f;
});
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();
Without modifying your code too much, this is how you do it:
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
const pages = 25;
var i = 1;
let f = null
while(i <= pages && f === null){
console.log(i);
const data = await makeRequest(i) // put in try/catch
const p = data['total_pages'];
const search = data['data'].find(o => o.alpha3Code === code);
f = search !== null ? search['name'] : null;
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();
This is the code to fetch all the results from the website.
const puppeteer = require('puppeteer');
let students = [];
let rollPrefix = '387EA';
let regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
let i = 0;
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
while(i < 4){
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
}catch{
currReg++;
continue;
}
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Here the variable value of currReg stays the same pls help
The code tries each roll no and reg no combinations but there are some reg no that doesnt match with roll no so in the code the roll no should stay the same but the reg no should increase by one..
Not really sure what should happen with each combination, but here's an implementation which inputs all combinations. Below a short explanation:
const puppeteer = require('puppeteer');
let students = [];
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let i = 0;
let j = 0;
const rollPrefix = '387EA';
const regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
while(i < 4){
while(j < 4) {
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
console.log("rolltemp = ", rolltemp, " regtemp = ", regTemp);
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
} catch {
continue;
}
currReg++;
j++;
}
currReg = 80;
j = 0;
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Explanation
so, assuming you want all combinations of pairs {currentRol, currentReg}, you'll definitely need two loops. There are gonna be 4x4=16 combinations in total (I assume, basing on i < 4 condition). First mistake which you made was assigning regTemp before while loop, effectively not changing the strings entered to the inputs, only some unused, temporary values (currentRoll, currentReg). So, first and foremost is to move rollTemp and regTemp definitions into the while loop. Now, as I said, you're gonna need two nested loops, as you need to generate all possible combinations (for each currentRol, all currentRegs). One more thing to remember is that you'll have to reset currentReg with each outer loop iteration, as you want to test each reg for given roll.
Note about variables' scopes
This is a great example why variables scopes are critical when programming. Not only it increases readability and comprehensibility of the given code - it prevents other functions/scopes from using symbols which do not really belong to them. Please notice where the variables definitions are within my snippet. Probably it's not perfect, but why would you pollute global namespace as in your example?
I have a firebase cloud function that is unable to finish executing. I suspect my code can be dramatically improved but I'm not quite sure how.
I've made the query as specific as possible to try and reduce the number of documents required to iterate through but that didn't solve the issue.
I get a Deadline Exceeded error which I suspect is due to the fact that I'm iterating through so many documents and trying to update them.
I increased the timeout (9 minutes)and memory allocation (2GB) in Google cloud console but that didn't help either.
exports.updatePollWinner = functions.runWith(runtimeOpts).firestore.document('triggerAccuracyCalculation/{id}').onCreate(trigger => {
const week = trigger.get('week');
const scoringTags = ["STD", "0.25PPR", "0.5PPR", "PPR", "0.10PPC", "0.25PPC", "0.5PPC", "4PTPASS", "5PTPASS", "6PTPASS", "-2INT", "TEPREMIUM"]
let winningChoiceIds = [];
let totalPollIds = [];
return db.collection("polls").where("sport", "==", 1).where("week", "==", week).where("pollType", "==", "WDIS").get()
.then((querySnapshot) => {
console.log("A");
querySnapshot.forEach((doc) => {
totalPollIds.push(doc.id);
let pollData = doc.data();
// extract relevant scoring tags
let tags = pollData.tags.filter(tag => scoringTags.includes(tag.code)).map(tag => tag.code);
// if no scoring setting is tagged, then use STD - determine what STD is
// extract player from each option
let winner = {score: 0, choice: {}, choiceId: null};
let cnt = 0;
pollData.choices.forEach((choice) => {
let choiceId = choice.id
let mappedChoices = choice.players.map(player => {
return { displayName: player.displayName, playerId: player.playerId, position: player.position, team: player.team }
});
// ToDo: What happens if someone posts a poll with two players in one option? This poll should be ignoree from accuracy calculation
// ignmore if option has more than one player
// if (mappedChoices.length > 1) return;
const player = mappedChoices[0]
// We can't score defense
if (player.position === "DEF") {
return;
}
const playerId = player.playerId;
// Make FFN API call to retrieve stats for that player in that weekconst statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}${req.url}`;
const statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}/${playerId}`;
const json = {"Stats": {"2019": ""}, "Player": {}};
https.get(statsEndpoint, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const weekString = week.toString();
const fetchedStats = JSON.parse(data).Stats
if (!fetchedStats) return;
const response = fetchedStats["2019"]
if (!response) return;
// TODO SCORE KICKERS AND DEFENSES
const stats = response[weekString];
let score = 0;
stats["recYards"] ? score += parseInt(stats["recYards"]) / 10 : false
stats["recTD"] ? score += parseInt(stats["recTD"]) * 6 : false
stats["rushYards"] ? score += parseInt(stats["rushYards"]) / 10 : false
stats["rushTD"] ? score += parseInt(stats["rushTD"]) * 6 : false
stats["xpMade"] ? score += parseInt(stats["xpMade"]) : false
stats["fgMade"] ? score += parseInt(stats["fgMade"]) * 3 : false
stats["kickoffRet"] ? score += parseInt(stats["kickoffRet"]) / 10 : false
stats["SackYards"] ? score -= parseInt(stats["SackYards"]) / 10 : false
stats["fumbleLost"] ? score -= parseInt(stats["fumbleLost"]) * 2 : false
// Determine winner
// ToDo: handle ties
if (score > winner.score) {
winner.score = score;
winner.choiceId = choiceId;
winner.choice = choice;
}
if (cnt>=pollData.choices.length-1){
// Save player object on the poll Document (include choice ID)
winningChoiceIds.push(winner.choiceId);
const pollDoc = db.doc(`polls/${doc.id}`);
pollDoc.update({winner: winner});
}
cnt++;
});
}).on("error", (err) => {
console.log("Error: ", err.message);
});
});
});
console.log("B");
return false;
}).then(() => {
console.log("C");
let dateToQueryAfter = new Date(new Date("08/22/19").setHours(0,0,0,0))
return db.collection("users").where("recentVote", ">", dateToQueryAfter).get()
})
.then((querySnapshot) => {
console.log("D");
const promises = [];
querySnapshot.forEach((doc) => {
const p = db.collection("votes").where("uid", "==", doc.id).where("week", "==", week).where("pollType", "==", "WDIS").get()
promises.push(p)
});
return Promise.all(promises)
})
.then((querySnapshots) => {
console.log("E");
querySnapshots.forEach((querySnapshot) => {
if (querySnapshot.docs.length <= 0) return;
const uid = querySnapshot.docs[0].data().uid
const retrieveUserDoc = db.doc(`users/${uid}`);
let correctVotes = 0;
let cnt = 0;
let totalVotes = 0;
let pollVoteIds = [];
let pollVoteIdsCorrect = [];
querySnapshot.docs.forEach((doc) => {
const voteData = doc.data();
if (totalPollIds.includes(voteData.poll)) {
pollVoteIds.push(voteData.poll)
totalVotes++;
if (winningChoiceIds.includes(voteData.choice)) {
pollVoteIdsCorrect.push(voteData.poll)
correctVotes++;
}
}
if (cnt>=querySnapshot.size-1){
console.log("Updating user ID: ", uid);
retrieveUserDoc.update({
['accuracyWeeks.week'+week]: true,
['accuracy.week'+week]: {
totalVotes: totalVotes,
correct: correctVotes,
accuracy: correctVotes/totalVotes,
correctVoteIds: pollVoteIdsCorrect,
totalVoteIds: pollVoteIds
}
});
}
cnt++;
})
});
console.log("F");
return false;
})
.catch((error) => {
console.log("Error getting documents: ", error);
});
});
Imagine for example that you want to store paginated data from an API to a database.
let db;
let pageitems = 35
var offset = 0;
dbConnect //establish connection to database
.then( fetch(apiLink+?offset=2)
.then( res => res.json())
.then( res => {
var total = res.count
return collection.insertMany(res.data, {ordered: false})
// If offset is less than total, I want to increase offset and go back to the fetch-event.
.catch( err => {
if(err.code !== 11000){log(err)}
else{log({completed: err.result.nInserted, duplicates:
err.result.result.writeErrors.length});}
})
.then(() => {
connection.close();
})
You could just use a regular loop:
(async function() {
const conn = await dbConnect;
for(let offset = 0; true; offset++) {
const { data, count } = await (await fetch(`api?page=${offset}`)).json();
// Exit if the page is empty
if(count === 0) break;
await collection.insertMany(data, { ordered: false });
}
})();
To speed that up you could execute multiple requests in parallel:
const chunkSize = 10; // 10 in parallel
for(let offset = 0; offset < chunkSize; offset++) {
(async function() {
const conn = await dbConnect;
for(let offset2 = 0; true; offset2 += chunkSize) {
const { data, count } = await (await fetch(`api?page=${offset + offset2}`)).json();
// Exit if the page is empty
if(count === 0) break;
await collection.insertMany(data, { ordered: false });
}
})();
}
Basically, you will want to wrap your fetch and insert into a function that you will call many times. See the below as an example to illustrate my point...
let db;
let pageitems = 35
var offset = 0;
var db = dbConnect() //establish connection to database
function fetch_and_insert(offset) {
db
.then(fetch(apiLink + "?" + offset))
.then(res => res.json())
.then(res => {
var total = res.count
collection.insertMany(res.data, { ordered: false })
.catch(err => {
if (err.code !== 11000) { log(err) }
else {
log({
completed: err.result.nInserted, duplicates: err.result.result.writeErrors.length
});
}
})
if (offset < total) return fetch_and_insert(offset + pageitems)
return null;
})
}
fetch_and_insert(offset)
.then(() => {
connection.close();
})