We Keep Coding

Are there multicall functions to query the token balance history of significant user or can get response with blockNumber?

I'm trying to query the token balance history of significant user with web3's batchRequest.
But the function can't guarantee the order of functions, so I get the balance datas not in order.
So I want to know other multicall functions which can get response with blockNumber to organaize the order after querying datas.
This is my current code
// this succeeded to query, and I'm using this
const Web3 = require('web3');
const { getTokens } = require('./utils/getTokens.js');
const { convertToNumber } = require('./utils/convertToNumber.js')
const { abi, rpcEndpoint, walletAddress, wsEndpoint } = require('./constant.js');
const { formatUnits } = require('ethers/lib/utils');
const web3 = new Web3(new Web3.providers.HttpProvider(rpcEndpoint));
const queryHistoryBalance = async (userAddress, tokenAddress, startBlock, endBlock, decimal) => {
const batch = new web3.BatchRequest(); // https://stackoverflow.com/questions/69788660/web3-batchrequest-always-returning-undefined-what-am-i-doing-wrong
const contract = new web3.eth.Contract(abi, tokenAddress);
/*
let step = Math.floor((endBlock - startBlock) / eachNumber);
if (step < 1) {
step = 1;
}
for (let n = 0; n < step; n++) {
}*/
let data = { [userAddress]: [] }
for (let n = startBlock; n <= endBlock; n++) {
batch.add(
contract.methods.balanceOf(userAddress).call.request({}, n, (err, res) => {
if (res) {
data[userAddress].push({ 'blockNumber': n, 'balance': res })
}
})
)
/*
batch.add(
contract.methods.balanceOf(userAddress).call.request({}, n, async (err, res) => {
const timestamp = await web3.eth.getBlock(n).timestamp;
data[userAddress].push({ 'blockNumber': n, 'blockTime': timestamp, 'balance': res })
//data[userAddress].push({ 'blockNumber': n,'balance': res })
})
)*/
}
await batch.execute();
//console.log(data);
let data2 = { [userAddress]: [] }
let formarNumber
await Promise.all(data[userAddress].map((element, i) => {
//console.log(element['balance'] / 1)
//data[userAddress][i]['balance'] = convertToNumber(element['balance'], 6)
let formatBalance = element['balance'] / (10 ** decimal);
data[userAddress][i]['balance'] = formatBalance;
if (formarNumber !== element['balance']) {
data2[userAddress].push({ 'blockNumber': element['blockNumber'], 'balance': formatBalance });
formarNumber = element['balance'];
}
if (i === 0) formarNumber = element['balance'];
}))
//console.log("data", data)
return data2
}
const main = async () => {
const data = await queryHistoryBalance('0x8d9e0f65569b3cc2f602a7158d02ab52976a4535', '0xdAC17F958D2ee523a2206206994597C13D831ec7', 14954947, 14958046, 6)
console.log(data)
}
main()
module.exports.queryHistoryBalance = queryHistoryBalance;
I appreciate any help and advice

Unable to find memory leak despite using proper tensor disposal (tfjs)

I've tried all sorts of ways to dispose of tensors (tf.dispose(), start/endscope).
The closest I've got was through this code, where 1 unused tensor remains after each execution. It takes about 2 hours for this program to run enough to use up 64 GB of RAM (big memory leak).
I also suspect that other factors besides TFJS-based operations are contributing to the memory leak, though (in theory) garbage collection should clean this up.
The piece of code below is one event that gets processed by an Event Listener handler. Any help with this issue would be greatly appreciated!
'use strict';
global.fetch = require("node-fetch");
const { MessageActionRow, MessageButton, Permissions } = require('discord.js');
const { mod, eco, m, n } = require(`../../index.js`);
const { Readable } = require('stream');
const PImage = require('pureimage');
const tf = require('#tensorflow/tfjs');
const tfnode = require('#tensorflow/tfjs-node');
const wait = require('util').promisify(setTimeout);
let bufferToStream = (binary) => {
let readableInstanceStream = new Readable({
read() {
this.push(binary);
this.push(null);
}
});
return readableInstanceStream;
}
const predict = async (imageUrl, modelFile) => {
let model = await tf.loadLayersModel(modelFile);
let modelClasses = [ "NSFW", "SFW" ];
let data = await fetch(imageUrl);
let fileType = data.headers.get("Content-Type");
let buffer = await data.buffer();
let stream = bufferToStream(buffer);
let image;
if ((/png/).test(fileType)) {
image = await PImage.decodePNGFromStream(stream);
}
else if ((/jpe?g/).test(fileType)) {
image = await PImage.decodeJPEGFromStream(stream);
}
else {
return;
}
let rawArray;
rawArray = tf.tidy(() => {
let tensorImage;
tensorImage = tf.browser.fromPixels(image).toFloat();
tensorImage = tf.image.resizeNearestNeighbor(tensorImage, [model.inputs[0].shape[1], model.inputs[0].shape[2]]);
tensorImage = tensorImage.reshape([1, model.inputs[0].shape[1], model.inputs[0].shape[2], model.inputs[0].shape[3]]);
return model.predict(tensorImage);
});
rawArray = await rawArray.data();
rawArray = Array.from(rawArray);
tf.disposeVariables();
model.layers.forEach(l => {
l.dispose();
});
if (rawArray[1] > rawArray[0]) {
return [`SFW`, rawArray[1]];
}
else {
return [`NSFW`, rawArray[0]];
}
};
const getResults = async (imageLink, imageNumber) => {
let image = `${imageLink}`;
let prediction = await predict(image, `file://D:/retake7/sfwmodel/model.json`);
let className = `SFW`;
if (prediction[0] == `NSFW`) {
className = `**NSFW**`;
}
return [`[Image ${imageNumber+1}](${imageLink}): ${className} (${(prediction[1]*100).toFixed(2)}% Certainty)`, ((prediction[1]*100).toFixed(2))*1];
}
const main = async (message, client, Discord) => {
if (message.attachments.size == 0 || message.author.bot || message.channel.nsfw) return;
await client.shard.broadcastEval(c => {
console.log(`Scanning...`);
}).catch(e => {
return;
});
let inChannel = await eco.seid.get(`${message.guild.id}.${message.channel.id}.active`);
let sfwImage = await eco.seid.get(`${message.guild.id}.sfwAlerts`);
if (inChannel == `no`) return;
let atmentArr = Array.from(message.attachments);
let msgArr = [];
if (message.attachments.size > 1) {
msgArr.push(`**Images Scanned**`);
} else {
msgArr.push(`**Image Scanned**`);
}
let hasNSFW = false;
let uncertain = false;
for (i = 0; i < message.attachments.size; i++) {
let msg = await getResults(atmentArr[i][1][`proxyURL`], i);
if (msg[1] < 80) {
uncertain = true;
}
if (msg[0].includes(`NSFW`)) {
hasNSFW = true;
}
msgArr.push(msg[0]);
}
if (uncertain == false && hasNSFW == false) {
let cont = `${msgArr.join(`\n`)}`;
msgArr = null;
client.seid.set(`${message.channel.id}.previousScan`, cont);
return;
}
let embed = new Discord.MessageEmbed()
.setColor(`GREEN`)
.setDescription(msgArr.join(`\n`));
let cont2 = `${msgArr.join(`\n`)}`;
client.seid.set(`${message.channel.id}.previousScan`, cont2);
msgArr = null;
if (sfwImage != `no` || hasNSFW || msg[1] <= 80) {
embed.setColor(`RED`);
await message.delete();
let msgSent = await message.channel.send({embeds: [embed], components: [row]});
};
};
module.exports = {
event: 'messageCreate',
run: async (message, client, Discord) => {
await main(message, client, Discord);
},
};

first, separate model loading and inference - in your current code, you'd reload a model each time you need to run prediction on a new image.
and then look at any possible leaks in prediction function - so once model is loaded.
you're loading a model and disposing each layer, but that doesn't mean model itself gets unloaded so there more than a chance that part of model remains in memory.
but leak itself is this line:
rawArray = await rawArray.data();
that variable is already used and its a tensor.
now you're overwriting the same variable with a data array and tensor never gets disposed.

Pulling from cryptowatch API

I am trying to pull price data from the API for cryptowatch, when I go to the URL with my API key it works fine, but my program isn't successfully pulling it so I am getting my error of Could not set price feed for cryptowatch:" + cryptowatchMarketId
I'm pretty stuck on where to go from here.
// Set initial prices
const cryptowatchApiKey = process.env.CRYPTOWATCH_API_KEY || MM_CONFIG.cryptowatchApiKey;
const cryptowatchMarkets = await fetch("https://api.cryptowat.ch/markets?apikey=" + cryptowatchApiKey).then(r => r.json());
const cryptowatchMarketPrices = await fetch("https://api.cryptowat.ch/markets/prices?apikey=" + cryptowatchApiKey).then(r => r.json());
for (let i in cryptowatchMarketIds) {
const cryptowatchMarketId = cryptowatchMarketIds[i];
try {
const cryptowatchMarket = cryptowatchMarkets.result.find(row => row.id == cryptowatchMarketId);
const exchange = cryptowatchMarket.exchange;
const pair = cryptowatchMarket.pair;
const key = `market:${exchange}:${pair}`;
PRICE_FEEDS['cryptowatch:'+cryptowatchMarketIds[i]] = cryptowatchMarketPrices.result[key];
} catch (e) {
console.error("Could not set price feed for cryptowatch:" + cryptowatchMarketId);
}
}
const subscriptionMsg = {
"subscribe": {
"subscriptions": []
}
}
for (let i in cryptowatchMarketIds) {
const cryptowatchMarketId = cryptowatchMarketIds[i];
// first get initial price info
subscriptionMsg.subscribe.subscriptions.push({
"streamSubscription": {
"resource": `markets:${cryptowatchMarketId}:book:spread`
}
})
}
let cryptowatch_ws = new WebSocket("wss://stream.cryptowat.ch/connect?apikey=" + cryptowatchApiKey);
cryptowatch_ws.on('open', onopen);
cryptowatch_ws.on('message', onmessage);
cryptowatch_ws.on('close', onclose);
cryptowatch_ws.on('error', console.error);
function onopen() {
cryptowatch_ws.send(JSON.stringify(subscriptionMsg));
}
function onmessage (data) {
const msg = JSON.parse(data);
if (!msg.marketUpdate) return;
const marketId = "cryptowatch:" + msg.marketUpdate.market.marketId;
let ask = msg.marketUpdate.orderBookSpreadUpdate.ask.priceStr;
let bid = msg.marketUpdate.orderBookSpreadUpdate.bid.priceStr;
let price = ask / 2 + bid / 2;
PRICE_FEEDS[marketId] = price;
}
function onclose () {
setTimeout(cryptowatchWsSetup, 5000, cryptowatchMarketIds);
}
}

Variable value is not incremented in the catch block

This is the code to fetch all the results from the website.
const puppeteer = require('puppeteer');
let students = [];
let rollPrefix = '387EA';
let regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
let i = 0;
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
while(i < 4){
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
}catch{
currReg++;
continue;
}
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Here the variable value of currReg stays the same pls help
The code tries each roll no and reg no combinations but there are some reg no that doesnt match with roll no so in the code the roll no should stay the same but the reg no should increase by one..

Not really sure what should happen with each combination, but here's an implementation which inputs all combinations. Below a short explanation:
const puppeteer = require('puppeteer');
let students = [];
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let i = 0;
let j = 0;
const rollPrefix = '387EA';
const regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
while(i < 4){
while(j < 4) {
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
console.log("rolltemp = ", rolltemp, " regtemp = ", regTemp);
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
} catch {
continue;
}
currReg++;
j++;
}
currReg = 80;
j = 0;
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Explanation
so, assuming you want all combinations of pairs {currentRol, currentReg}, you'll definitely need two loops. There are gonna be 4x4=16 combinations in total (I assume, basing on i < 4 condition). First mistake which you made was assigning regTemp before while loop, effectively not changing the strings entered to the inputs, only some unused, temporary values (currentRoll, currentReg). So, first and foremost is to move rollTemp and regTemp definitions into the while loop. Now, as I said, you're gonna need two nested loops, as you need to generate all possible combinations (for each currentRol, all currentRegs). One more thing to remember is that you'll have to reset currentReg with each outer loop iteration, as you want to test each reg for given roll.
Note about variables' scopes
This is a great example why variables scopes are critical when programming. Not only it increases readability and comprehensibility of the given code - it prevents other functions/scopes from using symbols which do not really belong to them. Please notice where the variables definitions are within my snippet. Probably it's not perfect, but why would you pollute global namespace as in your example?

Break the loop in the map function and move

So basically im working on a cron job in my app that fires every 3 hours and updating users 'score' by calling the RiotApi
basically the function so far
exports.updatePlayersPoints = async () => {
console.log('STARTED UPDATING');
try {
const players = await UserLadder.findAll();
await Promise.all(
players.map(async (player) => {
const p = await RiotAccount.findOne({
where: {
userId: player.userId,
},
include: RiotRegions,
});
const beginTime = new Date(player.dataValues.createdAt);
let data;
try {
const res = await axios.get(
`https://${
p.dataValues.riot_region.dataValues.name
}.api.riotgames.com/lol/match/v4/matchlists/by-account/${
p.dataValues.accountId
}?queue=420&beginTime=${beginTime.getTime()}&api_key=${
process.env.RIOT_KEY
}`
);
data = res.data;
} catch (error) {
if (!error.response.status === 404) {
console.error(error);
}
}
if (!data) {
return;
}
let totalScore = player.dataValues.userPoints;
await Promise.all(
data.matches.map(async (match, i) => {
if (i < 15) {
const { data } = await axios.get(
`https://${p.dataValues.riot_region.dataValues.name}.api.riotgames.com/lol/match/v4/matches/${match.gameId}?api_key=${process.env.RIOT_KEY}`
);
const calculateScore = () => {
return new Promise((resolve) => {
const { stats } = _.find(
data.participants,
(o) => o.championId === match.champion
);
const killsPts = stats.kills * 2;
const deathPts = stats.deaths * -1.5;
const assistsPts = stats.assists;
const wardsPts = stats.wardsPlaced / 4;
const firstBloodPts = stats.firstBloodKill ? 3 : 0;
const firstBloodAssistPts = stats.firstBloodAssist ? 3 : 0;
const firstTowerPts = stats.firstTowerKill ? 2 : 0;
const firstTowerAssistPts = stats.firstTowerAssist ? 2 : 0;
const score =
killsPts +
deathPts +
assistsPts +
wardsPts +
firstBloodPts +
firstBloodAssistPts +
firstTowerPts +
firstTowerAssistPts;
totalScore += score;
resolve();
});
};
await calculateScore();
}
})
);
const user = await UserLadder.findOne({
where: {
userId: player.userId,
},
});
user.userPoints = parseFloat(totalScore);
user.lastGameId = data.matches[0].gameId;
await user.save();
})
);
console.log('FINISHED UPDATING');
} catch (error) {
console.error(error);
}
};
Basically it just looks up the table userladder to find the players that are signed to the ladder and for each one of these players it fires a map function that makes a request to the riotapi to get the match history of this player and then later make an inside map function to map each one of these matches.
but basically I updated it to now keep track of the game id of the last call before 3 hours so it doesn't have to make request that was already done.
user.lastGameId = data.matches[0].gameId;
but now in my second map function that maps the matches I wasn't it so that if the last game from my database matches the game id that currently being mapped I want to stop the map function and not continue this record or the ones after because it also means they all have been already counted.
but I can not seem to find a way to do it.
i tried using break; but it didn't work
any ideas?
using for loop
I tried a small test with for loop so I tried
for (let i = 0; i < 15; i++) {
await new Promise(async (resolve, reject) => {
const match = data.matches[i];
console.log(match);
resolve();
if (i === 1) {
break;
}
});
}
but I still go the same error
SyntaxError: Illegal break statement

Instead of trying to "break" a map, you should filter the matches that you want to process before you execute the map.
Something like this:
await Promise.all(
const filteredMatches = data.matches.filter(match => match.gameId > previousId);
filteredMatches.map(async (match, i) => { ...
More on filter() in javascript.
Edit: If generated id's are random and are not ordered, you can store all previous id's in a Set, and then just ask if it has been previously added
await Promise.all(
const filteredMatches = data.matches.filter(match => mySet.has(match.gameId));
filteredMatches.map(async (match, i) => { ...
More on Set in javascript.