We Keep Coding

Unable to find memory leak despite using proper tensor disposal (tfjs)

I've tried all sorts of ways to dispose of tensors (tf.dispose(), start/endscope).
The closest I've got was through this code, where 1 unused tensor remains after each execution. It takes about 2 hours for this program to run enough to use up 64 GB of RAM (big memory leak).
I also suspect that other factors besides TFJS-based operations are contributing to the memory leak, though (in theory) garbage collection should clean this up.
The piece of code below is one event that gets processed by an Event Listener handler. Any help with this issue would be greatly appreciated!
'use strict';
global.fetch = require("node-fetch");
const { MessageActionRow, MessageButton, Permissions } = require('discord.js');
const { mod, eco, m, n } = require(`../../index.js`);
const { Readable } = require('stream');
const PImage = require('pureimage');
const tf = require('#tensorflow/tfjs');
const tfnode = require('#tensorflow/tfjs-node');
const wait = require('util').promisify(setTimeout);
let bufferToStream = (binary) => {
let readableInstanceStream = new Readable({
read() {
this.push(binary);
this.push(null);
}
});
return readableInstanceStream;
}
const predict = async (imageUrl, modelFile) => {
let model = await tf.loadLayersModel(modelFile);
let modelClasses = [ "NSFW", "SFW" ];
let data = await fetch(imageUrl);
let fileType = data.headers.get("Content-Type");
let buffer = await data.buffer();
let stream = bufferToStream(buffer);
let image;
if ((/png/).test(fileType)) {
image = await PImage.decodePNGFromStream(stream);
}
else if ((/jpe?g/).test(fileType)) {
image = await PImage.decodeJPEGFromStream(stream);
}
else {
return;
}
let rawArray;
rawArray = tf.tidy(() => {
let tensorImage;
tensorImage = tf.browser.fromPixels(image).toFloat();
tensorImage = tf.image.resizeNearestNeighbor(tensorImage, [model.inputs[0].shape[1], model.inputs[0].shape[2]]);
tensorImage = tensorImage.reshape([1, model.inputs[0].shape[1], model.inputs[0].shape[2], model.inputs[0].shape[3]]);
return model.predict(tensorImage);
});
rawArray = await rawArray.data();
rawArray = Array.from(rawArray);
tf.disposeVariables();
model.layers.forEach(l => {
l.dispose();
});
if (rawArray[1] > rawArray[0]) {
return [`SFW`, rawArray[1]];
}
else {
return [`NSFW`, rawArray[0]];
}
};
const getResults = async (imageLink, imageNumber) => {
let image = `${imageLink}`;
let prediction = await predict(image, `file://D:/retake7/sfwmodel/model.json`);
let className = `SFW`;
if (prediction[0] == `NSFW`) {
className = `**NSFW**`;
}
return [`[Image ${imageNumber+1}](${imageLink}): ${className} (${(prediction[1]*100).toFixed(2)}% Certainty)`, ((prediction[1]*100).toFixed(2))*1];
}
const main = async (message, client, Discord) => {
if (message.attachments.size == 0 || message.author.bot || message.channel.nsfw) return;
await client.shard.broadcastEval(c => {
console.log(`Scanning...`);
}).catch(e => {
return;
});
let inChannel = await eco.seid.get(`${message.guild.id}.${message.channel.id}.active`);
let sfwImage = await eco.seid.get(`${message.guild.id}.sfwAlerts`);
if (inChannel == `no`) return;
let atmentArr = Array.from(message.attachments);
let msgArr = [];
if (message.attachments.size > 1) {
msgArr.push(`**Images Scanned**`);
} else {
msgArr.push(`**Image Scanned**`);
}
let hasNSFW = false;
let uncertain = false;
for (i = 0; i < message.attachments.size; i++) {
let msg = await getResults(atmentArr[i][1][`proxyURL`], i);
if (msg[1] < 80) {
uncertain = true;
}
if (msg[0].includes(`NSFW`)) {
hasNSFW = true;
}
msgArr.push(msg[0]);
}
if (uncertain == false && hasNSFW == false) {
let cont = `${msgArr.join(`\n`)}`;
msgArr = null;
client.seid.set(`${message.channel.id}.previousScan`, cont);
return;
}
let embed = new Discord.MessageEmbed()
.setColor(`GREEN`)
.setDescription(msgArr.join(`\n`));
let cont2 = `${msgArr.join(`\n`)}`;
client.seid.set(`${message.channel.id}.previousScan`, cont2);
msgArr = null;
if (sfwImage != `no` || hasNSFW || msg[1] <= 80) {
embed.setColor(`RED`);
await message.delete();
let msgSent = await message.channel.send({embeds: [embed], components: [row]});
};
};
module.exports = {
event: 'messageCreate',
run: async (message, client, Discord) => {
await main(message, client, Discord);
},
};

first, separate model loading and inference - in your current code, you'd reload a model each time you need to run prediction on a new image.
and then look at any possible leaks in prediction function - so once model is loaded.
you're loading a model and disposing each layer, but that doesn't mean model itself gets unloaded so there more than a chance that part of model remains in memory.
but leak itself is this line:
rawArray = await rawArray.data();
that variable is already used and its a tensor.
now you're overwriting the same variable with a data array and tensor never gets disposed.

Javascript How to use Async/Await with a mysql

I'm trying to query my mysql database for a value using Async/Await but my constant 'map' always returns undefined, I am not too sure why
const get_map = () =>{
db.query('SELECT game,game2 FROM battles WHERE player1=? OR player2=?',[result.char,result.char],(err,results)=>{
if(err){
return(err)
}
if(!results[0]){
return('No such battle?')
}
console.log(results[0])
return(results[0])
})
}
const proccessTurn = async ()=>{
const map = await get_map()
console.log(map)
let game = JSON.parse(map.game)
let game2 = JSON.parse(map.game2)
const char = result.char
const playerTurn = game2.turn[0]
}
The console doesn't even log
console.log(results[0])
This line so why does the await function resolve before it actually returns something? Shouldn't it wait for the returns?
Thank you for the help

As the mysql library doesn't support promises, you can use the promisify util function from Node.js like this:
const { promisify } = require('util');
const query = promisify(db.query).bind(db);
const get_map = async () => {
try {
const result = await query('SELECT game,game2 FROM battles WHERE player1=? OR player2=?',[result.char,result.char]);
if(!results[0]){
return 'No such battle?';
}
console.log(results[0])
return results[0];
}
catch(e) {
return e;
}
}
const proccessTurn = async ()=>{
const map = await get_map()
console.log(map)
let game = JSON.parse(map.game)
let game2 = JSON.parse(map.game2)
const char = result.char
const playerTurn = game2.turn[0]
}

The third parameter to db.query() is a callback. If you're returning anything in callback it's actually returned to the function itself, (db.query in this case). You're not returning anything from get_map(). So return a new promise from get_map like this,
const get_map = () =>{
return new Promise((resolve, reject) => {
db.query('SELECT game,game2 FROM battles WHERE player1=? OR player2=?',[result.char,result.char],(err,results)=>{
if(err){
reject(err)
}
if(!results[0]){
reject('No such battle?')
}
console.log(results[0])
resolve(results[0])
}
});
}
const proccessTurn = async ()=>{
const map = await get_map()
console.log(map)
let game = JSON.parse(map.game)
let game2 = JSON.parse(map.game2)
const char = result.char
const playerTurn = game2.turn[0]
});

node js async function won't stop

I am trying to break from the loop in an async function in node js, but it seems not to work. Once the script it's done, it starts executing again from the beginning. I can't use process.exit(), because then it won't save my logs.
Any idea how can I terminate this function?
Here is the code:
async function program() {
const xlsxFile = require('read-excel-file/node');
let rows = await xlsxFile('./file.xlsx', { sheet: 'sheet' });
const userRolesArray = [rows[0][1], rows[0][2]];
let roles = await getAllRoles();
let users = await getAllUsers();
let count = 0;
roleId = null;
for (userRole in userRolesArray) {
for (i in rows){
let someObject = {
test1: rows[i][0],
test1: rows[i][2],
};
if (someObject.test1 =='undefined') {
fs = require('fs')
await fs.appendFile('log_file.txt', i+")"+JSON.stringify(someObject)+"\n", function (err) {
if (err) throw err;
// console.log('Saved!');
});
} else {
//do something else;
}
}
}
count++;
if (userRole === '1') {
// loop has reached the end...Now exit
break;
}
};
program().then(() => {
return "Finished....!"
});

Node.js How to retrieve data from http.get response

I am doing some practice in node.js. In this exercise I been asked to find a country name through a GET Http Request to an endpoint passing a page integer as a parameter.
Where the important response structs are these {page, total_pages, data}.
page is the current page,
total_pages is the last page,
data is an array of 10 country object.
In getCountryName func I am able to retrieve the right answer only if the answer is on the 1st page, the 1 iteration of the loop. So, why the loop only happens once?
Aditional, I wanted to retrieve the total_pages to replace the hardcode '25' value but I do not figure it out how to return it along with the search.
Any hint you wanna give me? The whole problem is in getCountryCode func.
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
var res = '';
var pages = 25;
var i = 1;
while(i <= pages && res == ''){
console.log(i);
res = makeRequest(i)
.then(data => {
let f = ''
let p = data['total_pages'];
let search = data['data'].find(o => o.alpha3Code === code);
f = search != null ? search['name'] : f;
return f;
});
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();

Without modifying your code too much, this is how you do it:
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
const pages = 25;
var i = 1;
let f = null
while(i <= pages && f === null){
console.log(i);
const data = await makeRequest(i) // put in try/catch
const p = data['total_pages'];
const search = data['data'].find(o => o.alpha3Code === code);
f = search !== null ? search['name'] : null;
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();

Variable value is not incremented in the catch block

This is the code to fetch all the results from the website.
const puppeteer = require('puppeteer');
let students = [];
let rollPrefix = '387EA';
let regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
let i = 0;
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
while(i < 4){
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
}catch{
currReg++;
continue;
}
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Here the variable value of currReg stays the same pls help
The code tries each roll no and reg no combinations but there are some reg no that doesnt match with roll no so in the code the roll no should stay the same but the reg no should increase by one..

Not really sure what should happen with each combination, but here's an implementation which inputs all combinations. Below a short explanation:
const puppeteer = require('puppeteer');
let students = [];
(async () => {
const browser = await puppeteer.launch({
headless: false, // Show the window for debugging
slowMo: 150 // slow down by 50ms
});
const page = await browser.newPage();
let i = 0;
let j = 0;
const rollPrefix = '387EA';
const regPrefix = 'EA87S18';
let currRoll = 80;
let currReg = 80;
while(i < 4){
while(j < 4) {
let rolltemp = rollPrefix + pad(currRoll,3);
let regTemp = regPrefix + pad(currReg,3);
console.log("rolltemp = ", rolltemp, " regtemp = ", regTemp);
await page.goto('http://orissaresults.nic.in/CHSE');
await page.type('#txtRollNo', rolltemp);
await page.type('#txtRegNo', regTemp);
const element = await page.$("#divCaptch");
const text = await (await element.getProperty('textContent')).jsonValue();
await page.type('#txt_UserCaptcha', text);
await page.click('#btnSubmit');
page.on('dialog', async (dialog) => {
await dialog.dismiss().catch(() => {
console.log(dialog.message());
return new Result(TestStatus.FAIL, dialog.message());
})})
try{
await page.waitForNavigation()
await page.waitForSelector('table');
const RollNO = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[0].cells[1].innerText.trim();
});
const Name = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[2].cells[1].innerText.trim();
});
const RegNo = await page.evaluate(() => {
return document.querySelectorAll('table')[2].rows[1].cells[1].innerText.trim();
});
const Total = await page.evaluate(() => {
return document.querySelectorAll('table')[3].rows[8].cells[0].innerText.trim();
});
let student = new Student(RollNO,Name,RegNo,Total)
students.push(student)
} catch {
continue;
}
currReg++;
j++;
}
currReg = 80;
j = 0;
currRoll++;
i++;
}
await browser.close()
// let json = JSON.stringify(students);
// storeData(json,'test.json')
})();
// function delay(time) {
// return new Promise(function(resolve) {
// setTimeout(resolve, time)
// });
// }
function pad(num, size) {
var s = num+"";
while (s.length < size) s = "0" + s;
return s;
}
class Student {
constructor(roll,name,reg,total){
this.roll = roll;
this.name = name;
this.reg = reg;
this.total = total;
}
}
const fs = require('fs')
const storeData = (data, path) => {
try {
fs.writeFileSync(path, data)
} catch (err) {
console.error(err)
}
}
Explanation
so, assuming you want all combinations of pairs {currentRol, currentReg}, you'll definitely need two loops. There are gonna be 4x4=16 combinations in total (I assume, basing on i < 4 condition). First mistake which you made was assigning regTemp before while loop, effectively not changing the strings entered to the inputs, only some unused, temporary values (currentRoll, currentReg). So, first and foremost is to move rollTemp and regTemp definitions into the while loop. Now, as I said, you're gonna need two nested loops, as you need to generate all possible combinations (for each currentRol, all currentRegs). One more thing to remember is that you'll have to reset currentReg with each outer loop iteration, as you want to test each reg for given roll.
Note about variables' scopes
This is a great example why variables scopes are critical when programming. Not only it increases readability and comprehensibility of the given code - it prevents other functions/scopes from using symbols which do not really belong to them. Please notice where the variables definitions are within my snippet. Probably it's not perfect, but why would you pollute global namespace as in your example?