"Cleanup called..." Message log Tensorflow.js (Node) - javascript

I am using around 60000 + 10000 Images for creating the training and validation dataset using a generator
( Images used are the MNIST Images of Handwritten Digits in the PNG Format ).
But when I fit the model using them, "Cleanup called" message logs are constantly getting printed.
function* dataGenerator(type) {
const dataRoot = `MNIST/${type}-Data`;
const labels = fs.readdirSync(dataRoot);
for (let _idx = 0; _idx < labels.length; _idx++) {
const label = labels[_idx];
const files = fs.readdirSync(`${dataRoot}/${label}`);
for (let idx = 0; idx < files.length; idx++) {
const img = fs.readFileSync(`${dataRoot}/${label}/${files[idx]}`);
const imageTensor = tf.node.decodePng(img, 1);
const oneHotArr = new Array(labels.length).fill(0);
oneHotArr[label] = 1;
const oneHotTensor = tf.tensor1d(oneHotArr);
yield { xs: imageTensor, ys: oneHotTensor };
};
};
}
const trainingDataset = tf.data.generator(() => dataGenerator("Training"))
.shuffle(100)
.batch(100);
const validationDataset = tf.data.generator(() => dataGenerator("Validation"))
.shuffle(100)
.batch(100);
// Fitting the model
await model.fitDataset(trainingDataset, {
epochs: 5,
validationData: validationDataset
});
What am I doing wrong ?

nothing. message is info-only and comes from underlying tensorflow c library (where it was introduced at the wrong message level) used by tfjs-node.
planned upgrade of shared library from 2.7.3 to 2.9.1 will take care of that - it should be part of tfjs 3.21 once its released.

Related

Problem in using vanilla JavaScript in React js

I am trying to learn React.
One of the tasks I am trying to do is to convert my existing html/js/css files to React.
My question is:
Is there a simple, universal way to include JavaScript routines into React without changing anything in the JavaScript routine?
I have tried various ways given in various tutorials but am yet to find some method to simple use the routines without any changes.
Specifically, I have tried the following methods:
Using React-script-tag
Using Helmet
JavaScript DOM method.
I personally liked the third method but it gave me an error as follows:
Uncaught SyntaxError: Unexpected token '<' (at newfunction.js:1:1)
The javascript file was named newfunction.js and the script was as follows:
const loveMe = document.querySelector('.loveMe');
const times = document.querySelector('#times');
let clickTime = 0;
let timesClicked = 0;
loveMe.addEventListener('click', (e) => {
if (clickTime === 0) {
clickTime = new Date().getTime();
} else {
if (new Date().getTime() - clickTime < 800) {
createHeart(e);
clickTime = 0;
} else {
clickTime = new Date().getTime();
}
}
});
const createHeart = (e) => {
const heart = document.createElement('i');
heart.classList.add('fas');
heart.classList.add('fa-heart');
const x = e.clientX;
const y = e.clientY;
const leftOffset = e.target.offsetLeft;
const topOffset = e.target.offsetTop;
const xInside = x - leftOffset;
const yInside = y - topOffset;
heart.style.top = `${yInside}px`;
heart.style.left = `${xInside}px`;
loveMe.appendChild(heart);
times.innerHTML = ++timesClicked;
setTimeout(() => heart.remove(), 1000);
};
Would really appreciate some help in getting around this issue.
Many thanks

how to check if a github repository has unused packages?

I wrote the following code in Node.js:
const axios = require("axios");
const prompt = require("prompt-sync")();
let numRepo = prompt("Welcome, How many repositories to search? ");
numRepo = parseInt(numRepo);
while (!Number.isInteger(numRepo))
numRepo = parseInt(prompt("Please insert integer: "));
let n;
if (numRepo < 100) n = 1;
else n = numRepo % 100;
axios
.get(
`https://api.github.com/search/repositories?q=language:js&sort=stars&order=desc&per_page=${numRepo}&page=${n}`
)
.then((response) => {
let repo = [];
response.data.items.forEach((e) => repo.push(e));
console.log(repo);
})
.catch((error) => {
console.log(error);
});
I used the GitHub Search APi, and my goal is to write a function whose purpose is to check for each repository how much unused packaged it has.
How can this be done?

Adding inputs to a `tf.data.generator` in tensorflow.js

I'm trying to create a data generator, which I verified was working by itself in pure js. TFJS documentation for it is here, with two examples:
https://js.tensorflow.org/api/latest/#data.generator
I'd like to use a tf.data.generator as this datasets requires elaborate preprocessing. A minimal example is as follows:
const tf = require('#tensorflow/tfjs-node');
class dataGeneratorGenerator {
constructor(test) {
this.test = test
}
* dataGenerator() {
let len = this.test.length
let idx = 0
while (idx < len) {
idx++
console.log(idx)
yield this.test[idx]
}
}
}
let dgg = new dataGeneratorGenerator(['hi', 'hi2', 'hi3'])
let trainDs = tf.data.generator(dgg.dataGenerator);
trainDs.forEachAsync(e => console.log(e));
The error is as follows:
TypeError: Error thrown while iterating through a dataset: Cannot read property 'test' of undefined
Iterating through our datagenerator in pure javascript works:
let dgg = new dataGeneratorGenerator(['hi', 'hi2', 'hi3'])
let dg = dgg.dataGenerator()
console.log(dgg.next())
console.log(dgg.next())
console.log(dgg.next())
My understanding is that we are only passing dataGenerator into tf.data.generator instead of the entire class. Then, how is it possible to input variables into tf.data.generator? Thanks.
One can simply use an arrow function.
const tf = require('#tensorflow/tfjs-node');
function* dataGenerator(test) {
let len = test.length
let idx = 0
while (idx < len) {
idx++
console.log(idx)
}
}
let trainDs = tf.data.generator(() => dataGenerator(['hi', 'hi2', 'hi3']));
trainDs.forEachAsync(e => console.log(e));

Standard error of estimated parameters with tensorflow.js

I am using tensorflow.js to get the parameters of an exponential regression:
y(x) = c0*e^(kx)
I attach the code:
x = tf.tensor1d(x);
y = tf.tensor1d(y);
const c0 = tf.scalar().variable();
const k = tf.scalar(this.regression_parameters.array[index][0]).variable();
// y = c0*e^(k*x)
const fun = (x) => x.mul(k).exp().mul(c0);
const cost = (pred, label) => pred.sub(label).square().mean();
const learning_rate = 0.1;
const optimizer = tf.train.adagrad(learning_rate);
// Train the model.
for (let i = 0; i < 500; i++) {
optimizer.minimize(() => cost(fun(x), y));
}
which fits well the experimental signal. However, I need to report a standard error of the estimations (c0 and k), as it is given in SciPy by curve_fit(). I am wondering if this can be done with tensorflow.js. If not, is there any other JavaScript library that could be useful? Thanks!
For anyone interested, I end up estimating the Hessian matrix of the exponential function myself to calculate the standard error of the parameters after tensorflow.js has optimised them. I followed the code on:
https://www.cpp.edu/~pbsiegel/javascript/curvefitchi.html
which is as follows:
function hess_exp_errors(c0, k, x, y){
var sum1=0,sum2=0,sum3=0,sum4=0,sum5=0, expon1,
he11, he12, he22, dett, dof, k_err, c0_error;
for (i=0;i<x.length;i++){
expon1=Math.exp(k*x[i]);
sum1+=expon1*expon1;
sum2+=y[i]*x[i]*x[i]*expon1;
sum3+=x[i]*x[i]*expon1*expon1;
sum4+=y[i]*x[i]*expon1;
sum5+=x[i]*expon1*expon1;
}
he11=4*c0*c0*sum3-2*c0*sum2; he22=2*sum1;
he12=4*c0*sum5-2*sum4; dett=he11*he22-he12*he12;
c0_err=Math.sqrt(he11/dett); k_err=Math.sqrt(he22/dett);
return [c0_err, k_err];
};

Classification of Images using Nodes JS

I am kinda new to NodeJs. I have been trying to make an API which can classify an image and give the resultant in a Json object. The resultant is score prediction using NSFWJS.
My task is when I pass a JSON object with a {key:value} pair where key is "Codes" and the values is a list of Codes which is using to fetch url object, it should parse through every image and give the resultant output.
* Basic flow of the code *
Input : A Json object something like this :- * {'Codes':['Code1','Code2','Code3']} *
Loop to all the objects from the value :
1. parse through a function to store the image
2. pas through the model to get the prediction
3. Stored the resultant in a Json object
Return the final Json object
This the code of my implementation :-
const express = require('express')
const multer = require('multer')
const jpeg = require('jpeg-js')
const tf = require('#tensorflow/tfjs-node')
const nsfw = require('nsfwjs')
const bodyParser=require("body-parser")
const axios = require('axios');
const cheerio = require('cheerio');
var fs = require('fs');
var http = require('http');
var request=require('request');
const app = express()
const upload = multer()
app.use(bodyParser.json());
let _model
let file_content;
let test_data;
function convertBuffer()
{
return new Promise(function(resolve,reject)
{
try
{
file_content=fs.readFileSync('ans1.png');
resolve(file_content);
}
catch
{
reject('Null String')
}
})
}
app.post('/nsfw',async(req,res) =>{
var arrayCodes=req.body.Codes;
var jsonObj={};
var i=0;
var iterator=function(i){
if(i>arrayCodes.length)
{
console.log("Testing",jsonObj);
res.json(jsonObj);
return;
}
}
while(i<arrayCodes.length)
{
var imgUrl="http://url/to/image"+String(Codes[i])
//getImageUrl(imgUrl);
await axios.get(imgUrl)
.then(response=>{
var result=response.data;
var resultJson=result.facets.Media.productImage.url;
var localPath="ans1.png"
var fullUrl = resultJson;
var download=function(uri,filename,callback){
request.head(uri,function(err,res,body){
// console.log('Content-Type:',res.headers['content-type']);
// console.log('Content-length:',res.headers['content-length']);
request(uri).pipe(fs.createWriteStream(filename).on('close',callback));
});
}
download(fullUrl,localPath,function(){console.log("Done")});
var convertBufferdata=convertBuffer();
convertBufferdata.then(async function(result){
var image =await jpeg.decode(file_content, true)
const numChannels = 3
const numPixels = image.width * image.height
const values = new Int32Array(numPixels * numChannels)
for (let i = 0; i < numPixels; i++)
for (let c = 0; c < numChannels; ++c)
values[i * numChannels + c] = image.data[i * 4 + c]
test_data=tf.tensor3d(values, [image.height, image.width, numChannels], 'int32')
var predictions=await _model.classify(test_data);
jsonObj[String(arrayAsins[i])]=predictions;
console.log('JsonObj',jsonObj);
i=i+1;
console.log("I going out is",i);
iterator(i);
},function(err){console.log(err)})
})
.catch(error=>{
i=arrayCodes.length+1;
console.log(error);
})
}
})
const load_model = async () => {
_model = await nsfw.load()
}
load_model().then(() => app.listen(8080))
What problem I am facing is that the code is not completely synchronous due to which the resultant Json object is maybe having more keys than it should be or maybe not all the values retrieved are correct (meaning that it is getting overwritten).
* After every time the single image gets fetched and processed , the next image is then download/overwritten to the same image *
Any kind of help in refactoring of the code or making some changes to fix the issue will be appreciated.

Categories