I used tf.expandDims() to add dimensions. Since I'm able to go into model.fit(), but got stuck due to this error Cannot start training because another fit() call is ongoing. and Cannot read property 'length' of undefined.
You can find my code here
// Train the model using the data.
let tesnor_dim =[];
let tensr;for(var j=0; j<2; j++){
console.log('resize_image',resize_image);
tensr = tf.expandDims(ysarr[j], 0);
tesnor_dim.push(tensr);
console.log('tesnor_dim',tesnor_dim);
model.fit(resize_image[j], tesnor_dim[j], {epochs: 100}).then((loss) => {
console.log('resize_image[j]',resize_image[j]);
console.log('tesnor_dim[j]',tesnor_dim[j]);
console.log('loss',loss);
const t = model.predict(resize_image[j]);
console.log('Prediction:::'+t);
pred = t.argMax(1).dataSync(); // get the class of highest probability
const labelsPred = Array.from(pred).map(e => setLabel[e])
console.log('labelsPred:::'+labelsPred);
//const saveResults = model.save('downloads://my-model-1');
//console.log(saveResults);
}).catch((e) => {
console.log(e.message);
})
}
When multiple fit are called on the same model, they have to be done sequentially. It means that the second call has to start only when the first one has completed. Using async and await will prevent your second call to happen unless the first has completed.
loss = await model.fit(resize_image[j], tesnor_dim[j], {epochs: 100})
// continue rest of processing
you may use this code,
await model.fit(resize_image[j], tesnor_dim[j], {epochs: 100}).then((loss) => {
console.log('resize_image[j]',resize_image[j]);
console.log('tesnor_dim[j]',tesnor_dim[j]);
console.log('loss',loss);
const t = model.predict(resize_image[j]);
console.log('Prediction:::'+t);
pred = t.argMax(1).dataSync(); // get the class of highest probability
const labelsPred = Array.from(pred).map(e => setLabel[e])
console.log('labelsPred:::'+labelsPred);
//const saveResults = model.save('downloads://my-model-1');
//console.log(saveResults);
}).catch((e) => {
console.log(e.message);
})
}
Related
Context
I'm retrieving data from the ESPN API to fetch weekly NFL matchup data. So, I'm making 18 api calls each time I need to fetch this data to account for all 18 weeks in the NFL season. I'm then creating an array with the data I need from the responses to those calls and writing out 18 files that align with each week in the NFL season (week1.json, week2.json, etc.).
Problem
The problem is that when I call my endpoint, I am seeing 2 things intermittently, and not necessarily at the same time:
(1) Some of the json files(week1.json, week2.json, etc.) include only a portion of the expected array. So, instead of 16 objects in the array, I may see only 4, or only 6, etc. Why would I only see a portion of the response data written to the array that's ultimately written to the .json files?
(2) Not all files are written to each time the endpoint is called. So, I may see that only week1-week5's .json files are written. Why aren't all of them updated?
Problem Code
// iterate 18 times
for (let i = 0; i < 18; i++) {
let weekNumber;
weekNumber = i + 1;
const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
const weekJson = JSON.parse(week);
// empty weekJson.games array
weekJson.games = []
// get all items
axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en®ion=us`)
.then(response => {
const schedule = [];
// get all items from response
const items = response.data.items
// console.log(response.data.items)
items.forEach(item => {
// make get call to $ref
axios.get(item.$ref)
.then(response => {
// get name
const name = response.data.name
// get date
const date = response.data.date
// get event id
const eventid = response.data.id
// get team ids
let team1 = response.data.competitions[0].competitors[0].id
let team2 = response.data.competitions[0].competitors[1].id
// create new object
const newObject = {
name: name,
date: date,
eventid: eventid,
team1: team1,
team2: team2
}
// add games for week
weekJson.games.push(newObject);
fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));
})
.catch(error => {
console.log(error)
})
})
}).catch(error => {
console.log(error)
})
}
Updated Code
router.get('/getschedules', (req, res) => {
async function writeGames() {
// iterate 18 times
for (let i = 0; i < 18; i++) {
let weekNumber;
weekNumber = i + 1;
const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
const weekJson = JSON.parse(week);
// empty weekJson.games array
weekJson.games = []
// get all items
// Add await keyword to wait for a week to be processed before going to the next one
await axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en®ion=us`)
.then(async (response) => { // add async to be able to use await
const schedule = [];
// get all items from response
const items = response.data.items
console.log(response.data.items)
// Use standard loop to be able to benefit from async/await
for (let item of items) {
// make get call to $ref
// wait for an item to be processed before going to the next one
await axios.get(item.$ref)
.then(response => {
// get name
const name = response.data.name
// get date
const date = response.data.date
// get event id
const eventid = response.data.id
// get team ids
let team1 = response.data.competitions[0].competitors[0].id
let team2 = response.data.competitions[0].competitors[1].id
// create new object
const newObject = {
name: name,
date: date,
eventid: eventid,
team1: team1,
team2: team2
}
// add games for week
weekJson.games.push(newObject);
})
.catch(error => {
console.log(error)
})
}
// moved out of the for loop since you only need to write this once
fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));
}).catch(error => {
console.log(error)
})
}
}
writeGames();
})
Your issue might come from the fact that you are looping over an array of item that triggers parallel asynchronous calls and write weekJson before you get the entire data. (But theoretically your code should work if writeSyncFile is really synchronous, maybe there are locks on the file system that prevents node to write properly?)
You could try to make everything sequential and only write weekJson once instead of everytime you go over an item:
EDIT
I updated my original code proposition by keeping parallel calls and it worked for me (it's similar to OP's code but I only write the json file once per week).
Then I tried to run OP's code and it was working fine as well. So this makes me think that the problem isn't from the code itself but rather how it's called. As a pure node script, there doesn't seem to be any issue. But I just noticed that OP is using it server side as the result of an API call.
Having an API write so many JSON concurrently is probably not the best idea (especially if the api is called multiple times almost simultaneously). You could either
just return the games in the response
or precompute the results
or fetch and write them only once then cache the result to be reused
Then I wonder if due to the server context, there is not some kind of timeout since OP said that with my initial solution, only the first week was created.
const axios = require("axios");
const fs = require("fs");
async function writeGames() {
const writeWeekGamesPromises = [];
// iterate 18 times
for (let weekNumber = 1; weekNumber < 19; weekNumber++) {
// give week a default value in case the json file doesn't exist (for repro purpose)
let week = "{}";
try {
week = fs.readFileSync(`./pickem/week${weekNumber}.json`, "utf8");
} catch (e) {
console.log(`error reading week ${weekNumber} json file:`, e);
// file doesn't exist yet
}
const weekJson = JSON.parse(week);
// empty weekJson.games array
const games = [];
weekJson.games = games;
// get all items
// Add await keyword to wait for a week to be processed before going to the next one
writeWeekGamesPromises.push(axios
.get(
`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en®ion=us`
)
.then(async (eventListResponse) => {
// add async to be able to use await
const schedule = [];
console.log(JSON.stringify(eventListResponse.data),'\n');
// get all items from response
const items = eventListResponse.data.items;
// console.log(eventListResponse.data.items); // this seems to be useless since we log the whole data just above
// parallelize calls and wait for all games from a week to be fetched before writing the file
await Promise.all(
items.map((item) => {
// we return the promise so that Promise.all will wait for all games to be pushed before going on writing the file
return axios
.get(item.$ref)
.then((response) => {
// get name, date and eventid
const {name, date, id: eventid} = response.data;
// get team ids
let team1 = response.data.competitions[0].competitors[0].id;
let team2 = response.data.competitions[0].competitors[1].id;
games.push({ name, date, eventid, team1, team2 });
})
.catch((error) => {
console.log(error);
});
})
);
// Now that all game data is ready, write in the file
fs.writeFileSync(
`./pickem/week${weekNumber}.json`,
JSON.stringify(weekJson)
);
})
.catch((error) => {
console.log(error);
}));
}
// Waiting for all games from all weeks to be processed
await Promise.all(writeWeekGamesPromises);
}
async function runAndLogTime() {
const start = Date.now();
await writeGames();
console.log(`took ${(Date.now() - start) / 1000}s to write all json files`);
}
runAndLogTime();
Good Afternoon,
I am using the MERN stack to making a simple invoice application.
I have a function that runs 2 forEach() that goes through the invoices in the DB and the Users. if the emails match then it gives the invoices for that user.
When I log DBElement to the console it works, it has the proper data, but when I log test1 to the console (app.get()) it only has one object not both.
// forEach() function
function matchUserAndInvoice(dbInvoices, dbUsers) {
dbInvoices.forEach((DBElement) => {
dbUsers.forEach((userElement) => {
if(DBElement.customer_email === userElement.email){
const arrayNew = [DBElement];
arrayNew.push(DBElement);
app.set('test', arrayNew);
}
})
})
}
// end point that triggers the function and uses the data.
app.get('/test', async (req,res) => {
const invoices = app.get('Invoices');
const users = await fetchUsersFromDB().catch((e) => {console.log(e)});
matchUserAndInvoice(invoices,users,res);
const test1 = await app.get('test');
console.log(test1);
res.json(test1);
})
function matchUserAndInvoice(dbInvoices, dbUsers) {
let newArray = [];
dbInvoices.forEach((DBElement) => {
dbUsers.forEach(async(userElement) => {
if(DBElement.customer_email === userElement.email){
newArray.push(DBElement);
app.set('test', newArray);
}
})
})
}
app.set('test', DBElement); overrides the existing DBElement, so only the last matching DBElement is shown in test1.
If you want to have test correspond to all matching DBElement, you should set it to an array, and then append a new DBElement to the array each time it matches inside the for-loop:
if(DBElement.customer_email === userElement.email){
let newArray = await app.get('test');
newArray.push(DBElement);
app.set('test', newArray);
}
I have created an async function that will extra the data from the argument, create a Postgres query based on a data, then did some processing using the retrieved query data. Yet, when I call this function inside a map function, it seemed like it has looped through all the element to extra the data from the argument first before it proceed to the second and the third part, which lead to wrong computation on the second element and onwards(the first element is always correct). I am new to async function, can someone please take at the below code? Thanks!
async function testWeightedScore(test, examData) {
var grade = [];
const testID = examData[test.name];
console.log(testID);
var res = await DefaultPostgresPool().query(
//postgres query based on the score constant
);
var result = res.rows;
for (var i = 0; i < result.length; i++) {
const score = result[i].score;
var weightScore = score * 20;
//more computation
const mid = { "testID": testID, "score": weightScore, more values...};
grade.push(mid);
}
return grade;
}
(async () => {
const examSession = [{"name": "Sally"},{"name": "Bob"},{"name": "Steph"}]
const examData = {
"Sally": 384258,
"Bob": 718239,
"Steph": 349285,
};
var test = [];
examSession.map(async sesion => {
var result = await testWeightedScore(sesion,examData);
let counts = result.reduce((prev, curr) => {
let count = prev.get(curr.testID) || 0;
prev.set(curr.testID, curr.score + count);
return prev;
}, new Map());
let reducedObjArr = [...counts].map(([testID, score]) => {
return {testID, score}
})
console.info(reducedObjArr);
}
);
})();
// The console log printed out all the tokenID first(loop through all the element in examSession ), before it printed out reducedObjArr for each element
The async/await behaviour is that the code pause at await, and do something else (async) until the result of await is provided.
So your code will launch a testWeightedScore, leave at the postgresql query (second await) and in the meantime go to the other entries in your map, log the id, then leave again at the query level.
I didn't read your function in detail however so I am unsure if your function is properly isolated or the order and completion of each call is important.
If you want each test to be fully done one after the other and not in 'parallel', you should do a for loop instead of a map.
I'm trying to iterate and print out in order an array in Javascript that contains the title of 2 events that I obtained from doing web scraping to a website but it prints out in disorder. I know Javascript is asynchronous but I'm new in this world of asynchronism. How can I implement the loop for to print the array in order and give customized info?
agent.add('...') is like console.log('...'). I'm doing a chatbot with DialogFlow and NodeJs 8 but that's not important at this moment. I used console.log() in the return just for debug.
I tried the next:
async function printEvent(event){
agent.add(event)
}
async function runLoop(eventsTitles){
for (let i = 0; i<eventsTitles.length; i++){
aux = await printEvent(eventsTitles[i])
}
}
But i got this error error Unexpected await inside a loop no-await-in-loop
async function showEvents(agent) {
const cheerio = require('cheerio');
const rp = require('request-promise');
const options = {
uri: 'https://www.utb.edu.co/eventos',
transform: function (body) {
return cheerio.load(body);
}
}
return rp(options)
.then($ => {
//** HERE START THE PROBLEM**
var eventsTitles = [] // array of event's titles
agent.add(`This mont we have these events available: \n`)
$('.product-title').each(function (i, elem) {
var event = $(this).text()
eventsTitles.push(event)
})
agent.add(`${eventsTitles}`) // The array prints out in order but if i iterate it, it prints out in disorder.
// *** IMPLEMENT LOOP FOR ***
agent.add(`To obtain more info click on this link https://www.utb.edu.co/eventos`)
return console.log(`Show available events`);
}).catch(err => {
agent.add(`${err}`)
return console.log(err)
})
}
I would like to always print out Event's title #1 and after Event's title #2. Something like this:
events titles.forEach((index,event) => {
agent.add(`${index}. ${event}`) // remember this is like console.log(`${index}. ${event}`)
})
Thanks for any help and explanation!
There no async case here but if you still face difficultly than use this loop
for (let index = 0; index < eventsTitles.length; index++) {
const element = eventsTitles[index];
agent.add(${index}. ${element})
}
So I'm trying to connect to external server called Pexels to get some photos. I'm doing that from node.js but it is just a javascript issue. Pexels unfortunately lets user to download object with only 40 pictures per page.
https://api.pexels.com/v1/curated?per_page=40&page=1 // 40 is maximum
But actually I need more then that. I'd like to get 160 results, ie. to combine all first four pages. In order to do that I tried looping the request:
let pexelsData = [];
for(let i = 1; i < 5; i++) {
const randomPage = getRandomFromRange(1, 100); //pages should be randomized
const moreData = await axios.get(`https://api.pexels.com/v1/curated?per_page=40&page=${randomPage}`,
createHeaders('bearer ', keys.pexelsKey));
pexelsData = [ ...moreData.data.photos, ...pexelsData ];
}
Now I can use pexelsData but it work very unstable, sometimes it is able to get all combined data, sometimes it crashes. Is there a correct and stable way of looping requests?
You work with 3rd party API, which has rate limits. So you should add rate limits to your code. The simplest solution for you is using p-limit or similar approach form promise-fun
It will looks like that:
const pLimit = require('p-limit');
const limit = pLimit(1);
const input = [
limit(() => fetchSomething('foo')),
limit(() => fetchSomething('bar')),
limit(() => doSomething())
];
(async () => {
// Only one promise is run at once
const result = await Promise.all(input);
console.log(result);
})();
you can break it into functions like..
let images=[];
const getResponse = async i=> {
if(i<5)
return await axios.get(`https://api.pexels.com/v1/curated?per_page=40&page=${i}`)
}
const getImage = (i)=>{
if(i<5){
try {
const request = getResponse(i);
images = [...images,...request];
// here you will get all the images in an array
console.log(images)
getImage(++i)
} catch (error) {
console.log("catch error",error)
// getImage(i)
}
}
}
getImage(0); //call initail