We Keep Coding

(CLOSED) Return statement never gets called. NodeJS loop is still finite and doesn't go forever. What's happening?

I was able to fix this by changing the contents of the .push() of allTeams (second to last instance of .push() in the code)
The final return statement should return an array of objects. (allObjectTeams, right after await browser.close()), BUT it returns undefined
Placing the variable right above the return statement still does not solve the issue.
Commenting the biggest loop (the one that iterates over blocks.length) makes that final return statement work but I don't know what part of said loop is messing it up, Commenting out anything that does an action to this said variable doesn't solve the issue
makeTeam() is an instance of recursion but it looks fine to me. Essentially taking a random set of elements from the people array and placing it as a property of an object, then to add than object to an array,
The function also removes the specific elements that were chosen from that array so if there are still more, It calls itself again. A heads-up but I don’t think it does anything.
I am using Puppeteer.
Code (JS):
const start = async () => {
let teamSize = 2
let allObjectTeams = [] //This is the variable I am trying to return
let allTeams = []
const browser = await require("puppeteer").launch()
const page = await browser.newPage()
await page.goto("https://www.when2meet.com/?16521246-5ySm2")
const blocks = await page.$$("#GroupGridSlots > div")
for (let i = 0; i < blocks.length; i++) { // This is the loop I'm talking about, blocks.length is finite. This loop does not go on forever for the return statement never to be called.
await blocks[i].hover()
const team = await page.$eval("#Available", ((availableList, teamSize) => {
const people = availableList.innerHTML.split("<br>")
people.pop()
function getMultipleRandom(arr, num) { // Grabs random elements from array
const shuffled = [...arr].sort(() => 0.5 - Math.random());
return shuffled.slice(0, num);
}
if (people.length == teamSize) {
const availableTeam = {}
availableTeam.people = people
return availableTeam
} else if (people.length > teamSize) {
const teamArray = []
const makeTeam = (currentLength) => {
const availableTeam = {}
const teamPeople = getMultipleRandom(people, teamSize)
availableTeam.people = teamPeople
teamArray.push(availableTeam)
if ((currentLength - teamSize) > teamSize) {
teamArray[teamArray.length - 1].people.forEach(person => {
people.forEach(availablePerson => {
if (availablePerson == person) {
people.splice(people.indexOf(person), 1)
}
})
})
makeTeam(currentLength - teamSize)
}
}
makeTeam(people.length)
return teamArray // returns array of Objects, But THIS is not the one
} else {
return
}
}), teamSize)
if (!team) {
} else if (!Array.isArray(team) && allTeams.includes(team.people.toString())) {
return
} else if (Array.isArray(team)) {
team.forEach(t => {
allTeams.forEach(registeredTeam => {
if (t == registeredTeam) {
team.splice(team.indexOf(t), 1)
}
})
})
for (let i = 0; i < team.length; i++) {
allTeams.push(team[i].toString())
const availableDate = await page.$eval("#AvailableDate", (availableDate => {
return availableDate.textContent.split("Sunday ").pop()
}))
team[i].time = availableDate
allObjectTeams.push(team[i])
}
} else if (team.hasOwnProperty("people")) {
allTeams.push(team.people.toString()) // this was changed to allTeams.push(team.toString())
const availableDate = await page.$eval("#AvailableDate", (availableDate => {
return availableDate.textContent.split("Sunday ").pop()
}))
team.time = availableDate
allObjectTeams.push(team)
}
}
await browser.close()
return allObjectTeams // This return statement returns undefined. THE OTHERS DO NOT
}
start().then((arr) => {
console.log(arr)
})

Function moving on before AXIOS promise response is returned

I am having a problem with the responses of this function. The function is supposed to return a concatenated list of all ITEMID for a game. This list can change depending on what is being sold at a time. The first axios call is used to grab the first 1000 item numbers and the number of pages. I would then have to make axios calls for the amount of pages to retrieve all of the item numbers. Seems a bit tedious but I don't think there is a way around this.
The problem is the function isn't running the other pages and only returns the first 1000 items before the other responses come back. Is there a better way to implement this function?
export const getAllItemID = (region) => {
let finalList = [];
let pages = 0;
let currentPage = 1;
let region_buy_id = `URL For Call`;
axios
.get(region_buy_id)
.then(json => {
pages = json.headers['x-pages'];
for(let k in Object.values(json.data)){
finalList.push(json.data[k]);
}
currentPage++;
})
.then(()=>{
while(currentPage <= pages) {
let region_buy_id = `URL For Call`;
axios
.get(region_buy_id)
.then((json) => {
for(let k in Object.keys(json.data)){
finalList.push(json.data[k]);
}
});
currentPage++;
}
})
.then(()=>{
return finalList;
});
}
Updated Function
export async function getAllItemID(region) {
let finalList = [];
let pages = 0;
let currentPage = 1;
let region_buy_id = `URL`;
await axios
.get(region_buy_id)
.then(json => {
pages = json.headers['x-pages'];
for(let k in Object.values(json.data)){
finalList.push(json.data[k]);
}
currentPage++;
});
while(currentPage <= pages) {
let region_buy_id = `URL`;
await axios
.get(region_buy_id)
.then((json) => {
for(let k in Object.keys(json.data)){
finalList.push(json.data[k]);
}
});
currentPage++;
}
}

JavaScript Asynchronous in Redux..?

Action in fuction (action for data imported from firstore)
Here you get index data.
export async function getUserIndex (data){
let db = loadFB().firestore();
console.log(data)
let date = moment(data).utc().format()
let query = db.collection('users').where("create_date", ">", date)
console.log(query)
return await query.get().then(docs=>{
let result = docs.size
console.log("result!!", result)
return result
})
}
components in function
async getuserIndex_component(date){
let number =""
number = await userAction.getUserIndex(date)
console.log("number",number)
return await number
}
const {user_list} = this.props; //user_list = [{id:aa, pw:bb},{id:cc, pw:dd}...} data
let data = user_list
let groups = {}
let number =0
for (let index_data of data) {
let date = moment(index_data.create_date).format("YYYY-MM-DD").slice(0, 10)
let index = this.getuserIndex_component(date) //<==here log Promise{<pendding>}
console.log(index)
if (groups[date]) {
let group = groups[date];
group.push({ index: index-number, ...index_data });
} else {
number =0;
groups[date] = [{ index: index-number, ...index_data }]
}
number++;
}
const dates = Object.keys(groups)
const user_list_result = []
for (let date of dates) {
user_list_result.push(...(groups[date]))
}
return(...)
I am using redux. in action
Made a "getUserIndex()."
The component will then access the "getUserIndex" and retrieve data. But there is only pending in the console.log, and the data I want does not appear.
In this situation, I would like to know how to deal with async in for of.
I want to know what part I'm making a mistake.
In summary, functions in components are executed later than the render. So the index in the render does not contain the data. What is the solution to this?

As getuserIndex_component is asynchronous, it will return a promise. You will need to wait for that promise to resolve before the index result is available.
Firstly, you should be able to simplify getuserIndex_component to
async getuserIndex_component(date) {
let number = await userAction.getUserIndex(date)
console.log("number", number)
return number
}
And when you remove the console.log it could simply be
async getuserIndex_component(date) {
return userAction.getUserIndex(date)
}
As I mentioned in the comments. You can send off all of the async calls, then just wait for all of the promises to resolve via Promise.all.
Something like this. (I couldn't test any of this, but it should demonstrate the general idea.)
const { user_list } = this.props; //user_list = [{id:aa, pw:bb},{id:cc, pw:dd}...} data
let data = user_list
let groups = {}
let number = 0
const dates = []
const promises = data.map((index_data) => {
const date = moment(index_data.create_date).format("YYYY-MM-DD").slice(0, 10)
dates.push(date) // To use after all promises resolve.
return this.getuserIndex_component(date) // The index promise.
})
const indexes = await Promise.all(promises)
for (let i = 0; i < dates.length; i++) {
const index = indexes[i]
const date = dates[i]
console.log(index)
if (groups[date]) {
let group = groups[date];
group.push({
index: index - number,
...index_data
});
} else {
number = 0;
groups[date] = [{
index: index - number,
...index_data
}]
}
number++;
}
const dates = Object.keys(groups)
const user_list_result = []
for (let date of dates) {
user_list_result.push(...(groups[date]))
}
return (...)

parallel async Mongoose queries || eval(return) SyntaxError : Illegal return statement

I'm trying to make a lot of requests to Mongo with Mongoose. I'd like them to be parallel and async. I don't know how many I might have to query so I created this helper function. It works fine, although I'm not sure about my eval usage. The problem is the eval(return${returnStr}) giving me the SyntaxError.
const batchRetrieve = async (query, models) => {
models.forEach((model, i) => {
eval(`task${i} = ${model}.findOne(query)`);
});
let str = [];
for (let i = 0; i < models.length; i++) {
str.push(`res${i}: await task${i}`);
}
const joinedStr = Array.prototype.join.call(str, ', '); //output is 'res0: await task0, res1: await task1'
// return { res0: await task0, res1: await task1 }; works fine
eval(`return { ${joinedStr} };`);
};

Rule of thumb: If you're using eval, you're doing it wrong. Try this:
const batchRetrieve = async (query, models) => {
const promises = [];
models.forEach(model => {
promises.push(model.findOne(query));
});
const results = [];
for (let i = 0; i < models.length; i++) {
results.push(await promises[i]);
}
return results;
};

replacing the eval(return { ${joinedStr} };); with
return eval(`(async () => {
return { ${joinedStr} }
})();`);
resolves my issue.

Nested for loop combined with fetching data from an API in Nodejs?

My scenario is as follows:
There is an API I would like to fetch from. The API returns a json that has an array named "assets". This array size will always be 20.
Now, I call the endpoint like so:
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
where if offset is 0 it will return the array of the 0 - 20 assets, if the offset is 20 it will return 20 to 40 and so on.
I want to check 1000 items which means I would like to call this fetch 1000/20 = 50 times.
Whenever I call the fetch I would like loop over these 20 items and insert them into my DB.
The problem is that I can't do something like this:
let offset=0;
for(let i = 0; i < 50; i++ {
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
for(let j = 0; j < 20; j++){
// Insert into DB
}
offset+=20;
}
Due to JS asynchronous nature. Whenever I try to do this it will call the fetch with the value 0 for offset a couple of times, it won't wait until the nested for loop finishes and than call it for 20 and later 40 and so on...
What is the correct way to achive this behavior?

I've nicked the following from one of my nodeJS repos as it employs async code for solving a very similar problem:
// Throttling is important because you don't want to
// overload the API
const createThrottle = require('async-throttle');
const throttle = createThrottle(2);
// First push all the links into an array using
// the offset
const links = [];
for (let offset = 0; offset < 100; offset += 20) {
links.push(`https://api.example.io/api/v1/assets/?offset=${offset}`);
}
// Next create an array of promises by `map`ing
// over the links using `fetch`.
// Notice I've used throttle here to slow down the hits on
// the API
const promises = links.map(link => throttle(async => () {
const res = await fetch(link);
return await res.json();
}));
// Once all the promises are complete, iterate over their datasets
Promise.all(promises).then(datasets => {
// iterate over datasets
});

Why not use Promise.all?
const promises = [];
for (let i = 0; i < 10; i++) {
promises.push(
fetch(`https://api.example.io/api/v1/assets/?offset=${i}`)
);
}
Promise.all(promises).then( _results => {
// All fetch calls have completed at this point. _results will be in array of your fetch results.
console.log(results);
// Do db insert here
});
One thing you could do is to make a function that is a promise. Within that function, have it execute the fetch and then execute the db insert all in one function (using .then). If you did it this way, the single Promise.all call would handle everything. If you don't, you'll have to loop through the promises again and insert those values into the db. It might look something like this:
const promises = [];
for (let i = 0; i < 10; i++) {
promises.push(fetchAndInsert(i));
}
Promise.all(promises).then( _results => {
console.log(results);
});
function fetchAndInsert(offset) {
return new Promise( (resolve, reject) => {
fetch(`https://api.example.io/api/v1/assets/?offset=${i}`).then (_result => {
// db.insert().then( _result => {
//resolve();
//})
})
})
}

You could use async and await. This should work:
async function fetchExample() {
for (let i = 0; i < 50; i++) {
let fetchedData = await fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
);
for(data of fetchedData) {
// Insert into DB
}
offset+=20;
}
}

Instead of for..loop, you could use recursion or Promises.
Recursion::
let insertToDB = function (records, cb) {
if (!records.length) return cb();
let record = records.shift();//assuming records is an array
// Insert record into DB1
insertToDB(records, cb);
};
let loop = function (count, offset, cb) {
if (!count) return cb();
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
insertToDB(recordsFromFetch, function () {
offset += 20;
--count;
loop(count, offset, cb)
})
};
loop(50, 0, function () {
console.log("All Done");
})
Promise:: I have not run it, so might be some syntactical error, but you get the idea
let insertToDB = function (record) {
return new Promise(function (resolve, reject) {
// Insert record into DB then call resolve
resolve();
})
};
let fetchPhase = function (offset) {
fetch(
`https://api.example.io/api/v1/assets/?offset=${offset}`
)
let dbAll = [];
for (let j = 0; j < 20; j++) {
// assuming you get records from fetch , pass record to be added to db in the parameter
dbAll.push(insertToDB(records[j]))
}
return Promise.all(dbAll)
};
let all = [];
let offset = 0;
for (let i = 0; i < 50; i++) {
all.push(fetchPhase(i));
}
Promise.all(all)
.then(function () {
console.log("ALL DONe");
})

Promise.all(Array(50).fill(null).map((v,i) => {
const url = `https://api.example.io/api/v1/assets/?offset=${i*20}`;
return fetch(url).then(results => {
for (let result of results) {
// insert into DB
}
);
}).then(() => console.log("done"));