Imagine for example that you want to store paginated data from an API to a database.
let db;
let pageitems = 35
var offset = 0;
dbConnect //establish connection to database
.then( fetch(apiLink+?offset=2)
.then( res => res.json())
.then( res => {
var total = res.count
return collection.insertMany(res.data, {ordered: false})
// If offset is less than total, I want to increase offset and go back to the fetch-event.
.catch( err => {
if(err.code !== 11000){log(err)}
else{log({completed: err.result.nInserted, duplicates:
err.result.result.writeErrors.length});}
})
.then(() => {
connection.close();
})
You could just use a regular loop:
(async function() {
const conn = await dbConnect;
for(let offset = 0; true; offset++) {
const { data, count } = await (await fetch(`api?page=${offset}`)).json();
// Exit if the page is empty
if(count === 0) break;
await collection.insertMany(data, { ordered: false });
}
})();
To speed that up you could execute multiple requests in parallel:
const chunkSize = 10; // 10 in parallel
for(let offset = 0; offset < chunkSize; offset++) {
(async function() {
const conn = await dbConnect;
for(let offset2 = 0; true; offset2 += chunkSize) {
const { data, count } = await (await fetch(`api?page=${offset + offset2}`)).json();
// Exit if the page is empty
if(count === 0) break;
await collection.insertMany(data, { ordered: false });
}
})();
}
Basically, you will want to wrap your fetch and insert into a function that you will call many times. See the below as an example to illustrate my point...
let db;
let pageitems = 35
var offset = 0;
var db = dbConnect() //establish connection to database
function fetch_and_insert(offset) {
db
.then(fetch(apiLink + "?" + offset))
.then(res => res.json())
.then(res => {
var total = res.count
collection.insertMany(res.data, { ordered: false })
.catch(err => {
if (err.code !== 11000) { log(err) }
else {
log({
completed: err.result.nInserted, duplicates: err.result.result.writeErrors.length
});
}
})
if (offset < total) return fetch_and_insert(offset + pageitems)
return null;
})
}
fetch_and_insert(offset)
.then(() => {
connection.close();
})
Related
I'm trying to query the token balance history of significant user with web3's batchRequest.
But the function can't guarantee the order of functions, so I get the balance datas not in order.
So I want to know other multicall functions which can get response with blockNumber to organaize the order after querying datas.
This is my current code
// this succeeded to query, and I'm using this
const Web3 = require('web3');
const { getTokens } = require('./utils/getTokens.js');
const { convertToNumber } = require('./utils/convertToNumber.js')
const { abi, rpcEndpoint, walletAddress, wsEndpoint } = require('./constant.js');
const { formatUnits } = require('ethers/lib/utils');
const web3 = new Web3(new Web3.providers.HttpProvider(rpcEndpoint));
const queryHistoryBalance = async (userAddress, tokenAddress, startBlock, endBlock, decimal) => {
const batch = new web3.BatchRequest(); // https://stackoverflow.com/questions/69788660/web3-batchrequest-always-returning-undefined-what-am-i-doing-wrong
const contract = new web3.eth.Contract(abi, tokenAddress);
/*
let step = Math.floor((endBlock - startBlock) / eachNumber);
if (step < 1) {
step = 1;
}
for (let n = 0; n < step; n++) {
}*/
let data = { [userAddress]: [] }
for (let n = startBlock; n <= endBlock; n++) {
batch.add(
contract.methods.balanceOf(userAddress).call.request({}, n, (err, res) => {
if (res) {
data[userAddress].push({ 'blockNumber': n, 'balance': res })
}
})
)
/*
batch.add(
contract.methods.balanceOf(userAddress).call.request({}, n, async (err, res) => {
const timestamp = await web3.eth.getBlock(n).timestamp;
data[userAddress].push({ 'blockNumber': n, 'blockTime': timestamp, 'balance': res })
//data[userAddress].push({ 'blockNumber': n,'balance': res })
})
)*/
}
await batch.execute();
//console.log(data);
let data2 = { [userAddress]: [] }
let formarNumber
await Promise.all(data[userAddress].map((element, i) => {
//console.log(element['balance'] / 1)
//data[userAddress][i]['balance'] = convertToNumber(element['balance'], 6)
let formatBalance = element['balance'] / (10 ** decimal);
data[userAddress][i]['balance'] = formatBalance;
if (formarNumber !== element['balance']) {
data2[userAddress].push({ 'blockNumber': element['blockNumber'], 'balance': formatBalance });
formarNumber = element['balance'];
}
if (i === 0) formarNumber = element['balance'];
}))
//console.log("data", data)
return data2
}
const main = async () => {
const data = await queryHistoryBalance('0x8d9e0f65569b3cc2f602a7158d02ab52976a4535', '0xdAC17F958D2ee523a2206206994597C13D831ec7', 14954947, 14958046, 6)
console.log(data)
}
main()
module.exports.queryHistoryBalance = queryHistoryBalance;
I appreciate any help and advice
My use case demands me to call an sql recursively till no rows are returned for which I have written the below code which due to async nature doesn't work as expected.
The piece of code which does this invocation is:
let Response = await getData(userId);
async function getData(userId) {
console.log("Invoking Get Data Function");
let arrayOfUserId = [userId];
let fetchMore = true,
j = 1;
let keyWithQoutes = -1;
return new Promise((resolve, reject) => {
do {
console.log(arrayOfUserId, j)
j++;
if (arrayOfUserId.length > 0) {
keyWithQoutes = arrayOfUserId.map((it) => {
return `'${it}'`;
});
}
const sql = ` Select userId from USER where reportingTo in (${arrayOfUserId})`;
console.log(' SQL Query ', sql);
con.query(sql, [], async(error, response) => {
if (error) {
fetchMore = false;
reject(error);
}
console.log(
" Response for ",
userId,
response,
response.length
);
if (response.length == 0) {
fetchMore = false;
resolve(arrayOfUserId);
}
else {
for (let i = 0; i < response.length; i++) {
console.log(response[i].userId);
arrayOfUserId.push(response[i].userId);
}
}
});
} while (fetchMore);
});
}
I'm still trying to learn about global and local identifiers on nodejs, but still not clear for me.
I'm trying to use those variables from some query, then use them again with the results from database in page.evaluate().
Someone can help me? I've tried many ways.
let initial_time = null;
let end_time = null;
(async () => {
const query_time = await query('select time_i, time_f from time_config where config_id = 1', function(err, results) {
if (err) throw err;
for (var i in results) {
var time = results[0];
initial_time = time['time_i'];
end_time = time['time_f'];
}
})
})
()
const result_time = await page.evaluate(() => {
return [].concat.apply([], document.querySelectorAll('.ipo-Fixture.ipo-Fixture_Time'))
.filter(e => {
const timeString = e.innerText;
const minute = +timeString.split(':')[0];
return (minute > initial_time && minute < end_time);
})
.map(e => e.innerText);
})
One of the solutions is to pass the variables to your function something like this
(async () => {
const query_time = await query('select time_i, time_f from time_config where config_id = 1', function(err, results){
if(err) throw err;
for (var i in results){
var time = results[0];
var initial_time = time['time_i'];
var end_time = time['time_f'];
}
page.evaluate(initial_time,end_time)
})
})
()
const result_time = await page.evaluate(initial_time,end_time,() => {
return [].concat.apply([], document.querySelectorAll('.ipo-Fixture.ipo-Fixture_Time'))
.filter(e => {
const timeString = e.innerText;
const minute = +timeString.split(':')[0];
return (minute > initial_time && minute < end_time);
})
.map(e => e.innerText);
})
So basically im working on a cron job in my app that fires every 3 hours and updating users 'score' by calling the RiotApi
basically the function so far
exports.updatePlayersPoints = async () => {
console.log('STARTED UPDATING');
try {
const players = await UserLadder.findAll();
await Promise.all(
players.map(async (player) => {
const p = await RiotAccount.findOne({
where: {
userId: player.userId,
},
include: RiotRegions,
});
const beginTime = new Date(player.dataValues.createdAt);
let data;
try {
const res = await axios.get(
`https://${
p.dataValues.riot_region.dataValues.name
}.api.riotgames.com/lol/match/v4/matchlists/by-account/${
p.dataValues.accountId
}?queue=420&beginTime=${beginTime.getTime()}&api_key=${
process.env.RIOT_KEY
}`
);
data = res.data;
} catch (error) {
if (!error.response.status === 404) {
console.error(error);
}
}
if (!data) {
return;
}
let totalScore = player.dataValues.userPoints;
await Promise.all(
data.matches.map(async (match, i) => {
if (i < 15) {
const { data } = await axios.get(
`https://${p.dataValues.riot_region.dataValues.name}.api.riotgames.com/lol/match/v4/matches/${match.gameId}?api_key=${process.env.RIOT_KEY}`
);
const calculateScore = () => {
return new Promise((resolve) => {
const { stats } = _.find(
data.participants,
(o) => o.championId === match.champion
);
const killsPts = stats.kills * 2;
const deathPts = stats.deaths * -1.5;
const assistsPts = stats.assists;
const wardsPts = stats.wardsPlaced / 4;
const firstBloodPts = stats.firstBloodKill ? 3 : 0;
const firstBloodAssistPts = stats.firstBloodAssist ? 3 : 0;
const firstTowerPts = stats.firstTowerKill ? 2 : 0;
const firstTowerAssistPts = stats.firstTowerAssist ? 2 : 0;
const score =
killsPts +
deathPts +
assistsPts +
wardsPts +
firstBloodPts +
firstBloodAssistPts +
firstTowerPts +
firstTowerAssistPts;
totalScore += score;
resolve();
});
};
await calculateScore();
}
})
);
const user = await UserLadder.findOne({
where: {
userId: player.userId,
},
});
user.userPoints = parseFloat(totalScore);
user.lastGameId = data.matches[0].gameId;
await user.save();
})
);
console.log('FINISHED UPDATING');
} catch (error) {
console.error(error);
}
};
Basically it just looks up the table userladder to find the players that are signed to the ladder and for each one of these players it fires a map function that makes a request to the riotapi to get the match history of this player and then later make an inside map function to map each one of these matches.
but basically I updated it to now keep track of the game id of the last call before 3 hours so it doesn't have to make request that was already done.
user.lastGameId = data.matches[0].gameId;
but now in my second map function that maps the matches I wasn't it so that if the last game from my database matches the game id that currently being mapped I want to stop the map function and not continue this record or the ones after because it also means they all have been already counted.
but I can not seem to find a way to do it.
i tried using break; but it didn't work
any ideas?
using for loop
I tried a small test with for loop so I tried
for (let i = 0; i < 15; i++) {
await new Promise(async (resolve, reject) => {
const match = data.matches[i];
console.log(match);
resolve();
if (i === 1) {
break;
}
});
}
but I still go the same error
SyntaxError: Illegal break statement
Instead of trying to "break" a map, you should filter the matches that you want to process before you execute the map.
Something like this:
await Promise.all(
const filteredMatches = data.matches.filter(match => match.gameId > previousId);
filteredMatches.map(async (match, i) => { ...
More on filter() in javascript.
Edit: If generated id's are random and are not ordered, you can store all previous id's in a Set, and then just ask if it has been previously added
await Promise.all(
const filteredMatches = data.matches.filter(match => mySet.has(match.gameId));
filteredMatches.map(async (match, i) => { ...
More on Set in javascript.
I have a firebase cloud function that is unable to finish executing. I suspect my code can be dramatically improved but I'm not quite sure how.
I've made the query as specific as possible to try and reduce the number of documents required to iterate through but that didn't solve the issue.
I get a Deadline Exceeded error which I suspect is due to the fact that I'm iterating through so many documents and trying to update them.
I increased the timeout (9 minutes)and memory allocation (2GB) in Google cloud console but that didn't help either.
exports.updatePollWinner = functions.runWith(runtimeOpts).firestore.document('triggerAccuracyCalculation/{id}').onCreate(trigger => {
const week = trigger.get('week');
const scoringTags = ["STD", "0.25PPR", "0.5PPR", "PPR", "0.10PPC", "0.25PPC", "0.5PPC", "4PTPASS", "5PTPASS", "6PTPASS", "-2INT", "TEPREMIUM"]
let winningChoiceIds = [];
let totalPollIds = [];
return db.collection("polls").where("sport", "==", 1).where("week", "==", week).where("pollType", "==", "WDIS").get()
.then((querySnapshot) => {
console.log("A");
querySnapshot.forEach((doc) => {
totalPollIds.push(doc.id);
let pollData = doc.data();
// extract relevant scoring tags
let tags = pollData.tags.filter(tag => scoringTags.includes(tag.code)).map(tag => tag.code);
// if no scoring setting is tagged, then use STD - determine what STD is
// extract player from each option
let winner = {score: 0, choice: {}, choiceId: null};
let cnt = 0;
pollData.choices.forEach((choice) => {
let choiceId = choice.id
let mappedChoices = choice.players.map(player => {
return { displayName: player.displayName, playerId: player.playerId, position: player.position, team: player.team }
});
// ToDo: What happens if someone posts a poll with two players in one option? This poll should be ignoree from accuracy calculation
// ignmore if option has more than one player
// if (mappedChoices.length > 1) return;
const player = mappedChoices[0]
// We can't score defense
if (player.position === "DEF") {
return;
}
const playerId = player.playerId;
// Make FFN API call to retrieve stats for that player in that weekconst statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}${req.url}`;
const statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}/${playerId}`;
const json = {"Stats": {"2019": ""}, "Player": {}};
https.get(statsEndpoint, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const weekString = week.toString();
const fetchedStats = JSON.parse(data).Stats
if (!fetchedStats) return;
const response = fetchedStats["2019"]
if (!response) return;
// TODO SCORE KICKERS AND DEFENSES
const stats = response[weekString];
let score = 0;
stats["recYards"] ? score += parseInt(stats["recYards"]) / 10 : false
stats["recTD"] ? score += parseInt(stats["recTD"]) * 6 : false
stats["rushYards"] ? score += parseInt(stats["rushYards"]) / 10 : false
stats["rushTD"] ? score += parseInt(stats["rushTD"]) * 6 : false
stats["xpMade"] ? score += parseInt(stats["xpMade"]) : false
stats["fgMade"] ? score += parseInt(stats["fgMade"]) * 3 : false
stats["kickoffRet"] ? score += parseInt(stats["kickoffRet"]) / 10 : false
stats["SackYards"] ? score -= parseInt(stats["SackYards"]) / 10 : false
stats["fumbleLost"] ? score -= parseInt(stats["fumbleLost"]) * 2 : false
// Determine winner
// ToDo: handle ties
if (score > winner.score) {
winner.score = score;
winner.choiceId = choiceId;
winner.choice = choice;
}
if (cnt>=pollData.choices.length-1){
// Save player object on the poll Document (include choice ID)
winningChoiceIds.push(winner.choiceId);
const pollDoc = db.doc(`polls/${doc.id}`);
pollDoc.update({winner: winner});
}
cnt++;
});
}).on("error", (err) => {
console.log("Error: ", err.message);
});
});
});
console.log("B");
return false;
}).then(() => {
console.log("C");
let dateToQueryAfter = new Date(new Date("08/22/19").setHours(0,0,0,0))
return db.collection("users").where("recentVote", ">", dateToQueryAfter).get()
})
.then((querySnapshot) => {
console.log("D");
const promises = [];
querySnapshot.forEach((doc) => {
const p = db.collection("votes").where("uid", "==", doc.id).where("week", "==", week).where("pollType", "==", "WDIS").get()
promises.push(p)
});
return Promise.all(promises)
})
.then((querySnapshots) => {
console.log("E");
querySnapshots.forEach((querySnapshot) => {
if (querySnapshot.docs.length <= 0) return;
const uid = querySnapshot.docs[0].data().uid
const retrieveUserDoc = db.doc(`users/${uid}`);
let correctVotes = 0;
let cnt = 0;
let totalVotes = 0;
let pollVoteIds = [];
let pollVoteIdsCorrect = [];
querySnapshot.docs.forEach((doc) => {
const voteData = doc.data();
if (totalPollIds.includes(voteData.poll)) {
pollVoteIds.push(voteData.poll)
totalVotes++;
if (winningChoiceIds.includes(voteData.choice)) {
pollVoteIdsCorrect.push(voteData.poll)
correctVotes++;
}
}
if (cnt>=querySnapshot.size-1){
console.log("Updating user ID: ", uid);
retrieveUserDoc.update({
['accuracyWeeks.week'+week]: true,
['accuracy.week'+week]: {
totalVotes: totalVotes,
correct: correctVotes,
accuracy: correctVotes/totalVotes,
correctVoteIds: pollVoteIdsCorrect,
totalVoteIds: pollVoteIds
}
});
}
cnt++;
})
});
console.log("F");
return false;
})
.catch((error) => {
console.log("Error getting documents: ", error);
});
});