I'm using a webworker to calculate coordinates and values belonging to those places. The calculations happen in the background perfectly, keeping the DOM responsive. However, when I send the data from the webworker back to the main thread the DOM becomes unresponsive for a part of the transfer time.
My webworker (sending part):
//calculates happen before; this is the final step to give the calculated data back to the mainthread.
var processProgressGEO = {'cmd':'geoReport', 'name': 'starting transfer to main', 'current': c, 'total': polys}
postMessage(processProgressGEO);
postMessage({
'cmd':'heatmapCompleted',
'heatdata': rehashedMap,
'heatdatacount': p,
'current': c,
'total': polys,
'heatmapPeak': peakHM,
});
self.close();
The variable rehashedMap in the code snippet above is an object with numerical keys. Each key contains an array with another object in.
My mainthread (only the relevant part:)
var heatMaxforAuto = 1000000; //maximum amount of datapoints allowed in the texdata. This takes into account the spread of a singel datapoint.
async function fetchHeatData(){
return new Promise((resolve, reject) => {
var numbercruncher = new Worker('calculator.js');
console.log("Performing Second XHR request:");
var url2 = 'backend.php?datarequest=geodata'
$.ajax({
type: "GET",
url: url2,
}).then(async function(RAWGEOdata) {
data.georaw = RAWGEOdata;
numbercruncher.onmessage = async function(e){
var w = (e.data.current/e.data.total)*100+'%';
if (e.data.cmd === 'geoReport'){
console.log("HEAT: ", e.data.name, end(),'Sec.' );
}else if (e.data.cmd === 'heatmapCompleted') {
console.log("received Full heatmap data: "+end());
data.heatmap = e.data.heatdata;
console.log("heatData transfered", end());
data.heatmapMaxValue = e.data.heatmapPeak;
data.pointsInHeatmap = e.data.heatdatacount;
console.log("killing worker");
numbercruncher.terminate();
resolve(1);
}else{
throw "Unexpected command received by worker: "+ e.data.cmd;
}
}
console.log('send to worker')
numbercruncher.postMessage({'mode':'geo', 'data':data});
}).catch(function(error) {
reject(0);
throw error;
})
});
}
async function makemap(){
let heatDone = false;
if (data.texdatapoints<= heatMaxforAuto){
heatDone = await fetchHeatData();
}else{
var manualHeatMapFetcher = document.createElement("BUTTON");
var manualHeatMapFetcherText = document.createTextNode('Fetch records');
manualHeatMapFetcher.appendChild(manualHeatMapFetcherText);
manualHeatMapFetcher.id='manualHeatTriggerButton';
manualHeatMapFetcher.addEventListener("click", async function(){
$(this).toggleClass('hidden');
heatDone = await fetchHeatData();
console.log(heatDone, 'allIsDone', end());
});
document.getElementById("toggleIDheatmap").appendChild(manualHeatMapFetcher);
}
}
makemap();
The call to the end() function is needed to calculate the seconds since the start of the webworker. It returns the difference between a global set starttime and the time of calling.
What shows in my console:
HEAT: starting transfer to main 35 Sec. animator.js:44:19
received Full heatmap data: 51 animator.js:47:19
heatData transfered 51 animator.js:49:19
killing worker animator.js:52:19
1 allIsDone 51
The issue:
My DOM freezes between the start of the data transfer and the message after receiving the full heatmap data. This is the phase between the first and second message in my console. It takes 16 seconds to transfer, but the DOM only goes unresponsive once for a part of that time. Webworkers can't share data with the mainthread, so a transfer is needed.
Question:
Firstly, how to prevent the freeze of the the DOM during the onmessage phase of the webworker? Secondly, more out of curiosity: how can this freeze only occur during a part of that phase, as these are triggered by two consecutive steps with nothing going on in between?
What I tried so far:
Doing a for-loop over the rehashedMap and return key by key. This still triggers DOM freezes; shorter, but more than once. In rare occurrences it takes the tab down.
Looking for a way to buffer the onmessage phase; however, there's no such option specified in the documentation (https://developer.mozilla.org/en-US/docs/Web/API/Worker/onmessage) as compared to the postMessage phase (https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage). Am I missing something here?
As a test I replaced the rehashedMap with an empty object; this didn't cause any freezes in the DOM. Of course, this is leaves me without access to the calculate data.
I looked at this thread on SO:Javascript WebWorker - Async/Await But I'm not sure how to compare that context to mine.
Options
It's understandable you should associate this with the web worker, but it probably doesn't have anything to do with it. I was wrong, it does. I see two possible reasons for the problem:
(We know this is not true for the OP, but may still be relevant for others.) The problem is probably that you have a lot of DOM manipulation to do once you've received the heat map. If you do that in a tight loop that never lets the main thread do anything else, the page will be unresponsive during that time.
If that's what's going on, you have to either find a way to do the DOM manipulation more quickly (sometimes that's possible, other times not) or find a way to carve it up into chunks and process each chunk separately, yielding back to the browser between chunks so that the browser can handle any pending UI work (including rendering the new elements).
You haven't included the DOM work being done with the heat map so it's not really possible to give you code to solve the problem, but the "carving up" would be done by processing a subset of the data and then using setTimeout(fn, 0) (possibly combined with requestAnimationFrame to ensure that a repaint has occurred) to schedule continuing the work (using fn) after briefly yielding to the browser.
If it's really the time spent transferring the data between the worker and the main thread, you might be able to use a transferable object for your heat map data rather than your current object, although doing so may require significantly changing your data structure. With a transferable object, you avoid copying the data from the worker to the main thread; instead, the worker transfers the actual memory to the main thread (the worker loses access to the transferable object, and the main thread gains access to it — all without copying it). For instance, the ArrayBuffer used by typed arrays (Int32Array, etc.) is transferable.
If it's really the time spent receiving the data from the worker (and from your experiments it sounds like it is), and using a transferable isn't an option (for instance, because you need the data to be in a format that isn't compatible with a transferable), the only remaining option I can see is to have the worker send the main script smaller blocks of data spaced out enough for the main thread to remain responsive. (Perhaps even sending the data as it becomes available.)
Closer look at #3
You've described an array of 1,600 entries, where each entry is an array with between 0 and "well over 7,000" objects, each with three properties (with number values). That's over 5.6 million objects. It's no surprise that cloning that data takes a fair bit of time.
Here's an example of the problem you've described:
const workerCode = document.getElementById("worker").textContent;
const workerBlob = new Blob([workerCode], { type: "text/javascript" });
const workerUrl = (window.webkitURL || window.URL).createObjectURL(workerBlob);
const worker = new Worker(workerUrl);
worker.addEventListener("message", ({data}) => {
if ((data && data.action) === "data") {
console.log(Date.now(), `Received ${data.array.length} rows`);
if (data.done) {
stopSpinning();
}
}
});
document.getElementById("btn-go").addEventListener("click", () => {
console.log(Date.now(), "requesting data");
startSpinning();
worker.postMessage({action: "go"});
});
const spinner = document.getElementById("spinner");
const states = [..."▁▂▃▄▅▆▇█▇▆▅▄▃▂▁"];
let stateIndex = 0;
let spinHandle = 0;
let maxDelay = 0;
let intervalStart = 0;
function startSpinning() {
if (spinner) {
cancelAnimationFrame(spinHandle);
maxDelay = 0;
queueUpdate();
}
}
function queueUpdate() {
intervalStart = Date.now();
spinHandle = requestAnimationFrame(() => {
updateMax();
spinner.textContent = states[stateIndex];
stateIndex = (stateIndex + 1) % states.length;
if (spinHandle) {
queueUpdate();
}
});
}
function stopSpinning() {
updateMax();
cancelAnimationFrame(spinHandle);
spinHandle = 0;
if (spinner) {
spinner.textContent = "Done";
console.log(`Max delay between frames: ${maxDelay}ms`);
}
}
function updateMax() {
if (intervalStart !== 0) {
const elapsed = Date.now() - intervalStart;
if (elapsed > maxDelay) {
maxDelay = elapsed;
}
}
}
<div>(Look in the real browser console.)</div>
<input type="button" id="btn-go" value="Go">
<div id="spinner"></div>
<script type="worker" id="worker">
const r = Math.random;
self.addEventListener("message", ({data}) => {
if ((data && data.action) === "go") {
console.log(Date.now(), "building data");
const array = Array.from({length: 1600}, () =>
Array.from({length: Math.floor(r() * 7000)}, () => ({lat: r(), lng: r(), value: r()}))
);
console.log(Date.now(), "data built");
console.log(Date.now(), "sending data");
postMessage({
action: "data",
array,
done: true
});
console.log(Date.now(), "data sent");
}
});
</script>
Here's an example of the worker sending the data in chunks as fast as it can but in separate messages. It makes the page responsive (though still jittery) when receiving the data:
const workerCode = document.getElementById("worker").textContent;
const workerBlob = new Blob([workerCode], { type: "text/javascript" });
const workerUrl = (window.webkitURL || window.URL).createObjectURL(workerBlob);
const worker = new Worker(workerUrl);
let array = null;
let clockTimeStart = 0;
worker.addEventListener("message", ({data}) => {
if ((data && data.action) === "data") {
if (clockTimeStart === 0) {
clockTimeStart = Date.now();
console.log(Date.now(), "Receiving data");
}
array.push(...data.array);
if (data.done) {
console.log(Date.now(), `Received ${array.length} row(s) in total, clock time to receive data: ${Date.now() - clockTimeStart}ms`);
stopSpinning();
}
}
});
document.getElementById("btn-go").addEventListener("click", () => {
console.log(Date.now(), "requesting data");
array = [];
clockTimeStart = 0;
startSpinning();
worker.postMessage({action: "go"});
});
const spinner = document.getElementById("spinner");
const states = [..."▁▂▃▄▅▆▇█▇▆▅▄▃▂▁"];
let stateIndex = 0;
let spinHandle = 0;
let maxDelay = 0;
let intervalStart = 0;
function startSpinning() {
if (spinner) {
cancelAnimationFrame(spinHandle);
maxDelay = 0;
queueUpdate();
}
}
function queueUpdate() {
intervalStart = Date.now();
spinHandle = requestAnimationFrame(() => {
updateMax();
spinner.textContent = states[stateIndex];
stateIndex = (stateIndex + 1) % states.length;
if (spinHandle) {
queueUpdate();
}
});
}
function stopSpinning() {
updateMax();
cancelAnimationFrame(spinHandle);
spinHandle = 0;
if (spinner) {
spinner.textContent = "Done";
console.log(`Max delay between frames: ${maxDelay}ms`);
}
}
function updateMax() {
if (intervalStart !== 0) {
const elapsed = Date.now() - intervalStart;
if (elapsed > maxDelay) {
maxDelay = elapsed;
}
}
}
<div>(Look in the real browser console.)</div>
<input type="button" id="btn-go" value="Go">
<div id="spinner"></div>
<script type="worker" id="worker">
const r = Math.random;
self.addEventListener("message", ({data}) => {
if ((data && data.action) === "go") {
console.log(Date.now(), "building data");
const array = Array.from({length: 1600}, () =>
Array.from({length: Math.floor(r() * 7000)}, () => ({lat: r(), lng: r(), value: r()}))
);
console.log(Date.now(), "data built");
const total = 1600;
const chunks = 100;
const perChunk = total / chunks;
if (perChunk !== Math.floor(perChunk)) {
throw new Error(`total = ${total}, chunks = ${chunks}, total / chunks has remainder`);
}
for (let n = 0; n < chunks; ++n) {
postMessage({
action: "data",
array: array.slice(n * perChunk, (n + 1) * perChunk),
done: n === chunks - 1
});
}
}
});
</script>
Naturally it's a tradeoff. The total clock time spent receiving the data is longer the smaller the chunks; the smaller the chunks, the less jittery the page is. Here's really small chunks (sending each of the 1,600 arrays separately):
const workerCode = document.getElementById("worker").textContent;
const workerBlob = new Blob([workerCode], { type: "text/javascript" });
const workerUrl = (window.webkitURL || window.URL).createObjectURL(workerBlob);
const worker = new Worker(workerUrl);
let array = null;
let clockTimeStart = 0;
worker.addEventListener("message", ({data}) => {
if ((data && data.action) === "data") {
if (clockTimeStart === 0) {
clockTimeStart = Date.now();
}
array.push(data.array);
if (data.done) {
console.log(`Received ${array.length} row(s) in total, clock time to receive data: ${Date.now() - clockTimeStart}ms`);
stopSpinning();
}
}
});
document.getElementById("btn-go").addEventListener("click", () => {
console.log(Date.now(), "requesting data");
array = [];
clockTimeStart = 0;
startSpinning();
worker.postMessage({action: "go"});
});
const spinner = document.getElementById("spinner");
const states = [..."▁▂▃▄▅▆▇█▇▆▅▄▃▂▁"];
let stateIndex = 0;
let spinHandle = 0;
let maxDelay = 0;
let intervalStart = 0;
function startSpinning() {
if (spinner) {
cancelAnimationFrame(spinHandle);
maxDelay = 0;
queueUpdate();
}
}
function queueUpdate() {
intervalStart = Date.now();
spinHandle = requestAnimationFrame(() => {
updateMax();
spinner.textContent = states[stateIndex];
stateIndex = (stateIndex + 1) % states.length;
if (spinHandle) {
queueUpdate();
}
});
}
function stopSpinning() {
updateMax();
cancelAnimationFrame(spinHandle);
spinHandle = 0;
if (spinner) {
spinner.textContent = "Done";
console.log(`Max delay between frames: ${maxDelay}ms`);
}
}
function updateMax() {
if (intervalStart !== 0) {
const elapsed = Date.now() - intervalStart;
if (elapsed > maxDelay) {
maxDelay = elapsed;
}
}
}
<div>(Look in the real browser console.)</div>
<input type="button" id="btn-go" value="Go">
<div id="spinner"></div>
<script type="worker" id="worker">
const r = Math.random;
self.addEventListener("message", ({data}) => {
if ((data && data.action) === "go") {
console.log(Date.now(), "building data");
const array = Array.from({length: 1600}, () =>
Array.from({length: Math.floor(r() * 7000)}, () => ({lat: r(), lng: r(), value: r()}))
);
console.log(Date.now(), "data built");
array.forEach((chunk, index) => {
postMessage({
action: "data",
array: chunk,
done: index === array.length - 1
});
});
}
});
</script>
That's building all the data and then sending it, but if building the data times time, interspersing building and sending it may make the page responsiveness smoother, particularly if you can send the inner arrays in smaller pieces (as even sending ~7,000 objects still causes jitter, as we can see in the last example above).
Combining #2 and #3
Each entry in your main array is an array of objects with three numeric properties. We could instead send Float64Arrays with those values in lat/lng/value order, using the fact they're transferable:
const workerCode = document.getElementById("worker").textContent;
const workerBlob = new Blob([workerCode], { type: "text/javascript" });
const workerUrl = (window.webkitURL || window.URL).createObjectURL(workerBlob);
const worker = new Worker(workerUrl);
let array = null;
let clockTimeStart = 0;
worker.addEventListener("message", ({data}) => {
if ((data && data.action) === "data") {
if (clockTimeStart === 0) {
clockTimeStart = Date.now();
}
const nums = data.array;
let n = 0;
const entry = [];
while (n < nums.length) {
entry.push({
lat: nums[n++],
lng: nums[n++],
value: nums[n++]
});
}
array.push(entry);
if (data.done) {
console.log(Date.now(), `Received ${array.length} row(s) in total, clock time to receive data: ${Date.now() - clockTimeStart}ms`);
stopSpinning();
}
}
});
document.getElementById("btn-go").addEventListener("click", () => {
console.log(Date.now(), "requesting data");
array = [];
clockTimeStart = 0;
startSpinning();
worker.postMessage({action: "go"});
});
const spinner = document.getElementById("spinner");
const states = [..."▁▂▃▄▅▆▇█▇▆▅▄▃▂▁"];
let stateIndex = 0;
let spinHandle = 0;
let maxDelay = 0;
let intervalStart = 0;
function startSpinning() {
if (spinner) {
cancelAnimationFrame(spinHandle);
maxDelay = 0;
queueUpdate();
}
}
function queueUpdate() {
intervalStart = Date.now();
spinHandle = requestAnimationFrame(() => {
updateMax();
spinner.textContent = states[stateIndex];
stateIndex = (stateIndex + 1) % states.length;
if (spinHandle) {
queueUpdate();
}
});
}
function stopSpinning() {
updateMax();
cancelAnimationFrame(spinHandle);
spinHandle = 0;
if (spinner) {
spinner.textContent = "Done";
console.log(`Max delay between frames: ${maxDelay}ms`);
}
}
function updateMax() {
if (intervalStart !== 0) {
const elapsed = Date.now() - intervalStart;
if (elapsed > maxDelay) {
maxDelay = elapsed;
}
}
}
<div>(Look in the real browser console.)</div>
<input type="button" id="btn-go" value="Go">
<div id="spinner"></div>
<script type="worker" id="worker">
const r = Math.random;
self.addEventListener("message", ({data}) => {
if ((data && data.action) === "go") {
for (let n = 0; n < 1600; ++n) {
const nums = Float64Array.from(
{length: Math.floor(r() * 7000) * 3},
() => r()
);
postMessage({
action: "data",
array: nums,
done: n === 1600 - 1
}, [nums.buffer]);
}
}
});
</script>
That dramatically reduces the clock time to receive the data, while keeping the UI fairly responsive.
Related
I've tried all sorts of ways to dispose of tensors (tf.dispose(), start/endscope).
The closest I've got was through this code, where 1 unused tensor remains after each execution. It takes about 2 hours for this program to run enough to use up 64 GB of RAM (big memory leak).
I also suspect that other factors besides TFJS-based operations are contributing to the memory leak, though (in theory) garbage collection should clean this up.
The piece of code below is one event that gets processed by an Event Listener handler. Any help with this issue would be greatly appreciated!
'use strict';
global.fetch = require("node-fetch");
const { MessageActionRow, MessageButton, Permissions } = require('discord.js');
const { mod, eco, m, n } = require(`../../index.js`);
const { Readable } = require('stream');
const PImage = require('pureimage');
const tf = require('#tensorflow/tfjs');
const tfnode = require('#tensorflow/tfjs-node');
const wait = require('util').promisify(setTimeout);
let bufferToStream = (binary) => {
let readableInstanceStream = new Readable({
read() {
this.push(binary);
this.push(null);
}
});
return readableInstanceStream;
}
const predict = async (imageUrl, modelFile) => {
let model = await tf.loadLayersModel(modelFile);
let modelClasses = [ "NSFW", "SFW" ];
let data = await fetch(imageUrl);
let fileType = data.headers.get("Content-Type");
let buffer = await data.buffer();
let stream = bufferToStream(buffer);
let image;
if ((/png/).test(fileType)) {
image = await PImage.decodePNGFromStream(stream);
}
else if ((/jpe?g/).test(fileType)) {
image = await PImage.decodeJPEGFromStream(stream);
}
else {
return;
}
let rawArray;
rawArray = tf.tidy(() => {
let tensorImage;
tensorImage = tf.browser.fromPixels(image).toFloat();
tensorImage = tf.image.resizeNearestNeighbor(tensorImage, [model.inputs[0].shape[1], model.inputs[0].shape[2]]);
tensorImage = tensorImage.reshape([1, model.inputs[0].shape[1], model.inputs[0].shape[2], model.inputs[0].shape[3]]);
return model.predict(tensorImage);
});
rawArray = await rawArray.data();
rawArray = Array.from(rawArray);
tf.disposeVariables();
model.layers.forEach(l => {
l.dispose();
});
if (rawArray[1] > rawArray[0]) {
return [`SFW`, rawArray[1]];
}
else {
return [`NSFW`, rawArray[0]];
}
};
const getResults = async (imageLink, imageNumber) => {
let image = `${imageLink}`;
let prediction = await predict(image, `file://D:/retake7/sfwmodel/model.json`);
let className = `SFW`;
if (prediction[0] == `NSFW`) {
className = `**NSFW**`;
}
return [`[Image ${imageNumber+1}](${imageLink}): ${className} (${(prediction[1]*100).toFixed(2)}% Certainty)`, ((prediction[1]*100).toFixed(2))*1];
}
const main = async (message, client, Discord) => {
if (message.attachments.size == 0 || message.author.bot || message.channel.nsfw) return;
await client.shard.broadcastEval(c => {
console.log(`Scanning...`);
}).catch(e => {
return;
});
let inChannel = await eco.seid.get(`${message.guild.id}.${message.channel.id}.active`);
let sfwImage = await eco.seid.get(`${message.guild.id}.sfwAlerts`);
if (inChannel == `no`) return;
let atmentArr = Array.from(message.attachments);
let msgArr = [];
if (message.attachments.size > 1) {
msgArr.push(`**Images Scanned**`);
} else {
msgArr.push(`**Image Scanned**`);
}
let hasNSFW = false;
let uncertain = false;
for (i = 0; i < message.attachments.size; i++) {
let msg = await getResults(atmentArr[i][1][`proxyURL`], i);
if (msg[1] < 80) {
uncertain = true;
}
if (msg[0].includes(`NSFW`)) {
hasNSFW = true;
}
msgArr.push(msg[0]);
}
if (uncertain == false && hasNSFW == false) {
let cont = `${msgArr.join(`\n`)}`;
msgArr = null;
client.seid.set(`${message.channel.id}.previousScan`, cont);
return;
}
let embed = new Discord.MessageEmbed()
.setColor(`GREEN`)
.setDescription(msgArr.join(`\n`));
let cont2 = `${msgArr.join(`\n`)}`;
client.seid.set(`${message.channel.id}.previousScan`, cont2);
msgArr = null;
if (sfwImage != `no` || hasNSFW || msg[1] <= 80) {
embed.setColor(`RED`);
await message.delete();
let msgSent = await message.channel.send({embeds: [embed], components: [row]});
};
};
module.exports = {
event: 'messageCreate',
run: async (message, client, Discord) => {
await main(message, client, Discord);
},
};
first, separate model loading and inference - in your current code, you'd reload a model each time you need to run prediction on a new image.
and then look at any possible leaks in prediction function - so once model is loaded.
you're loading a model and disposing each layer, but that doesn't mean model itself gets unloaded so there more than a chance that part of model remains in memory.
but leak itself is this line:
rawArray = await rawArray.data();
that variable is already used and its a tensor.
now you're overwriting the same variable with a data array and tensor never gets disposed.
i get error
let executor = await this.members.fetch(executorID);
^^^^^
SyntaxError: await is only valid in async function
when using the code below (use is to check if user breaks any of set filters and if so remove roles or ban user whatever they set option to)
ive tried my best to lable what parts of code does please not english isnt my first language
ive only recieved this error since trying to add a check whitelist feature - everything else works without the whitelist check code
without the code for whitelist the code works and performs as intended and the whitelist code succesfully logs ids for that guild
if(whitelisted && whitelisted.length) {
whitelisted.forEach(x => {
if (executorID === x.user) return;
const { Structures } = require('discord.js');
let whitelisted = db.get(`whitelist_${message.guild.id}`)
const { limits, defaultPrefix } = require('../config.js');
Structures.extend('Guild', Guild => {
class GuildExt extends Guild {
constructor(...args) {
super(...args);
}
get prefix() {
return this.get('prefix', defaultPrefix);
}
get(key, fallback) {
return this.client.db.get(`${this.id}_${key}`) || fallback;
}
set(key, data) {
return this.client.db.set(`${this.id}_${key}`, data);
}
delete(key) {
return this.client.db.delete(`${this.id}_${key}`);
}
resolveChannel(channelID) {
const channel = this.channels.cache.get(channelID);
return channel;
}
get limits() {
var obj = {};
for (var k in limits) {
obj[k] = {
minute: this.get(
`limits.${k}.minute`,
limits[k].per_minute
),
hour: this.get(`limits.${k}.hour`, limits[k].per_hour)
};
}
return obj;
}
getActions(limit = 10, filter = () => true) {
var obj = {};
var l = limits;
for (var k in limits) {
obj[k] = {
name: this.client.Utils.toProperCase(k),
actions: this.client.Utils.convertEntries(
[
...this.get(
this.client.Utils.convertLimitNameToActionType(
k
),
[]
),
...this.get(
`archive.${this.client.Utils.convertLimitNameToActionType(
k
)}`,
[]
)
]
.filter(filter)
.slice(0, limit)
)
};
}
return obj;
}
find_entry(action, filter) {
let guild = this;
return new Promise(resolve => {
(async function search(iter) {
//console.log(`ACTION = ${action} | ITER = ${iter}`);
if (!guild.me) return resolve(null);
if (guild.me.hasPermission('VIEW_AUDIT_LOG')) {
let logs = await guild.fetchAuditLogs({
limit: 10,
type: action
});
let entries = logs.entries;
let entry = null;
entries = entries.filter(filter);
for (var e of entries)
if (!entry || e[0] > entry.id) entry = e[1];
if (entry) return resolve(entry);
}
if (++iter === 5) return resolve(null);
else return setTimeout(search, 200, iter);
})(0);
});
}
push_entry(entry, displayName) {
const action = ['MEMBER_KICK', 'MEMBER_BAN_ADD'].includes(
entry.action
)
? 'MEMBER_REMOVE'
: entry.action;
const oneHourAgo = Date.now() - 1000 * 60 * 60;
// Fetch Entries for a sepcific action (Last Hour)
let entries = this.get(action, []);
// Filter entries older than one hour to a new variable
let olderThanOneHour = entries.filter(
i => !(i.timestamp > oneHourAgo)
);
// Prepend entries older than one hour to the archive
if (olderThanOneHour.length > 0)
this.set(`archive.${action}`, [
...olderThanOneHour,
...this.get(`archive.${action}`, [])
]);
// Filter entries older than one hour from old variable
entries = entries.filter(i => i.timestamp > oneHourAgo);
// Prepend new entry if not already found
if (
!entries.find(
i =>
i.target.id === entry.target.id &&
i.executor.id === entry.executor.id
)
)
entries.unshift({
timestamp: entry.createdTimestamp,
action: entry.action,
target: {
id: entry.target.id,
displayName,
targetType: entry.targetType
},
executor: {
id: entry.executor.id,
displayName: entry.executor.tag
}
});
// Update entries newer than one hour
return this.set(action, entries);
}
async check_limits(entries, executorID, configAction) {
// Ignore if executor is the owner or is whitelisted
if (executorID === this.ownerID) return;
if(whitelisted && whitelisted.length) {
whitelisted.forEach(x => {
if (executorID === x.user) retrun;
// Filter actions relating to executor
const oneMinuteAgo = Date.now() - 1000 * 60;
let executorActionsHour = entries.filter(
i => i.executor.id === executorID
);
let executorActionsMinute = executorActionsHour.filter(
i => i.timestamp > oneMinuteAgo
);
console.log(
`${configAction}/${executorID}: LAST_HOUR: ${executorActionsHour.length} LAST_MINUTE: ${executorActionsMinute.length} `
);
let limits = this.limits;
let limitReached = null;
if (executorActionsHour.length >= limits[configAction].hour)
limitReached = 'Hour';
if (executorActionsMinute.length >= limits[configAction].minute)
limitReached = 'Minute';
// Check if the amount of actions is greater than or equal to the limit
if (limitReached) {
// Remove all of the executor's roles
let executor = await this.members.fetch(executorID);
executor.roles.remove(executor.roles.cache);
// Handle managed roles
let managed = executor.roles.cache
.filter(r => r.managed)
.array();
for (var i = 0; i < managed.length; i++)
managed[i].setPermissions(0, 'Guardian Action');
// Notify owner, executor, and logging channel
const embed = this.client.util
.embed()
.setTitle(`Limit Reached - ${limitReached}`)
.setDescription(
this.client.Utils.convertEntries(
limitReached === 'Hour'
? executorActionsHour
: executorActionsMinute
)
)
.setColor(0x7289da);
await this.owner.send(
embed.setFooter(
"This message was sent to you because you're the Guild owner."
)
);
await executor.send(
embed.setFooter(
'This message was sent to you because you were the executor.'
)
);
const loggingChannel = this.resolveChannel(
this.get(`loggingChannelID`)
);
if (loggingChannel)
await loggingChannel.send(embed.setFooter(''));
}
})
}
}
}
return GuildExt;
});
i am new to JS and any help would be appreciated
please dont hate if i do have bad syntax !!
i am new - sorry if i dont get things the first time
You forgot to make your forEach function async, just change it to:
/* ... */
whitelisted.forEach(async (x) => {
/* ... */
let executor = await this.members.fetch(executorID);
/* ... */
}
/* ... */
Not part of your question but you misspelled return here
if (executorID === x.user) retrun;
Your line
let executor = await this.members.fetch(executorID);
is inside a non-async anonymous function in:
if (whitelisted && whitelisted.length) {
whitelisted.forEach(x => { // <- This line
if (executorID === x.user) return;
// ...
Try changing it with:
if (whitelisted && whitelisted.length) {
whitelisted.forEach(async (x) => { // <- Use async here
if (executorID === x.user) return;
// ...
Also, avoid using forEach to make asynchronous calls.
I´m really having problems with this.
The function im using is checking a counter for every component of an array, to push an array to a variable only if the counter is less than the number y state.
The problem here is that when the counter reaches the limit y send the variable to an api, and in a succesfull response I restart the counter and empty the variable to do it all over again.
My problem is when i send the first one the api takes a time to give me a response but the rest of the array im cheking has already past and becouse I only reset the counter in the response, only one time the api is consulted
The code im currently using:
cargaLotes() {
let totallengthchange = this.Arraydata.length;
this.cargaloader = true;
this.Step = true;
for (const values of this.Arraydata) {
this.check(values);
}
// this.lenghtloader = this.Arraydata.length;
let lengthafterPush;
let lengthdivided;
let lengthdisplay;
const porcetaje = 1;
const totaldisplay = 100;
let lengtharray = this.Arraydata.length;
const lengtharraystatico = this.Arraydata.length;
const temp: any = this.Arraydata;
if (this.Step === true) {
let contador1 = 0;
(contador1 < 500) {
temp.forEach(data => {
if (contador1 < 499) {
this.arraytosend.push(data);
contador1 = contador1 + 1;
if (contador1 === lengtharray) {
this.service.(service)(this.arraytosend).subscribe(resp => {
this.notificacion.showNotification('top', 'right', 'Se Cargaron los Productos', 'warning');
this.getProductoPrice();
this.lenghtloader = 100;
}, err => {
this.notificacion.showNotification('top', 'right', 'Ocurrio un error', 'danger');
});
}
if (contador1 === 499) {
this.arraytosend.push(data);
this.service.(service)(this.arraytosend).subscribe(resp => {
console.log(resp);
totallengthchange = totallengthchange - 500;
lengthafterPush = totallengthchange;
console.log(lengthafterPush);
lengthdivided = lengthafterPush / lengtharraystatico;
lengthdisplay = porcetaje - lengthdivided;
this.lenghtloader = lengthdisplay * totaldisplay;
}, err => {
console.log(err);
this.notificacion.showNotification('top', 'right', 'Ocurrio un error', 'danger');
});
lengtharray = lengtharray - 500;
this.arraytosend = [];
contador1 = 0;
}
}
});
}
} else {
this.ArraydataDuplicates = [];
for (const values of this.Arraydata) {
this.showDuplicates(values);
}
this._snackBar.open(`El Archivo Contiene Productos que ya han sido registrados!.`, 'Ocultar', {
duration: 5000
});
}
}
In very simple terms, your code is buffering values into an array, which then gets sent to an API. This will repeat until there are no values remaining in the array. You are sending batches of values to an API.
I will give a very simplistic example of the kind of pattern you could use below. I have ignored a lot of the side-effects that your code is performing - these should be simple enough to integrate once you have the main pattern sorted.
component.ts
private batchSize = 500;
private data: number[];
// handle some event that determines when the batches should be sent
onSend() {
// assume this.data has been set somewhere
this.sendBatches().subscribe(() => {
console.log('done');
});
}
private sendBatches(): Observable<any> {
// splice the data to get the batch of data to be sent
const batch: number[] = this.data.splice(0, this.batchSize);
if (batch.length === 0) {
return of(null);
}
return this.sendData(batch).pipe(
// recursive chain the observable calls
switchMap(() => this.sendBatches(batchSize))
);
}
private sendData(data: number[]): Observable<any> {
// mock call to the API
return of({}).pipe(delay(1000));
}
DEMO: https://stackblitz.com/edit/angular-60605895
There is a page with two images where you can vote to one of the pictures.
After the voting two new images is loading randomly and so on. The votes should be
incremented and saved to the server, but here comes the problem that I can not put the incremented
number, because I get an array: [{"votes":2},null]
I'd like to increment, save, and load the two new images with one onClick event.
Here it is what I tried to do:
handelVote(id) {
this.setState(
prevState => ({ //I get an array
voteNr: prevState.castles.map(c =>
c.id === id ? { votes: c.votes + 1 } : console.log(this.state.voteNr)
)
})
this.getCastles(),
axios
.put("............." + id, {
vote: this.state.voteNr
})
.then(response => {
console.log(response.data);
})
.catch(error => {
console.log(error);
})
);
}
here I'm getting the data from the server and load only two pictures and the belonging two votes:
componentDidMount() {
if (this.state.castles.length === 0) this.getCastles();
}
async getCastles() {
let castles = [];
let voteNr = [];
let i = [];
for (i = 0; i < 1; i++) {
const f = z(4); //random nr from Choise.js
const uniqueSet = new Set(f); //to avoid duplicates
const b = [...uniqueSet];
if (b.length < 2) {
b.shift();
b.push(1, 2); //in case of duplicates load the first two images
}
let res = await axios.get(".....");
let { data } = res;
let castle = data;
this.setState({ castles: castle });
for (i = 0; i < 2; i++) {
//load only two
var t;
t = b[i];
let name = castle[t].name;
let id = castle[t]._id;
let image = castle[t].image;
let votes = castle[t].vote;
castles.push({ image, name, id, votes });
voteNr.push({votes});
}
}
this.setState(prevState => ({
loading: false,
castles: [...castles]
}));
}
and after this comes the already qouted onClick event what makes the problems.
I have a firebase cloud function that is unable to finish executing. I suspect my code can be dramatically improved but I'm not quite sure how.
I've made the query as specific as possible to try and reduce the number of documents required to iterate through but that didn't solve the issue.
I get a Deadline Exceeded error which I suspect is due to the fact that I'm iterating through so many documents and trying to update them.
I increased the timeout (9 minutes)and memory allocation (2GB) in Google cloud console but that didn't help either.
exports.updatePollWinner = functions.runWith(runtimeOpts).firestore.document('triggerAccuracyCalculation/{id}').onCreate(trigger => {
const week = trigger.get('week');
const scoringTags = ["STD", "0.25PPR", "0.5PPR", "PPR", "0.10PPC", "0.25PPC", "0.5PPC", "4PTPASS", "5PTPASS", "6PTPASS", "-2INT", "TEPREMIUM"]
let winningChoiceIds = [];
let totalPollIds = [];
return db.collection("polls").where("sport", "==", 1).where("week", "==", week).where("pollType", "==", "WDIS").get()
.then((querySnapshot) => {
console.log("A");
querySnapshot.forEach((doc) => {
totalPollIds.push(doc.id);
let pollData = doc.data();
// extract relevant scoring tags
let tags = pollData.tags.filter(tag => scoringTags.includes(tag.code)).map(tag => tag.code);
// if no scoring setting is tagged, then use STD - determine what STD is
// extract player from each option
let winner = {score: 0, choice: {}, choiceId: null};
let cnt = 0;
pollData.choices.forEach((choice) => {
let choiceId = choice.id
let mappedChoices = choice.players.map(player => {
return { displayName: player.displayName, playerId: player.playerId, position: player.position, team: player.team }
});
// ToDo: What happens if someone posts a poll with two players in one option? This poll should be ignoree from accuracy calculation
// ignmore if option has more than one player
// if (mappedChoices.length > 1) return;
const player = mappedChoices[0]
// We can't score defense
if (player.position === "DEF") {
return;
}
const playerId = player.playerId;
// Make FFN API call to retrieve stats for that player in that weekconst statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}${req.url}`;
const statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}/${playerId}`;
const json = {"Stats": {"2019": ""}, "Player": {}};
https.get(statsEndpoint, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const weekString = week.toString();
const fetchedStats = JSON.parse(data).Stats
if (!fetchedStats) return;
const response = fetchedStats["2019"]
if (!response) return;
// TODO SCORE KICKERS AND DEFENSES
const stats = response[weekString];
let score = 0;
stats["recYards"] ? score += parseInt(stats["recYards"]) / 10 : false
stats["recTD"] ? score += parseInt(stats["recTD"]) * 6 : false
stats["rushYards"] ? score += parseInt(stats["rushYards"]) / 10 : false
stats["rushTD"] ? score += parseInt(stats["rushTD"]) * 6 : false
stats["xpMade"] ? score += parseInt(stats["xpMade"]) : false
stats["fgMade"] ? score += parseInt(stats["fgMade"]) * 3 : false
stats["kickoffRet"] ? score += parseInt(stats["kickoffRet"]) / 10 : false
stats["SackYards"] ? score -= parseInt(stats["SackYards"]) / 10 : false
stats["fumbleLost"] ? score -= parseInt(stats["fumbleLost"]) * 2 : false
// Determine winner
// ToDo: handle ties
if (score > winner.score) {
winner.score = score;
winner.choiceId = choiceId;
winner.choice = choice;
}
if (cnt>=pollData.choices.length-1){
// Save player object on the poll Document (include choice ID)
winningChoiceIds.push(winner.choiceId);
const pollDoc = db.doc(`polls/${doc.id}`);
pollDoc.update({winner: winner});
}
cnt++;
});
}).on("error", (err) => {
console.log("Error: ", err.message);
});
});
});
console.log("B");
return false;
}).then(() => {
console.log("C");
let dateToQueryAfter = new Date(new Date("08/22/19").setHours(0,0,0,0))
return db.collection("users").where("recentVote", ">", dateToQueryAfter).get()
})
.then((querySnapshot) => {
console.log("D");
const promises = [];
querySnapshot.forEach((doc) => {
const p = db.collection("votes").where("uid", "==", doc.id).where("week", "==", week).where("pollType", "==", "WDIS").get()
promises.push(p)
});
return Promise.all(promises)
})
.then((querySnapshots) => {
console.log("E");
querySnapshots.forEach((querySnapshot) => {
if (querySnapshot.docs.length <= 0) return;
const uid = querySnapshot.docs[0].data().uid
const retrieveUserDoc = db.doc(`users/${uid}`);
let correctVotes = 0;
let cnt = 0;
let totalVotes = 0;
let pollVoteIds = [];
let pollVoteIdsCorrect = [];
querySnapshot.docs.forEach((doc) => {
const voteData = doc.data();
if (totalPollIds.includes(voteData.poll)) {
pollVoteIds.push(voteData.poll)
totalVotes++;
if (winningChoiceIds.includes(voteData.choice)) {
pollVoteIdsCorrect.push(voteData.poll)
correctVotes++;
}
}
if (cnt>=querySnapshot.size-1){
console.log("Updating user ID: ", uid);
retrieveUserDoc.update({
['accuracyWeeks.week'+week]: true,
['accuracy.week'+week]: {
totalVotes: totalVotes,
correct: correctVotes,
accuracy: correctVotes/totalVotes,
correctVoteIds: pollVoteIdsCorrect,
totalVoteIds: pollVoteIds
}
});
}
cnt++;
})
});
console.log("F");
return false;
})
.catch((error) => {
console.log("Error getting documents: ", error);
});
});