I am doing some practice in node.js. In this exercise I been asked to find a country name through a GET Http Request to an endpoint passing a page integer as a parameter.
Where the important response structs are these {page, total_pages, data}.
page is the current page,
total_pages is the last page,
data is an array of 10 country object.
In getCountryName func I am able to retrieve the right answer only if the answer is on the 1st page, the 1 iteration of the loop. So, why the loop only happens once?
Aditional, I wanted to retrieve the total_pages to replace the hardcode '25' value but I do not figure it out how to return it along with the search.
Any hint you wanna give me? The whole problem is in getCountryCode func.
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
var res = '';
var pages = 25;
var i = 1;
while(i <= pages && res == ''){
console.log(i);
res = makeRequest(i)
.then(data => {
let f = ''
let p = data['total_pages'];
let search = data['data'].find(o => o.alpha3Code === code);
f = search != null ? search['name'] : f;
return f;
});
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();
Without modifying your code too much, this is how you do it:
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
const pages = 25;
var i = 1;
let f = null
while(i <= pages && f === null){
console.log(i);
const data = await makeRequest(i) // put in try/catch
const p = data['total_pages'];
const search = data['data'].find(o => o.alpha3Code === code);
f = search !== null ? search['name'] : null;
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();
Related
I've tried all sorts of ways to dispose of tensors (tf.dispose(), start/endscope).
The closest I've got was through this code, where 1 unused tensor remains after each execution. It takes about 2 hours for this program to run enough to use up 64 GB of RAM (big memory leak).
I also suspect that other factors besides TFJS-based operations are contributing to the memory leak, though (in theory) garbage collection should clean this up.
The piece of code below is one event that gets processed by an Event Listener handler. Any help with this issue would be greatly appreciated!
'use strict';
global.fetch = require("node-fetch");
const { MessageActionRow, MessageButton, Permissions } = require('discord.js');
const { mod, eco, m, n } = require(`../../index.js`);
const { Readable } = require('stream');
const PImage = require('pureimage');
const tf = require('#tensorflow/tfjs');
const tfnode = require('#tensorflow/tfjs-node');
const wait = require('util').promisify(setTimeout);
let bufferToStream = (binary) => {
let readableInstanceStream = new Readable({
read() {
this.push(binary);
this.push(null);
}
});
return readableInstanceStream;
}
const predict = async (imageUrl, modelFile) => {
let model = await tf.loadLayersModel(modelFile);
let modelClasses = [ "NSFW", "SFW" ];
let data = await fetch(imageUrl);
let fileType = data.headers.get("Content-Type");
let buffer = await data.buffer();
let stream = bufferToStream(buffer);
let image;
if ((/png/).test(fileType)) {
image = await PImage.decodePNGFromStream(stream);
}
else if ((/jpe?g/).test(fileType)) {
image = await PImage.decodeJPEGFromStream(stream);
}
else {
return;
}
let rawArray;
rawArray = tf.tidy(() => {
let tensorImage;
tensorImage = tf.browser.fromPixels(image).toFloat();
tensorImage = tf.image.resizeNearestNeighbor(tensorImage, [model.inputs[0].shape[1], model.inputs[0].shape[2]]);
tensorImage = tensorImage.reshape([1, model.inputs[0].shape[1], model.inputs[0].shape[2], model.inputs[0].shape[3]]);
return model.predict(tensorImage);
});
rawArray = await rawArray.data();
rawArray = Array.from(rawArray);
tf.disposeVariables();
model.layers.forEach(l => {
l.dispose();
});
if (rawArray[1] > rawArray[0]) {
return [`SFW`, rawArray[1]];
}
else {
return [`NSFW`, rawArray[0]];
}
};
const getResults = async (imageLink, imageNumber) => {
let image = `${imageLink}`;
let prediction = await predict(image, `file://D:/retake7/sfwmodel/model.json`);
let className = `SFW`;
if (prediction[0] == `NSFW`) {
className = `**NSFW**`;
}
return [`[Image ${imageNumber+1}](${imageLink}): ${className} (${(prediction[1]*100).toFixed(2)}% Certainty)`, ((prediction[1]*100).toFixed(2))*1];
}
const main = async (message, client, Discord) => {
if (message.attachments.size == 0 || message.author.bot || message.channel.nsfw) return;
await client.shard.broadcastEval(c => {
console.log(`Scanning...`);
}).catch(e => {
return;
});
let inChannel = await eco.seid.get(`${message.guild.id}.${message.channel.id}.active`);
let sfwImage = await eco.seid.get(`${message.guild.id}.sfwAlerts`);
if (inChannel == `no`) return;
let atmentArr = Array.from(message.attachments);
let msgArr = [];
if (message.attachments.size > 1) {
msgArr.push(`**Images Scanned**`);
} else {
msgArr.push(`**Image Scanned**`);
}
let hasNSFW = false;
let uncertain = false;
for (i = 0; i < message.attachments.size; i++) {
let msg = await getResults(atmentArr[i][1][`proxyURL`], i);
if (msg[1] < 80) {
uncertain = true;
}
if (msg[0].includes(`NSFW`)) {
hasNSFW = true;
}
msgArr.push(msg[0]);
}
if (uncertain == false && hasNSFW == false) {
let cont = `${msgArr.join(`\n`)}`;
msgArr = null;
client.seid.set(`${message.channel.id}.previousScan`, cont);
return;
}
let embed = new Discord.MessageEmbed()
.setColor(`GREEN`)
.setDescription(msgArr.join(`\n`));
let cont2 = `${msgArr.join(`\n`)}`;
client.seid.set(`${message.channel.id}.previousScan`, cont2);
msgArr = null;
if (sfwImage != `no` || hasNSFW || msg[1] <= 80) {
embed.setColor(`RED`);
await message.delete();
let msgSent = await message.channel.send({embeds: [embed], components: [row]});
};
};
module.exports = {
event: 'messageCreate',
run: async (message, client, Discord) => {
await main(message, client, Discord);
},
};
first, separate model loading and inference - in your current code, you'd reload a model each time you need to run prediction on a new image.
and then look at any possible leaks in prediction function - so once model is loaded.
you're loading a model and disposing each layer, but that doesn't mean model itself gets unloaded so there more than a chance that part of model remains in memory.
but leak itself is this line:
rawArray = await rawArray.data();
that variable is already used and its a tensor.
now you're overwriting the same variable with a data array and tensor never gets disposed.
i get error
let executor = await this.members.fetch(executorID);
^^^^^
SyntaxError: await is only valid in async function
when using the code below (use is to check if user breaks any of set filters and if so remove roles or ban user whatever they set option to)
ive tried my best to lable what parts of code does please not english isnt my first language
ive only recieved this error since trying to add a check whitelist feature - everything else works without the whitelist check code
without the code for whitelist the code works and performs as intended and the whitelist code succesfully logs ids for that guild
if(whitelisted && whitelisted.length) {
whitelisted.forEach(x => {
if (executorID === x.user) return;
const { Structures } = require('discord.js');
let whitelisted = db.get(`whitelist_${message.guild.id}`)
const { limits, defaultPrefix } = require('../config.js');
Structures.extend('Guild', Guild => {
class GuildExt extends Guild {
constructor(...args) {
super(...args);
}
get prefix() {
return this.get('prefix', defaultPrefix);
}
get(key, fallback) {
return this.client.db.get(`${this.id}_${key}`) || fallback;
}
set(key, data) {
return this.client.db.set(`${this.id}_${key}`, data);
}
delete(key) {
return this.client.db.delete(`${this.id}_${key}`);
}
resolveChannel(channelID) {
const channel = this.channels.cache.get(channelID);
return channel;
}
get limits() {
var obj = {};
for (var k in limits) {
obj[k] = {
minute: this.get(
`limits.${k}.minute`,
limits[k].per_minute
),
hour: this.get(`limits.${k}.hour`, limits[k].per_hour)
};
}
return obj;
}
getActions(limit = 10, filter = () => true) {
var obj = {};
var l = limits;
for (var k in limits) {
obj[k] = {
name: this.client.Utils.toProperCase(k),
actions: this.client.Utils.convertEntries(
[
...this.get(
this.client.Utils.convertLimitNameToActionType(
k
),
[]
),
...this.get(
`archive.${this.client.Utils.convertLimitNameToActionType(
k
)}`,
[]
)
]
.filter(filter)
.slice(0, limit)
)
};
}
return obj;
}
find_entry(action, filter) {
let guild = this;
return new Promise(resolve => {
(async function search(iter) {
//console.log(`ACTION = ${action} | ITER = ${iter}`);
if (!guild.me) return resolve(null);
if (guild.me.hasPermission('VIEW_AUDIT_LOG')) {
let logs = await guild.fetchAuditLogs({
limit: 10,
type: action
});
let entries = logs.entries;
let entry = null;
entries = entries.filter(filter);
for (var e of entries)
if (!entry || e[0] > entry.id) entry = e[1];
if (entry) return resolve(entry);
}
if (++iter === 5) return resolve(null);
else return setTimeout(search, 200, iter);
})(0);
});
}
push_entry(entry, displayName) {
const action = ['MEMBER_KICK', 'MEMBER_BAN_ADD'].includes(
entry.action
)
? 'MEMBER_REMOVE'
: entry.action;
const oneHourAgo = Date.now() - 1000 * 60 * 60;
// Fetch Entries for a sepcific action (Last Hour)
let entries = this.get(action, []);
// Filter entries older than one hour to a new variable
let olderThanOneHour = entries.filter(
i => !(i.timestamp > oneHourAgo)
);
// Prepend entries older than one hour to the archive
if (olderThanOneHour.length > 0)
this.set(`archive.${action}`, [
...olderThanOneHour,
...this.get(`archive.${action}`, [])
]);
// Filter entries older than one hour from old variable
entries = entries.filter(i => i.timestamp > oneHourAgo);
// Prepend new entry if not already found
if (
!entries.find(
i =>
i.target.id === entry.target.id &&
i.executor.id === entry.executor.id
)
)
entries.unshift({
timestamp: entry.createdTimestamp,
action: entry.action,
target: {
id: entry.target.id,
displayName,
targetType: entry.targetType
},
executor: {
id: entry.executor.id,
displayName: entry.executor.tag
}
});
// Update entries newer than one hour
return this.set(action, entries);
}
async check_limits(entries, executorID, configAction) {
// Ignore if executor is the owner or is whitelisted
if (executorID === this.ownerID) return;
if(whitelisted && whitelisted.length) {
whitelisted.forEach(x => {
if (executorID === x.user) retrun;
// Filter actions relating to executor
const oneMinuteAgo = Date.now() - 1000 * 60;
let executorActionsHour = entries.filter(
i => i.executor.id === executorID
);
let executorActionsMinute = executorActionsHour.filter(
i => i.timestamp > oneMinuteAgo
);
console.log(
`${configAction}/${executorID}: LAST_HOUR: ${executorActionsHour.length} LAST_MINUTE: ${executorActionsMinute.length} `
);
let limits = this.limits;
let limitReached = null;
if (executorActionsHour.length >= limits[configAction].hour)
limitReached = 'Hour';
if (executorActionsMinute.length >= limits[configAction].minute)
limitReached = 'Minute';
// Check if the amount of actions is greater than or equal to the limit
if (limitReached) {
// Remove all of the executor's roles
let executor = await this.members.fetch(executorID);
executor.roles.remove(executor.roles.cache);
// Handle managed roles
let managed = executor.roles.cache
.filter(r => r.managed)
.array();
for (var i = 0; i < managed.length; i++)
managed[i].setPermissions(0, 'Guardian Action');
// Notify owner, executor, and logging channel
const embed = this.client.util
.embed()
.setTitle(`Limit Reached - ${limitReached}`)
.setDescription(
this.client.Utils.convertEntries(
limitReached === 'Hour'
? executorActionsHour
: executorActionsMinute
)
)
.setColor(0x7289da);
await this.owner.send(
embed.setFooter(
"This message was sent to you because you're the Guild owner."
)
);
await executor.send(
embed.setFooter(
'This message was sent to you because you were the executor.'
)
);
const loggingChannel = this.resolveChannel(
this.get(`loggingChannelID`)
);
if (loggingChannel)
await loggingChannel.send(embed.setFooter(''));
}
})
}
}
}
return GuildExt;
});
i am new to JS and any help would be appreciated
please dont hate if i do have bad syntax !!
i am new - sorry if i dont get things the first time
You forgot to make your forEach function async, just change it to:
/* ... */
whitelisted.forEach(async (x) => {
/* ... */
let executor = await this.members.fetch(executorID);
/* ... */
}
/* ... */
Not part of your question but you misspelled return here
if (executorID === x.user) retrun;
Your line
let executor = await this.members.fetch(executorID);
is inside a non-async anonymous function in:
if (whitelisted && whitelisted.length) {
whitelisted.forEach(x => { // <- This line
if (executorID === x.user) return;
// ...
Try changing it with:
if (whitelisted && whitelisted.length) {
whitelisted.forEach(async (x) => { // <- Use async here
if (executorID === x.user) return;
// ...
Also, avoid using forEach to make asynchronous calls.
So basically im working on a cron job in my app that fires every 3 hours and updating users 'score' by calling the RiotApi
basically the function so far
exports.updatePlayersPoints = async () => {
console.log('STARTED UPDATING');
try {
const players = await UserLadder.findAll();
await Promise.all(
players.map(async (player) => {
const p = await RiotAccount.findOne({
where: {
userId: player.userId,
},
include: RiotRegions,
});
const beginTime = new Date(player.dataValues.createdAt);
let data;
try {
const res = await axios.get(
`https://${
p.dataValues.riot_region.dataValues.name
}.api.riotgames.com/lol/match/v4/matchlists/by-account/${
p.dataValues.accountId
}?queue=420&beginTime=${beginTime.getTime()}&api_key=${
process.env.RIOT_KEY
}`
);
data = res.data;
} catch (error) {
if (!error.response.status === 404) {
console.error(error);
}
}
if (!data) {
return;
}
let totalScore = player.dataValues.userPoints;
await Promise.all(
data.matches.map(async (match, i) => {
if (i < 15) {
const { data } = await axios.get(
`https://${p.dataValues.riot_region.dataValues.name}.api.riotgames.com/lol/match/v4/matches/${match.gameId}?api_key=${process.env.RIOT_KEY}`
);
const calculateScore = () => {
return new Promise((resolve) => {
const { stats } = _.find(
data.participants,
(o) => o.championId === match.champion
);
const killsPts = stats.kills * 2;
const deathPts = stats.deaths * -1.5;
const assistsPts = stats.assists;
const wardsPts = stats.wardsPlaced / 4;
const firstBloodPts = stats.firstBloodKill ? 3 : 0;
const firstBloodAssistPts = stats.firstBloodAssist ? 3 : 0;
const firstTowerPts = stats.firstTowerKill ? 2 : 0;
const firstTowerAssistPts = stats.firstTowerAssist ? 2 : 0;
const score =
killsPts +
deathPts +
assistsPts +
wardsPts +
firstBloodPts +
firstBloodAssistPts +
firstTowerPts +
firstTowerAssistPts;
totalScore += score;
resolve();
});
};
await calculateScore();
}
})
);
const user = await UserLadder.findOne({
where: {
userId: player.userId,
},
});
user.userPoints = parseFloat(totalScore);
user.lastGameId = data.matches[0].gameId;
await user.save();
})
);
console.log('FINISHED UPDATING');
} catch (error) {
console.error(error);
}
};
Basically it just looks up the table userladder to find the players that are signed to the ladder and for each one of these players it fires a map function that makes a request to the riotapi to get the match history of this player and then later make an inside map function to map each one of these matches.
but basically I updated it to now keep track of the game id of the last call before 3 hours so it doesn't have to make request that was already done.
user.lastGameId = data.matches[0].gameId;
but now in my second map function that maps the matches I wasn't it so that if the last game from my database matches the game id that currently being mapped I want to stop the map function and not continue this record or the ones after because it also means they all have been already counted.
but I can not seem to find a way to do it.
i tried using break; but it didn't work
any ideas?
using for loop
I tried a small test with for loop so I tried
for (let i = 0; i < 15; i++) {
await new Promise(async (resolve, reject) => {
const match = data.matches[i];
console.log(match);
resolve();
if (i === 1) {
break;
}
});
}
but I still go the same error
SyntaxError: Illegal break statement
Instead of trying to "break" a map, you should filter the matches that you want to process before you execute the map.
Something like this:
await Promise.all(
const filteredMatches = data.matches.filter(match => match.gameId > previousId);
filteredMatches.map(async (match, i) => { ...
More on filter() in javascript.
Edit: If generated id's are random and are not ordered, you can store all previous id's in a Set, and then just ask if it has been previously added
await Promise.all(
const filteredMatches = data.matches.filter(match => mySet.has(match.gameId));
filteredMatches.map(async (match, i) => { ...
More on Set in javascript.
I am trying to trigger an another function in Firebase Cloud function with javascript. But i always getting an error of Can't set headers after they are sent. Please take a look at my code below: ................. ................. ............ ................ ................. ............... ....................... .................. ..............
exports.productIndexShuffleOne = functions.https.onRequest(async (req, res) => {
const interval = req.query.interval;
console.log("interval: "+interval);
const productRef = admin.firestore().collection("Products");
const adminRef = admin.firestore().collection("Admin").doc("totalProd").get();
const dateRef = admin.firestore().collection("Admin").doc("totalProd").collection("indexShuffle").doc("productShuffle").get();
return dateRef.then(documentSnapshot => {
const setDate = documentSnapshot.get('date').seconds;
var nextDay = setDate;
console.log("Date: "+nextDay);
const x = setInterval(function() {
clearInterval(x);
return Promise.all([adminRef]).then(result => {
const totalNum = result[0].data().totalNumber;
console.log("totalNum: "+totalNum);
var numberList = [];
var index = 1;
while(index <= totalNum){
numberList.push(index);
index++;
}
var cidx, ridx, tmp;
cidx = numberList.length;
while (cidx !== 0) {
ridx = Math.floor(Math.random() * cidx);
cidx--;
tmp = numberList[cidx];
numberList[cidx] = numberList[ridx];
numberList[ridx] = tmp;
}
console.log(numberList);
var counter = 0;
return productRef.get().then(snapshot => {
snapshot.forEach(doc => {
const prodID = doc.get('productID');
const index = doc.get('index');
var newIndex = numberList[counter];
counter++;
console.log("oldIndex: "+index);
console.log("newIndex: "+newIndex);
productRef.doc(prodID).update({
index: newIndex
}, {merge: true});
});
return res.redirect('https://us-central1-myfunction-123456.cloudfunctions.net/productIndexShuffleTwo?interval='+interval);
})
.catch(err => {
console.log('Error getting documents', err);
});
});
}, interval);
return res.status(203).send(interval);
}).catch(function(err) {
console.error(err);
});
});
This is because you've sent multiple responses while the rule is that you only allowed sending one response. Please try to look at your code and optimize it in such a way that it contains only one response.
I can see you have multiple responses as below:
1 -> return res.redirect('https://us-central1-myfunction-123456.cloudfunctions.net/productIndexShuffleTwo?interval='+interval);
2 -> return res.status(203).send(interval);
I believe that you can have res.redirect and then res.status.send called one after another. When you writing endpoints there rule of a thumb: always send response and only do that once. Refactor your code so there no way you can make those two calls, but only one of them.
I am trying to read some data from 2 different tables and parse a CSV file before rendering an ejs file.
I can get the data from both tables and from the CSV file but I seem to be unable to return the result.
Pretty sure this is a problem with the way I handle async execution but I fail to see what I am doing wrong.
I've spent the last 2 days reading about this (including the threads around here) and browsing but somehow the answer still escapes me.
First file - usercms.js
app.get('/userscms', function(req, res)
{
existingUsers.getExistingUsers()
.then(function(appUsers)
{
//global users array
//I can display these in my ejs file
globalAppUsers = appUsers;
})
.then(existingUsersAttributesQlik.getExistingUsersAttributesQlik())
.then(function(usersQlikAttributes)
{
//global user attributes array
//undefined data
globalUsersQlikAttributes = usersQlikAttributes;
})
.then(existingSuppliers.parseSuppliersCSV())
.then(function(supplierData)
{
//the result I am expecting
//this prints undefined
console.log(supplierData);
}).then(function()
{
res.render('userscms.ejs',
{
users: globalAppUsers,
attributes: globalUsersQlikAttributes
});
});
});
Second function - getxistingUsers.js (identical to the getExistingUsersAttributesQlik, except for the query)
var userData = [];
var appUsers = [];
(function (exports)
{
exports.getExistingUsers = function ()
{
return promisemysql.createConnection(dbconfig.development).then(function(conn)
{
var result = conn.query("SELECT id, username, firstName, lastName, email, phone, lastLogin, isAdmin, isValid, isPhoneValid, accountCreationDateTime FROM Users");
conn.end();
return result;
}).then(function(rows)
{
return rows;
}).then(function(rows)
{
if (rows.length)
{
userData = [];
appUsers = [];
rows.forEach(function (elem)
{
userData.push(_.toArray(elem));
});
for (i = 0; i < userData.length; i++)
{
var appUser = new appUserModel.AppUser(
userData[i][0],
userData[i][1],
userData[i][2],
userData[i][3],
userData[i][4],
userData[i][5],
userData[i][6],
userData[i][7],
userData[i][8],
userData[i][9],
userData[i][10]);
appUsers.push(_.toArray(appUser));
}
return appUsers;
}
else
{
console.log("NOPE");
return null;
}
}).then(function(appUsers)
{
console.log(appUsers);
return appUsers;
});
};
})(typeof exports === 'undefined' ? this['getExistingUsers'] = {} : exports);
Third file - parseSuppliersCSV.js
var supplierData = [];
var suppliersData = [];
var csvCount = 0;
(function (exports)
{
exports.parseSuppliersCSV = function ()
{
return new Promise(function(resolve, reject)
{
var fileStream = fs.createReadStream("myCSV.csv");
var parser = fastCsv();
csvCount = 0;
supplierData = [];
suppliersData = [];
fileStream
.on("readable", function ()
{
var data;
while ((data = fileStream.read()) !== null)
{
parser.write(data);
}
})
.on("end", function ()
{
parser.end();
});
parser
.on("readable", function ()
{
var data;
while ((data = parser.read()) !== null)
{
if(csvCount >= 1)
{
csvCount++;
var arrayOfStrings = data[0].split(';');
var supplier = new supplierModel.Supplier(arrayOfStrings[0],arrayOfStrings[1]);
suppliersData.push(_.toArray(supplier));
}
else
{
csvCount++;
}
}
})
.on("end", function ()
{
console.log("done");
//all OK here
console.log(suppliersData);
//this doesn't seem to return anything
return suppliersData;
});
});
};
})(typeof exports === 'undefined' ? this['parseSuppliersCSV'] = {} : exports);
Any ideas what I am doing wrong? Am I approaching this the wrong way?
I'll take a guess here and assume the promise you created should resolve to something...instead of returning a value.
.on("end", function ()
{
console.log("done");
//all OK here
console.log(suppliersData);
//this doesn't seem to return anything
return resolve(suppliersData);
});