I have following code which process a queue and I need to exist the function when there are no messages in the queue and there is no enough time to process more messages. My problem is, it doesn't jump out of the function upon failing the condition and I think it's due to that this a recursive function but I cannot figure it out.
/**
* Check if there is enough time to process more messages
*
* #param {} context
* #returns {boolean}
*/
async function enoughTimeToProcess(context) {
return context.getRemainingTimeInMillis() > 230000;
}
/**
* Consume the queue and increment usages
*
* #param context
*
* #returns {boolean}
*/
async function process(context) {
const messagesPerRequest = queueConst.messagesPerRequest;
const messagesToBeDeleted = [];
const queue = new queueClient();
const messages = await queue.getMessages(messagesPerRequest);
if (messages === undefined) {
if (await enoughTimeToProcess(context) === true) {
await process(context);
} else {
return false;
}
}
const responses = messages.map(async(messageItem) => {
const messageBody = JSON.parse(messageItem.Body);
const parsedMessage = JSON.parse(messageBody.Message);
const accountId = parsedMessage[0].context.accountId;
let code = parsedMessage[0].context.code;
// Our DB support only lowercase characters in the path
code = code.toLowerCase();
const service = parsedMessage[0].name;
const count = parsedMessage[0].increment;
const storageResponse = await incrementUsage(
{ storageClient: storage, code, accountId, service, count }
);
if (storageResponse) {
messagesToBeDeleted.push({
Id: messageItem.MessageId,
ReceiptHandle: messageItem.ReceiptHandle,
});
}
return 1;
});
const processedMessages = await Promise.all(responses);
const processedMessagesCount = processedMessages.length;
if (messagesToBeDeleted.length > 0) {
console.log(`${processedMessagesCount} messages processed.`);
await queue.deleteMessageBatch(messagesToBeDeleted);
}
if (await enoughTimeToProcess(context) === true) {
await process(context);
}
return true;
}
I think the problem can be when messages are undefined and there is still enough time, because the recursive function is going to be called infinite times, because it always accomplishes both conditions, and probably it exceeds the available resources.
Try to sleep some time before calling process function again, just to be sure it is the problem
Related
I'm a new developer and am currently asked to set up a login / signup page for a Single Page Application in JavaScript. This SPA is in the early stages, since the published version is a multipage application.
Our database already contains the user information. I suppose I would have to work with post and get requests in order to authenticate users?
So far, I have managed to fetch data from the database in json, but I don't know how to handle user input and send it to the database.
A general overview: Our Single Page Application is written in JavaScript. A separate Python file retrieves data from the database, creates specific routes (urls) (which we access in our js files) and stores the data in json.
P.S. the code might be pretty janky, and any suggestions on how to improve or simplify the code are appreciated
rest_server.py:
#app.route(url_prefix + '/json/team/<int:team_id>')
def get_team(team_id):
team = adapter.team_info(team_id)
resp = flask.jsonify(team)
resp.headers['Access-Control-Allow-Origin'] = '*'
return resp
#app.route(url_prefix + '/json/team_ids')
def get_team_ids():
team_ids = adapter.team_ids()
resp = flask.jsonify(team_ids)
resp.headers['Access-Control-Allow-Origin'] = '*'
return resp
main.js (reduced version here, I hope it makes sense):
async function renderPage(path) {
if (!path) {
path = window.location.pathname;
}
console.log(`render page ${path}`);
let renderedPath = await pathHandler.createContent(path);
if (renderedPath && renderedPath != path) {
// if page was rendered under a different name, override history
console.log(`different rendered path ${renderedPath}`);
window.history.replaceState(renderedPath,'', renderedPath);
}
}
/**
* handles back button events on window object
* #param {Event} event event object emitted by window.
*/
function onBackButton(event) {
event.preventDefault();
renderPage();
}
/**
* handles click events on the document interface
* #param {Event} event object on the document interface
*/
function onBodyClick(event) {
var tag = event.target;
// use default behavior if this was not a hyperlink
if (tag.tagName != 'A' || !tag.href || event.button != 0) {
return;
}
// use default behavior if link points to other server
if (tag.origin != window.location.origin) {
return;
}
// use default behavior if path is not supported
if (!pathHandler.isSupported(tag.pathname)) {
return;
}
event.preventDefault();
redirect(tag.pathname);
}
/**
* determines the path prefix, the page is loaded with. at the moment, a path prefix requires an
* indicator string. in future, the path could be transmitted within the HTML page and extracted here
* #returns {String|null} path prefix or null if there is none.
*/
function determinePathPrefix() {
let pathPrefixIndicator = '/sp/';
let path = window.location.pathname;
let index = path.lastIndexOf(pathPrefixIndicator);
if (index >= 0) {
return path.slice(0, index + pathPrefixIndicator.length - 1);
}
return null;
}
function displayTeams() {
contentCreator.createDisplayTeams();
}
const contentElement = document.querySelector('.content');
const headerElement = document.querySelector('.details');
const teamsLi = document.querySelector('#teams');
const baseElements = {
headerElement: headerElement,
contentElement: contentElement
};
const functions = {
displayTeams: displayTeams
}
const pathPrefix = determinePathPrefix();
console.log(`path prefix: ${pathPrefix}`);
const pathHandler = new PathHandler(pathPrefix);
const contentCreator = new ContentCreator(pathHandler, baseElements, functions);
const createDisplayTeams = contentCreator.createDisplayTeams.bind(contentCreator);
const createTeams = contentCreator.createTeams.bind(contentCreator);
pathHandler.addPath('createTeams', '/team_<int:itemId>', createTeams);
pathHandler.addPath('teams', '/teams', createDisplayTeams);
document.body.addEventListener('click', onBodyClick);
teamsLi.addEventListener('click', displayTeams);
window.addEventListener('popstate', onBackButton);
renderPage();
Then there is a PathHandler.js file, which might not be necessary to show here.
Finally, ContentCreator.js, which takes care of the UI (also reduced):
class ContentCreator {
#pathHandler;
#baseElements;
#functions;
/**
* #param {PathHandler} pathHandler path handler to be used for generating links.
* #param {Object} baseElements base HTML elements of which children can be replaced.
* #param {Object} functions functions for reloading or redirecting.
*/
constructor(pathHandler, baseElements, functions) {
this.#pathHandler = pathHandler;
this.#baseElements = baseElements;
this.#functions = functions;
}
// the displayTeams function in main.js calls this function
async createDisplayTeams() {
let paragraphElement = document.createElement('p');
paragraphElement.setAttribute('class', 'teamsList');
const response = await fetch('http://localhost:5000/json/teams');
const teams = await response.json();
for (let i = 0; i < teams.length; i++) {
const id = teams[i].TeamId;
const li = document.createElement('li');
const link = document.createElement('a');
const name = teams[i].Name;
link.innerHTML = name;
li.appendChild(link);
li.setAttribute('class', 'teamLi');
paragraphElement.appendChild(li);
link.setAttribute('href', this.#pathHandler.generatePath('createTeams', {itemId: id}));
}
this.#baseElements.contentElement.replaceChildren(paragraphElement);
}
async createTeams(args) {
const id = args['itemId'];
const infoResponse = await fetch(`http://localhost:5000/json/team/${id}`);
const teamInfo = await infoResponse.json();
const roleInfo = teamInfo['RoleInfoList'];
const members = document.createElement('ul');
const container = document.createElement('div');
container.setAttribute('class', 'teamContainer');
const header = document.createElement('h2');
header.innerHTML = teamInfo['Name'];
container.appendChild(header);
const paragraphElement = document.createElement('p');
if (teamInfo['Email']) {
paragraphElement.innerHTML = teamInfo['Email'];
}
for (let p = 0; p < roleInfo.length; p++) {
const member = document.createElement('li');
member.innerHTML = `${roleInfo[p]['Person Name']}`;
members.appendChild(member);
}
container.appendChild(paragraphElement);
container.appendChild(members);
this.#baseElements.contentElement.replaceChildren(container);
}
}
Now I would like to create a form in ContentCreator.js, which somehow fetches the user input and then makes a database request in order to authenticate the user. The UI code won't be a problem, but how could I fetch and send the data?
Thank you!!
Mina
I am trying to create a script that pulls from the coin market cap API and displays the current price. The script is working fine on the back end when I assign the variable a value. However, when I try to run the function on sheets the returned value is null.
function marketview(ticker) {
var url = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?CMC_PRO_API_KEY=XXX&symbol=" + ticker;
var data = UrlFetchApp.fetch(url);
const jsondata = JSON.parse(data);
Logger.log(jsondata.data[ticker].quote['USD'].price)
}
My execution logs show that the scripts are running, but when when I use the function and try and quote ETH for example, the script is running for BTC.
When I do this on the backend and assign ETH the script works fine and returns the right quote. Any ideas on what I'm missing?
I did the same with coingecko API and add an issue having all my requests being rejected with quota exceeded error.
I understood that Google sheets servers IPs address were already spamming coingecko server. (I was obviously not the only one to try this).
This is why I used an external service like apify.com to pull the data and re-expose data over their API.
This is my AppScripts coingecko.gs:
/**
* get latest coingecko market prices dataset
*/
async function GET_COINGECKO_PRICES(key, actor) {
const coinGeckoUrl = `https://api.apify.com/v2/acts/${actor}/runs/last/dataset/items?token=${key}&status=SUCCEEDED`
return ImportJSON(coinGeckoUrl);
}
You need ImportJSON function, available here: https://github.com/bradjasper/ImportJSON/blob/master/ImportJSON.gs
Then in a cell I write: =GET_COINGECKO_PRICES(APIFY_API_KEY,APIFY_COINGECKO_MARKET_PRICES), you will have to create two field named APIFY_API_KEY and APIFY_COINGECKO_MARKET_PRICES in order for this to work.
Then register on apify.com, then you'll have to create an actor by forking apify-webscraper actor.
I set the StartURLs with https://api.coingecko.com/api/v3/coins/list, this will give me the total number of existing crypto (approx 11000 as of today), and number of page so I can run the request concurrently (rate limit is 10 concurrent requests on coingecko), then I just replace /list with /market and set the proper limit to get all the pages I need.
I use the following for the tasks page function:
async function pageFunction(context) {
let marketPrices = [];
const ENABLE_CONCURRENCY_BATCH = true;
const PRICE_CHANGE_PERCENTAGE = ['1h', '24h', '7d'];
const MAX_PAGE_TO_SCRAP = 10;
const MAX_PER_PAGE = 250;
const MAX_CONCURRENCY_BATCH_LIMIT = 10;
await context.WaitFor(5000);
const cryptoList = readJson();
const totalPage = Math.ceil(cryptoList.length / MAX_PER_PAGE);
context.log.info(`[Coingecko total cryptos count: ${cryptoList.length} (${totalPage} pages)]`)
function readJson() {
try {
const preEl = document.querySelector('body > pre');
return JSON.parse(preEl.innerText);
} catch (error) {
throw Error(`Failed to read JSON: ${error.message}`)
}
}
async function loadPage($page) {
try {
const params = {
vs_currency: 'usd',
page: $page,
per_page: MAX_PER_PAGE,
price_change_percentage: PRICE_CHANGE_PERCENTAGE.join(','),
sparkline: true,
}
let pageUrl = `${context.request.url.replace(/\/list$/, '/markets')}?`;
pageUrl += [
`vs_currency=${params.vs_currency}`,
`page=${params.page}`,
`per_page=${params.per_page}`,
`price_change_percentage=${params.price_change_percentage}`,
].join('&');
context.log.info(`GET page ${params.page} URL: ${pageUrl}`);
const page = await fetch(pageUrl).then((response) => response.json());
context.log.info(`Done GET page ${params.page} size ${page.length}`);
marketPrices = [...marketPrices, ...page];
return page
} catch (error) {
throw Error(`Fail to load page ${$page}: ${error.message}`)
}
}
try {
if (ENABLE_CONCURRENCY_BATCH) {
const fetchers = Array.from({ length: totalPage }).map((_, i) => {
const pageIndex = i + 1;
if (pageIndex > MAX_PAGE_TO_SCRAP) {
return null;
}
return () => loadPage(pageIndex);
}).filter(Boolean);
while (fetchers.length) {
await Promise.all(
fetchers.splice(0, MAX_CONCURRENCY_BATCH_LIMIT).map((f) => f())
);
}
} else {
let pageIndex = 1
let page = await loadPage(pageIndex)
while (page.length !== 0 && page <= MAX_PAGE_TO_SCRAP) {
pageIndex += 1
page = await loadPage(pageIndex)
}
}
} catch (error) {
context.log.info(`Fetchers failed: ${error.message}`);
}
context.log.info(`End: Updated ${marketPrices.length} prices for ${cryptoList.length} cryptos`);
const data = marketPrices.sort((a, b) => a.id.toLowerCase() > b.id.toLowerCase() ? 1 : -1);
context.log.info(JSON.stringify(data.find((item) => item.id.toLowerCase() === 'bitcoin')));
function sanitizer(item) {
item.symbol = item.symbol.toUpperCase()
return item;
}
return data.map(sanitizer)
}
I presume you are hiting the same issue I had with coinmarketcap, and that you could do the same with it.
You're not return ing anything to the sheet, but just logging it. Return it:
return jsondata.data[ticker].quote['USD'].price
I am trying to create a serverless React app to make recommendations by processing some data from Spotify's API. I am using spotify web api js as a wrapper to make the API calls. My problem is that one of the results I get from my functions appears when I call console.log on it, but not when I pass it to another function. Here's the code for the submit handler on my page:
handleSubmit(e) {
e.preventDefault();
this.setState({recOutput: {}});
spotify.setAccessToken(this.props.vars.token);
spotify.searchArtists(this.state.artist)
.then(res => this.getRecs(res))
.then(output => this.processResults(output))
.then(processed => this.setState({resultReceived: true, recOutput: processed}))
.catch(err => this.handleError(err));
}
And here are all the functions it's calling:
/**
* Gets recommendations for a specific artist and outputs recOutput value in redux store
* #param {string} name The name of the artist
* #return Promise with output
* TODO: catch errors again lol
* /
**/
async getRecs(name) {
const MAX_SONGS = 50;
var output = {};
spotify.searchPlaylists(name, {limit: 1})
.then(searchTotal => {spotify.searchPlaylists(name, {limit: 50, offset: Math.floor(Math.random() * (searchTotal.playlists.total/50))}).then(
res => {
for (let i of res.playlists.items) {
spotify.getPlaylistTracks(i.id).then(
pt => {
let curSongs = 0;
if (pt == undefined) {
return;
}
for (let track of pt.items) {
if (curSongs > MAX_SONGS) break;
if (track.track != null
&& track.track.artists[0].name !== name
&& track.track.artists[0].name !== "") {
if (track.track.artists[0].name in output) {
output[track.track.artists[0].name]++;
} else {
output[track.track.artists[0].name] = 1;
}
}
curSongs++;
}
})
}
}
)
}).catch(err => this.handleError(err));
return output;
}
/**
* Processes results from our query to spotify, removing entries beyond a certain threshhold then sorting the object.
* #return Promise for updated results update
*/
async processResults(input) {
debugger;
let processed = {};
for (let key in input) {
console.log(key);
if (input[key]> 10) {
processed.key = input.key;
}
}
return processed;
}
My problem is that when I call .then(output => this.processResults(output)), the process method receives an empty output in the debugger, but when I call .then(output => console.log(output)), I see the expected output for the function.
Here is the context of my component:
I'm trying to make an array of sets to make something like this
{
'user1': ["value#1", "value#2",..."value#N"],
'user2': ["value#2",..."value#N"],
'userN': [..."value#N"]
}
and then remove the value#x after 5 seconds (for example).
here is my code:
var myset = new Set();
var ran = myset[USERID] = commandNumber;
//i'm trying to make "if myset contains userNumber AND commandName" return,
//if its not, run someFunction() and continue
if (myset.has(ran)) return;
someFunction();
myset.add(ran);
setTimeout(() => {
myset.delete(ran);
}, 5000);
instead of getting output like the first code, i get this output instead
Set { 'command1', 'command2',
'USER1': 'command3',
'USER2': 'command4'
'USERN': 'commandN'
}
Feel free to comment if you have a question, so sorry if my question is hard to understand
A Set for this purpose is not necessary but I did a small POC that could help you to implement the solution you need:
'use strict';
const mySet = new Set();
const mySetMetadata = {};
const removeFromSet = (userKey, commandName) => {
const commands = mySetMetadata[userKey] || [];
if (commands.includes(commandName)) {
mySetMetadata[userKey] = commands.filter(c => c !== commandName);
if (mySetMetadata[userKey].length === 0) {
mySet.delete(userKey);
mySetMetadata[userKey] = undefined;
}
}
};
/**
* Add relation between an userKey and a command
* #param {String} userKey
* #param {Array} commands Array of commands
*/
const addToSet = (userkey, commands) => {
mySet.add(userkey);
if (typeof mySetMetadata[userkey] === 'undefined') {
mySetMetadata[userkey] = commands;
} else {
mySetMetadata[userKey] = [...mySetMetadata[userKey], ...commands]
}
}
// Populate with demo data
addToSet('user1', ['value#1', 'value#2', 'value#N']);
addToSet('user2', ['value#2', 'value#N']);
addToSet('user3', ['value#N']);
// Set up a timeout for a given user + key
setTimeout(() => {
removeFromSet('user1', 'value#2');
}, 5000);
Hey so im making a leaderboard for a discord bot using discord.js And I want to display users by their names instead of their ID's so using discord.js I use the function .fetchUser(ID)
.fetchUser(ID) is a promise which can take a some of time depending on the bandwidth.
So because discord.js uses a promise my code is no longer Async, I thought that by putting the code in a promise it would run Async.
And I was wrong.
my code:
//This is ran inside a .prototype function so (this) is defined
return new Promise((resolve, reject) => {
this.list = [];
//users is an object with user's IDs as the key
//Currently it only has one key in it (mine)
for (let i in users) {
let pos = 0;
let score = this.getScore(users[i]);
if (score === 0) {
client.fetchUser(i).then((user)=> {
console.log(`pushed`);//logs way after the "finish" is logged
this.list.push([user.username.substring(0,13), score])
});
continue;
}
for (let h = 0; h < this.list.length; h++) {
if (score >= this.list[h][1]) {
pos = h;
break;
}
}
client.fetchUser(users[i].id).then((user) => {
this.list.splice(pos, 0, [user.username.substring(0,13), score])
})
}
console.log(`Finished: `+this.list.length);
resolve(this.list);
})
You have to chain off of Promises you receive. Client#fetchUser() returns a Promise which you are waiting on, but not enough. You have to propagate up Promises. If something in your function call chain is asynchronous, you should consider the whole chain async.
You fill this.list from within the fetchUser(...).then(...), which isn't necessarily bad, as long as you don't try to use list until after fetchUser's resolution chain is done. You aren't doing that; you immediately resolve(this.list).
Consider this abbreviated form of your original function:
return new Promise((resolve, reject) => {
this.list = [];
for (let i in users) {
// A promise is created right here
client.fetchUser(i).then((user) => {
// This will populate list AFTER the then callback
this.list.push([user.username.substring(0, 13), score])
});
}
// You aren't waiting until the promise created by fetchUser completes
resolve(this.list);
})
this.list can't be considered "complete" until all the users involved have had their profiles loaded and their scores retrieved. Considering that, we can use Promise.all() which takes an array of Promises and then resolves once all of the provided promises have resolved. So to wait that way, we would do something like this, which still isn't ideal, but waits correctly:
return new Promise((resolve, reject) => {
this.list = [];
// This is an array of Promises
const discordUsersPromise = users.map(user => client.fetchUser(user));
// Wait till all the fetchUser calls are done
const listIsPopulatedPromise = Promise.all(discordUsersPromise).then(dUsers => {
// This replaces your for (let i in users) {}
Object.entries(users).forEach((user, idx) => {
const score = this.getScore(user);
const discordUser = dUsers[idx];
this.list.push([discordUser.username.substring(0, 13), score])
});
});
// We still have to wait for the list to be completely populated
return listIsPopulatedPromise.then(() => this.list);
})
Consider this implementation. I have made some assumptions about your code since you use this.list but don't include what this is an instance of, but most of it should be the same:
/**
* Object to composite certain user properties
* #typedef {RealUser}
* #property {String} user The local string for the user
* #property {User} realUser The user that Discord gives us
* #property {Number} score The score this user has
*/
/**
* Class to encapsulate user and score and data
*/
class Game {
/**
* Constructs a game
*/
constructor() {
/**
* The users we are keeping score of
* #type {Object}
*/
this.users = {};
}
/**
* Get the score of a particular user
* #param {String} user User to get score of
* #returns {Number} User's score
*/
getScore(user) {
return this.users[user] || 0;
}
/**
* Get a composite of users and their status
* #param {String[]} users The users to put on our leaderboard
* #returns {Promise<RealUser[]>} Sorted list of users that we included in our leaderboard
*/
getLeaderBoard(users) {
// Map all the users that we are given to Promises returned bye fetchUser()
const allRealUsersPromise = Promise.all(users.map(user => client.fetchUser(user)
/*
* Create an object that will composite the string that we use
* to note the user locally, the Discord User Object, and the
* current score of the user that we are tracking locally
*/
.then(realUser => ({
user,
realUser,
score: this.getScore(user)
}))));
/*
* Once we have all the data we need to construct a leaderboard,
* we should sort the users by score, and hand back an array
* of RealUsers which should contain all the data we want to
* print a leaderboard
*/
return allRealUsersPromise
.then(scoredUsers => scoredUsers.sort((a, b) => a.score - b.score));
}
/**
* Prints out a leaderboard
* #param {String[]} users The users to include on our leaderboard
*/
printLeaderBoard(users) {
// Go get a leaderboard to print
this.getLeaderBoard(users).then(sortedScoredUsers => {
// Iterate our RealUsers
sortedScoredUsers.forEach((sortedScoredUser, idx) => {
const username = sortedScoredUser.realUser.username;
const score = sortedScoredUser.score;
// Print out their status
console.log(`${username.substring(0, 13)} is in position ${idx + 1} with ${score} points`);
});
});
}
}
const game = new Game();
game.users["bob"] = 5;
game.users["sue"] = 7;
game.users["tim"] = 3;
game.printLeaderBoard(Object.keys(game.users));