We Keep Coding

API host's rate-limit header not read properly

Here is my code (not the entire code, but I think this is the only really relevant part):
const timer = ms => new Promise(resolve => setTimeout(resolve, ms));
const createThrottler = (limitHeader) => {
let requestTimestamp = 0;
let rateLimit = 0;
return (requestHandler) => {
return async (...params) => {
const currentTimestamp = Number(Date.now());
if (currentTimestamp < requestTimestamp + rateLimit) {
const timeOut = rateLimit - (currentTimestamp - requestTimestamp);
requestTimestamp = Number(Date.now()) + timeOut;
await timer(timeOut)
}
requestTimestamp = Number(Date.now());
const response = await requestHandler(...params);
if (!rateLimit > 0) {
rateLimit = Math.floor((60 / response.headers.get(limitHeader)) * 1000);
}
console.log(limitHeader);
console.log(rateLimit);
return response;
}
}
}
const throttle = createThrottler("X-***-Ratelimit");
const throttleFetch = throttle(fetch);
function getRelease(idFiltered) {
return throttleFetch(`https://api.***.com/releases/${idFiltered}`, {
headers: {
'User-Agent': '***/0.1',
},
}).then(response => response.json())
.then(data => {
if (data.message === 'Release not found.') {
return { error: `Release with ID ${idFiltered} does not exist` };
} else {
const id = data.id;
const delimiter = document.getElementById("delimiter").value || "|";
const artists = data.artists ? data.artists.map(artist => artist.name) : [];
const barcode = data.identifiers.filter(id => id.type === 'Barcode')
.map(barcode => barcode.value);
var formattedBarcode = barcode.join(delimiter);
const country = data.country || 'Unknown';
const genres = data.genres || [];
const formattedGenres = genres.join(delimiter);
const labels = data.labels ? data.labels.map(label => label.name) : [];
const formattedLabels = labels.join(delimiter);
const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
const formattedCatNo = catno.join(delimiter);
const styles = data.styles || [];
const formattedStyles = styles.join(delimiter);
const tracklist = data.tracklist ? data.tracklist
.map(track => track.title) : [];
const formattedTracklist = tracklist.join(delimiter);
const year = data.year || 'Unknown';
const format = data.formats ? data.formats.map(format => format.name) : [];
const qty = data.formats ? data.formats.map(format => format.qty) : [];
const descriptions = data.formats ? data.formats
.map(descriptions => descriptions.descriptions) : [];
const preformattedDescriptions = descriptions.toString()
.replace('"', '""').replace(/,/g, ', ');
const formattedDescriptions = '"' + preformattedDescriptions + '"';
console.log(idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
)
return [idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
];
}
});
}
But the "X-***-Ratelimit" header is clearly not being read correctly, as when I do
console.log(limitHeader);
console.log(rateLimit);
I initially get back
object
and thereafter
X-***-Ratelimit
Infinity
From the host's documentation:
We attach the following headers to responses to help you track your rate limit use:
X-***-Ratelimit: The total number of requests you can make in a one minute window.
X-***-Ratelimit-Used : The number of requests you’ve made in your existing rate limit window.
X-***-Ratelimit-Remaining: The number of remaining requests you are able to make in the existing rate limit window.
Any help please? TIA.
Edit: amazingly, I managed to greatly increase the rate-limit by getting my app authenticated thusly:
headers: {
'User-Agent': '***/0.1',
'Authorization': '*** key=***, secret=***',
},
However, I just took the key and secret from the site documentation, and I now get back this JSON response:
message - "Invalid consumer key/secret. Please register an app before making requests."
Edit2: OK, I worked out how to register my app now. I'm proceeding to further tests.
Edit3: the rate-limit in effect from the host is much better now, but my app is still not reading the header from the response correctly, so limitHeader is still coming back as "Infinity", instead of some meaningful/appropriate value.

How to avoid empty object error in a loop using Google Apps Script?

I'm calling an API and getting data going through its pagination. When I get to the last page, though, the obejct giving me the last page is empty and it's throwing the following error: TypeError: Cannot convert undefined or null to object Besides, I don't any data from that last page.
Here's the pagination information I get:
{"count":100,"total":545,"_links":
{
"self":{
"href":"\/candidates?page=0&per_page=100"
},
"next":{
"href":"\/candidates?per_page=100"
},
"last":{
"href":"\/candidates?page=6&per_page=100"
}
},
Here's the code I'm using to get the data:
function allcandidates() {
const url = "https://api.catsone.com/v3/candidates";
const params = {
'muteHttpExceptions': true,
'method': 'GET',
'redirect': 'follow',
'headers': {
'Content-Type': 'application/json',
'Authorization': 'Token ' + API_KEY
}
};
let pageNum = 1;
let lastPgNo;
let data = {}, output = [];
do {
let currentUrl = url + '?' + 'per_page=100' + '&' + 'page=' + pageNum;
//One of their URL parameter is "per_page", which is 25 result/page and it go up to 100. I'm not sure if the fact that the last page doesn't have all 100 results may result in an error, too.
const response = UrlFetchApp.fetch(currentUrl, params);
const respCode = response.getResponseCode();
if (respCode != 200) {
Browser.msgBox('The server seems to be temporarily down. Please try again later.');
return;
}
//Gets the last page number
const getText = response.getContentText();
const lastPageObj = JSON.parse(getText)['_links']['last'];
const lastPgVal = Object.values(lastPageObj); //This is where the error occurs
const lastPgText = lastPgVal.toString();
lastPgNo = Number(lastPgText.split('?page=').pop().split('&')[0])
//Gets current page
const currentPageObj = JSON.parse(getText)['_links']['self'];
const currentPgVal = Object.values(currentPageObj);
const nextPgText = currentPgVal.toString();
var currentPgNo = Number(nextPgText.split('?page=').pop().split('&')[0])
const dataSet = JSON.parse(getText)['_embedded']['candidates'];
for (let i = 0; i < dataSet.length; i++) {
data = dataSet[i];
output.push([data.id]);
}
pageNum = pageNum + 1;
} while (pageNum <= lastPgNo);
}

You might use an if statement and continue. I.E. replace
const lastPgVal = Object.values(lastPageObj);
by
if(lastPageObj){
const lastPgVal = Object.values(lastPageObj);
} else {
continue;
}
Another option is to use try...catch
Resources
https://developer.mozilla.org/en-US/docs/Glossary/Truthy
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/continue
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/try...catch

How to assign a JS variable to a JSON route

So I'm coding this
https://vercel.com/eduardodevolmedo/jsd-aily
function executeWeekly() {
fetch("/data.json")
.then(function (resp) {
return resp.json();
})
.then(function (data) {
currentW = data[0].timeframes.weekly.current
previousW = data[0].timeframes.weekly.previous
currentWork.innerHTML = `${currentW}${hrs}`
previousWork.innerHTML = `${lastWeek} - ${previousW}${hrs}`
//DATA FROM PLAY
currentP = data[1].timeframes.weekly.current
previousP = data[1].timeframes.weekly.previous
currentPlay.innerHTML = `${currentP}${hrs}`
previousPlay.innerHTML = `${lastWeek} - ${previousP}${hrs}`
//DATA FROM STUDY
currentS = data[2].timeframes.weekly.current
previousS = data[2].timeframes.weekly.previous
currentStudy.innerHTML = `${currentS}${hrs}`
previousStudy.innerHTML = `${lastWeek} - ${previousS}${hrs}`
//DATA FROM EXERCISE
currentE = data[3].timeframes.weekly.current
previousE = data[3].timeframes.weekly.previous
console.log(currentE, previousE)
currentExercise.innerHTML = `${currentE}${hrs}`
previousExercise.innerHTML = `${lastWeek} - ${previousE}${hrs}`
//DATA FROM SOCIAL
currentSO = data[4].timeframes.weekly.current;
previousSO = data[4].timeframes.weekly.previous;
currentSocial.innerHTML = `${currentSO}${hrs}`
previousSocial.innerHTML = `${lastWeek} - ${previousSO}${hrs}`
//DATA FROM SELFCARE
currentSE = data[5].timeframes.weekly.current;
previousSE = data[5].timeframes.weekly.previous;
currentSelfcare.innerHTML = `${currentSE}${hrs}`
previousSelfcare.innerHTML = `${lastWeek} - ${previousSE}${hrs}`
})
eachBlock.forEach(el => el.classList.toggle('animatedBox'))
}
function executeDaily() {
fetch("/data.json")
.then(function (resp) {
return resp.json();
})
.then(function (data) {
//DATA FROM WORK
currentW = data[0].timeframes.daily.current
previousW = data[0].timeframes.daily.previous
currentWork.innerHTML = `${currentW}${hrs}`
previousWork.innerHTML = `${lastDay} - ${previousW}${hrs}`
//DATA FROM PLAY
currentP = data[1].timeframes.daily.current
previousP = data[1].timeframes.daily.previous
currentPlay.innerHTML = `${currentP}${hrs}`
previousPlay.innerHTML = `${lastDay} - ${previousP}${hrs}`
//DATA FROM STUDY
currentS = data[2].timeframes.daily.current
previousS = data[2].timeframes.daily.previous
currentStudy.innerHTML = `${currentS}${hrs}`
previousStudy.innerHTML = `${lastDay} - ${previousS}${hrs}`
//DATA FROM EXERCISE
currentE = data[3].timeframes.daily.current
previousE = data[3].timeframes.daily.previous
currentExercise.innerHTML = `${currentE}${hrs}`
previousExercise.innerHTML = `${lastDay} - ${previousE}${hrs}`
//DATA FROM SOCIAL
currentSO = data[4].timeframes.daily.current;
previousSO = data[4].timeframes.daily.previous;
currentSocial.innerHTML = `${currentSO}${hrs}`
previousSocial.innerHTML = `${lastDay} - ${previousSO}${hrs}`
//DATA FROM SELFCARE
currentSE = data[5].timeframes.daily.current;
previousSE = data[5].timeframes.daily.previous;
currentSelfcare.innerHTML = `${currentSE}${hrs}`
previousSelfcare.innerHTML = `${lastDay} - ${previousSE}${hrs}`
})
eachBlock.forEach(el => el.classList.toggle('animatedBox'))
}
So, im running the exact same function at monthly, and daily.
Which is now working, but i had to copy the same code for the same functions, so I was thinking in a way I could do it in less lines of code. So, my idea was to assign a string to a variable, and then making it a string, using JSON.stringify()
let z = weekly
let x = JSON.stringify(z)
Then i thought, that I would assing a variable to each button on html, using an if statement, for example if the button value was "daily" then z would be daily, and the function would run using daily as that argument.
And then, I just add it like a variable, depending on what i want:
currentW = data[0].timeframes.z.current
instead of:
currentW = data[0].timeframes.daily.current
In that way, I would only need to use one function.
But that doesn't seems to work.
How can I do this? Is there any way?
If you want to check the code further:
https://github.com/EduardoDevOlmedo/JSDaily

You have to access the object member as if the object was an associative array:
currentW = data[0].timeframes[z].current

Solved:
Accessing to
this.innerHTML
and assigning the value to z.
function check(){
value = this.innerHTML
execute(value)
}

How can I receive data from firebase under a certain variable in jquery or javascript

I am trying to receive data from realtime database every time when I load the page. I don't know why it's not working. I am trying to receive data under the word "email". I don't know how to do this with a variable because firebase doesn't let me do this. But can you help me with receiving the data?
Thank you!
$('.save').on('click', function() {
var Email = localStorage.getItem("email");
window.alert(Email)
todoArray = [];
for (i = 1; i < 25; i++) {
let todoValue = $('#' + i).val();
let todoObject = {
todoHour: i,
todoItem: todoValue
}
firebase.database().ref('email').set({
todoArray
});
todoArray.push(todoObject);
}
})
// save data
const issuesRef = firebase.database().ref('email');
function loadTodos() {
issuesRef.on("value", function(snapshot) {
snapshot.forEach(snap => {
const issue = snap.val();
todoObject = issue
})
})
}
//get data

Node / MySQL - code: 'ER_PARSE_ERROR', when trying to insert ~800 records

I am working on small idea to collect errors from pages and to store them in DB and then use graph API to display information visually.
There is 8 sites and on each of them there is 100 entries - so 800 transactions per time.
I loop through each site and then sub-loop through table of errors and collect them.
I got it working if I make insert query on each of those sub-loops for all 800 entries but I am getting some sort of memory leak from so many transactions and after few minutes - Node breaks due to memory exceeding.
So I tried queuing all 800 entries into Array of Arrays and then performing multi-insert at the end of every iteration but I am getting ER_PARSE_ERROR.
var tabletojson = require('tabletojson');
var mysql = require("mysql");
var striptag = require("striptags");
var fs = require("fs");
var path = require('path');
var startCollector;
var iterations = 0;
var insertions = 0;
var duplicated = 0;
var datas = [];
var clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
var appDir = path.dirname(require.main.filename);
var errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
var con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
function CollectErrors() {
startCollector = new Date();
for(var a = 0; a < clients.length; a++) {
(function(a) {
tabletojson.convertUrl("http://example.com" + clients[a] + "/page.php?limit=100", { stripHtmlFromCells: false }, function(response) {
var rs = response[0];
for(var l = rs.length-1; l > -1; l--) {
var newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
var user = getUser(striptag(rs[l]["User"]));
var msg = striptag(rs[l]["Error"]);
var splitError = rs[l]["Error"].split("<a href=\"");
var link = getUrl(splitError[1]);
var id = getId(link);
var type = getType(striptag(splitError[0]));
var temp = [newDate, link, type, user, clients[a], id, msg];
datas.push(temp);
}
});
})(a);
}
con.getConnection(function(err, connection) {
connection.query("INSERT IGNORE INTO entries (time, url, type, author, client, uid, message) VALUES ?", [datas], function(err, rows) {
console.log(err);
});
connection.release();
datas = [];
});
setTimeout(CollectErrors, 10000);
}
function formatDate(date, time) {
var newdate = date.split("/").reverse().join("-");
var newtime = time+":00";
return newdate + " " + newtime;
}
function getUrl(uri) {
return "http://example.com/"+uri.split("\">Details")[0];
}
function getId(url) {
return decodeURIComponent((new RegExp('[?|&]' + "id" + '=' + '([^&;]+?)(&|#|;|$)').exec(url) || [null, ''])[1].replace(/\+/g, '%20')) || null;
}
function getType(error) {
for(var a = 0; a < errorList.length; a++) {
if(error.indexOf(errorList[a]) !== -1) {
return errorList[a];
}
}
return "Other";
}
function getUser(user) {
if(user == "" || user == " " || user == null) {
return "System";
}
return user;
}
CollectErrors();
I've tried mysql.createConnection too but that also gave me same issue.
I've been stuck for past 12 hours and I can't see what's wrong, I've even tried populating Datas table with just strings but got same error.

I've changed your code to use ES6 and correct modules features.
Useful links: correct pooling with mysql, correct insert query, async/await, IIFE, enhanced object
const tabletojson = require('tabletojson'),
mysql = require("mysql"),
striptag = require("striptags"),
fs = require("fs"),
path = require('path');
const startCollector,
iterations = 0,
insertions = 0,
duplicated = 0;
let datas = [];
const clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
const appDir = path.dirname(require.main.filename);
const errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
const con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
// We'll use async/await from ES6
const collectErrors = async() => {
// Up to here I've only changed syntax to ES6
let startCollector = new Date();
// We'll try to iterate through each client. And we use here for..of syntax to allow us using await
for (let client of clients) {
// Please, check that client value return correct data. If not, change for..of to your for..each and client variable to clients[a]
const tbj = await tabletojson.convertUrl("http://example.com" + client + "/page.php?limit=100", {
stripHtmlFromCells: false
});
const result = tgj[0];
for (rs of result) {
// I can't check this part, but I hope your example was with correct values.
let newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
let user = getUser(striptag(rs[l]["User"]));
let link = getUrl(splitError[1]);
let msg = striptag(rs[l]["Error"]);
let id = getId(link);
let splitError = rs[l]["Error"].split("<a href=\"");
let getType = getType(striptag(splitError[0]));
// ES6 enhanced object syntax
datas.push({
newDate,
user,
msg,
id,
splitError,
link,
getType,
temp: [newDate, link, type, user, client, id, msg]
});
}
}
// OK, here we have fulfilled datas array. And we want to save it.
con.getConnection((err, connection) => {
// Please, notice, here I've changed your insert query to prepared statement.
connection.query("INSERT IGNORE INTO entries SET ?", datas, (err, rows) => {
console.log(err);
connection.release();
datas = [];
});
});
// I don't see why do you need timeout here, so I've left it commented.
// setTimeout(CollectErrors, 10000);
};
// Here your other methods go....
// And to call your async function we'll use IIFE
(async() => {
await collectErrors();
})();
Probably there may be errors with mysql insert, but that's not for sure. If occurred, please write in comments and I'll help you with that.