Fetch new data from sqlite database on reload - javascript

I managed to get sensor data into a sqlite3 database. Every 2 minutes new values appear in the database. Now I want to display it with chart.js and therefore make use of sql.js
The code I use is:
const config_sqljs = {
locateFile: filename => (
'https://cdnjs.cloudflare.com/ajax/libs/sql.js/1.5.0/sql-wasm.wasm'
)
}
const sqlPromise = initSqlJs(config_sqljs)
const dataPromise = fetch((
'/sensors.db'
))
.then(res => res.arrayBuffer())
Promise.all([sqlPromise, dataPromise])
.then(([SQL, buf]) => {
const db = new SQL.Database(new Uint8Array(buf))
//const res = db.exec('select time_,temp,humi from sensors where topic=\"/x/sensors\"')
const res = db.exec('select time_,temp,humi from sensors where topic =\"/y/sensors\" and (date=\"29.11.2021\" or date=\"30.11.2021\" or date=\"01.12.2021\");')
const values = res[0].values;
let labelx = values.map((arr) => arr[0]);
let y = values.map((arr) => arr[1]);
let y2 = values.map((arr) => arr[2]);
})
It works well at first, but then I realized that the values in the arrays labelx, y, and y2 are not the newest one, they lack beheind, about half a day. The new values alwasy appear in the database. I think it has todo something with Promise.all() and .then but unfortunally I don't know much about js. The code is executet each time the site loads or reloads, even I I use Strg+F5 to force a complete, uncached reload it stays the same. Can you help me? Thanks!
=== EDIT ===
I adapted your suggestion:
const dataPromise = fetch((
'/db.db', {
method: "GET",
"cache-control": "no-store"}
))
.then(res => res.arrayBuffer())ยด
Sadly I get the error:
Uncaught (in promise) Error: file is not a database
the relevant line:
const res = db.exec('select x from db;')

Fetch might be caching the responses. Try:
fetch('/sensors.db', {
method: "GET",
"cache-control": "no-store"
})

Related

How to render RapidAPI data on another HTML page?

I am new to JavaScript and this is my first question here. I've been trying for week to render my RapidApi data on another HTML page. I made search form on my index page and then put its values as my api call parameters in order to influence my API response. I used fetch to do so. The issue is that my API data keeps rendering on the same index page which is understandable since I don't know how to render it on a separate page. This also means that my CSS styling options are limited since I cannot design API data as I want without messing up my index page. If you have any sort of solution that is not way too complicated I would really appreciate your help.
Here is part of my code:
const input = document.getElementById(`location`);
const guests = document.getElementById(`guests`);
const check = document.querySelectorAll(".date");
let id;
document.getElementById(`submit`).addEventListener(`click`, function (e) {
e.preventDefault();
locationId();
});
async function locationId () {
let hotelId = input.value;
const options = {
method: 'GET',
headers: {
'X-RapidAPI-Key': '//API key goes here',
'X-RapidAPI-Host': 'tripadvisor16.p.rapidapi.com'
}
};
let response = await fetch(`https://tripadvisor16.p.rapidapi.com/api/v1/hotels/searchLocation?query=${hotelId}`, options);
if (!response.ok) throw new Error(`Woops something went wrong`);
let data = await response.json();
let geoId = await (data.data[0].geoId);
id= parseInt(geoId);
return (fetch(`https://tripadvisor16.p.rapidapi.com/api/v1/hotels/searchHotels?geoId=${id}&checkIn=${check[0].value}&checkOut=${check[1].value}&pageNumber=1&adults=${guests.value}currencyCode=USD`, options))
.then(response => response.json())
.then(data => {
let list = data.data.data;
displayObjectElements(list)
function displayObjectElements (object) {
let display = ``;
let price = ``;
object.forEach(element => {
display+= `<div class = "objectResults">
<ul class="hotel__lists">
<li><h2 class = "title">${element.title}</h2></li>
<li><img class= "hotels--photo "src="${element.cardPhotos[0].sizes.urlTemplate.split("?")[0] + `?w=500&h=500`}" alt=image--photo/></li>
<li><p>Ranking:${element.bubbleRating.rating}&#9734 out of 5&#9734</p></li>`
if(!element.priceForDisplay) {
display+= `<li><p>There is no price to display</p></li>`
display+= `<li><button class="booking-btn">Click to book</button></li>`
} else {
price =element.priceForDisplay.substring(1);
price= parseInt(price);
// console.log(price);
display+= `<li><p>Price: $${price} in total</p></li>`
display+= `<li><button class = "booking-btn">Click to book</button></li>
</ul>
</div>`
// console.log(display);
}});
document.body.innerHTML = display;
}
})
.catch(err => console.error(err));
}
I already tried with localStorage and sessionStorage but as a newbie I am just now sure how to put the whole API data in storage. Also, I desperately tried with window.location object as well but as I assumed that did nothing but open a new tab. Again, thanks in advance for any help!

How do I fix: SyntaxError: Unexpected token u in JSON at position 1

function json_data () {
const url = "https://in.investing.com/economic-calendar/historical-data-table?eventId=201"
const res = UrlFetchApp.fetch(url,{muteHttpExceptions: true});
const data = JSON.parse(JSON.parse(res.getContentText));
const result = data.map(post => {
return [post["release_date"],post["actual"]]
})
return result
}
this is my first time writing in a programming language, I am running this program code in Google-Sheet App-Script. the result I am receiving either comes as null or undefined, I am not even sure if it is the proper JSON file I am requesting. a segment of source URL I am trying to request:
{"release_date":"Oct 24, 2022 (Oct)","time":"13:30","actual":"46.6","forecast":"47.8","previous":"48.4","metaData":{"timestamp":1666598400,"perliminary":true,"color":{"actual":"u-down","previous":""}
change this line :
const data = JSON.parse(JSON.parse(res.getContentText));
to
const data = JSON.parse(JSON.stringify(res.getContentText()));
( also you can remove extra JSON.parse(JSON.stringify()) entirely )

Why am I intermittently writing partial arrays to files in Node with fs.readFileSync?

Context
I'm retrieving data from the ESPN API to fetch weekly NFL matchup data. So, I'm making 18 api calls each time I need to fetch this data to account for all 18 weeks in the NFL season. I'm then creating an array with the data I need from the responses to those calls and writing out 18 files that align with each week in the NFL season (week1.json, week2.json, etc.).
Problem
The problem is that when I call my endpoint, I am seeing 2 things intermittently, and not necessarily at the same time:
(1) Some of the json files(week1.json, week2.json, etc.) include only a portion of the expected array. So, instead of 16 objects in the array, I may see only 4, or only 6, etc. Why would I only see a portion of the response data written to the array that's ultimately written to the .json files?
(2) Not all files are written to each time the endpoint is called. So, I may see that only week1-week5's .json files are written. Why aren't all of them updated?
Problem Code
// iterate 18 times
for (let i = 0; i < 18; i++) {
let weekNumber;
weekNumber = i + 1;
const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
const weekJson = JSON.parse(week);
// empty weekJson.games array
weekJson.games = []
// get all items
axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`)
.then(response => {
const schedule = [];
// get all items from response
const items = response.data.items
// console.log(response.data.items)
items.forEach(item => {
// make get call to $ref
axios.get(item.$ref)
.then(response => {
// get name
const name = response.data.name
// get date
const date = response.data.date
// get event id
const eventid = response.data.id
// get team ids
let team1 = response.data.competitions[0].competitors[0].id
let team2 = response.data.competitions[0].competitors[1].id
// create new object
const newObject = {
name: name,
date: date,
eventid: eventid,
team1: team1,
team2: team2
}
// add games for week
weekJson.games.push(newObject);
fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));
})
.catch(error => {
console.log(error)
})
})
}).catch(error => {
console.log(error)
})
}
Updated Code
router.get('/getschedules', (req, res) => {
async function writeGames() {
// iterate 18 times
for (let i = 0; i < 18; i++) {
let weekNumber;
weekNumber = i + 1;
const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
const weekJson = JSON.parse(week);
// empty weekJson.games array
weekJson.games = []
// get all items
// Add await keyword to wait for a week to be processed before going to the next one
await axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`)
.then(async (response) => { // add async to be able to use await
const schedule = [];
// get all items from response
const items = response.data.items
console.log(response.data.items)
// Use standard loop to be able to benefit from async/await
for (let item of items) {
// make get call to $ref
// wait for an item to be processed before going to the next one
await axios.get(item.$ref)
.then(response => {
// get name
const name = response.data.name
// get date
const date = response.data.date
// get event id
const eventid = response.data.id
// get team ids
let team1 = response.data.competitions[0].competitors[0].id
let team2 = response.data.competitions[0].competitors[1].id
// create new object
const newObject = {
name: name,
date: date,
eventid: eventid,
team1: team1,
team2: team2
}
// add games for week
weekJson.games.push(newObject);
})
.catch(error => {
console.log(error)
})
}
// moved out of the for loop since you only need to write this once
fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));
}).catch(error => {
console.log(error)
})
}
}
writeGames();
})
Your issue might come from the fact that you are looping over an array of item that triggers parallel asynchronous calls and write weekJson before you get the entire data. (But theoretically your code should work if writeSyncFile is really synchronous, maybe there are locks on the file system that prevents node to write properly?)
You could try to make everything sequential and only write weekJson once instead of everytime you go over an item:
EDIT
I updated my original code proposition by keeping parallel calls and it worked for me (it's similar to OP's code but I only write the json file once per week).
Then I tried to run OP's code and it was working fine as well. So this makes me think that the problem isn't from the code itself but rather how it's called. As a pure node script, there doesn't seem to be any issue. But I just noticed that OP is using it server side as the result of an API call.
Having an API write so many JSON concurrently is probably not the best idea (especially if the api is called multiple times almost simultaneously). You could either
just return the games in the response
or precompute the results
or fetch and write them only once then cache the result to be reused
Then I wonder if due to the server context, there is not some kind of timeout since OP said that with my initial solution, only the first week was created.
const axios = require("axios");
const fs = require("fs");
async function writeGames() {
const writeWeekGamesPromises = [];
// iterate 18 times
for (let weekNumber = 1; weekNumber < 19; weekNumber++) {
// give week a default value in case the json file doesn't exist (for repro purpose)
let week = "{}";
try {
week = fs.readFileSync(`./pickem/week${weekNumber}.json`, "utf8");
} catch (e) {
console.log(`error reading week ${weekNumber} json file:`, e);
// file doesn't exist yet
}
const weekJson = JSON.parse(week);
// empty weekJson.games array
const games = [];
weekJson.games = games;
// get all items
// Add await keyword to wait for a week to be processed before going to the next one
writeWeekGamesPromises.push(axios
.get(
`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`
)
.then(async (eventListResponse) => {
// add async to be able to use await
const schedule = [];
console.log(JSON.stringify(eventListResponse.data),'\n');
// get all items from response
const items = eventListResponse.data.items;
// console.log(eventListResponse.data.items); // this seems to be useless since we log the whole data just above
// parallelize calls and wait for all games from a week to be fetched before writing the file
await Promise.all(
items.map((item) => {
// we return the promise so that Promise.all will wait for all games to be pushed before going on writing the file
return axios
.get(item.$ref)
.then((response) => {
// get name, date and eventid
const {name, date, id: eventid} = response.data;
// get team ids
let team1 = response.data.competitions[0].competitors[0].id;
let team2 = response.data.competitions[0].competitors[1].id;
games.push({ name, date, eventid, team1, team2 });
})
.catch((error) => {
console.log(error);
});
})
);
// Now that all game data is ready, write in the file
fs.writeFileSync(
`./pickem/week${weekNumber}.json`,
JSON.stringify(weekJson)
);
})
.catch((error) => {
console.log(error);
}));
}
// Waiting for all games from all weeks to be processed
await Promise.all(writeWeekGamesPromises);
}
async function runAndLogTime() {
const start = Date.now();
await writeGames();
console.log(`took ${(Date.now() - start) / 1000}s to write all json files`);
}
runAndLogTime();

Javascript API fetch loop using offset

I'm fetching data from an API that has a limit of 50 records per call. There are 10,000 records in total that I need to retrieve so looking at using pagination via the offset parameter to achieve this.
I'm not too familiar with the syntax I need to use but I would like to keep looping through the fetch until the number of records returned for a single call is < 50 to signal the final page.
I need my JSON responses from each call to be appended together, in addition to a final 'count' that gives me 10,000
let offset = 0
fetch(`https://api.opensea.io/api/v1/assets?collection=womenandweapons&format=json&offset=${offset}&limit=50&order_direction=desc`,
{
method: 'GET',
headers: {
'Accept': 'application/json'
}
})
.then(response => response.json())
.then(data => {
let len = Object.keys(data.assets).length
console.log(len)
console.log(data.assets)
})
offset += 50
I've had some luck doing something like this:
const main = async () => {
const url = new URL(AIRTABLE_URL);
let data = await fetcher(url, AIRTABLE_API_KEY);
storiesTotal = mapData(data.records);
while (data.offset) {
data = await fetcher(url, AIRTABLE_API_KEY, data.offset);
storiesTotal = [...storiesTotal, ...mapData(data.records)];
}
}
main();

Pokemon API, Promises javascript get data inside of array json

I have been working to make a small app to get the images and info from the pokemon api at http://pokeapi.co/ , I am following course using promises however I can get a group (example: water type) which give me 78 objects , each one with a resource_uri which is the data which give me the information of each pokemon.
Until now I have make this and I can get all the objects, however, now how can I make in order to get for each object push in the console.log (or later use them) all the 78pokemons with the data of each.
My code until now is this>
var $ = window.jQuery
var base = 'http://pokeapi.co/api/v1/egg/'
Promise.resolve($.get(base + '2'))
.then((results) =>
.then((results) => {
var pokechara = results.pokemon
var basechara = 'http://pokeapi.co'
var promises = []
for (let chara of pokechara){
var pokech = pokechara[i]
'${something}'
var pokechurl = basechara + pokechara[0].resource_uri
promises.push(Promise.resolve($.get(pokechurl)))
}
debugger
return Promise.all(promises)
})
.then((poke) => {
console.log(poke)
})
.catch((err) => {
debugger
})
I really wanted to make something with the api of yugioh.wikia but I dont see how I can get work because the problem with the Cros-server or header. To see a preview of what I am doing you can see at http://www.kengreg.com/yugiohapp/
Thanks in advance
Actually, in the code used in your reply to Alex, the string variables you use to call the api are the only thing wrong, the 'http://' part at the start is needed. Apart from that, your code works fine.
var $ = window.jQuery;
var base = 'http://pokeapi.co/api/v1/egg/';
Promise.resolve($.get(base + '2'))
.then((results) => {
var pokechara = results.pokemon;
var basechara = 'http://pokeapi.co/';
var promises = [];
for (var i in pokechara){
var pokech = pokechara[i];
var pokechurl = basechara + pokechara[0].resource_uri;
promises.push(Promise.resolve($.get(pokechurl)));
}
debugger
return Promise.all(promises);
})
.then((poke) => {
debugger;
console.log(poke);
})
.catch((err) => { debugger })
I left the debugger so you can checkout the results of the promises. The poke variable at the end should have the objects array you wanted.

Categories