I have collections and their sub-collections. I loop over them and collect data. It works fine but it is too slow. Can anyone suggest to improve performance issue?
static async getSubCategories(category_id) {
const db = Firebase.firestore(),
subCategories = [];
activeRef = await db.collection("sub_categories").where("category_id", "==", category_id).orderBy("id").get();
for (let doc of activeRef.docs) {
const subCategory = doc.data();
if (_.isObject(subCategory)) {
const subRef = await doc.ref.collection('sub_sub_categories').orderBy("id").get(),
subSubCategories = [];
for (let subDoc of subRef.docs) {
const subSubCategory = subDoc.data();
if (_.isObject(subCategory)) {
subSubCategories.push(subSubCategory);
// If SubCategory has image, download storage uri
if (subSubCategory.image_storage_uri) {
const imageRef = Firebase.storage().refFromURL(subSubCategory.image_storage_uri),
imageUri = await imageRef.getDownloadURL();
subSubCategory.image_uri = imageUri;
}
}
}
if (subSubCategories.length > 0)
subCategory.sub_sub_categories = subSubCategories;
subCategories.push(subCategory);
}
}
return subCategories;
}
Related
Loop over a list of ids and foreach id get data from api but, i can make only 40 requests per 10 seconds, i'm getting data from tbdb but i'm getting kicked by the api here is my code
Popular.findAll({ limit: 25 }).then((filmes) => {
for (let index = 0; index < filmes.length; index++) {
const element = filmes[index];
const id_filme = element.id_filme;
i get each movie id so i can make requests to the api with this function..
function fetchdata(id_filme) {
setTimeout(function () {
axios
.all([
axios.get(
`https://api.themoviedb.org/3/movie/${id_filme}?api_key=${process.env.API_KEY}&language=pt-BR`
),
axios.get(
`https://api.themoviedb.org/3/movie/${id_filme}/videos?api_key=${process.env.API_KEY}&language=pt-BR`
),
])
.then(
axios.spread(function (detalhes, linksyt) {
var filme = detalhes.data;
var youtube = linksyt.data;
// console.log(filme, youtube);
var filmesLista = new Array();
const title = filme.title;
const filmeid = filme.id;
if (Object.keys(filme.genres).length === 0) {
var genre = filme.genres;
} else {
var genre = filme.genres[0].name;
}
const overview = filme.overview;
const poster_path =
"https://image.tmdb.org/t/p/w500" + filme.poster_path;
const release_date = filme.release_date;
const vote_average = parseFloat(filme.vote_average);
const backdrop_path =
"https://image.tmdb.org/t/p/original" + filme.backdrop_path;
const imdb_id = filme.imdb_id;
const runtime = filme.runtime;
if (Object.keys(youtube.results).length === 0) {
var trailer = "";
} else {
var trailer =
"http://www.youtube.com/watch?v=" + youtube.results[0].key;
}
filmesLista.push([
filmeid,
title,
genre,
overview,
poster_path,
release_date,
vote_average,
backdrop_path,
imdb_id,
runtime,
trailer,
]);
console.log(filmesLista);
filmesLista.forEach((i) => {
const filmeid = i[0];
const title = i[1];
const genre = i[2];
const overview = i[3];
const poster_path = i[4];
const release_date = i[5];
const vote_average = i[6];
const backdrop_path = i[7];
const imdb_id = i[8];
const runtime = i[9];
const trailer = i[10];
Detalhes.create({
id_filme: `${filmeid}`,
title: `${title}`,
genre: `${genre}`,
release_date: `${release_date}`,
vote_average: `${vote_average}`,
imdb_id: `${imdb_id}`,
runtime: `${runtime}`,
overview: `${overview}`,
poster_path: `${poster_path}`,
backdrop_path: `${backdrop_path}`,
trailer: `${trailer}`,
});
});
})
);
o++;
if (o < 25) {
fetchdata(id_filme);
}
}, 10000);
console.log("INSERIDO NO BANCO");
}
fetchdata(id_filme);
the code works but, works so fast and i get kicked every time i run with more than 40 ids.. i'm strugling on just make 40 request every 10 secconds. anyone can help me please?
I’m building a site using sapper and requesting data from an API. It has been working smooth until now.
When I’m going from site.com/title/id1 to site.com/title/id2 the new information is not loaded until I hit a manual refresh. Any ideas?
import { stores, goto } from "#sapper/app";
import Card from "../_titlecard.svelte";
const { page } = stores();
const { slug } = $page.params;
import { onMount } from "svelte";
let looper = [];
let artistName = "";
let titleName = "";
let dvdCover = "";
let titleCover = "";
let genre = "";
let tracks = [];
onMount(async () => {
const res = await fetch(`https://.com/api/title/${slug}`);
const data = await res.json();
artistName = data.artistName;
titleName = data.name;
dvdCover = data.graphics.dvd;
titleCover = data.graphics.landscape;
genre = data.genre;
tracks = data.tracks.length;
const res2 = await fetch(`https://.com/api/artists/all`);
const data2 = await res2.json();
let moreTitles = [];
const more = data2.map((x) => {
if (x.titles.length > 0 && x.genre === genre) {
looper.push(x.titles[0]);
looper = moreTitles;
}
});
});
And then I have this in the html
{#each looper.slice(0, 4) as item, i}
<Card imgurl={item.graphics.dvd} concert={item.name} id={item.id} />
{/each}
A page component is not unmounted and mounted again if the navigation results in the same page component being used, so your onMount will only be run once with the first id.
You could use a reactive statement to make sure you run the desired code every time $page.params.slug changes.
Example
import { stores, goto } from "#sapper/app";
import Card from "../_titlecard.svelte";
const { page } = stores();
let looper = [];
let artistName = "";
let titleName = "";
let dvdCover = "";
let titleCover = "";
let genre = "";
let tracks = [];
$: (async () => {
const { slug } = $page.params;
const res = await fetch(`https://.com/api/title/${slug}`);
const data = await res.json();
artistName = data.artistName;
titleName = data.name;
dvdCover = data.graphics.dvd;
titleCover = data.graphics.landscape;
genre = data.genre;
tracks = data.tracks.length;
const res2 = await fetch(`https://.com/api/artists/all`);
const data2 = await res2.json();
let moreTitles = [];
data2.forEach((x) => {
if (x.titles.length > 0 && x.genre === genre) {
moreTitles.push(x.titles[0]);
}
});
looper = moreTitles;
})();
I have a javascript app that reads invoices. It's working perfectly but I would like to improve it's performance a bit.
The way the app works is this. There are Next and Submit buttons. After either, a new image is loaded and a function doOcr is called. In each instance of doOCR, a worker is created which must take time. I am trying to figure out how to move the worker outside the function so it doesn't need to be created each time. Code is below. Thoughts?
//ocr the image and fill values
function doOCR(){(async () => {
worker = Tesseract.createWorker();
await worker.load();
await worker.loadLanguage('eng');
await worker.initialize('eng');
const values = [];
for (let i = 0; i < rectangles.length; i++) {
const { data: { text } } = await worker.recognize(imgFile, { rectangle: rectangles[i] });
values.push(text);
}
console.log(values);
//calculate and display result
dir = values[0].split(' ');
$("#inDir").val(dir[1]);
//get invoice number
$("#inVendor").val(dir[0]);
//get invoice #
var str = values[0];
var n = str.indexOf("Invoice:", 0);
var n1 = str.indexOf("Invoice", n+5);
invoice = str.substr(n+9, n1-n-9);
$("#inInvoice").val(invoice);
await worker.terminate();
$('#status').attr('src','check.png').show();
})()};
Thank you.
I think I got it by changing the top half of the function to:
function doOCR(){(async () => {
if (typeof(worker) == "undefined") {
worker = Tesseract.createWorker();
await worker.load();
await worker.loadLanguage('eng');
await worker.initialize('eng');
}
const values = [];
for (let i = 0; i < rectangles.length; i++) {
const { data: { text } } = await worker.recognize(imgFile, {rectangle: rectangles[i]});
values.push(text);
}
console.log(values);
[ ... ]
and removed
await worker.terminate();
I have written a program to extract links to download photos in three steps:
The getPersons() function get the complete list of people to traverse.
Get the photos download links from the list of persons.
Download from the list created in step 2.
I am trying to refactor step 2 into an async function.
Is there an easy way to refactor second step into a function to make the code more readable?
Ideally, I would like to retrieve all the links and only then, start the download.
const axios = require("axios");
const cheerio = require("cheerio");
const url = "https://www.website.com";
const persons = [];
async function getPersons() {
await axios.get(url).then(response => {
const html = response.data;
const $ = cheerio.load(html);
const personList = $(".bio-btn");
console.log(personList.length);
personList.each(function() {
const link_raw = $(this).attr("href");
const link = url + link_raw;
const name = link_raw.replace("/bio/", "");
person.push({
name,
link
});
});
});
}
getPersons().then(function() {
persons.forEach(async function(person) {
var personLink = person.link;
await axios.get(personLink).then(response => {
const html = response.data;
const $ = cheerio.load(html);
const snapshots = $(".ratio-4-3");
snapshots.each(function() {
const pic = $(this).attr("style");
if (pic != undefined && pic.includes("biopicture")) {
var bioPhoto = s[1];
}
});
});
});
});
You are hardly getting much benefit out of your asynchronicity, as you end up making serial requests. I'd write it this way (untested):
async function getPersons() {
const response = await axios.get(url);
const html = response.data;
const $ = cheerio.load(html);
const personList = $('.bio-btn');
const persons = [];
personList.each(function() {
const link_raw = $(this).attr('href');
const link = url + link_raw;
const name = link_raw.replace("/bio/", "");
persons.push({
name,
link,
});
});
return persons;
};
async function getSnapshots() {
const persons = await getPersons();
const linkPromises = persons.map(person => axios.get(person.link));
const linkResponses = await Promise.all(linkPromises);
linkResults.forEach(response => {
const html = response.data;
const $ = cheerio.load(html);
const snapshots = $(".ratio-4-3");
// ...
});
}
I would refactor it like this. Removing .then() methods and the function keyword on anonymous functions makes the code look cleaner.
Using Promise.all() enables you to start all the downloads asynchronously which could be better than downloading images one by one.
const axios = require('axios');
const cheerio = require('cheerio');
const url = 'https://www.website.com';
async function getPersons() {
const response = await axios.get(url);
return extractPersonList(response);
}
// Step 1
function extractPersonList(response) {
const persons = [];
const html = response.data;
const $ = cheerio.load(html);
const personList = $('.bio-btn');
console.log(personList.length);
personList.each(() => {
const link_raw = $(this).attr('href');
const link = url + link_raw;
const name = link_raw.replace('/bio/', '');
persons.push({
name,
link
});
});
return persons;
}
async function getPhotos() {
const persons = await getPersons();
const promisies = persons.map(p => axios.get(p.link));
// Step 2
const responses = await Promise.all(promisies);
// Step 3
responses.forEach(response => {
const html = response.data;
const $ = cheerio.load(html);
const snapshots = $('.ratio-4-3');
snapshots.each(() => {
const pic = $(this).attr('style');
if (pic && pic.includes('biopicture')) {
var bioPhoto = s[1];
}
});
});
}
// Call getPhotos to start the process
getPhotos();
I am trying to get API data so i can render it on Webpage using ReactJS. I tried many different ways to fetch api where its stored in array. But i am unable to do it.
Note:https://www.hatchways.io/api/assessment/workers/<worker_id> Here i'm looping so the
worker id gets add the of url.
const fetchAPI = e => {
let array = [];
const api2 = `https://www.hatchways.io/api/assessment/workers/`;
for (var i = 0; i <= 4; i++) {
array.push(api2 + i);
}
return array;
};
console.log(fetchAPI());
Thanks in advance.
You need to hit the url first. Async/Await would be a good choice.
<script>
async function check()
{
var arrayData =[];
var url = "https://www.hatchways.io/api/assessment/workers/";
for(var i=1;i<=4;i++)
{
const response = await fetch(url+""+i);
const myJson = await response.json();
arrayData.push(myJson);
}
console.log(arrayData)
}
check();
</script>
Use Promise.all for example:
const baseURL = 'https://jsonplaceholder.typicode.com';
const fetchUsers = fetch(`${baseURL}/users`);
const fetchPosts = fetch(`${baseURL}/posts`);
Promise.all([fetchUsers, fetchPosts]).then((responses) => {
const responsesToJson = responses.map(response => response.json());
return Promise.all(responsesToJson);
}).then((jsonResponse) => {
const [userResponse, postResponse] = jsonResponse;
console.log(userResponse);
console.log(postResponse);
});