I have written a program to extract links to download photos in three steps:
The getPersons() function get the complete list of people to traverse.
Get the photos download links from the list of persons.
Download from the list created in step 2.
I am trying to refactor step 2 into an async function.
Is there an easy way to refactor second step into a function to make the code more readable?
Ideally, I would like to retrieve all the links and only then, start the download.
const axios = require("axios");
const cheerio = require("cheerio");
const url = "https://www.website.com";
const persons = [];
async function getPersons() {
await axios.get(url).then(response => {
const html = response.data;
const $ = cheerio.load(html);
const personList = $(".bio-btn");
console.log(personList.length);
personList.each(function() {
const link_raw = $(this).attr("href");
const link = url + link_raw;
const name = link_raw.replace("/bio/", "");
person.push({
name,
link
});
});
});
}
getPersons().then(function() {
persons.forEach(async function(person) {
var personLink = person.link;
await axios.get(personLink).then(response => {
const html = response.data;
const $ = cheerio.load(html);
const snapshots = $(".ratio-4-3");
snapshots.each(function() {
const pic = $(this).attr("style");
if (pic != undefined && pic.includes("biopicture")) {
var bioPhoto = s[1];
}
});
});
});
});
You are hardly getting much benefit out of your asynchronicity, as you end up making serial requests. I'd write it this way (untested):
async function getPersons() {
const response = await axios.get(url);
const html = response.data;
const $ = cheerio.load(html);
const personList = $('.bio-btn');
const persons = [];
personList.each(function() {
const link_raw = $(this).attr('href');
const link = url + link_raw;
const name = link_raw.replace("/bio/", "");
persons.push({
name,
link,
});
});
return persons;
};
async function getSnapshots() {
const persons = await getPersons();
const linkPromises = persons.map(person => axios.get(person.link));
const linkResponses = await Promise.all(linkPromises);
linkResults.forEach(response => {
const html = response.data;
const $ = cheerio.load(html);
const snapshots = $(".ratio-4-3");
// ...
});
}
I would refactor it like this. Removing .then() methods and the function keyword on anonymous functions makes the code look cleaner.
Using Promise.all() enables you to start all the downloads asynchronously which could be better than downloading images one by one.
const axios = require('axios');
const cheerio = require('cheerio');
const url = 'https://www.website.com';
async function getPersons() {
const response = await axios.get(url);
return extractPersonList(response);
}
// Step 1
function extractPersonList(response) {
const persons = [];
const html = response.data;
const $ = cheerio.load(html);
const personList = $('.bio-btn');
console.log(personList.length);
personList.each(() => {
const link_raw = $(this).attr('href');
const link = url + link_raw;
const name = link_raw.replace('/bio/', '');
persons.push({
name,
link
});
});
return persons;
}
async function getPhotos() {
const persons = await getPersons();
const promisies = persons.map(p => axios.get(p.link));
// Step 2
const responses = await Promise.all(promisies);
// Step 3
responses.forEach(response => {
const html = response.data;
const $ = cheerio.load(html);
const snapshots = $('.ratio-4-3');
snapshots.each(() => {
const pic = $(this).attr('style');
if (pic && pic.includes('biopicture')) {
var bioPhoto = s[1];
}
});
});
}
// Call getPhotos to start the process
getPhotos();
Related
I have a script that makes multiple API calls, and the result of one call effects the others.
async function getPlugin(id){
const fetchCardPlugin = `https://*********..amazonaws.com/*****/cardPlugin?id=${id}`
const cp = await fetch(fetchCardPlugin)
let pluginData = await cp.text();
pluginData = JSON.parse(pluginData);
if (typeof pluginData[0] != undefined){
return pluginData[0]['value'].split(':').pop().split('')[0]
}
else {
return ''
}
}
elm.addEventListener('click',()=>{
return t.get('board','shared','bid')
.then(data=>{
(async function(){
const sheetId = data;
const cardList = [];
const mm = {};
const fetchCardData = `https://*******.execute-api.******.amazonaws.com/******/cardData?id=${boardId}`
const cd = await fetch(fetchCardData)
let cardData = await cd.text();
cardData = JSON.parse(cardData);
await cardData.map(x=>{
const cardDict = {};
// console.log(getPlugin(x.shortLink));
cardDict['points'] = getPlugin(x.shortLink);
cardDict['id'] = x.id;
cardDict['title'] = x.name;
})
My first call goes to my fetchCardData. I then use the data from this return to both create an object, and make another call.
await cardData.map(x=>{
const cardDict = {};
// console.log(getPlugin(x.shortLink));
cardDict['points'] = getPlugin(x.shortLink);
cardDict['id'] = x.id;
My big sticking point is that, I need to use data from the second call to populate the object. Currently when I do this I am getting a Promise for the objects point value.
comments:0
description:""
id:"6354b75ddared4ba013aa06440"
labels:""
last activity:"Sat Oct 22 2022"
list:"To Do"
members:""
points:Promise
[[Prototype]]:Promise
[[PromiseState]]:"fulfilled"
[[PromiseResult]]:"5"
title:"test"
url:"https://trello.com/c/*******/**"
What I would like is this:
comments:0
description:""
id:"6354b75ddared4ba013aa06440"
labels:""
last activity:"Sat Oct 22 2022"
list:"To Do"
members:""
points:5
title:"test"
url:"https://trello.com/c/*******/**"
const cardListPromises = cardData.map(async (x) => {
const cardDict = {};
cardDict['points'] = await getPlugin(x.shortLink);
cardDict['id'] = x.id;
cardDict['title'] = x.name;
return cardDict;
})
const cardList = await Promise.all(cardListPromises);
console.log(cardList) // this is the final result you need.
I have written a function to pull stream data from the fetch API. I want to convert the function to a recursive one and would like to write it as an async function.
Here is the code which works but not asynchronous:
const productsStream = async () => {
const request = {
searchRange: ['fragrance', 'perfumery', 'cologne'],
exclusionCategory: ['discounted']
}
const response = await fetch('/products/stream', {
method: 'POST',
body: JSON.stringify(request)
});
const decoder = new TextDecoderStream();
const reader = response.body.pipeThrough(decoder).getReader();
let line = "";
while (true) {
const { done, value } = await reader.read();
let chunk = remainder + value;
let products = []
let newlineIndex;
while ((newlineIndex = chunk.indexOf("\n")) != -1) {
let item = chunk.substring(0, newlineIndex);
chunk = chunk.substring(newlineIndex);
if (chunk.length > 1) chunk = chunk.substring(1);
}
line = chunk;
products = []
if ( done ) break;
}
}
I'd welcome any help in converting this.
I’m building a site using sapper and requesting data from an API. It has been working smooth until now.
When I’m going from site.com/title/id1 to site.com/title/id2 the new information is not loaded until I hit a manual refresh. Any ideas?
import { stores, goto } from "#sapper/app";
import Card from "../_titlecard.svelte";
const { page } = stores();
const { slug } = $page.params;
import { onMount } from "svelte";
let looper = [];
let artistName = "";
let titleName = "";
let dvdCover = "";
let titleCover = "";
let genre = "";
let tracks = [];
onMount(async () => {
const res = await fetch(`https://.com/api/title/${slug}`);
const data = await res.json();
artistName = data.artistName;
titleName = data.name;
dvdCover = data.graphics.dvd;
titleCover = data.graphics.landscape;
genre = data.genre;
tracks = data.tracks.length;
const res2 = await fetch(`https://.com/api/artists/all`);
const data2 = await res2.json();
let moreTitles = [];
const more = data2.map((x) => {
if (x.titles.length > 0 && x.genre === genre) {
looper.push(x.titles[0]);
looper = moreTitles;
}
});
});
And then I have this in the html
{#each looper.slice(0, 4) as item, i}
<Card imgurl={item.graphics.dvd} concert={item.name} id={item.id} />
{/each}
A page component is not unmounted and mounted again if the navigation results in the same page component being used, so your onMount will only be run once with the first id.
You could use a reactive statement to make sure you run the desired code every time $page.params.slug changes.
Example
import { stores, goto } from "#sapper/app";
import Card from "../_titlecard.svelte";
const { page } = stores();
let looper = [];
let artistName = "";
let titleName = "";
let dvdCover = "";
let titleCover = "";
let genre = "";
let tracks = [];
$: (async () => {
const { slug } = $page.params;
const res = await fetch(`https://.com/api/title/${slug}`);
const data = await res.json();
artistName = data.artistName;
titleName = data.name;
dvdCover = data.graphics.dvd;
titleCover = data.graphics.landscape;
genre = data.genre;
tracks = data.tracks.length;
const res2 = await fetch(`https://.com/api/artists/all`);
const data2 = await res2.json();
let moreTitles = [];
data2.forEach((x) => {
if (x.titles.length > 0 && x.genre === genre) {
moreTitles.push(x.titles[0]);
}
});
looper = moreTitles;
})();
I am trying to get API data so i can render it on Webpage using ReactJS. I tried many different ways to fetch api where its stored in array. But i am unable to do it.
Note:https://www.hatchways.io/api/assessment/workers/<worker_id> Here i'm looping so the
worker id gets add the of url.
const fetchAPI = e => {
let array = [];
const api2 = `https://www.hatchways.io/api/assessment/workers/`;
for (var i = 0; i <= 4; i++) {
array.push(api2 + i);
}
return array;
};
console.log(fetchAPI());
Thanks in advance.
You need to hit the url first. Async/Await would be a good choice.
<script>
async function check()
{
var arrayData =[];
var url = "https://www.hatchways.io/api/assessment/workers/";
for(var i=1;i<=4;i++)
{
const response = await fetch(url+""+i);
const myJson = await response.json();
arrayData.push(myJson);
}
console.log(arrayData)
}
check();
</script>
Use Promise.all for example:
const baseURL = 'https://jsonplaceholder.typicode.com';
const fetchUsers = fetch(`${baseURL}/users`);
const fetchPosts = fetch(`${baseURL}/posts`);
Promise.all([fetchUsers, fetchPosts]).then((responses) => {
const responsesToJson = responses.map(response => response.json());
return Promise.all(responsesToJson);
}).then((jsonResponse) => {
const [userResponse, postResponse] = jsonResponse;
console.log(userResponse);
console.log(postResponse);
});
exports.respublished =
functions.database.ref('/Posts/{postid}').onWrite(event => {
const snapshot = event.data;
const postid = event.params.examid;
const uid = snapshot.child('uid').val();
const ispublic = snapshot.child('Public').val();
firebase.database.ref('Users/' + uid).once(event => {
const snapshot = event.data;
const name = snapshot.child('name').val();
});
});
The event is triggered by another node and i want to retrive data from another node of firebase database. I have tried the above code but it produces an error saying TypeError: firebase.database.ref(...).once is not a function.
yes i got the answer we can use this code
exports.respublished = functions.database.ref('/Posts/{postid}').onWrite(event => {
const snapshot = event.data;
const postid = event.params.examid;
const uid = snapshot.child('uid').val();
const ispublic = snapshot.child('Public').val();
return admin.database().ref('Users/' + uid).once(event => {
const snapshot = event.data;
const name = snapshot.child('name').val();
});
});