We Keep Coding

How to modify to promise.all JS

I'm new to javascript, and I'm trying to make some changes.
I have a search code in searchBar and with it a call to an API.json to fetch the information and put it on the site. Everything works fine, however I would like to modify it to add a Promise.all and be able to call more than one API and be able to display it on the site. But all my attempts have failed.
what do i have in js:
const localList = document.getElementById('localList');
const searchBar = document.getElementById('searchBar');
let hpWords = [];
searchBar.addEventListener('keyup', (e) => {
const searchString = e.target.value.toLowerCase();
const filteredWords = hpWords.filter((words) => {
return (
words.name.toLowerCase().includes(searchString)
);
});
displayCharacters(filteredWords);
});
const loadCharacters = async () => {
try {
const res = await fetch('https://duhnunes.github.io/api/local.json');
hpWords = await res.json();
displayCharacters(hpWords);
} catch (err) {
console.error(err);
}
};
const displayCharacters = (words) => {
const htmlString = words
.map((character) => {
return `
<div class="voc-box-content">
<h5>${character.name} - ${character.trans} [${character.type} - ${character.type2}]</h5>
<p>${character.description}</p>
</div>
`;
})
.join('');
localList.innerHTML = htmlString;
};
loadCharacters();
To display on the page I am using <div id="localList"></div>. In the example I am displaying local.json and I would also like to display item.json with <div id ="itemList"></div>.
Both have the same display structure on the <div class="voc-box-content">...</div> site.
Cheers,
DuH

To fetch words from multiple apis you can use this:
const searchBar = document.getElementById('searchBar');
const lists = {
localList: {
api: 'api1'
},
miscList: {
api: 'api2'
},
...
};
Object.entries(lists).forEach(([id, list]) => {
list.element = document.getElementById(id);
});
searchBar.addEventListener('keyup', (e) => {
const searchString = e.target.value.toLowerCase();
Object.values(lists).forEach((list) => {
list.filteredWords = list.words.filter((word) => {
return word.name.toLowerCase().includes(searchString);
});
});
displayCharacters();
});
const loadCharacters = async () => {
try {
await Promise.all(
Object.values(lists).map(async (list) => {
const response = await fetch(list.api);
list.words = await response.json();
})
);
displayCharacters();
} catch (err) {
console.error(err);
}
};
const displayCharacters = () => {
Object.values(lists).forEach((list) => {
list.element.innerHTML = (list.filteredWords ?? list.words)
.map((word) => `
<div class="voc-box-content">
<h5>${word.name} - ${word.trans} [${word.type} - ${word.type2}]</h5>
<p>${word.description}</p>
</div>`)
.join('');
});
};
loadCharacters();

Resolving async function after a loop end

I expect when I call an async function to resolve promise at the end, not before.
const urls = await uploadImages({ imageChanges, questions });
// ...next step
// I will use urls
But after calling await uploadImages() it continues to run until const data = await fetch(image.src);
And then ...next step starts. How can I make it wait for imageChanges.forEach loop finish ? Should I create another nested function inside ?
const uploadImages = async ({ imageChanges, questions }) => {
if (!imageChanges.length) return null;
const storage = firebase.storage();
let urls;
try {
//** convert each new image's src from blob to downloadUrl. */
imageChanges.forEach(async image => {
const questionId = questions.findIndex(q => q.id === image.questionId);
const imagePath = `${questionId}.jpg`;
const storageRef = storage.ref(imagePath);
// **
const data = await fetch(image.src);
const blob = await data.blob();
const uploadTaskSnapshot = await storageRef.put(blob);
const downloadURL = await uploadTaskSnapshot.ref.getDownloadURL();
urls.push(downloadURL)
});
return urls;
} catch (error) {
console.log(error.message);
}
};

forEach with async doesn't work as expected. Read this answer for more info.
Try like this
const uploadImages = async ({ imageChanges, questions }) => {
if (!imageChanges.length) return null;
const storage = firebase.storage();
try {
const imageChangesUrlPromise = imageChanges.map(async () => {
const questionId = questions.findIndex(q => q.id === image.questionId);
const imagePath = `${questionId}.jpg`;
const storageRef = storage.ref(imagePath);
const data = await fetch(image.src);
const blob = await data.blob();
const uploadTaskSnapshot = await storageRef.put(blob);
const downloadURL = await uploadTaskSnapshot.ref.getDownloadURL();
return downloadURL;
})
return await Promise.all(imageChangesUrlPromise);
} catch (error) {
console.log(error.message);
}
};
and then
const urls = await uploadImages({ imageChanges, questions });
...

JavaScript does this because forEach is not promise-aware. It cannot support async and await. You cannot use await in forEach.
If you use await in a map, map will always return an array of promises. This is because asynchronous functions always return promises.
By littile modification to your code, this should work,
const uploadImages = async ({ imageChanges, questions }) => {
if (!imageChanges.length) return null;
const storage = firebase.storage();
let urls;
try {
//** convert each new image's src from blob to downloadUrl. */
await Promise.all(imageChanges.map(async image => {
const questionId = questions.findIndex(q => q.id === image.questionId);
const imagePath = `${questionId}.jpg`;
const storageRef = storage.ref(imagePath);
// **
const data = await fetch(image.src);
const blob = await data.blob();
const uploadTaskSnapshot = await storageRef.put(blob);
const downloadURL = await uploadTaskSnapshot.ref.getDownloadURL();
urls.push(downloadURL)
}));
return urls;
} catch (error) {
console.log(error.message);
}
};
const urls = await uploadImages({ imageChanges, questions });

Not able to read the dom content in puppeteer [node.js] using class & objects

I'm trying to read the dom content from indian superleague for example goals, attacking, mins per goal, etc using class and object. It gives an error like this
Evaluation failed: TypeError: Cannot read property 'textContent' of undefined
at puppeteer_evaluation_script:7:64
Here's the code
config.js
const puppeteer = require('puppeteer')
class Puppeteer{
constructor(){
this.param = {
path: 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe',
url: 'https://indiansuperleague.com',
}
}
async connect(){
this.param.browser = await puppeteer.launch({executablePath: this.param.path, headless: false})
this.param.page = await this.param.browser.newPage()
await this.param.page.goto(this.param.url, {timeout: 0})
}
async disconnect(){
await this.param.browser.close()
}
}
module.exports = Puppeteer
states.js
class States{
constructor(param){
this.param = param
}
async fetchData(){
const page = this.param.page
const res = await page.evaluate(() => {
const title = 'si-fkt-sctn-title', value = 'si-fkt-sctn-number'
// const titleArray = document.getElementsByClassName(title)
// const valueArray = document.getElementsByClassName(value)
let key = document.getElementsByClassName(title)[0].textContent.trim()
let num = document.getElementsByClassName(value)[0].textContent.trim()
/* for(let i=0; i<titleArray.length; i++){
key[i] = titleArray[i].textContent.trim()
num[i] = valueArray[i].textContent.trim()
// Object.defineProperty(temp, key, {value:num,writable: true,configurable: true,enumerable: true})
} */
return {key, num}
})
console.log(res)
}
}
module.exports = States
app.js
const Puppeteer = require('./config')
const States = require('./modules/states')
const puppeteer = new Puppeteer()
const states = new States(puppeteer.param)
puppeteer.connect().then(async() => {
let res = await states.fetchData()
console.log(res)
await puppeteer.disconnect()
}).catch(e => console.log(e))
What is the solution?

The elements may be created dynamically after some time. You can try to use page.waitForSelector() before retrieving the data from them. For example:
'use strict';
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto('https://indiansuperleague.com');
await Promise.all([
page.waitForSelector('.si-fkt-sctn-title'),
page.waitForSelector('.si-fkt-sctn-number'),
]);
const data = await page.evaluate(() => {
return [
document.querySelector('.si-fkt-sctn-title').textContent,
document.querySelector('.si-fkt-sctn-number').textContent,
];
});
console.log(data);
await browser.close();
} catch (err) {
console.error(err);
}
})();
Output:
[ ' Goals', ' 63' ]

Loop through async requests with nested async requests

I have a scenario where I am calling an API that has pagination.
What I'd like to do is the following, 1 page at a time.
Call API Page 1
For each of the items in the response, call a Promise to get more data and store in an array
Send the array to an API
Repeat until all pages are complete
What I currently have is the following, however I think I am possibly complicating this too much, although unsure on how to proceed.
export const importData = async() {
const pSize = 15;
const response = await getItems(pSize, 1);
const noPage = Math.ceil(response.totalMerchandiseCount/pSize);
for (let i = 1; i < noPage; i++) {
const items = [];
const data = await getItems(pSize, i);
await async.each(data.merchandiseList, async(i, cb) => {
const imageURL = await getImageURL(i.id, i.type);
items.push({
id: i.id,
imageURL: imageURL,
});
cb();
}, async() => {
return await api.mockable('sync', items);
});
}
}
export const getImageURL = async(id, type) => {
let url = `https://example.com/${id}`;
return axios.get(url)
.then((response) => {
const $ = cheerio.load(response.data);
// do stuff to get imageUrl
return image;
})
.catch((e) => {
console.log(e);
return null;
})
};
The issue I have at the moment is that it seems to wait until all pages are complete before calling api.mockable. Items is also empty at this point.
Can anyone suggest a way to make this a bit neater and help me get it working?

If this is all meant to be serial, then you can just use a for-of loop:
export const importData = async() {
const pSize = 15;
const response = await getItems(pSize, 1);
const noPage = Math.ceil(response.totalMerchandiseCount/pSize);
for (let i = 1; i < noPage; i++) { // Are you sure this shouldn't be <=?
const items = [];
const data = await getItems(pSize, i);
for (const {id, type} of data.merchandiseList) {
const imageURL = await getImageURL(id, type);
items.push({id, imageURL});
}
await api.mockable('sync', items);
}
}
I also threw some destructuring and shorthand properties in there. :-)
If it's just the pages in serial but you can get the items in parallel, you can replace the for-of with map and Promise.all on the items:
export const importData = async() {
const pSize = 15;
const response = await getItems(pSize, 1);
const noPage = Math.ceil(response.totalMerchandiseCount/pSize);
for (let i = 1; i < noPage; i++) { // Are you sure this shouldn't be <=?
const data = await getItems(pSize, i);
const items = await Promise.all(data.merchandiseList.map(async ({id, type}) => {
const imageURL = await getImageURL(id, type);
return {id, imageURL};
}));
await api.mockable('sync', items);
}
}
That async function call to map can be slightly more efficient as a non-async function:
export const importData = async() {
const pSize = 15;
const response = await getItems(pSize, 1);
const noPage = Math.ceil(response.totalMerchandiseCount/pSize);
for (let i = 1; i < noPage; i++) {
const data = await getItems(pSize, i);
const items = await Promise.all(data.merchandiseList.map(({id, type}) =>
getImageURL(id, type).then(imageURL => ({id, imageURL}))
));
await api.mockable('sync', items);
}
}

Problems with async code in Jest tests

I am having problem with getting the code into the beforeAll function finish and wait for the promise that resolves the storyLinks. The console log at the end of the snippet returns undefined but I need it to return the hrefs of the stories in my storybook. I cannot wrap this into an async function because of the testing pipeline being clogged on fail.
const puppeteer = require('puppeteer');
const { toMatchImageSnapshot } = require('jest-image-snapshot');
expect.extend({ toMatchImageSnapshot });
const timeout = 5000;
describe('visual tests', () => {
let page, browser, storyLinks;
const selector = `a[href*="selectedStory="]`;
beforeAll(async() => {
browser = await puppeteer.connect({browserWSEndpoint});
page = await browser.newPage();
await page.goto('http://localhost:8080');
await page.evaluate(() => {
const components = Array.from(document.querySelectorAll('div[data-name]'));
for(let i = 1; i < components.length; i++) {
components[i].addEventListener('click',() => {});
components[i].click();
}
});
storyLinks = await page.evaluate((selector) => {
const stories = Array.from(document.querySelectorAll(selector));
const links = stories.map(story => {
let href = story.href;
let name = story.text.replace(/[^A-Z0-9]/ig, '-').replace(/-{2,}/,'-');
let component = href.match(/selectedKind=(.*?)\&/).pop();
return {href: href, name: component + '-' + name};
});
return links;
}, selector);
}, timeout);
afterAll(async () => {
await page.close();
await browser.disconnect();
})
console.log(storyLinks);
}, timeout);

There's a few things I notice might be causing your issues. You need to add async to your describe block. Also, "describe" groups together multiple tests so you're missing an it or test block. Jest docs also note adding the expect.assertions(NUM_OF_ASSERTIONS); I'd do something like:
const puppeteer = require('puppeteer');
const { toMatchImageSnapshot } = require('jest-image-snapshot');
expect.extend({ toMatchImageSnapshot });
const timeout = 5000;
async function myStoryLinkTest(page) {
const selector = `a[href*="selectedStory="]`;
await page.goto('http://localhost:8080');
await page.evaluate(() => {
Array.from(document.querySelectorAll('div[data-name]'), item => {
item.addEventListener('click', () => {});
item.click();
});
});
const storyLinks = await page.evaluate(selector => {
return Array.from(document.querySelectorAll(selector), story => {
let href = story.href;
let name = story.text.replace(/[^A-Z0-9]/gi, '-').replace(/-{2,}/, '-');
let component = href.match(/selectedKind=(.*?)\&/).pop();
return { href: href, name: component + '-' + name };
});
});
return storyLinks;
}
describe('visual tests', async () => {
let page, browser;
beforeAll(async () => {
browser = await puppeteer.connect({ browserWSEndpoint });
page = await browser.newPage();
});
afterAll(async () => {
await page.close();
await browser.disconnect();
});
it('should do something with storyLinks', async () => {
expect.assertions(1);
const storyLinkResult = await myStoryLinkTest(page);
expect(storyLinkResult).toEqual('Some value you expect');
}, timeout);
});