We Keep Coding

webdriver io js parsing

I need to copy links from (h5 a) elements on all pages using pagination.
My code collects links only from the first page, it goes through the rest of the pages but does not collect links. What is the problem here?
Thanks!
const linksCollector = async () => {
let browser;
try {
browser = await remote({
capabilities: { browserName: "chrome" },
});
await browser.navigateTo(link);
const links = [];
const pages = await browser.$$(".pagination__item");
const paginationBtn = await browser.$(".pagination__next-icon");
for (let i = 0; i < pages.length - 2; i++) {
const linksArr = await browser.$$("h5 a");
for (let i = 0; i < linksArr.length; i++) {
const link = await linksArr[i].getAttribute("href");
links.push(await link);
}
await paginationBtn.click();
}
await writeData(links);
await browser.deleteSession();
} catch (err) {
console.error(err);
return browser.deleteSession();
}
}
};

Node.js How to retrieve data from http.get response

I am doing some practice in node.js. In this exercise I been asked to find a country name through a GET Http Request to an endpoint passing a page integer as a parameter.
Where the important response structs are these {page, total_pages, data}.
page is the current page,
total_pages is the last page,
data is an array of 10 country object.
In getCountryName func I am able to retrieve the right answer only if the answer is on the 1st page, the 1 iteration of the loop. So, why the loop only happens once?
Aditional, I wanted to retrieve the total_pages to replace the hardcode '25' value but I do not figure it out how to return it along with the search.
Any hint you wanna give me? The whole problem is in getCountryCode func.
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
var res = '';
var pages = 25;
var i = 1;
while(i <= pages && res == ''){
console.log(i);
res = makeRequest(i)
.then(data => {
let f = ''
let p = data['total_pages'];
let search = data['data'].find(o => o.alpha3Code === code);
f = search != null ? search['name'] : f;
return f;
});
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();

Without modifying your code too much, this is how you do it:
'use strict';
const { Console } = require('console');
const https = require('https');
function makeRequest(page){
return new Promise(resolve => {
let obj='';
https.get('https://jsonmock.hackerrank.com/api/countries?page='+page, res => {
let data ='';
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
obj=JSON.parse(data);
resolve(obj);
});
});
});
}
async function getCountryName(code) {
const pages = 25;
var i = 1;
let f = null
while(i <= pages && f === null){
console.log(i);
const data = await makeRequest(i) // put in try/catch
const p = data['total_pages'];
const search = data['data'].find(o => o.alpha3Code === code);
f = search !== null ? search['name'] : null;
i++;
}
return res;
}
async function main() {
const name = await getCountryName('ARG');
console.log(`${name}\n`);
}
main();

Express app.use does not send correct fake data

I was practicing Express 4.x and noticed the following:
app.get('/fake', function(req, res) {
var obj = [];
for (let i = 0; i < 3; i++) {
jsf.resolve(fakeSchema).then(function(iter) {
obj.push(iter);
});
}
res.send(obj);
});
So, going to that route, I get "[ ]", while I was expecting to receive an array of 3 (fake) documents.
FYI, when logging each loop, I can clearly see the documents generated, even inside the array.
Any explanation?

Your jsf.resolve functiion is async so you can use async/await for this to perform task in sync manner.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
app.get('/fake', async function(req, res) {
var obj = [];
for (let i = 0; i < 3; i++) {
try {
var iter = await jsf.resolve(fakeSchema);
obj.push(iter);
} catch (e) {}
}
res.send(obj);
});

Although #Nishant's provided answer works, I suggest using this approach.
let jsf = {};
// faking your jsf.resolve method
jsf.resolve = (param) => {
return new Promise((resolve) => {
setTimeout(() => {
resolve(Math.random());
}, 1000);
})
};
let fakeSchema = {};
let obj = [];
let promises = [];
for (let i = 0; i !== 3; i++) {
promises.push(jsf.resolve(fakeSchema).then(function (iter) {
obj.push(iter);
}));
}
Promise.all(promises).then(() => {
console.log(obj);
});
This allows all the promises to run concurrently, imagine your jsx.resolve takes a long time to complete, using await would freeze your entire appp.
As opposed to this. Note the runtime.
(async () => {
let jsf = {};
// faking your jsf.resolve method
jsf.resolve = (param) => {
return new Promise((resolve) => {
setTimeout(() => {
resolve(Math.random());
}, 1000);
})
};
let fakeSchema = {};
let obj = [];
for (let i = 0; i !== 3; i++) {
obj.push(await jsf.resolve(fakeSchema));
}
console.log(obj);
})();

#Nishant Dixit's answer also correct!
You can try this simple solution also, if you like :
app.get('/fake', function(req, res) {
var obj = [];
for (let i = 0; i < 3; i++) {
try {
jsf.resolve(fakeSchema).then(function(iter) {
obj.push(iter);
res.send(obj);
} catch (e) {
res.send(e);
}
});
};
});

Get pdf buffer with puppeteer with session generated url

I've been using puppeteer to try and get pdfs - or its buffer response - from a website which does two requests after clicking on the link for the document (which open in a new tab):
The first request (http://epicdocs.planningni.gov.uk/ViewDocument.pa?uri=4157826&ext=PDF) retrieves the session guid to access the document
The second request (http://epicdocs.planningni.gov.uk/ViewDocument.aspx?guid=4ecd1fe5-43c6-4202-96e3-66b393fb819c) uses that guid to access the document and render the pdf on the browser.
The result of my attempts has been a blank pdf being generated, even if it was created after the page been loaded (checked with Fiddler).
I've tried
Intercepting targetcreated event to get the page
Get the second request url and use page.goto to get the pdf
Wait on a the page response to get the buffer
Set Page.setDownloadBehaviour to allow download instead of rendering it in the browser
Any guidance and help is appreciated.
The code tried is below:
const puppeteer = require("puppeteer");
let browser;
async function getDocument(index, title, page) {
if (index != 19) return "";
console.log("getDocument START");
console.log("#repDocuments__ctl" + index + "_lnkViewDoc\ntitle: " + title);
let docPagePromise = new Promise((resolve, reject) =>
browser.once("targetcreated", async target => {
let targetUrl = await target.url();
if (targetUrl.indexOf("ViewDocument.aspx?") !== -1) {
console.log(targetUrl);
return resolve(target.page());
} else {
console.log("Failed to detect the ViewDocument page");
}
})
);
/* Tried to set the download behaviour to download automatically the pdf but it didn't work */
// await page._client.send("Page.setDownloadBehaviour", {
// behaviour: "allow",
// downloadPath: "./"
// });
await page.click(`#repDocuments__ctl${index}_lnkViewDoc`);
let pdfResults = "";
let pdfPage = await docPagePromise;
/* If I get the target from the page returned from the promise I get the correct ur, however the page url is blank */
// let target = await pdfPage.target();
// let url = await target.url();
// let response = await pdfPage.goto(url);
// console.log(response);
pdfPage.on("console.log", msg => console.log(msg));
/* This is never called */
await pdfPage.on("response", async response => {
console.log("PDF PAGE Response");
let responseBuffer = await response.buffer();
let responseHeaders = response.headers();
console.log("PDF PAGE Response Header: " + responseHeaders);
console.log("PDF PAGE Response Buffer: " + responseBuffer);
return {
responseHeaders,
responseBuffer
};
});
console.log(pdfResults);
let pdfTitle = await pdfPage.title();
console.log("PDFPage URL: " + pdfPage.url());
console.log("PDFPage Title: " + pdfTitle);
let pdfTarget = await pdfPage.target();
console.log("PDFTarget URL: " + (await pdfTarget.url()));
console.log("PDFTarget Type: " + pdfTarget.type());
pdfPage = await pdfTarget.page();
console.log("PDFPage URL: " + pdfPage.url());
await pdfPage.waitFor(3000);
let pdf = await pdfPage.pdf({ path: title + ".pdf" });
console.log(pdf);
return pdf;
}
async function getAdditionalDocumentation(page) {
console.log("getAdditionalDocumentation START");
await page.waitForSelector("#repGroupSummary__ctl1_lnkGroupName");
await page.click("#repGroupSummary__ctl1_lnkGroupName");
await page.waitForSelector("#pnlDocumentList > table > tbody > tr");
await page.waitFor(2000);
const documents = await page.$$eval(
"#pnlDocumentList > table > tbody > tr",
docs =>
docs.map((doc, i) => ({
type: doc.querySelector(".tdl-subgroup > span").innerText,
datePublished: doc.querySelector(
".tdl-date > span[id*='DatePublished']"
).innerText,
dateReceived: doc.querySelector(".tdl-date > span[id*='DateReceived']")
.innerText,
docType: doc.querySelector(".tdl-doctype > span").innerText,
description: doc.querySelector(".tdl-description > span").innerText
// 'docBuffer': window.getDocument(i + 1, doc.querySelector('.tdl-description > span').innerText)
}))
);
for (let i = 0; i < documents.length; i++) {
documents[i].docBuffer = await getDocument(i + 1, documents[i].description, page);
}
await page.click("#btnSummary");
console.log("getAdditionalDocumentation FINISH");
return documents;
}
async function getDocuments(page, browser) {
console.log("getDocuments");
let newPagePromise = new Promise((resolve, reject) =>
browser.once("targetcreated", async target => {
let targetUrl = await target.url();
if (targetUrl.indexOf("ShowCaseFile.aspx?") !== -1) {
console.log(targetUrl);
return resolve(target.page());
} else {
console.log("Failed to detect the ShowCaseFile page");
}
})
);
await page.click("#tab_externalDocuments > span");
await page.waitForSelector("#hp-doc-link");
await page.click("#hp-doc-link");
const newPage = await newPagePromise;
const additionalDocumentation = await getAdditionalDocumentation(newPage);
return {
additionalDocumentation
};
}
async function run() {
try {
browser = await puppeteer.launch();
const page = await browser.newPage();
page.on("console", msg => console.log("PAGE LOG:", ...msg.args));
const planningReference = "LA04/2017/1388/F";
await page.goto(
"http://epicpublic.planningni.gov.uk/publicaccess/search.do?action=simple&searchType=Application"
);
await page.waitForSelector("#simpleSearchString");
await page.type("#simpleSearchString", planningReference);
await page.click("#simpleSearchForm > div.row3 > input.button.primary");
await page.waitForSelector("#simpleDetailsTable");
console.log("getDocuments START");
const documents = await getDocuments(page, browser);
console.log("getDocuments FINISH");
console.log(documents);
console.log(documents.additionalDocumentation.length);
} finally {
browser.close();
}
}
run();

Use exposefunction to write the buffer data to disk with:
page.exposeFunction("writeABString", async (strbuf, targetFile) => {
var str2ab = function _str2ab(str) { // Convert a UTF-8 String to an ArrayBuffer
var buf = new ArrayBuffer(str.length); // 1 byte for each char
var bufView = new Uint8Array(buf);
for (var i=0, strLen=str.length; i < strLen; i++) {
bufView[i] = str.charCodeAt(i);
}
return buf;
}
console.log("In 'writeABString' function...");
return new Promise((resolve, reject) => {
// Convert the ArrayBuffer string back to an ArrayBufffer, which in turn is converted to a Buffer
let buf = Buffer.from(str2ab(strbuf));
// Try saving the file.
fs.writeFile(targetFile, buf, (err, text) => {
if(err) reject(err);
else resolve(targetFile);
});
});
});
With the download link that you have use it in tandem with fetch api to get it as blob and convert it with:
page.evaluate( async () => {
function arrayBufferToString(buffer){ // Convert an ArrayBuffer to an UTF-8 String
var bufView = new Uint8Array(buffer);
var length = bufView.length;
var result = '';
var addition = Math.pow(2,8)-1;
for(var i = 0;i<length;i+=addition){
if(i + addition > length){
addition = length - i;
}
result += String.fromCharCode.apply(null, bufView.subarray(i,i+addition));
}
return result;
}
let geturl = "https://whateverurl.example.com";
return fetch(geturl, {
credentials: 'same-origin', // usefull when we are logged into a website and want to send cookies
responseType: 'arraybuffer', // get response as an ArrayBuffer
})
.then(response => response.arrayBuffer())
.then( arrayBuffer => {
var bufstring = arrayBufferToString(arrayBuffer);
return window.writeABString(bufstring, '/tmp/downloadtest.pdf');
})
.catch(function (error) {
console.log('Request failed: ', error);
});
});
For more info look at this issue on the github puppeteer page. The above solution was also suggested in the issue.
Source

Why array.length is not working while array is filled by a async function in javascript

I'm learing WebAudio API. I'm facing a problem with it. Basically things are asynchronous here...so im getting a bit confused. Please help.Here is my code:-
//"use strict";
var sources = new Array();
var actx;
var songs = ['src1.mp3', 'src2.mp3'];
async function start() {
console.log("WELCOME!!");
try {
actx = new AudioContext();
} catch (e) {
console.log('WebAudio api is not supported!!');
}
await getBuffers(actx, songs);
console.log(sources);
console.log(sources.length);
}
function load_song(url) {
let promise = new Promise((resolve, reject) => {
let request = new XMLHttpRequest();
request.open('GET', url, true);
request.responseType = 'arraybuffer';
request.onload = () => {
let audioData = request.response;
resolve(audioData);
}
request.onerror = () => {
reject(new error("Could not load the song:- " + url));
}
request.send();
});
return promise;
}
//creats buffers
async function getBuffers(actx, songs) {
// let buffer_list = new Array();
for (let x = 0; x < songs.length; x++) {
let temp = actx.createBufferSource();
await load_song(songs[x]).then((audioData) => {
actx.decodeAudioData(audioData).then((decodedAudioData) => {
temp.buffer = decodedAudioData;
sources.push(temp);
}).catch((error) => {
console.error(error);
});
});
}
//console.log(buffers.length);
}
async function play() {
//start();
sources[0].start(0);
//sources[1].start(0);
}
function stop() {
sources[0].stop(0);
//sources[1].stop(0);
}
Here in the two lines console.log(sources) and console.log(sources.length). Here the results are. Why console.log(sources.length) is 0?
Please help me........Thank you.

You need to return
actx.decodeAudioData(audioData).then((decodedAudioData) => {
As you are not returning it, you dont await it. Therefore the log appears before the array gets filled, however console.log(sources) is live, so you see the latest change.
Such mistakes are more unlikely to happen when you use await everywhere, and thats also easier to read IMO:
async function getBuffers(actx, songs) {
const sources = []; //mutating a global variable isnt good, so lets make it local
for (const song of songs) { //for...of is so beautiful, why dont you use it?
try { //this is similar to .catch
let temp = actx.createBufferSource();
const audioData = await load_song(song);
const decodedAudioData = await actx.decodeAudioData(audioData);
temp.buffer = decodedAudioData;
sources.push(temp);
} catch(e){ console.error(e); } //okay that does not count as an error handler...
}
return sources; //getBuffers implies that it returns sth
}

You should change your getBuffers code for something like this
async function getBuffers(actx, songs) {
try {
for (let song of songs) {
let temp = actx.createBufferSource();
var audioData = await load_song(songs[x])
var decodedAudioData = await actx.decodeAudioData(audioData)
temp.buffer = decodedAudioData;
sources.push(temp);
}
} catch (e) { console.log(e) }
}