We Keep Coding

Execution context was destroyed, most likely because of a navigation Puppeteer

In my Puppeteer Node JS app I need to read localStorage and cookies from a browser web page, but for some reason I'm getting the following error:
UnhandledPromiseRejectionWarning: Error: Execution context was destroyed, most likely because of a navigation.
What am I doing wrong/missing from my JS:
const dayjs = require('dayjs');
const AdvancedFormat = require('dayjs/plugin/advancedFormat');
dayjs.extend(AdvancedFormat);
const puppeteer = require('puppeteer');
const { config } = require('./config');
const helpers = require('./helpers');
const logs = require('./logs');
const runEmulation = async (body) => {
logs.debug('starting emulation');
// vars
const argOptions = [], journey = [];
// sandbox config
if ((config.puppeteer.run_in_sandbox === 'true')) {
argOptions.push('--no-sandbox');
}
// initiate a Puppeteer instance with options and launch
const browser = await puppeteer.launch({
args: argOptions,
headless: (config.puppeteer.run_in_headless === 'true') ? true : false
});
// launch a new page
const page = await browser.newPage()
// initiate a new CDP session
const client = await page.target().createCDPSession();
await client.send('Network.enable');
await client.on('Network.requestWillBeSent', async (e) => {
// if not a document, skip
if (e.type !== "Document") return;
const scrapablePageData = async () => {
function getLocalStorage () {
const values = [];
const keys = Object.keys(localStorage);
let index = keys.length;
while (index--) {
values.push({
key: keys[index],
value: localStorage.getItem(keys[index])
});
}
return values ? values : [];
}
return {
localStorage: getLocalStorage()
}
}
const scrapable = await page.evaluate(scrapablePageData);
const cookies = await page.cookies();
// the data we want to log
journey.push({
url: e.documentURL,
type: e.redirectResponse ? e.redirectResponse.status : 'JS Redirection',
storage: {
cookies: cookies ?? [],
local: scrapable.localStorage ?? []
},
duration_in_ms: 0,
duration_in_sec: 0,
loaded_at: dayjs().valueOf()
})
})
// set userAgent and go to the URL
await page.setUserAgent(body.userAgent);
await page.goto(body.url);
await page.waitForNavigation();
console.log(journey)
}
exports.runEmulation = runEmulation

How can I send a custom url screenshot on discord.js?

So, I try to do a command. You need to write '!screenshot" + a custom url. It needs to take that url and take a screenshot of it and send it back to the respective channel. I tried many ways but I cant make it work.
const Discord = require("discord.js");
const fetch = require('node-fetch');
const cheerio = require("cheerio");
const puppeteer = require('puppeteer');
module.exports = {
name: 'screenshot',
description: 'makes a screenshot and sends it back',
usage: '[command name]',
cooldown: 5,
execute(message, args) {
const data = [];
//const url = message.client;
if (message.content.startsWith("!screenshot ")) {
const url = message.content.substr("!screenshot ".length);
};
},
async run(client, message, args) {
const browser = await puppeteer.launch();
const page = await browser.newPage();
//const page = await context.newPage();
//const url = args.join(' '); // or something similar?
await page.goto(url, { waitUntil: 'networkidle2' });
console.log(await page.content());
await page.screenshot({path: 'screenshot.png'});
let screenshot = await page.screenshot();
await browser.close();
message.channel.send("Screenshot", {files: [screenshot]});
}
}

How to make api endpoint target user's localhost when deployed to Heroku

I have this api which works fine when running locally. But, once it is deployed to Heroku i get a error 503 which is because it tries to target localhost on Heroku's server and not the user's localhost. Is there a way to make this target the user's localhost instead?
The frontend is React. Here's the code in React that fetches this api every 5sec.
axiosFunc = () => {
const { user } = this.props.auth;
console.log(user);
axios.get(`api/avaya/${user.id}`).then((res) => console.log(res));
};
timer = (time) => {
const date = new Date(time);
return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
};
componentDidMount() {
this.axiosFunc();
this.interval = setInterval(this.axiosFunc, 5000);
}
componentWillUnmount() {
clearInterval(this.interval);
}
and this is the API on the backend with express
const router = require("express").Router();
const xml2js = require("xml2js");
const Avaya = require("../../models/Avaya");
const User = require("../../models/User");
router.route("/:id").get(async (req, res) => {
const user = await User.findById(req.params.id);
const axios = require("axios");
axios({
method: "post",
baseURL: `http://127.0.0.1:60000/onexagent/api/registerclient?name=${user.username}`,
timeout: 2000,
})
.then((reg) => {
xml2js
.parseStringPromise(reg.data, { mergeAttrs: true })
.then((result) => {
if (result.RegisterClientResponse.ResponseCode[0] === "0") {
const clientId = result.RegisterClientResponse.ClientId[0];
user.avayaClientId = clientId;
user.save();
}
const clientId = user.avayaClientId;
axios({
method: "post",
url: `http://127.0.0.1:60000/onexagent/api/nextnotification?clientid=${clientId}`,
}).then((notification) => {
xml2js
.parseStringPromise(notification.data, { mergeAttrs: true })
.then((result) => {
const notifType = [];
const notifDetails = [];
for (let i in result.NextNotificationResponse) {
notifType.push(i);
}
const arranged = {
NotificationType: notifType[1],
ResponseCode:
result.NextNotificationResponse[notifType[0]][0],
};
for (let i in result.NextNotificationResponse[
notifType[1]
][0]) {
notifDetails.push(i);
}
for (let i = 0; i < notifDetails.length; i++) {
arranged[[notifDetails[i]][0]] =
result.NextNotificationResponse[notifType[1]][0][
notifDetails[i]
][0];
}
for (let i in arranged) {
if ("Outbound" in arranged) {
arranged.CallType = "Outbound";
} else if ("Inbound" in arranged)
arranged.CallType = "Inbound";
else {
arranged.CallType = " ";
}
}
if (
arranged.NotificationType === "VoiceInteractionCreated" ||
arranged.NotificationType === "VoiceInteractionMissed" ||
arranged.NotificationType === "VoiceInteractionTerminated"
) {
const newLogs = new Avaya({
notification: arranged,
});
newLogs.owner = user;
newLogs.save();
user.avayaNotifications.push(newLogs),
user
.save()
.then((logs) => res.json(logs))
.catch((err) => res.status(400).json("Error: " + err));
} else {
res.send("Nothing to record");
}
});
});
});
})
.catch((err) => res.status(503).json(err));
});
router.route("/history/:username").get(async (req, res) => {
const user = await User.findOne({ username: [`${req.params.username}`] });
Avaya.find({ owner: [`${await user.id}`] }).then((user) => res.json(user));
});
module.exports = router;
EDIT: I was able to fix thanks to #Molda
using fetch instead of axios doesn't result in cors error.
New frontend code
getLogs = async () => {
const { user } = this.props.auth;
const reg = await fetch(
`http://127.0.0.1:60000/onexagent/api/registerclient?name=${user.id}`
);
let regData = await reg.text();
let regxml = new XMLParser().parseFromString(regData);
if (regxml.attributes.ResponseCode === "0") {
axios.post(`/api/avaya/register/${user.id}`, regxml);
console.log(regxml.attributes.ResponseCode);
}
let resp = await fetch(`/api/avaya/getid/${user.id}`);
let clientId = await resp.text();
let logs = await fetch(
`http://127.0.0.1:60000/onexagent/api/nextnotification?clientid=${clientId}`
);
let data = await logs.text();
var xml = new XMLParser().parseFromString(data);
axios.post(`/api/avaya/getlogs/${user.id}`, xml);
};
timer = (time) => {
const date = new Date(time);
return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
};
componentDidMount() {
this.getLogs();
this.interval = setInterval(this.getLogs, 5000);
}
New backend code:
const router = require("express").Router();
const Avaya = require("../../models/Avaya");
const User = require("../../models/User");
router.route("/register/:id").post(async (req, res) => {
const user = await User.findById(req.params.id);
const clientId = req.body.attributes.ClientId;
user.avayaClientId = clientId;
user.save();
});
router.route("/getid/:id").get(async (req, res) => {
const user = await User.findById(req.params.id);
res.send(user.avayaClientId);
});
router.route("/getlogs/:id").post(async (req, res) => {
const user = await User.findById(req.params.id);
const arranged = {
NotificationType: req.body.children[0].name,
ResponseCode: req.body.attributes.ResponseCode,
CallType: " ",
};
for (let i in req.body.children[0].attributes) {
if (i === "Outbound") {
arranged.CallType = "Outbound";
}
if (i === "Inbound") {
arranged.CallType = "Inbound";
}
arranged[i] = req.body.children[0].attributes[i];
}
console.log(arranged);
if (
arranged.NotificationType === "VoiceInteractionCreated" ||
arranged.NotificationType === "VoiceInteractionMissed" ||
arranged.NotificationType === "VoiceInteractionTerminated"
) {
const newLogs = new Avaya({
notification: arranged,
});
newLogs.owner = user;
newLogs.save();
user.avayaNotifications.push(newLogs),
user
.save()
.then((logs) => res.json(logs))
.catch((err) => res.status(400).json("Error: " + err));
} else {
res.send("Nothing to record");
}
});
router.route("/history/:username").get(async (req, res) => {
const user = await User.findOne({ username: [`${req.params.username}`] });
Avaya.find({ owner: [`${await user.id}`] }).then((user) => res.json(user));
});
module.exports = router;

I really don't get the part of (requesting with Axios in API)
Is this a third party API ?
But I suggest you to use (.env) which is a file in your root folder contains the development config like base URLs, expire tokens, API keys ... etc
and when you upload to Heroku you have to make a (.env) in Heroku app and but your config
Let's take an example
in my development mode, my .env looks like
app_url = localhost:4000
port = 4000
db = development_api
db_username = root
db_password =
db_engine = mysql2
in my production mode, my .env looks like
app_url = http://appsomething.heroku.com
port = 80
db = production_api
db_username = root
db_password = 3210LDWAK#AALKQ
db_engine = mysql2
and read more about how to use .ENV

Fetch - Download Excel File does not work. File Size 1 kb

I am trying to download a file, it does not work after download. I am getting files but the size is 1kb which is not actual file size.
If I used fetchResp.text() I am not able to open a file name.
Here is full code.
I think the problem could be here: return await fetchResp.text();
This is example, it is also important to set cookies, because i want to download data behind login.
How to handle puppeteer cookies and fetch?
What if i put fetch function outside page.evaluation. Does { credentials: "include" } will work?
Thanks in advance for your help.
const puppeteer = require("puppeteer");
const cheerio = require("cheerio");
const fs = require("fs");
(async () => {
const browser = await puppeteer.launch({
args: ["--no-sandbox"],
headless: false,
slowMo: 30,
});
const page = await browser.newPage();
await page.goto(
"https://file-examples.com/index.php/sample-documents-download/sample-xls-download/"
);
const content = await page.content();
const $ = cheerio.load(content);
const listings = $("#table-files > tbody > tr:has(a)")
.map((index, element) => {
const URL = $(element).find("a").attr("href");
const Filename = $(element).find("a").attr("href").split("/").pop();
//.replace(/^.*[\\\/]/g, "");
const name = $(element)
.find("td:nth-child(1)")
.text()
.trim()
.replace("\n", "");
return {
Filename,
URL,
};
})
.get();
for (let val of listings) {
const downloadUrl = val.URL;
const Filename = val.Filename;
console.log(val);
const downloadedContent = await page.evaluate(async (downloadUrl) => {
const fetchResp = await fetch(downloadUrl, { credentials: "include" });
return await fetchResp.text();
}, downloadUrl);
fs.writeFile(`./${Filename}`, downloadedContent, () =>
console.log("Wrote file")
);
}
await page.close();
await browser.close();
})();

The main problem here is that you are getting the file contents as just text, which would be fine if you wanted a plain text file, but you need to write an excel file, so you will need blob or an arrayBuffer, both of which cannot be returned from the page.evaluate method. See https://github.com/puppeteer/puppeteer/issues/3722
So you don't need to fetch the excel files using the page.evaluate function from puppeteer, you can directly get them from node using https module after getting all the links and then stream the contents to the files, which is easier in this case and also less code. You'll need these modifications
First require the https module
const https = require('https');
Then close puppeteer after getting the links, since we don't need it anymore
.get();
await page.close();
await browser.close();
Call the function here, when looping throught the links
for (let val of listings) {
const downloadUrl = val.URL;
const Filename = val.Filename;
console.log(val);
var file = await getFile(downloadUrl, Filename);
}
Finally, you need to create a function to read/write the file, outside of your main code block
function getFile(downloadUrl, Filename) {
var data = '';
var writeStream = fs.createWriteStream(Filename);
var req = https.get(downloadUrl, function(res) {
res.pipe(writeStream);
res.on('end', () => {
console.log('No more data in response.');
});
});
req.end();
}
Full snippet
const puppeteer = require('puppeteer');
const cheerio = require("cheerio");
const fs = require("fs");
const https = require('https');
(async () => {
const browser = await puppeteer.launch({
args: ["--no-sandbox"],
headless: false,
slowMo: 30,
});
const page = await browser.newPage();
await page.goto(
"https://file-examples.com/index.php/sample-documents-download/sample-xls-download/"
);
const content = await page.content();
const $ = cheerio.load(content);
const listings = $("#table-files > tbody > tr:has(a)")
.map((index, element) => {
const URL = $(element).find("a").attr("href");
const Filename = $(element).find("a").attr("href").split("/").pop();
//.replace(/^.*[\\\/]/g, "");
const name = $(element)
.find("td:nth-child(1)")
.text()
.trim()
.replace("\n", "");
return {
Filename,
URL,
};
})
.get();
await page.close();
await browser.close();
for (let val of listings) {
const downloadUrl = val.URL;
const Filename = val.Filename;
console.log(val);
//call the function with each link and filename
var file = await getFile(downloadUrl, Filename);
}
})();
//send request and stream the response to a file
function getFile(downloadUrl, Filename) {
var writeStream = fs.createWriteStream(Filename);
var req = https.get(downloadUrl, function(res) {
res.pipe(writeStream);
res.on('end', () => {
console.log('No more data in response.');
});
});
req.end();
}
EDIT Saw your comment, you can send cookies by modifying the get request like this, but remember about the same domain policy for cookies
function getFile(downloadUrl, Filename) {
var url = new URL(downloadUrl)
var options = {
hostname: url.hostname,
path: url.pathname,
method: 'GET',
headers: {
'Cookie': 'myCookie=myvalue'
}
};
var writeStream = fs.createWriteStream(Filename);
var req = https.request(options, function(res) {
res.pipe(writeStream);
res.on('end', () => {
console.log('No more data in response.');
});
});
req.end();
}

Always throws registration failed error while subscribing push notifications

I am working on solutions using which i can send desktop push notification to subscribed clients.
I have created basic solution in where whenever user click on button i ask user for whether they want to allow notifications for my app or not!
I am getting an error of "Registration failed - permission denied" whenever i click on button for first time.
So that i am not able to get required endpoints to save at backend
Here is my code
index.html
<html>
<head>
<title>PUSH NOT</title>
<script src="index.js"></script>
</head>
<body>
<button onclick="main()">Ask Permission</button>
</body>
</html>
index.js
const check = () => {
if (!("serviceWorker" in navigator)) {
throw new Error("No Service Worker support!");
} else {
console.log("service worker supported")
}
if (!("PushManager" in window)) {
throw new Error("No Push API Support!");
} else {
console.log("PushManager worker supported")
}
};
const registerServiceWorker = async () => {
const swRegistration = await navigator.serviceWorker.register("/service.js?"+Math.random());
return swRegistration;
};
const requestNotificationPermission = async () => {
const permission = await window.Notification.requestPermission();
// value of permission can be 'granted', 'default', 'denied'
// granted: user has accepted the request
// default: user has dismissed the notification permission popup by clicking on x
// denied: user has denied the request.
if (permission !== "granted") {
throw new Error("Permission not granted for Notification");
}
};
const main = async () => {
check();
const swRegistration = await registerServiceWorker();
const permission = await requestNotificationPermission();
};
// main(); we will not call main in the beginning.
service.js
// urlB64ToUint8Array is a magic function that will encode the base64 public key
// to Array buffer which is needed by the subscription option
const urlB64ToUint8Array = base64String => {
const padding = "=".repeat((4 - (base64String.length % 4)) % 4);
const base64 = (base64String + padding)
.replace(/\-/g, "+")
.replace(/_/g, "/");
const rawData = atob(base64);
const outputArray = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
outputArray[i] = rawData.charCodeAt(i);
}
return outputArray;
};
const saveSubscription = async subscription => {
console.log("Save Sub")
const SERVER_URL = "http://localhost:4000/save-subscription";
const response = await fetch(SERVER_URL, {
method: "post",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(subscription)
});
return response.json();
};
self.addEventListener("activate", async () => {
try {
const applicationServerKey = urlB64ToUint8Array(
"BFPtpIVOcn2y25il322-bHQIqXXm-OACBtFLdo0EnzGfs-jIGXgAzjY6vNapPb4MM1Z1WuTBUo0wcIpQznLhVGM"
);
const options = { applicationServerKey, userVisibleOnly: true };
const subscription = await self.registration.pushManager.subscribe(options);
console.log(JSON.stringify(subscription))
const response = await saveSubscription(subscription);
} catch (err) {
console.log(err.code)
console.log(err.message)
console.log(err.name)
console.log('Error', err)
}
});
self.addEventListener("push", function(event) {
if (event.data) {
console.log("Push event!! ", event.data.text());
} else {
console.log("Push event but no data");
}
});
Also i have created a bit of backend as well
const express = require("express");
const cors = require("cors");
const bodyParser = require("body-parser");
const webpush = require('web-push')
const app = express();
app.use(cors());
app.use(bodyParser.json());
const port = 4000;
app.get("/", (req, res) => res.send("Hello World!"));
const dummyDb = { subscription: null }; //dummy in memory store
const saveToDatabase = async subscription => {
// Since this is a demo app, I am going to save this in a dummy in memory store. Do not do this in your apps.
// Here you should be writing your db logic to save it.
dummyDb.subscription = subscription;
};
// The new /save-subscription endpoint
app.post("/save-subscription", async (req, res) => {
const subscription = req.body;
await saveToDatabase(subscription); //Method to save the subscription to Database
res.json({ message: "success" });
});
const vapidKeys = {
publicKey:
'BFPtpIVOcn2y25il322-bHQIqXXm-OACBtFLdo0EnzGfs-jIGXgAzjY6vNapPb4MM1Z1WuTBUo0wcIpQznLhVGM',
privateKey: 'mHSKS-uwqAiaiOgt4NMbzYUb7bseXydmKObi4v4bN6U',
}
webpush.setVapidDetails(
'mailto:janakprajapati90#email.com',
vapidKeys.publicKey,
vapidKeys.privateKey
)
const sendNotification = (subscription, dataToSend='') => {
webpush.sendNotification(subscription, dataToSend)
}
app.get('/send-notification', (req, res) => {
const subscription = {endpoint:"https://fcm.googleapis.com/fcm/send/dLjyDYvI8yo:APA91bErM4sn_wRIW6xCievhRZeJcIxTiH4r_oa58JG9PHUaHwX7hQlhMqp32xEKUrMFJpBTi14DeOlECrTsYduvHTTnb8lHVUv3DkS1FOT41hMK6zwMvlRvgWU_QDDS_GBYIMRbzjhg",expirationTime:null,keys:{"p256dh":"BE6kUQ4WTx6v8H-wtChgKAxh3hTiZhpfi4DqACBgNRoJHt44XymOWFkQTvRPnS_S9kmcOoDSgOVD4Wo8qDQzsS0",auth:"CfO4rOsisyA6axdxeFgI_g"}} //get subscription from your databse here.
const message = 'Hello World'
sendNotification(subscription, message)
res.json({ message: 'message sent' })
})
app.listen(port, () => console.log(`Example app listening on port ${port}!`));
Please help me

Try the following code:
index.js
const check = () => {
if (!("serviceWorker" in navigator)) {
throw new Error("No Service Worker support!");
} else {
console.log("service worker supported")
}
if (!("PushManager" in window)) {
throw new Error("No Push API Support!");
} else {
console.log("PushManager worker supported")
}
};
const saveSubscription = async subscription => {
console.log("Save Sub")
const SERVER_URL = "http://localhost:4000/save-subscription";
const response = await fetch(SERVER_URL, {
method: "post",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(subscription)
});
return response.json();
};
const urlB64ToUint8Array = base64String => {
const padding = "=".repeat((4 - (base64String.length % 4)) % 4);
const base64 = (base64String + padding)
.replace(/\-/g, "+")
.replace(/_/g, "/");
const rawData = atob(base64);
const outputArray = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
outputArray[i] = rawData.charCodeAt(i);
}
return outputArray;
};
const registerServiceWorker = async () => {
return navigator.serviceWorker.register("service.js?"+Math.random()).then((swRegistration) => {
console.log(swRegistration);
return swRegistration;
});
};
const requestNotificationPermission = async (swRegistration) => {
return window.Notification.requestPermission().then(() => {
const applicationServerKey = urlB64ToUint8Array(
"BFPtpIVOcn2y25il322-bHQIqXXm-OACBtFLdo0EnzGfs-jIGXgAzjY6vNapPb4MM1Z1WuTBUo0wcIpQznLhVGM"
);
const options = { applicationServerKey, userVisibleOnly: true };
return swRegistration.pushManager.subscribe(options).then((pushSubscription) => {
console.log(pushSubscription);
return pushSubscription;
});
});
};
const main = async () => {
check();
const swRegistration = await registerServiceWorker();
const subscription = await requestNotificationPermission(swRegistration);
// saveSubscription(subscription);
};
service.js
self.addEventListener("push", function(event) {
if (event.data) {
console.log("Push event!! ", event.data.text());
} else {
console.log("Push event but no data");
}
});

I can think of three reasons that the permission is denied
1) your site is not on https (including localhost that is not on https), the default behaviour from chrome as far as i know is to block notifications on http sites. If that's the case, click on the info icon near the url, then click on site settings, then change notifications to ask
2) if you are on Safari, then safari is using the deprecated interface of the Request permission, that is to say the value is not returned through the promise but through a callback so instead of
Notification.requestPermission().then(res => console.log(res))
it is
Notification.requestPermission(res => console.log(res))
3) Your browser settings are blocking the notifications request globally, to ensure that this is not your problem run the following code in the console (on a secured https site)
Notification.requestPermission().then(res => console.log(res))
if you receive the alert box then the problem is something else, if you don't then make sure that the browser is not blocking notifications requests