interaction of a chrome extension based on React TSX UI chrome API - javascript

I'm attempting to build some extension which contains a form and an option to capture screen with desktopCapture, which looks like this:
The form is written in React TypeScript and the code for capturing the screen (taken from here) is the following:
chrome.runtime.onMessage.addListener(
(message, sender, senderResponse) => {
if (message.name === "stream" && message.streamId) {
let track, canvas;
navigator.mediaDevices
.getUserMedia({
video: {
mandatory: {
chromeMediaSource: "desktop",
chromeMediaSourceId: message.streamId,
},
},
})
.then((stream) => {
track = stream.getVideoTracks()[0];
const imageCapture = new ImageCapture(track);
return imageCapture.grabFrame();
})
.then((bitmap) => {
track.stop();
canvas = document.createElement("canvas");
canvas.width = bitmap.width;
canvas.height = bitmap.height;
let context = canvas.getContext("2d");
context.drawImage(bitmap, 0, 0, bitmap.width, bitmap.height);
return canvas
.toDataURL()
.then((url) => {
//TODO download the image from the URL
chrome.runtime.sendMessage(
{ name: "download", url },
(response) => {
if (response.success) {
alert("Screenshot saved");
} else {
alert("Could not save screenshot");
}
canvas.remove();
senderResponse({ success: true });
}
);
})
.catch((err) => {
alert("Could not take screenshot");
senderResponse({ success: false, message: err });
});
});
}
return true;
}
);
My intention is that when the user will click on "take screen shot", the code above will run, and then, on save, the image will be presented in that box.
I was able to 'grab' the two elements, both the box where I wish the image to appear after screenshooting, and the "TAKE SCREEN SHOT" button.
as far as I'm aware of, content_script only injects into web-pages (browser), and has no access to extension, therefor, that's not the way to add the code inside..
What am I missing? How could I add an eventListener, that if the button is clicked, the screenCapturing code will run, and I'll be able to set the box to be the captured image?
Best regards!

As i understand,
you want to take screenshot of tab's page content.
(I assume you don't need to grab playing video or audio content)
Fix 1:
Use chrome.tabs.captureVisibleTab api for capture screenshot.
API link
chrome.tabs
Add this in background.js
const takeShot = async (windowId) => {
try {
let imgUrl64 = await chrome.tabs.captureVisibleTab(windowId, { format: "jpeg", quality: 80 });
console.log(imgUrl64);
chrome.runtime.sendMessage({ msg: "update_screenshot",imgUrl64:imgUrl64});
} catch (error) {
console.error(error);
}
};
chrome.runtime.onMessage.addListener(async (req, sender, sendResponse) => {
if(req.msg === "take_screenshot") takeShot(sender.tab.windowId)
}
Fix 2:
Content_script has limited api access.
Check this page. Understand content script capabilities
Solution:
Send message from content_script to background and ask them to capture screenshot.
Background capture screenshot
content.js
chrome.runtime.sendMessage({ msg: "take_screenshot"});
popup.js
chrome.runtime.onMessage.addListener(async (req, sender, sendResponse) => {
if(req.msg === "update_screenshot") console.log(req.imgUrl64)
}

Related

Service worker activate and push event don't get triggered even though the registration is successful and it's activated

I have built a React progressive web application that makes use of service workers.
The service worker gets registered and is activated:
I have been trying to detect the "activate" event using this:
service-worker.js
navigator.serviceWorker.addEventListener("activate", function (event) {
console.log("service worker activated");
});
I added that at the end of the service-worker file. But, this event never gets triggered and I have no idea why.
I also tried to implement push notifications and trigger the from the backend. For this, I needed a "push" event listener that would listen to these events from the server:
navigator.serviceWorker.addEventListener("push", async function (event) {
const message = await event.data.json();
let { title, description, image } = message;
await event.waitUntil(showPushNotification(title, description, image));
});
This is how showPushNotification is defined:
export function showPushNotification(title, description, image) {
if (!("serviceWorker" in navigator)) {
console.log("Service Worker is not supported in this browser");
return;
}
navigator.serviceWorker.ready.then(function (registration) {
registration.showNotification(title, {
body: description,
icon: image,
actions: [
{
title: "Say hi",
action: "Say hi",
},
],
});
});
}
I tested calling that function manually and it successfully triggerss a push notification.
This is the server code that triggers the push notification:
const sendPushNotification = async (user_id, title, description, image) => {
const search_option = { user: user_id };
const users_subscriptions = await PushNotificationSubscription.find(
search_option
);
const number_of_users_subscriptions = users_subscriptions.length;
const options = {
vapidDetails: {
subject: "mailto:xxxx#xxxx.com",
publicKey: VAPID_PUBLIC_KEY,
privateKey: VAPID_PRIVATE_KEY,
},
};
let push_notif_sending_results = {};
for (let i = 0; i < number_of_users_subscriptions; i++) {
const user_subscription = users_subscriptions[i];
await webPush
.sendNotification(
user_subscription,
JSON.stringify({
title,
description,
image,
}),
options
)
.then((notif_send_result) => {
push_notif_sending_results[i] = { success: notif_send_result };
})
.catch((error) => {
push_notif_sending_results[i] = { error: error };
});
}
return push_notif_sending_results;
};
This is the part responsible for sending the push notification:
webPush
.sendNotification(
user_subscription,
JSON.stringify({
title,
description,
image,
}),
options
)
And it's successfully executed as it returns a 201 HTTP response.
So the "push" event listener is supposed to detect it and trigger a push notification.
I think everything regarding the push notification has been successfully implementing and the problem is how the "push" event listener is added since the "activate" event listener also doesn't work.
So I tried moving the two event listeners here right after the registration of the service worker is successful:
function registerValidSW(swUrl, config) {
navigator.serviceWorker.register(swUrl).then((registration) => {
registration.addEventListener("activate", (event) => {
console.log(
"🚀 ~ file: serviceWorker.js:159 ~ navigator.serviceWorker.register ~ event",
event
);
});
registration.addEventListener("push", async function (event) {
const message = await event.data.json();
let { title, description, image } = message;
await event.waitUntil(
showPushNotification(title, description, image)
);
});
});
}
But, it's still the same result.
Neither the "push" nor the "activate" event listeners get triggered.
Any idea what's going on?
Here's the whole service-worker file:
service-worker.js
import axios from "axios";
const isLocalhost = Boolean(
window.location.hostname === "localhost" ||
// [::1] is the IPv6 localhost address.
window.location.hostname === "[::1]" ||
// 127.0.0.0/8 are considered localhost for IPv4.
window.location.hostname.match(
/^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/
)
);
export function register(config) {
if (process.env.NODE_ENV === "production" && "serviceWorker" in navigator) {
// The URL constructor is available in all browsers that support SW.
const publicUrl = new URL(process.env.PUBLIC_URL, window.location.href);
if (publicUrl.origin !== window.location.origin) {
// Our service worker won't work if PUBLIC_URL is on a different origin
// from what our page is served on. This might happen if a CDN is used to
// serve assets;
return;
}
window.addEventListener("load", () => {
const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
if (isLocalhost) {
// This is running on localhost. Let's check if a service worker still exists or not.
checkValidServiceWorker(swUrl, config);
// Add some additional logging to localhost, pointing developers to the
// service worker/PWA documentation.
} else {
// Is not localhost. Just register service worker
console.log(
"Is not localhost. Just register a service worker, by calling registerValidSW"
);
registerValidSW(swUrl, config);
}
});
}
}
async function subscribeToPushNotifications(serviceWorkerReg) {
let subscription = await serviceWorkerReg.pushManager.getSubscription();
if (subscription === null) {
const dev_public_vapid_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
const prod_public_vapid_key =
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
const public_vapid_key = isLocalhost
? dev_public_vapid_key
: prod_public_vapid_key;
subscription = await serviceWorkerReg.pushManager.subscribe({
userVisibleOnly: true,
applicationServerKey: public_vapid_key,
});
axios
.post("/api/push_notif_subscription/subscribe", subscription)
.then((response) => {})
.catch((error) => {});
}
}
export function showPushNotification(title, description, image) {
if (!("serviceWorker" in navigator)) {
console.log("Service Worker is not supported in this browser");
return;
}
navigator.serviceWorker.ready.then(function (registration) {
registration.showNotification(title, {
body: description,
icon: image,
actions: [
{
title: "Say hi",
action: "Say hi",
},
],
});
});
}
function registerValidSW(swUrl, config) {
navigator.serviceWorker.register(swUrl).then((registration) => {
subscribeToPushNotifications(registration);
registration.onupdatefound = () => {
const installingWorker = registration.installing;
if (!installingWorker) {
return;
}
installingWorker.onstatechange = () => {
if (installingWorker.state === "installed") {
if (navigator.serviceWorker.controller) {
// At this point, the updated precached content has been fetched,
// but the previous service worker will still serve the older
// content until all client tabs are closed.
// Execute callback
if (config && config.onUpdate) {
config.onUpdate(registration);
}
} else {
// At this point, everything has been preached.
// It's the perfect time to display a
// "Content is cached for offline use." message.
// Execute callback
if (config && config.onSuccess) {
config.onSuccess(registration);
}
}
}
};
};
});
}
function checkValidServiceWorker(swUrl, config) {
// Check if the service worker can be found. If it can't reload the page.
fetch(swUrl, {
headers: { "Service-Worker": "script" },
}).then((response) => {
// Ensure the service worker exists, and that we really are getting a JS file.
const contentType = response.headers.get("content-type");
if (
response.status === 404 ||
(!!contentType && contentType.indexOf("javascript") === -1)
) {
// No service worker found. Probably a different app. Reload the page.
navigator.serviceWorker.ready.then((registration) => {
registration.unregister().then(() => {
window.location.reload();
});
});
} else {
// Service worker found. Proceed as normal.
console.log("Service worker found, calling registerValidSW");
registerValidSW(swUrl, config);
}
});
}
export function unregister() {
if ("serviceWorker" in navigator) {
navigator.serviceWorker.ready.then((registration) => {
registration.unregister();
});
}
}
navigator.serviceWorker.addEventListener("activate", function (event) {
console.log("service worker activated");
});
navigator.serviceWorker.addEventListener("push", async function (event) {
const message = await event.data.json();
let { title, description, image } = message;
await event.waitUntil(showPushNotification(title, description, image));
});
The events "push" and "activate" are part of the ServiceWorkerGlobalScope as within the Service Worker API.
Push notifications must be handled within the service worker itself.
Therefore only the service worker can register an "activate" event listener.
The same applies for a "push" listener.
Specially in terms of the "push" listener this makes sense.
The idea of push events is to receive them, even if the main app (in this case the website) has been closed.
The service worker is an exception, as it even runs without the page being loaded.
Therefore move the "push" event into your service worker.
Your code (within the service worker) may look like this:
this.addEventListener("push", async function (event) {
const message = await event.data.json();
let { title, description, image } = message;
await event.waitUntil(showPushNotification(title, description, image));
});
function showPushNotification(title, description, image) {
registration.showNotification(title, {
body: description,
icon: image,
actions: [
{
title: "Say hi",
action: "Say hi",
},
],
});
}
The rest seems fine to me.
Update (Some more explanation)
I took a more careful look at your service-worker.js and it seems it contains general methods for registering the service worker.
As mentioned above the main app and the service worker are two completely separate chunks of code, running in different spaces. So this means everything which is not supposed to run in the service worker itself must be put outside of the service-worker.js. The service worker (in your case) should only contain the code for handling push notifications. It's important that you do not include the "service-worker.js" within your application.
In your case, you may seperate these functions into service-worker-register.js which contain all functions which are for managing the service worker registration but should not be executed within the service worker itself (isLocalhost, register, subscribeToPushNotifications, registerValidSW, checkValidServiceWorker, and unregister). Please note the code snippet from above and make changes accordingly to the code left within the service worker.
MDN has a pretty in depth tutorial on service workers (and there are a lot more) I recommend having a look at:
developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API/Using_Service_Workers

Adding screen share option webrtc app. unable to get on other users

i was adding screen share functionality to my app but its is not working .. its only show screen share on my side but not on other user.
here is code :
try {
navigator.mediaDevices
.getDisplayMedia({
video: true,
audio: true
})
.then((stream) => {
const video1 = document.createElement("video");
video1.controls = true;
addVideoStream(video1, stream);
socket.on("user-connected", (userId) => {
const call = peer.call(userId, stream);
stream.getVideoTracks()[0].addEventListener("ended", () => {
video1.remove();
});
call.on("close", () => {});
});
stream.getVideoTracks()[0].addEventListener("ended", () => {
video1.remove();
});
});
} catch (err) {
console.log("Error: " + err);
}
Issue could be related to signaling and that depends on each project.
You could start from a working example that streams webcam/microphone and then switch source to screen.
In this HTML5 Live Streaming example, you can switch source between camera and desktop - transmission is the same. So you could achive somethign similar by starting from an example for camera streaming and testing that first.

How to determine if user picked video or image?

Using react native and expo, we want a user to be able to post a video or an image.
Here is the code:
import * as ImagePicker from 'expo-image-picker';
const openImagePickerAsync = async() => {
const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync();
if (permissionResult.granted === false) {
setImage(null);
setHasImage(false);
alert('Permission to access camera roll is required!');
return;
}
const pickerResult = await ImagePicker.launchImageLibraryAsync({
mediaTypes: ImagePicker.MediaTypeOptions.All, <---------------- allowing photos and video
allowsEditing: true,
});
try {
if (pickerResult.cancelled === true) {
setHasImage(false);
console.log('pickerResult is cancelled');
return;
}
if (pickerResult !== null) {
setHasImage(true);
setImage(pickerResult.uri);
console.log(image);
} else {
setImage(null);
setHasImage(false);
console.log('pickerResult is null');
return;
}
} catch (error) {
console.log(error);
}
};
How do we make it so that we can know if a user picked a photo or a video? Is it in the metadata?
You should use pickerResult.type which would have video or image
You can refer the documentation
Returns If the user cancelled the picking, returns { cancelled: true
}.
Otherwise, this method returns information about the selected media
item. When the chosen item is an image, this method returns {
cancelled: false, type: 'image', uri, width, height, exif, base64 };
when the item is a video, this method returns { cancelled: false,
type: 'video', uri, width, height, duration }.

"No Credentials" error thrown when trying to access user settings in AWS through React

I’m trying to make a system in React where users can log in and change their credentials. I’m also trying to allow for users to save a profile picture. Whenever they upload a picture, it appears on the settings page but I can’t seem to find a way to fetch it again. To save the image file, this is what I’m using.
Storage.put("profile.png", file, {
contentType: "image/png"
})
And then I use Storage.get("profile.png") to fetch the image. The problem is when I try to fetch the image, I get an error thrown that says “No Credentials” and in the console, I see a message that says “cannot get guest credentials when mandatory signin enabled.” I also get the same “No Credentials” error thrown when I try to save the image. Does anyone know what’s going on? Here’s the code I used for saving and retrieving the image.
onProcessFile = async (e) => {
e.preventDefault();
let reader = new FileReader();
const file = e.target.files[0];
try {
reader.readAsDataURL(file);
} catch (err) {
alert(err);
}
reader.onloadend = () => {
setImage(reader.result)
};
const { id } = await Auth.currentUserPoolUser();
Storage.put("profile.png", file, {
contentType: "image/png"
})
.then(result => console.log(result))
.catch(err => alert(err));
};
getCurrentUser = async () => {
Storage.get("profile.png")
.then(url => {
var myRequest = new Request(url);
fetch(myRequest).then(function(response) {
if (response.status === 200) {
setImage(url)
}
});
})
.catch(err => alert(err));
}
My guess is you would need to set the right credentials on your bucket to be able to fetch the images when the user is signed in. Here is a link that goes a little deeper into this subject. Set up bucket credentials

Play a sound from a Service Worker

Is there a way to play an audio file from a service worker?
I'm trying to use io.sound library but it is a JavaScript plugin that requires window, so it doesn't work.
EDIT
As suggested by Jeff I'm trying to open a new window and post a message to that window. this is my code:
function notifyClientToPlaySound() {
idbKeyval.get('pageToOpenOnNotification')
.then(url => {
console.log("notifyClientToPlaySound", url);
clients.matchAll({
type: "window"
//includeUncontrolled: true
})
.then((windowClients) => {
console.log("notifyClientToPlaySound - windowClients", windowClients);
for (var i = 0; i < windowClients.length; i++) {
var client = windowClients[i];
if (client.url === url && "focus" in client) {
notify({ event: "push" });
return client.focus();
}
}
//https://developer.mozilla.org/en-US/docs/Web/API/Clients/openWindow
if (clients.openWindow) {
return clients.openWindow("/")
.then(() => {
notify({ event: "push" });
});
}
})
});
}
This function is now called from event.waitUntil(..) inside self.addEventListener("push", (event) => { ... }
self.addEventListener("push", (event) => {
console.log("[serviceWorker] Push message received", event);
event.waitUntil(
idbKeyval.get('fetchNotificationDataUrl')
.then(url => {
console.log("[serviceWorker] Fetching notification data from -> " + url);
return fetch(url, {
credentials: "include"
});
})
.then(response => {
if (response.status !== 200) {
// Either show a message to the user explaining the error
// or enter a generic message and handle the
// onnotificationclick event to direct the user to a web page
console.log("[serviceWorker] Looks like there was a problem. Status Code: " + response.status);
throw new Error();
}
// Examine the text in the response
return response.json();
})
.then(data => {
if (!data) {
console.error("[serviceWorker] The API returned no data. Showing default notification", data);
//throw new Error();
showDefaultNotification({ url: "/" });
}
notifyClientToPlaySound(); <------ HERE
var title = data.Title;
var message = data.Message;
var icon = data.Icon;
var tag = data.Tag;
var url = data.Url;
return self.registration.showNotification(title, {
body: message,
icon: icon,
tag: tag,
data: {
url: url
},
requireInteraction: true
});
})
.catch(error => {
console.error("[serviceWorker] Unable to retrieve data", error);
var title = "An error occurred";
var message = "We were unable to get the information for this push message";
var icon = "/favicon.ico";
var tag = "notification-error";
return self.registration.showNotification(title, {
body: message,
icon: icon,
tag: tag,
data: {
url: "/"
},
requireInteraction: true
});
})
);
});
But when clients.openWindow is called, it returns the following exception:
Uncaught (in promise) DOMException: Not allowed to open a window.
How can I solve this?
The living specification for the Web Notifications API does reference a sound property that could be specified when showing a notification, and would theoretically allow you to play the sound of your choosing when showing a notification from a service worker.
However, while the specification references this property, as of the time of this writing, it's not supported in any browsers.
Update (Aug. '19): It looks like reference to sound has been removed from https://notifications.spec.whatwg.org/#alerting-the-user
Your best bet would be post a message along to an open window that's controlled by the current service worker, and have the window play the sound in response to the message event.
If there is no controlled client available (e.g. because your service worker has been awoken by a push event, and your site isn't currently open in a browser) then you'd have the option of opening a new window inside your notificationclick handler, which is triggered in response to a user clicking on the notification you display in your push event handler. You can then post a message to that new window.

Categories