Add offscreen user media sample

functional-samples/cookbook.offscreen-user-media/README.md

@@ -0,0 +1,34 @@
+This recipe shows how to use User Media in an Extension Service Worker using the [Offscreen document][1].
+
+## Context
+
+- The extension aims to capture audio from the user in the context of extension (globally) via a media recorder in the offscreen document.
+- Service worker no longer have access to window objects and APIs. Hence, it was difficult for the extension to fetch permissions and capture audio.
+- Offscreen document handles permission checks, audio devices management and recording using navigator API and media recorder respectively.
+
+## Steps
+
+1. User presses START/STOP recording from extension popup
+2. Popup sends message to background service worker.
+3. If STOP, Service worker sends message to offscreen to stop mediarecorder.
+4. If START, Service worker sends message to the active tab's content script to intiate recording process.
+5. The content script sends message to offscreen to check audio permissions.
+   - If GRANTED, send message to offscreen to start media recorder.
+   - If DENIED, show alert on window
+   - If PROMPT,
+     - inject an IFrame to request permission from the user.
+     - Listen to the user'e response on the iFrame
+     - If allowed, move to GRANTED step. Else, DENIED.
+
+## Running this extension
+
+1. Clone this repository.
+2. Load this directory in Chrome as an [unpacked extension][2].
+3. Open the Extension menu and click the extension named "Offscreen API - User media".
+
+Click on the extension popup for START and STOP recording buttons.
+
+Inspect the offscreen html page to view logs from media recorder and audio chunk management.
+
+[1]: https://developer.chrome.com/docs/extensions/reference/offscreen/
+[2]: https://developer.chrome.com/docs/extensions/mv3/getstarted/development-basics/#load-unpacked

functional-samples/cookbook.offscreen-user-media/background.js

@@ -0,0 +1,133 @@
+/**
+ * Path to the offscreen HTML document.
+ * @type {string}
+ */
+const OFFSCREEN_DOCUMENT_PATH = 'offscreen/offscreen.html';
+
+/**
+ * Reason for creating the offscreen document.
+ * @type {string}
+ */
+const OFFSCREEN_REASON = 'USER_MEDIA';
+
+/**
+ * Listener for extension installation.
+ */
+chrome.runtime.onInstalled.addListener(handleInstall);
+
+/**
+ * Listener for messages from the extension.
+ * @param {Object} request - The message request.
+ * @param {Object} sender - The sender of the message.
+ * @param {function} sendResponse - Callback function to send a response.
+ */
+chrome.runtime.onMessage.addListener((request) => {
+  switch (request.message.type) {
+    case 'TOGGLE_RECORDING':
+      switch (request.message.data) {
+        case 'START':
+          initateRecordingStart();
+          break;
+        case 'STOP':
+          initateRecordingStop();
+          break;
+      }
+      break;
+  }
+});
+
+/**
+ * Handles the installation of the extension.
+ */
+async function handleInstall() {
+  console.log('Extension installed...');
+  if (!(await hasDocument())) {
+    // create offscreen document
+    await createOffscreenDocument();
+  }
+}
+
+/**
+ * Sends a message to the offscreen document.
+ * @param {string} type - The type of the message.
+ * @param {Object} data - The data to be sent with the message.
+ */
+async function sendMessageToOffscreenDocument(type, data) {
+  // Create an offscreen document if one doesn't exist yet
+  try {
+    if (!(await hasDocument())) {
+      await createOffscreenDocument();
+    }
+  } finally {
+    // Now that we have an offscreen document, we can dispatch the message.
+    chrome.runtime.sendMessage({
+      message: {
+        type: type,
+        target: 'offscreen',
+        data: data
+      }
+    });
+  }
+}
+
+/**
+ * Initiates the stop recording process.
+ */
+function initateRecordingStop() {
+  console.log('Recording stopped at offscreen');
+  sendMessageToOffscreenDocument('STOP_OFFSCREEN_RECORDING');
+}
+
+/**
+ * Initiates the start recording process.
+ */
+function initateRecordingStart() {
+  chrome.tabs.query({ active: true, lastFocusedWindow: true }, ([tab]) => {
+    if (chrome.runtime.lastError || !tab) {
+      console.error('No valid webpage or tab opened');
+      return;
+    }
+
+    chrome.tabs.sendMessage(
+      tab.id,
+      {
+        // Send message to content script of the specific tab to check and/or prompt mic permissions
+        message: { type: 'PROMPT_MICROPHONE_PERMISSION' }
+      },
+      (response) => {
+        // If user allows the mic permissions, we continue the recording procedure.
+        if (response.message.status === 'success') {
+          console.log('Recording started at offscreen');
+          sendMessageToOffscreenDocument('START_OFFSCREEN_RECORDING');
+        }
+      }
+    );
+  });
+}
+
+/**
+ * Checks if there is an offscreen document.
+ * @returns {Promise<boolean>} - Promise that resolves to a boolean indicating if an offscreen document exists.
+ */
+async function hasDocument() {
+  // Check all windows controlled by the service worker if one of them is the offscreen document
+  const matchedClients = await clients.matchAll();
+  for (const client of matchedClients) {
+    if (client.url.endsWith(OFFSCREEN_DOCUMENT_PATH)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Creates the offscreen document.
+ * @returns {Promise<void>} - Promise that resolves when the offscreen document is created.
+ */
+async function createOffscreenDocument() {
+  await chrome.offscreen.createDocument({
+    url: OFFSCREEN_DOCUMENT_PATH,
+    reasons: [OFFSCREEN_REASON],
+    justification: 'To interact with user media'
+  });
+}

functional-samples/cookbook.offscreen-user-media/contentScript.js

@@ -0,0 +1,79 @@
+/**
+ * Listener for messages from the background script.
+ * @param {Object} request - The message request.
+ * @param {Object} sender - The sender of the message.
+ * @param {function} sendResponse - Callback function to send a response.
+ * @returns {boolean} - Whether the response should be sent asynchronously (true by default).
+ */
+chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
+  switch (request.message.type) {
+    case 'PROMPT_MICROPHONE_PERMISSION':
+      // Check for mic permissions. If not found, prompt
+      checkMicPermissions()
+        .then(() => {
+          sendResponse({ message: { status: 'success' } });
+        })
+        .catch(() => {
+          promptMicPermissions();
+          const iframe = document.getElementById('PERMISSION_IFRAME_ID');
+          window.addEventListener('message', (event) => {
+            if (event.source === iframe.contentWindow && event.data) {
+              if (event.data.type === 'permissionsGranted') {
+                sendResponse({
+                  message: { status: 'success' }
+                });
+              } else {
+                sendResponse({
+                  message: {
+                    status: 'failure'
+                  }
+                });
+              }
+              document.body.removeChild(iframe);
+            }
+          });
+        });
+      break;
+
+    default:
+      // Do nothing for other message types
+      break;
+  }
+  return true;
+});
+
+/**
+ * Checks microphone permissions using a message to the background script.
+ * @returns {Promise<void>} - Promise that resolves if permissions are granted, rejects otherwise.
+ */
+function checkMicPermissions() {
+  return new Promise((resolve, reject) => {
+    chrome.runtime.sendMessage(
+      {
+        message: {
+          type: 'CHECK_PERMISSIONS',
+          target: 'offscreen'
+        }
+      },
+      (response) => {
+        if (response.message.status === 'success') {
+          resolve();
+        } else {
+          reject(response.message.data);
+        }
+      }
+    );
+  });
+}
+
+/**
+ * Prompts the user for microphone permissions using an iframe.
+ */
+function promptMicPermissions() {
+  const iframe = document.createElement('iframe');
+  iframe.setAttribute('hidden', 'hidden');
+  iframe.setAttribute('allow', 'microphone');
+  iframe.setAttribute('id', 'PERMISSION_IFRAME_ID');
+  iframe.src = chrome.runtime.getURL('requestPermissions.html');
+  document.body.appendChild(iframe);
+}

functional-samples/cookbook.offscreen-user-media/manifest.json

@@ -0,0 +1,25 @@
+{
+  "name": "Offscreen API - User media",
+  "version": "1.0",
+  "manifest_version": 3,
+  "description": "Shows how to record audio in a chrome extension via offscreen document.",
+  "background": {
+    "service_worker": "background.js"
+  },
+  "content_scripts": [
+    {
+      "matches": ["<all_urls>"],
+      "js": ["contentScript.js"]
+    }
+  ],
+  "action": {
+    "default_popup": "popup/popup.html"
+  },
+  "permissions": ["offscreen", "activeTab", "tabs"],
+  "web_accessible_resources": [
+    {
+      "resources": ["requestPermissions.html", "requestPermissions.js"],
+      "matches": ["<all_urls>"]
+    }
+  ]
+}

functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Offscreen Document</title>
+    <script src="./offscreen.js"></script>
+  </head>
+  <body></body>
+</html>

functional-samples/cookbook.offscreen-user-media/offscreen/offscreen.js

@@ -0,0 +1,152 @@
+/**
+ * MediaRecorder instance for audio recording.
+ * @type {MediaRecorder}
+ */
+let mediaRecorder;
+
+/**
+ * Event listener for messages from the extension.
+ * @param {Object} request - The message request.
+ * @param {Object} sender - The sender of the message.
+ * @param {function} sendResponse - Callback function to send a response.
+ * @returns {boolean} - Indicates whether the response should be asynchronous.
+ */
+chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
+  if (request.message.target !== 'offscreen') {
+    return;
+  }
+
+  switch (request.message.type) {
+    case 'START_OFFSCREEN_RECORDING':
+      // Start recording
+      handleRecording();
+      sendResponse({});
+      break;
+    case 'STOP_OFFSCREEN_RECORDING':
+      // Stop recording
+      stopRecording();
+      sendResponse({});
+      break;
+    case 'CHECK_PERMISSIONS':
+      checkAudioPermissions()
+        .then((data) => sendResponse(data))
+        .catch((errorData) => sendResponse(errorData));
+      break;
+    default:
+      break;
+  }
+
+  return true;
+});
+
+/**
+ * Stops the recording if the MediaRecorder is in the recording state.
+ */
+function stopRecording() {
+  if (mediaRecorder && mediaRecorder.state === 'recording') {
+    console.log('Stopped recording in offscreen...');
+    mediaRecorder.stop();
+  }
+}
+
+/**
+ * Initiates the audio recording process using MediaRecorder.
+ */
+async function handleRecording() {
+  getAudioInputDevices().then((audioInputDevices) => {
+    const deviceId = audioInputDevices[0].deviceId;
+    navigator.mediaDevices
+      .getUserMedia({
+        audio: {
+          deviceId: { exact: deviceId }
+        }
+      })
+      .then((audioStream) => {
+        try {
+          mediaRecorder = new MediaRecorder(audioStream);
+          mediaRecorder.ondataavailable = (event) => {
+            if (mediaRecorder.state === 'recording') {
+              saveAudioChunks([event.data]);
+            }
+          };
+          mediaRecorder.onstop = handleStopRecording;
+
+          // Start MediaRecorder and capture chunks every 10s.
+          mediaRecorder.start(10000);
+
+          console.log('Started recording in offscreen...');
+        } catch (error) {
+          console.error(
+            'Unable to initiate MediaRecorder and/or streams',
+            error
+          );
+        }
+      });
+  });
+}
+
+/**
+ * Saves audio chunks captured by MediaRecorder.
+ * @param {Blob[]} chunkData - Array of audio chunks in Blob format.
+ */
+function saveAudioChunks(chunkData) {
+  console.log('Chunk captured from MediaRecorder');
+  // Manage audio chunks accordingly as per your needs
+}
+
+/**
+ * Event handler for when MediaRecorder is stopped.
+ */
+function handleStopRecording() {
+  // Handle cases when MediaRecorder is stopped if needed
+}
+
+/**
+ * Fetches audio input devices using the `navigator.mediaDevices.enumerateDevices` API.
+ * @returns {Promise<Object[]>} - Promise that resolves to an array of audio input devices.
+ */
+function getAudioInputDevices() {
+  return new Promise((resolve, reject) => {
+    navigator.mediaDevices
+      .enumerateDevices()
+      .then((devices) => {
+        // Filter the devices to include only audio input devices
+        const audioInputDevices = devices.filter(
+          (device) => device.kind === 'audioinput'
+        );
+        resolve(audioInputDevices);
+      })
+      .catch((error) => {
+        console.log('Error getting audio input devices', error);
+        reject(error);
+      });
+  });
+}
+
+/**
+ * Checks microphone permissions using the `navigator.permissions.query` API.
+ * @returns {Promise<Object>} - Promise that resolves to an object containing permission status.
+ */
+function checkAudioPermissions() {
+  return new Promise((resolve, reject) => {
+    navigator.permissions
+      .query({ name: 'microphone' })
+      .then((result) => {
+        if (result.state === 'granted') {
+          console.log('Mic permissions granted');
+          resolve({ message: { status: 'success' } });
+        } else {
+          console.log('Mic permissions missing', result.state);
+          reject({
+            message: { status: 'error', data: result.state }
+          });
+        }
+      })
+      .catch((error) => {
+        console.warn('Permissions error', error);
+        reject({
+          message: { status: 'error', data: { error: error } }
+        });
+      });
+  });
+}

functional-samples/cookbook.offscreen-user-media/popup/popup.css

@@ -0,0 +1,29 @@
+body {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100px;
+  margin: 0;
+}
+
+#popup-container {
+  text-align: center;
+}
+
+#start-button,
+#stop-button {
+  padding: 10px;
+  font-size: 14px;
+  cursor: pointer;
+  margin: 5px;
+}
+
+#start-button {
+  background-color: #4caf50; /* Green */
+  color: white;
+}
+
+#stop-button {
+  background-color: #f44336; /* Red */
+  color: white;
+}

functional-samples/cookbook.offscreen-user-media/popup/popup.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet" href="./popup.css" />
+    <title>Record Extension</title>
+  </head>
+  <body>
+    <div id="popup-container">
+      <button id="start-button">Start Recording</button>
+      <button id="stop-button">Stop Recording</button>
+    </div>
+    <script src="./popup.js"></script>
+  </body>
+</html>

functional-samples/cookbook.offscreen-user-media/popup/popup.js

@@ -0,0 +1,25 @@
+document.addEventListener('DOMContentLoaded', function () {
+  const startButton = document.getElementById('start-button');
+  const stopButton = document.getElementById('stop-button');
+
+  // Function to send a message to the background script
+  const sendMessageToBackground = (message) => {
+    chrome.runtime.sendMessage({
+      message: {
+        type: 'TOGGLE_RECORDING',
+        target: 'background',
+        data: message
+      }
+    });
+  };
+
+  // Add a click event listener to the start button
+  startButton.addEventListener('click', function () {
+    sendMessageToBackground('START');
+  });
+
+  // Add a click event listener to the stop button
+  stopButton.addEventListener('click', function () {
+    sendMessageToBackground('STOP');
+  });
+});

functional-samples/cookbook.offscreen-user-media/requestPermissions.html

@@ -0,0 +1,8 @@
+<!-- requestPermissions.html -->
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Request Permissions</title>
+    <script src="requestPermissions.js"></script>
+  </head>
+</html>

functional-samples/cookbook.offscreen-user-media/requestPermissions.js

@@ -0,0 +1,45 @@
+/**
+ * Requests user permission for microphone access and sends a message to the parent window.
+ */
+function getUserPermission() {
+  console.info('Getting user permission for microphone access...');
+
+  navigator.mediaDevices
+    .getUserMedia({ audio: true })
+    .then((response) => {
+      if (response.id !== null && response.id !== undefined) {
+        console.log('Microphone access granted');
+        // Post a message to the parent window indicating successful permission
+        window.parent.postMessage({ type: 'permissionsGranted' }, '*');
+        return;
+      }
+      // Post a message to the parent window indicating failed permission
+      window.parent.postMessage(
+        {
+          type: 'permissionsFailed'
+        },
+        '*'
+      );
+    })
+    .catch((error) => {
+      console.warn('Error requesting microphone permission: ', error);
+      if (error.message === 'Permission denied') {
+        // Show an alert if permission is denied
+        window.alert(
+          'Please allow microphone access. Highlight uses your microphone to record audio during meetings.'
+        );
+      }
+
+      // Post a message to the parent window indicating failed permission with an optional error message
+      window.parent.postMessage(
+        {
+          type: 'permissionsFailed',
+          message: error.message
+        },
+        '*'
+      );
+    });
+}
+
+// Call the function to request microphone permission
+getUserPermission();