Use local assets for blurring

package.json

@@ -32,6 +32,7 @@
     "@livekit/components-core": "^0.11.0",
     "@livekit/components-react": "^2.0.0",
     "@livekit/track-processors": "^0.3.2",
+    "@mediapipe/tasks-vision": "^0.10.18",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/core": "^1.25.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.55.0",

src/livekit/BlurBackgroundTransformer.ts

@@ -0,0 +1,62 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundTransformer,
+  VideoTransformer,
+  VideoTransformerInitOptions,
+} from "@livekit/track-processors";
+import { ImageSegmenter } from "@mediapipe/tasks-vision";
+
+interface WasmFileset {
+  /** The path to the Wasm loader script. */
+  wasmLoaderPath: string;
+  /** The path to the Wasm binary. */
+  wasmBinaryPath: string;
+}
+
+// n.b. this only includes the SIMD versions of the WASM files which have good support:
+// https://caniuse.com/?search=simd
+const wasmFileset: WasmFileset = {
+  wasmLoaderPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
+    import.meta.url,
+  ).href,
+  wasmBinaryPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
+    import.meta.url,
+  ).href,
+};
+
+const modelAssetPath = new URL(
+  "../mediapipe/imageSegmenter/selfie_segmenter.tflite",
+  import.meta.url,
+).href;
+
+export class BlurBackgroundTransformer extends BackgroundTransformer {
+  public async init({
+    outputCanvas,
+    inputElement: inputVideo,
+  }: VideoTransformerInitOptions): Promise<void> {
+    // call super.super.init()
+    await VideoTransformer.prototype.init.call(this, {
+      outputCanvas,
+      inputElement: inputVideo,
+    });
+
+    this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
+      baseOptions: {
+        modelAssetPath,
+        delegate: "GPU",
+        ...this.options.segmenterOptions,
+      },
+      runningMode: "VIDEO",
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    });
+  }
+}

src/livekit/TrackProcessorContext.tsx

@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import {
-  BackgroundBlur as backgroundBlur,
-  BackgroundOptions,
-  ProcessorWrapper,
-} from "@livekit/track-processors";
+import { BackgroundOptions, ProcessorWrapper } from "@livekit/track-processors";
 import {
   createContext,
   FC,
@@ -26,6 +22,7 @@ import {
   backgroundBlur as backgroundBlurSettings,
   useSetting,
 } from "../settings/settings";
+import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
 
 type ProcessorState = {
   supported: boolean | undefined;
@@ -87,7 +84,12 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
   useEffect(() => {
     if (!shouldCheckSupport) return;
     try {
-      if (!blur.current) blur.current = backgroundBlur(15, { delegate: "GPU" });
+      if (!blur.current) {
+        blur.current = new ProcessorWrapper(
+          new BlurBackgroundTransformer({ blurRadius: 15 }),
+          "background-blur",
+        );
+      }
       setProcessorState({
         checkSupported,
         supported: true,

src/mediapipe/imageSegmenter/README.md

@@ -0,0 +1,5 @@
+# Google AI Edge MediaPipe Selfie Segmentation
+
+- See: https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
+- Latest: https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite
+- License: Apache 2.0 as per https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf

yarn.lock

@@ -1840,6 +1840,11 @@
   resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.9.tgz#fbd669f50ac2e888b2c64c9c9863927c111da02f"
   integrity sha512-/gFguyJm1ng4Qr7VVH2vKO+zZcQd8wc3YafUfvBuYFX0Y5+CvrV+VNPEVkl5W/gUZF5KNKNZAiaHPULGPCIjyQ==
 
+"@mediapipe/tasks-vision@^0.10.18":
+  version "0.10.18"
+  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.18.tgz#9ea0f0bf7506378c55ee661fa70aa0910f21f9b5"
+  integrity sha512-NRIlyqhGUz1Jdgcs6YybwPRhLK6dgeGAqAMXepIczEQ7FmA/0ouFtgMO1g9SPf/HaDSO8pNVdP54dAb9s9wj/Q==
+
 "@nodelib/[email protected]":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"