Add background blur feature for supported devices

knip.ts

@@ -24,6 +24,10 @@ export default {
     // then Knip will flag it as a false positive
     // https://github.com/webpro-nl/knip/issues/766
     "@vector-im/compound-web",
+    // We need this so the eslint is happy with @livekit/track-processors.
+    // This might be a bug in the livekit repo but for now we fix it on the
+    // element call side.
+    "@types/dom-mediacapture-transform",
     "matrix-widget-api",
   ],
   ignoreExportsUsedInFile: true,

locales/en/app.json

@@ -150,6 +150,9 @@
       "effect_volume_description": "Volume for sound effects such as: joining or leaving a call, and reactions.",
       "effect_volume_label": "Sound effect volume"
     },
+    "background_blur_header": "Background",
+    "background_blur_label": "Blur the background of the video",
+    "blur_not_supported_by_browser": "(Background blur is not supported by this device)",
     "developer_settings_label": "Developer Settings",
     "developer_settings_label_description": "Expose developer settings in the settings window.",
     "developer_tab_title": "Developer",

package.json

@@ -31,6 +31,8 @@
     "@formatjs/intl-segmenter": "^11.7.3",
     "@livekit/components-core": "^0.11.0",
     "@livekit/components-react": "^2.0.0",
+    "@livekit/track-processors": "^0.3.2",
+    "@mediapipe/tasks-vision": "^0.10.18",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/core": "^1.25.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
@@ -50,6 +52,7 @@
     "@testing-library/react-hooks": "^8.0.1",
     "@testing-library/user-event": "^14.5.1",
     "@types/content-type": "^1.1.5",
+    "@types/dom-mediacapture-transform": "^0.1.10",
     "@types/grecaptcha": "^3.0.9",
     "@types/jsdom": "^21.1.7",
     "@types/lodash-es": "^4.17.12",

src/App.tsx

@@ -28,6 +28,7 @@ import { Initializer } from "./initializer";
 import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
+import { ProcessorProvider } from "./livekit/TrackProcessorContext";
 
 const SentryRoute = Sentry.withSentryRouting(Route);
 
@@ -82,27 +83,25 @@ export const App: FC<AppProps> = ({ history }) => {
           <TooltipProvider>
             {loaded ? (
               <Suspense fallback={null}>
-                <ClientProvider>
-                  <MediaDevicesProvider>
-                    <Sentry.ErrorBoundary fallback={errorPage}>
-                      <DisconnectedBanner />
-                      <Switch>
-                        <SentryRoute exact path="/">
-                          <HomePage />
-                        </SentryRoute>
-                        <SentryRoute exact path="/login">
-                          <LoginPage />
-                        </SentryRoute>
-                        <SentryRoute exact path="/register">
-                          <RegisterPage />
-                        </SentryRoute>
-                        <SentryRoute path="*">
-                          <RoomPage />
-                        </SentryRoute>
-                      </Switch>
-                    </Sentry.ErrorBoundary>
-                  </MediaDevicesProvider>
-                </ClientProvider>
+                <Providers>
+                  <Sentry.ErrorBoundary fallback={errorPage}>
+                    <DisconnectedBanner />
+                    <Switch>
+                      <SentryRoute exact path="/">
+                        <HomePage />
+                      </SentryRoute>
+                      <SentryRoute exact path="/login">
+                        <LoginPage />
+                      </SentryRoute>
+                      <SentryRoute exact path="/register">
+                        <RegisterPage />
+                      </SentryRoute>
+                      <SentryRoute path="*">
+                        <RoomPage />
+                      </SentryRoute>
+                    </Switch>
+                  </Sentry.ErrorBoundary>
+                </Providers>
               </Suspense>
             ) : (
               <LoadingView />
@@ -113,3 +112,16 @@ export const App: FC<AppProps> = ({ history }) => {
     </Router>
   );
 };
+
+const Providers: FC<{
+  children: JSX.Element;
+}> = ({ children }) => {
+  // We use this to stack all used providers to not make the App component to verbose
+  return (
+    <ClientProvider>
+      <MediaDevicesProvider>
+        <ProcessorProvider>{children}</ProcessorProvider>
+      </MediaDevicesProvider>
+    </ClientProvider>
+  );
+};

src/livekit/BlurBackgroundTransformer.ts

@@ -0,0 +1,62 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundTransformer,
+  VideoTransformer,
+  VideoTransformerInitOptions,
+} from "@livekit/track-processors";
+import { ImageSegmenter } from "@mediapipe/tasks-vision";
+
+interface WasmFileset {
+  /** The path to the Wasm loader script. */
+  wasmLoaderPath: string;
+  /** The path to the Wasm binary. */
+  wasmBinaryPath: string;
+}
+
+// n.b. this only includes the SIMD versions of the WASM files which have good support:
+// https://caniuse.com/?search=simd
+const wasmFileset: WasmFileset = {
+  wasmLoaderPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
+    import.meta.url,
+  ).href,
+  wasmBinaryPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
+    import.meta.url,
+  ).href,
+};
+
+const modelAssetPath = new URL(
+  "../mediapipe/imageSegmenter/selfie_segmenter.tflite",
+  import.meta.url,
+).href;
+
+export class BlurBackgroundTransformer extends BackgroundTransformer {
+  public async init({
+    outputCanvas,
+    inputElement: inputVideo,
+  }: VideoTransformerInitOptions): Promise<void> {
+    // call super.super.init()
+    await VideoTransformer.prototype.init.call(this, {
+      outputCanvas,
+      inputElement: inputVideo,
+    });
+
+    this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
+      baseOptions: {
+        modelAssetPath,
+        delegate: "GPU",
+        ...this.options.segmenterOptions,
+      },
+      runningMode: "VIDEO",
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    });
+  }
+}

src/livekit/TrackProcessorContext.tsx

@@ -0,0 +1,113 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { BackgroundOptions, ProcessorWrapper } from "@livekit/track-processors";
+import {
+  createContext,
+  FC,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+import { logger } from "matrix-js-sdk/src/logger";
+import { LocalVideoTrack } from "livekit-client";
+
+import {
+  backgroundBlur as backgroundBlurSettings,
+  useSetting,
+} from "../settings/settings";
+import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
+
+type ProcessorState = {
+  supported: boolean | undefined;
+  processor: undefined | ProcessorWrapper<BackgroundOptions>;
+  /**
+   * Call this method to try to initialize a processor.
+   * This only needs to happen if supported is undefined.
+   * If the backgroundBlur setting is set to true this does not need to be called
+   * and the processorState.supported will update automatically to the correct value.
+   */
+  checkSupported: () => void;
+};
+const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
+
+export const useTrackProcessor = (): ProcessorState | undefined =>
+  useContext(ProcessorContext);
+
+export const useTrackProcessorSync = (
+  videoTrack: LocalVideoTrack | null,
+): void => {
+  const { processor } = useTrackProcessor() || {};
+  useEffect(() => {
+    if (processor && !videoTrack?.getProcessor()) {
+      void videoTrack?.setProcessor(processor);
+    }
+    if (!processor && videoTrack?.getProcessor()) {
+      void videoTrack?.stopProcessor();
+    }
+  }, [processor, videoTrack]);
+};
+
+interface Props {
+  children: JSX.Element;
+}
+export const ProcessorProvider: FC<Props> = ({ children }) => {
+  // The setting the user wants to have
+  const [blurActivated] = useSetting(backgroundBlurSettings);
+
+  // If `ProcessorState.supported` is undefined the user can activate that we want
+  // to have it at least checked (this is useful to show the settings menu properly)
+  // We dont want to try initializing the blur if the user is not even looking at the setting
+  const [shouldCheckSupport, setShouldCheckSupport] = useState(blurActivated);
+
+  // Cache the processor so we only need to initialize it once.
+  const blur = useRef<ProcessorWrapper<BackgroundOptions> | undefined>(
+    undefined,
+  );
+
+  const checkSupported = useCallback(() => {
+    setShouldCheckSupport(true);
+  }, []);
+  // This is the actual state exposed through the context
+  const [processorState, setProcessorState] = useState<ProcessorState>(() => ({
+    supported: false,
+    processor: undefined,
+    checkSupported,
+  }));
+
+  useEffect(() => {
+    if (!shouldCheckSupport) return;
+    try {
+      if (!blur.current) {
+        blur.current = new ProcessorWrapper(
+          new BlurBackgroundTransformer({ blurRadius: 15 }),
+          "background-blur",
+        );
+      }
+      setProcessorState({
+        checkSupported,
+        supported: true,
+        processor: blurActivated ? blur.current : undefined,
+      });
+    } catch (e) {
+      setProcessorState({
+        checkSupported,
+        supported: false,
+        processor: undefined,
+      });
+      logger.error("disable background blur", e);
+    }
+  }, [blurActivated, checkSupported, shouldCheckSupport]);
+
+  return (
+    <ProcessorContext.Provider value={processorState}>
+      {children}
+    </ProcessorContext.Provider>
+  );
+};

src/livekit/useLiveKit.ts

@@ -9,6 +9,7 @@ import {
   ConnectionState,
   E2EEOptions,
   ExternalE2EEKeyProvider,
+  LocalVideoTrack,
   Room,
   RoomOptions,
   Track,
@@ -33,6 +34,11 @@ import {
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { useInitial } from "../useInitial";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
@@ -79,12 +85,15 @@ export function useLiveKit(
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
 
+  const { processor } = useTrackProcessor() || {};
+  const initialProcessor = useInitial(() => processor);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
+        processor: initialProcessor,
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -95,7 +104,7 @@ export function useLiveKit(
       },
       e2ee: e2eeOptions,
     }),
-    [e2eeOptions],
+    [e2eeOptions, initialProcessor],
   );
 
   // Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -120,6 +129,15 @@ export function useLiveKit(
     return r;
   }, [roomOptions, e2eeSystem]);
 
+  const videoTrack = useMemo(
+    () =>
+      Array.from(room.localParticipant.videoTrackPublications.values()).find(
+        (v) => v.source === Track.Source.Camera,
+      )?.track as LocalVideoTrack | null,
+    [room.localParticipant.videoTrackPublications],
+  );
+  useTrackProcessorSync(videoTrack);
+
   const connectionState = useECConnectionState(
     {
       deviceId: initialDevices.current.audioInput.selectedId,
@@ -195,13 +213,15 @@ export function useLiveKit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
+                  room.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
               case MuteDevice.Camera:
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
+                  room.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;

src/mediapipe/imageSegmenter/README.md

@@ -0,0 +1,5 @@
+# Google AI Edge MediaPipe Selfie Segmentation
+
+- See: https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
+- Latest: https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite
+- License: Apache 2.0 as per https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf