Implement screen track capabilities

rtvi-client-js-daily/src/transport.ts

@@ -3,6 +3,7 @@ import Daily, {
   DailyEventObjectAppMessage,
   DailyEventObjectAvailableDevicesUpdated,
   DailyEventObjectLocalAudioLevel,
+  DailyEventObjectNonFatalError,
   DailyEventObjectParticipant,
   DailyEventObjectParticipantLeft,
   DailyEventObjectRemoteParticipantsAudioLevel,
@@ -133,6 +134,8 @@ export class DailyTransport extends Transport {
     const tracks: Tracks = {
       local: {
         audio: participants?.local?.tracks?.audio?.persistentTrack,
+        screenAudio: participants?.local?.tracks?.screenAudio?.persistentTrack,
+        screenVideo: participants?.local?.tracks?.screenVideo?.persistentTrack,
         video: participants?.local?.tracks?.video?.persistentTrack,
       },
     };
@@ -242,6 +245,7 @@ export class DailyTransport extends Transport {
     );
     this._daily.on("app-message", this.handleAppMessage.bind(this));
     this._daily.on("left-meeting", this.handleLeftMeeting.bind(this));
+    this._daily.on("nonfatal-error", this.handleNonFatalError.bind(this));
   }
 
   async disconnect() {
@@ -256,6 +260,18 @@ export class DailyTransport extends Transport {
     this._daily.sendAppMessage(message, "*");
   }
 
+  public startScreenShare(): void {
+    this._daily.startScreenShare();
+  }
+
+  public stopScreenShare(): void {
+    this._daily.stopScreenShare();
+  }
+
+  public isSharingScreen(): boolean {
+    return this._daily.localScreenAudio() || this._daily.localScreenVideo();
+  }
+
   private handleAppMessage(ev: DailyEventObjectAppMessage) {
     // Bubble any messages with realtime-ai label
     if (ev.data.label === "rtvi-ai") {
@@ -296,17 +312,39 @@ export class DailyTransport extends Transport {
   }
 
   private handleTrackStarted(ev: DailyEventObjectTrack) {
-    this._callbacks.onTrackStarted?.(
-      ev.track,
-      ev.participant ? dailyParticipantToParticipant(ev.participant) : undefined
-    );
+    if (ev.type === "screenAudio" || ev.type === "screenVideo") {
+      this._callbacks.onScreenTrackStarted?.(
+        ev.track,
+        ev.participant
+          ? dailyParticipantToParticipant(ev.participant)
+          : undefined
+      );
+    } else {
+      this._callbacks.onTrackStarted?.(
+        ev.track,
+        ev.participant
+          ? dailyParticipantToParticipant(ev.participant)
+          : undefined
+      );
+    }
   }
 
   private handleTrackStopped(ev: DailyEventObjectTrack) {
-    this._callbacks.onTrackStopped?.(
-      ev.track,
-      ev.participant ? dailyParticipantToParticipant(ev.participant) : undefined
-    );
+    if (ev.type === "screenAudio" || ev.type === "screenVideo") {
+      this._callbacks.onScreenTrackStopped?.(
+        ev.track,
+        ev.participant
+          ? dailyParticipantToParticipant(ev.participant)
+          : undefined
+      );
+    } else {
+      this._callbacks.onTrackStopped?.(
+        ev.track,
+        ev.participant
+          ? dailyParticipantToParticipant(ev.participant)
+          : undefined
+      );
+    }
   }
 
   private handleParticipantJoined(ev: DailyEventObjectParticipant) {
@@ -357,6 +395,14 @@ export class DailyTransport extends Transport {
     this._botId = "";
     this._callbacks.onDisconnected?.();
   }
+
+  private handleNonFatalError(ev: DailyEventObjectNonFatalError) {
+    switch (ev.type) {
+      case "screen-share-error":
+        this._callbacks.onScreenShareError?.(ev.errorMsg);
+        break;
+    }
+  }
 }
 
 const dailyParticipantToParticipant = (p: DailyParticipant): Participant => ({

rtvi-client-js/src/core.ts

@@ -45,6 +45,15 @@ export type VoiceEventCallbacks = Partial<{
 
   onTrackStarted: (track: MediaStreamTrack, participant?: Participant) => void;
   onTrackStopped: (track: MediaStreamTrack, participant?: Participant) => void;
+  onScreenTrackStarted: (
+    track: MediaStreamTrack,
+    participant?: Participant
+  ) => void;
+  onScreenTrackStopped: (
+    track: MediaStreamTrack,
+    participant?: Participant
+  ) => void;
+  onScreenShareError: (errorMessage: string) => void;
   onLocalAudioLevel: (level: number) => void;
   onRemoteAudioLevel: (level: number, participant: Participant) => void;
 
@@ -126,6 +135,18 @@ export abstract class Client extends (EventEmitter as new () => TypedEmitter<Voi
         options?.callbacks?.onTrackStopped?.(track, p);
         this.emit(VoiceEvent.TrackedStopped, track, p);
       },
+      onScreenTrackStarted: (track, p) => {
+        options?.callbacks?.onScreenTrackStarted?.(track, p);
+        this.emit(VoiceEvent.ScreenTrackStarted, track, p);
+      },
+      onScreenTrackStopped: (track, p) => {
+        options?.callbacks?.onScreenTrackStopped?.(track, p);
+        this.emit(VoiceEvent.ScreenTrackStopped, track, p);
+      },
+      onScreenShareError: (errorMessage) => {
+        options?.callbacks?.onScreenShareError?.(errorMessage);
+        this.emit(VoiceEvent.ScreenShareError, errorMessage);
+      },
       onAvailableCamsUpdated: (cams) => {
         options?.callbacks?.onAvailableCamsUpdated?.(cams);
         this.emit(VoiceEvent.AvailableCamsUpdated, cams);
@@ -475,6 +496,18 @@ export abstract class Client extends (EventEmitter as new () => TypedEmitter<Voi
     return this._transport.tracks();
   }
 
+  public startScreenShare() {
+    return this._transport.startScreenShare();
+  }
+
+  public stopScreenShare() {
+    return this._transport.stopScreenShare();
+  }
+
+  public isSharingScreen() {
+    return this._transport.isSharingScreen();
+  }
+
   // ------ Config methods
 
   /**

rtvi-client-js/src/events.ts

@@ -11,6 +11,7 @@ import { Participant, TransportState } from "./transport";
 export enum VoiceEvent {
   MessageError = "messageError",
   Error = "error",
+  ScreenShareError = "screenShareError",
 
   Connected = "connected",
   Disconnected = "disconnected",
@@ -24,6 +25,8 @@ export enum VoiceEvent {
   ParticipantLeft = "participantLeft",
   TrackStarted = "trackStarted",
   TrackedStopped = "trackStopped",
+  ScreenTrackStarted = "screenTrackStarted",
+  ScreenTrackStopped = "screenTrackStopped",
 
   AvailableCamsUpdated = "availableCamsUpdated",
   AvailableMicsUpdated = "availableMicsUpdated",
@@ -63,6 +66,8 @@ export type VoiceEvents = Partial<{
   participantLeft: (p: Participant) => void;
   trackStarted: (track: MediaStreamTrack, p?: Participant) => void;
   trackStopped: (track: MediaStreamTrack, p?: Participant) => void;
+  screenTrackStarted: (track: MediaStreamTrack, p?: Participant) => void;
+  screenTrackStopped: (track: MediaStreamTrack, p?: Participant) => void;
 
   availableCamsUpdated: (cams: MediaDeviceInfo[]) => void;
   availableMicsUpdated: (cams: MediaDeviceInfo[]) => void;
@@ -86,6 +91,7 @@ export type VoiceEvents = Partial<{
 
   error: (message: VoiceMessage) => void;
   messageError: (message: VoiceMessage) => void;
+  screenShareError: (errorMessage: string) => void;
 
   llmFunctionCall: (func: LLMFunctionCallData) => void;
   llmFunctionCallStart: (functionName: string) => void;

rtvi-client-js/src/transport/core.ts

@@ -21,6 +21,8 @@ export type Participant = {
 export type Tracks = {
   local: {
     audio?: MediaStreamTrack;
+    screenAudio?: MediaStreamTrack;
+    screenVideo?: MediaStreamTrack;
     video?: MediaStreamTrack;
   };
   bot?: {
@@ -78,4 +80,8 @@ export abstract class Transport {
   abstract get expiry(): number | undefined;
 
   abstract tracks(): Tracks;
+
+  abstract startScreenShare(): void;
+  abstract stopScreenShare(): void;
+  abstract isSharingScreen(): boolean;
 }

rtvi-client-react/src/VoiceClientVideo.tsx

@@ -12,6 +12,11 @@ interface Props
   extends Omit<React.VideoHTMLAttributes<HTMLVideoElement>, "onResize"> {
   participant: "local" | "bot";
 
+  /**
+   * Defines the video track type to display. Default: 'video'.
+   */
+  trackType: "screenVideo" | "video";
+
   /**
    * Defines whether the video should be fully contained or cover the box. Default: 'contain'.
    */
@@ -36,12 +41,13 @@ export const VoiceClientVideo = forwardRef<HTMLVideoElement, Props>(
       mirror,
       onResize,
       style = {},
+      trackType = "video",
       ...props
     },
     ref
   ) {
     const videoTrack: MediaStreamTrack | null = useVoiceClientMediaTrack(
-      "video",
+      trackType,
       participant
     );
 

rtvi-client-react/src/useVoiceClientMediaTrack.ts

@@ -11,15 +11,27 @@ type TrackType = keyof Tracks["local"];
 
 const localAudioTrackAtom = atom<MediaStreamTrack | null>(null);
 const localVideoTrackAtom = atom<MediaStreamTrack | null>(null);
+const localScreenAudioTrackAtom = atom<MediaStreamTrack | null>(null);
+const localScreenVideoTrackAtom = atom<MediaStreamTrack | null>(null);
 const botAudioTrackAtom = atom<MediaStreamTrack | null>(null);
 const botVideoTrackAtom = atom<MediaStreamTrack | null>(null);
 
 const trackAtom = atomFamily<
   { local: boolean; trackType: TrackType },
   PrimitiveAtom<MediaStreamTrack | null>
 >(({ local, trackType }) => {
-  if (local)
-    return trackType === "audio" ? localAudioTrackAtom : localVideoTrackAtom;
+  if (local) {
+    switch (trackType) {
+      case "audio":
+        return localAudioTrackAtom;
+      case "screenAudio":
+        return localScreenAudioTrackAtom;
+      case "screenVideo":
+        return localScreenVideoTrackAtom;
+      case "video":
+        return localVideoTrackAtom;
+    }
+  }
   return trackType === "audio" ? botAudioTrackAtom : botVideoTrackAtom;
 });
 
@@ -60,6 +72,14 @@ export const useVoiceClientMediaTrack = (
     }, [])
   );
 
+  useVoiceClientEvent(
+    VoiceEvent.ScreenTrackStarted,
+    useCallback((track: MediaStreamTrack, participant?: Participant) => {
+      const trackType = track.kind === "audio" ? "screenAudio" : "screenVideo";
+      updateTrack(track, trackType, Boolean(participant?.local));
+    }, [])
+  );
+
   useEffect(() => {
     if (!voiceClient) return;
     const tracks = voiceClient.tracks();