webrtc: initial prep for negotiated intercom codecs

koush · Oct 19, 2023 · ed35811 · ed35811
1 parent ae2228f
commit ed35811
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 12 deletions.
diff --git a/plugins/webrtc/src/ffmpeg-to-wrtc.ts b/plugins/webrtc/src/ffmpeg-to-wrtc.ts
@@ -1,7 +1,7 @@
 import { MediaStreamTrack, PeerConfig, RTCPeerConnection, RTCRtpCodecParameters, RTCRtpTransceiver, RtpPacket } from "./werift";
 
 import { Deferred } from "@scrypted/common/src/deferred";
-import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk";
+import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCInputMediaObjectTrack, RTCMediaObjectTrack, RTCOutputMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk";
 import { ScryptedSessionControl } from "./session-control";
 import { requiredAudioCodecs, requiredVideoCodec } from "./webrtc-required-codecs";
 import { logIsLocalIceTransport } from "./werift-util";
@@ -362,7 +362,7 @@ export function parseOptions(options: RTCSignalingOptions) {
     };
 }
 
-class WebRTCTrack implements RTCMediaObjectTrack {
+class WebRTCTrack implements RTCOutputMediaObjectTrack, RTCInputMediaObjectTrack {
     control: ScryptedSessionControl;
     removed = new Deferred<void>();
 
@@ -414,8 +414,8 @@ class WebRTCTrack implements RTCMediaObjectTrack {
         return this.cleanup(false);
     }
 
-    setPlayback(options: { audio: boolean; video: boolean; }): Promise<void> {
-        return this.control.setPlayback(options);
+    setPlayback(options: { audio: boolean; video: boolean; }): Promise<MediaObject> {
+        return this.control.setPlaybackInternal(options);
     }
 }
 
@@ -532,9 +532,16 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement {
         }
     }
 
+    addInputTrack(options: { videoMid?: string; audioMid?: string; }): Promise<RTCInputMediaObjectTrack> {
+        throw new Error('not implemented');
+    }
+
     async addTrack(mediaObject: MediaObject, options?: {
         videoMid?: string,
         audioMid?: string,
+        /**
+         * @deprecated
+         */
         intercomId?: string,
     }) {
         const { atrack, vtrack, createTrackForwarder, intercom } = await this.createTracks(mediaObject, options?.intercomId);

diff --git a/plugins/webrtc/src/session-control.ts b/plugins/webrtc/src/session-control.ts
@@ -3,7 +3,7 @@ import { Deferred } from "@scrypted/common/src/deferred";
 import { listenZeroSingleClient } from "@scrypted/common/src/listen-cluster";
 import { RtspServer } from "@scrypted/common/src/rtsp-server";
 import { createSdpInput, parseSdp } from "@scrypted/common/src/sdp-utils";
-import sdk, { FFmpegInput, Intercom, RTCSessionControl } from "@scrypted/sdk";
+import sdk, { FFmpegInput, Intercom, MediaObject, RTCSessionControl } from "@scrypted/sdk";
 
 const { mediaManager } = sdk;
 
@@ -22,7 +22,11 @@ export class ScryptedSessionControl implements RTCSessionControl {
         });
     }
 
-    async setPlayback(options: { audio: boolean; video: boolean; }) {
+    async setPlayback(options: { audio: boolean; video: boolean; }): Promise<void> {
+        await this.setPlaybackInternal(options);
+    }
+
+    async setPlaybackInternal(options: { audio: boolean; video: boolean; }): Promise<MediaObject> {
         if (this.killed.finished)
             return;
 
@@ -52,7 +56,9 @@ export class ScryptedSessionControl implements RTCSessionControl {
                 video: null,
             },
             inputArguments: [
-                '-rtsp_transport', 'udp',
+                '-analyzeduration', '0',
+                '-probesize', '512',
+                '-rtsp_transport', 'tcp',
                 '-i', url,
             ],
         };
@@ -79,7 +85,7 @@ export class ScryptedSessionControl implements RTCSessionControl {
         sdp = createSdpInput(0, 0, sdp);
 
 
-        const rtspServer = new RtspServer(client, sdp, true);
+        const rtspServer = new RtspServer(client, sdp);
         this.rtspServer = rtspServer;
         // rtspServer.console = console;
         await rtspServer.handlePlayback();
@@ -90,6 +96,8 @@ export class ScryptedSessionControl implements RTCSessionControl {
             rtpPacket.header.payloadType = 110;
             rtspServer.sendTrack(audioTrack, rtpPacket.serialize(), false);
         });
+
+        return mo;
     }
 
     async getRefreshAt() {

diff --git a/sdk/types/src/types.input.ts b/sdk/types/src/types.input.ts
@@ -1,5 +1,5 @@
-import type { Worker as NodeWorker } from 'worker_threads';
 import type { Socket as NodeNetSocket } from 'net';
+import type { Worker as NodeWorker } from 'worker_threads';
 
 export type ScryptedNativeId = string | undefined;
 
@@ -2105,12 +2105,24 @@ export interface RTCSessionControl {
  */
 export interface RTCMediaObjectTrack {
   onStop(): Promise<void>;
-  replace(mediaObject: MediaObject): Promise<void>;
   stop(): Promise<void>;
+}
+
+/**
+ * @category WebRTC Reference
+ */
+export interface RTCOutputMediaObjectTrack extends RTCMediaObjectTrack{
+  replace(mediaObject: MediaObject): Promise<void>;
+}
+
+/**
+ * @category WebRTC Reference
+ */
+export interface RTCInputMediaObjectTrack extends RTCMediaObjectTrack{
   setPlayback(options: {
     audio: boolean,
     video: boolean,
-  }): Promise<void>;
+  }): Promise<MediaObject>;
 }
 
 /**
@@ -2121,8 +2133,15 @@ export interface RTCConnectionManagement {
   addTrack(mediaObject: MediaObject, options?: {
     videoMid?: string,
     audioMid?: string,
+    /**
+     * @deprecated
+     */
     intercomId?: string,
-  }): Promise<RTCMediaObjectTrack>;
+  }): Promise<RTCOutputMediaObjectTrack>;
+  addInputTrack(options: {
+    videoMid?: string,
+    audioMid?: string,
+  }): Promise<RTCInputMediaObjectTrack>;
   close(): Promise<void>;
   probe(): Promise<void>;
 }