From ed35811296f4861d1011f6d1de35080a7ac42596 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Thu, 19 Oct 2023 10:57:17 -0700 Subject: [PATCH] webrtc: initial prep for negotiated intercom codecs --- plugins/webrtc/src/ffmpeg-to-wrtc.ts | 15 +++++++++++---- plugins/webrtc/src/session-control.ts | 16 ++++++++++++---- sdk/types/src/types.input.ts | 27 +++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/plugins/webrtc/src/ffmpeg-to-wrtc.ts b/plugins/webrtc/src/ffmpeg-to-wrtc.ts index e1bd0de636..280f01a101 100644 --- a/plugins/webrtc/src/ffmpeg-to-wrtc.ts +++ b/plugins/webrtc/src/ffmpeg-to-wrtc.ts @@ -1,7 +1,7 @@ import { MediaStreamTrack, PeerConfig, RTCPeerConnection, RTCRtpCodecParameters, RTCRtpTransceiver, RtpPacket } from "./werift"; import { Deferred } from "@scrypted/common/src/deferred"; -import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk"; +import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCInputMediaObjectTrack, RTCMediaObjectTrack, RTCOutputMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk"; import { ScryptedSessionControl } from "./session-control"; import { requiredAudioCodecs, requiredVideoCodec } from "./webrtc-required-codecs"; import { logIsLocalIceTransport } from "./werift-util"; @@ -362,7 +362,7 @@ export function parseOptions(options: RTCSignalingOptions) { }; } -class WebRTCTrack implements RTCMediaObjectTrack { +class WebRTCTrack implements RTCOutputMediaObjectTrack, RTCInputMediaObjectTrack { control: ScryptedSessionControl; removed = new Deferred(); @@ -414,8 +414,8 @@ class WebRTCTrack implements RTCMediaObjectTrack { return this.cleanup(false); } - setPlayback(options: { audio: boolean; video: boolean; }): Promise { - return this.control.setPlayback(options); + setPlayback(options: { audio: boolean; video: boolean; }): Promise { + return this.control.setPlaybackInternal(options); } } @@ -532,9 +532,16 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement { } } + addInputTrack(options: { videoMid?: string; audioMid?: string; }): Promise { + throw new Error('not implemented'); + } + async addTrack(mediaObject: MediaObject, options?: { videoMid?: string, audioMid?: string, + /** + * @deprecated + */ intercomId?: string, }) { const { atrack, vtrack, createTrackForwarder, intercom } = await this.createTracks(mediaObject, options?.intercomId); diff --git a/plugins/webrtc/src/session-control.ts b/plugins/webrtc/src/session-control.ts index 0faf8c887a..3f0eb11322 100644 --- a/plugins/webrtc/src/session-control.ts +++ b/plugins/webrtc/src/session-control.ts @@ -3,7 +3,7 @@ import { Deferred } from "@scrypted/common/src/deferred"; import { listenZeroSingleClient } from "@scrypted/common/src/listen-cluster"; import { RtspServer } from "@scrypted/common/src/rtsp-server"; import { createSdpInput, parseSdp } from "@scrypted/common/src/sdp-utils"; -import sdk, { FFmpegInput, Intercom, RTCSessionControl } from "@scrypted/sdk"; +import sdk, { FFmpegInput, Intercom, MediaObject, RTCSessionControl } from "@scrypted/sdk"; const { mediaManager } = sdk; @@ -22,7 +22,11 @@ export class ScryptedSessionControl implements RTCSessionControl { }); } - async setPlayback(options: { audio: boolean; video: boolean; }) { + async setPlayback(options: { audio: boolean; video: boolean; }): Promise { + await this.setPlaybackInternal(options); + } + + async setPlaybackInternal(options: { audio: boolean; video: boolean; }): Promise { if (this.killed.finished) return; @@ -52,7 +56,9 @@ export class ScryptedSessionControl implements RTCSessionControl { video: null, }, inputArguments: [ - '-rtsp_transport', 'udp', + '-analyzeduration', '0', + '-probesize', '512', + '-rtsp_transport', 'tcp', '-i', url, ], }; @@ -79,7 +85,7 @@ export class ScryptedSessionControl implements RTCSessionControl { sdp = createSdpInput(0, 0, sdp); - const rtspServer = new RtspServer(client, sdp, true); + const rtspServer = new RtspServer(client, sdp); this.rtspServer = rtspServer; // rtspServer.console = console; await rtspServer.handlePlayback(); @@ -90,6 +96,8 @@ export class ScryptedSessionControl implements RTCSessionControl { rtpPacket.header.payloadType = 110; rtspServer.sendTrack(audioTrack, rtpPacket.serialize(), false); }); + + return mo; } async getRefreshAt() { diff --git a/sdk/types/src/types.input.ts b/sdk/types/src/types.input.ts index 96da2265b5..3639e72ff0 100644 --- a/sdk/types/src/types.input.ts +++ b/sdk/types/src/types.input.ts @@ -1,5 +1,5 @@ -import type { Worker as NodeWorker } from 'worker_threads'; import type { Socket as NodeNetSocket } from 'net'; +import type { Worker as NodeWorker } from 'worker_threads'; export type ScryptedNativeId = string | undefined; @@ -2105,12 +2105,24 @@ export interface RTCSessionControl { */ export interface RTCMediaObjectTrack { onStop(): Promise; - replace(mediaObject: MediaObject): Promise; stop(): Promise; +} + +/** + * @category WebRTC Reference + */ +export interface RTCOutputMediaObjectTrack extends RTCMediaObjectTrack{ + replace(mediaObject: MediaObject): Promise; +} + +/** + * @category WebRTC Reference + */ +export interface RTCInputMediaObjectTrack extends RTCMediaObjectTrack{ setPlayback(options: { audio: boolean, video: boolean, - }): Promise; + }): Promise; } /** @@ -2121,8 +2133,15 @@ export interface RTCConnectionManagement { addTrack(mediaObject: MediaObject, options?: { videoMid?: string, audioMid?: string, + /** + * @deprecated + */ intercomId?: string, - }): Promise; + }): Promise; + addInputTrack(options: { + videoMid?: string, + audioMid?: string, + }): Promise; close(): Promise; probe(): Promise; }