RTC2RTMP: Fix screen sharing stutter caused by packet loss. v5.0.216 v6.0.157 v7.0.18

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v7-changes"></a>
 
 ## SRS 7.0 Changelog
+* v7.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v7.0.18 (#4160)
 * v7.0, 2024-10-15, Merge [#3979](https://github.com/ossrs/srs/pull/3979): ST: Use clock_gettime to prevent time jumping backwards. v7.0.17 (#3979)
 * v7.0, 2024-09-09, Merge [#4158](https://github.com/ossrs/srs/pull/4158): Proxy: Support proxy server for SRS. v7.0.16 (#4158)
 * v7.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v7.0.15 (#4171)
@@ -29,6 +30,7 @@ The changelog for SRS.
 <a name="v6-changes"></a>
 
 ## SRS 6.0 Changelog
+* v6.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v6.0.157 (#4160)
 * v6.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v6.0.156 (#4171)
 * v6.0, 2024-09-01, Merge [#4165](https://github.com/ossrs/srs/pull/4165): FLV: Refine source and http handler. v6.0.155 (#4165)
 * v6.0, 2024-09-01, Merge [#4166](https://github.com/ossrs/srs/pull/4166): Edge: Fix flv edge crash when http unmount. v6.0.154 (#4166)
@@ -189,6 +191,7 @@ The changelog for SRS.
 <a name="v5-changes"></a>
 
 ## SRS 5.0 Changelog
+* v5.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v5.0.216 (#4160)
 * v5.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v5.0.215 (#4171)
 * v5.0, 2024-07-24, Merge [#4126](https://github.com/ossrs/srs/pull/4126): Edge: Improve stability for state and fd closing. v5.0.214 (#4126)
 * v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057)

trunk/research/players/js/srs.sdk.js

@@ -527,36 +527,56 @@ function SrsRtcWhipWhepAsync() {
     // @url The WebRTC url to publish with, for example:
     //      http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream
     // @options The options to control playing, supports:
-    //      videoOnly: boolean, whether only play video, default to false.
-    //      audioOnly: boolean, whether only play audio, default to false.
+    //      camera: boolean, whether capture video from camera, default to true.
+    //      screen: boolean, whether capture video from screen, default to false.
+    //      audio: boolean, whether play audio, default to true.
     self.publish = async function (url, options) {
         if (url.indexOf('/whip/') === -1) throw new Error(`invalid WHIP url ${url}`);
-        if (options?.videoOnly && options?.audioOnly) throw new Error(`The videoOnly and audioOnly in options can't be true at the same time`);
+        const hasAudio = options?.audio ?? true;
+        const useCamera = options?.camera ?? true;
+        const useScreen = options?.screen ?? false;
+
+        if (!hasAudio && !useCamera && !useScreen) throw new Error(`The camera, screen and audio can't be false at the same time`);
 
-        if (!options?.videoOnly) {
+        if (hasAudio) {
             self.pc.addTransceiver("audio", {direction: "sendonly"});
         } else {
             self.constraints.audio = false;
         }
 
-        if (!options?.audioOnly) {
+        if (useCamera || useScreen) {
             self.pc.addTransceiver("video", {direction: "sendonly"});
-        } else {
+        }
+
+        if (!useCamera) {
             self.constraints.video = false;
         }
 
         if (!navigator.mediaDevices && window.location.protocol === 'http:' && window.location.hostname !== 'localhost') {
             throw new SrsError('HttpsRequiredError', `Please use HTTPS or localhost to publish, read https://github.com/ossrs/srs/issues/2762#issuecomment-983147576`);
         }
-        var stream = await navigator.mediaDevices.getUserMedia(self.constraints);
 
-        // @see https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addStream#Migrating_to_addTrack
-        stream.getTracks().forEach(function (track) {
-            self.pc.addTrack(track);
+        if (useScreen) {
+            const displayStream = await navigator.mediaDevices.getDisplayMedia({
+                video: true
+            });
+            // @see https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addStream#Migrating_to_addTrack
+            displayStream.getTracks().forEach(function (track) {
+                self.pc.addTrack(track);
+				// Notify about local track when stream is ok.
+                self.ontrack && self.ontrack({track: track});
+            });
+        }
 
-            // Notify about local track when stream is ok.
-            self.ontrack && self.ontrack({track: track});
-        });
+       if (useCamera || hasAudio) {
+            const userStream = await navigator.mediaDevices.getUserMedia(self.constraints);
+
+            userStream.getTracks().forEach(function (track) {
+                self.pc.addTrack(track);
+                // Notify about local track when stream is ok.
+                self.ontrack && self.ontrack({track: track});
+            });
+       }
 
         var offer = await self.pc.createOffer();
         await self.pc.setLocalDescription(offer);

trunk/research/players/whip.html

@@ -62,10 +62,16 @@
     <div class="form-inline">
         Controls:
         <label>
-            <input type="checkbox" id="ch_videoonly" style="margin-bottom: 8px"> Video Only
+            <input type="radio" id="ra_camera" name="video" value="Camera" checked style="margin-bottom: 8px"> Camera
         </label>
         <label>
-            <input type="checkbox" id="ch_audioonly" style="margin-bottom: 8px"> Audio Only
+            <input type="radio" id="ra_screen" name="video" value="Screen" style="margin-bottom: 8px"> Screen
+        </label>
+        <label>
+            <input type="radio" id="ra_none" name="video" value="None" style="margin-bottom: 8px"> No Video
+        </label>
+        <label>
+            <input type="checkbox" id="ch_audio" checked style="margin-bottom: 8px"> Audio
         </label>
     </div>
 
@@ -113,8 +119,9 @@
         // For example: webrtc://r.ossrs.net/live/livestream
         var url = $("#txt_url").val();
         sdk.publish(url, {
-            videoOnly: $('#ch_videoonly').prop('checked'),
-            audioOnly: $('#ch_audioonly').prop('checked'),
+            camera: $('#ra_camera').prop('checked'),
+            screen: $('#ra_screen').prop('checked'),
+            audio: $('#ch_audio').prop('checked')
         }).then(function(session){
             $('#sessionid').html(session.sessionid);
             $('#simulator-drop').attr('href', session.simulator + '?drop=1&username=' + session.sessionid);

trunk/src/app/srs_app_rtc_source.cpp

@@ -1775,6 +1775,23 @@ srs_error_t SrsRtcFrameBuilder::packet_video_rtmp(const uint16_t start, const ui
 
     if (0 == nb_payload) {
         srs_warn("empty nalu");
+
+        // The chrome web browser send RTP packet with empty payload frequently,
+        // reset header_sn_, lost_sn_ and continue to found next frame in this case,
+        // otherwise, all the cached RTP packets are dropped before next key frame arrive.
+        header_sn_ = end + 1;
+        uint16_t tail_sn = 0;
+        int sn = find_next_lost_sn(header_sn_, tail_sn);
+        if (-1 == sn) {
+            if (check_frame_complete(header_sn_, tail_sn)) {
+                err = packet_video_rtmp(header_sn_, tail_sn);
+            }
+        } else if (-2 == sn) {
+            return srs_error_new(ERROR_RTC_RTP_MUXER, "video cache is overflow");
+        } else {
+            lost_sn_ = sn;
+        }
+
         return err;
     }
 

trunk/src/core/srs_core_version5.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    215
+#define VERSION_REVISION    216
 
 #endif

trunk/src/core/srs_core_version6.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    156
+#define VERSION_REVISION    157
 
 #endif

trunk/src/core/srs_core_version7.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       7
 #define VERSION_MINOR       0
-#define VERSION_REVISION    17
+#define VERSION_REVISION    18
 
 #endif