open-webrtc-toolkit · jianjunz · Aug 13, 2021 · Oct 13, 2021 · Oct 19, 2021 · Oct 27, 2021
@@ -6,6 +6,8 @@ Change Log
 * Add a new property `rtpTransceivers` to `TransportSettings` and `TransportConstraints`.
 * Add a new property `peerConnection` to `ConferenceClient`.
 * The second argument of `ConferenceClient.publish` could be a list of `RTCRtpTransceiver`s.
+* Add support to publish a MediaStream over WebTransport.
+
 # 5.0
 * Add WebTransport support for conference mode, see [this design doc](../../design/webtransport.md) for detailed information.
 * All publications and subscriptions for the same conference use the same `PeerConnection`.

@@ -142,6 +142,8 @@ WebTransport is supported in conference mode as an experimental feature. QUIC ag
 - [JavaScript SDK design doc for WebTransport support](https://github.com/open-webrtc-toolkit/owt-client-javascript/blob/master/docs/design/webtransport.md)
 - [QUIC programming guide for OWT server](https://github.com/open-webrtc-toolkit/owt-server/blob/master/doc/design/quic-programming-guide.md)
 
+Publishing a MediaStream over WebTransport requires an additional worker for I/O. The worker is a standalone ES module, not included in owt.js. As we're moving the SDK from traditional JavaScript script to ES module, there is no plan to support this worker in old browsers.
+
 # 7 Events
 
 The JavaScript objects fires events using `Owt.Base.EventDispatchers`. For more detailed events, please refer to the specific class description page.

@@ -76,6 +76,10 @@ window.L = L;\n\
           watch: true
         },
       },
+      worker:{
+        src: ['dist/sdk/conference/webtransport/media-worker.js'],
+        dest: 'dist/sdk/media-worker.js',
+      },
       sinon: {
           src: ['node_modules/sinon/lib/sinon.js'],
           dest: 'test/unit/resources/scripts/gen/sinon-browserified.js',
@@ -102,7 +106,15 @@ window.L = L;\n\
         options: {
           base: '.',
           port: 7080,
-          keepalive: true
+          keepalive: true,
+          middleware: function(connect, options, middlewares) {
+            middlewares.unshift((req, res, next) => {
+              res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
+              res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
+              next();
+            });
+            return middlewares;
+          }
         },
       },
     },
@@ -189,7 +201,8 @@ window.L = L;\n\
           {expand: true,cwd:'src/extension/',src:['**'],dest:'dist/',flatten:false},
           {expand: true,cwd:'dist/sdk/',src:['owt.js'],dest:'dist/samples/conference/public/scripts/',flatten:false},
           {expand: true,cwd:'dist/samples/conference/public/scripts',src:['rest.js'],dest:'dist/samples/conference/',flatten:false},
-          {expand: true,cwd:'dist/sdk/',src:['owt.js'],dest:'dist/samples/p2p/js/',flatten:false}
+          {expand: true,cwd:'dist/sdk/',src:['owt.js'],dest:'dist/samples/p2p/js/',flatten:false},
+          {expand: true,cwd: 'dist/sdk/',src: ['media-worker.js'],dest: 'dist/samples/conference/public/scripts/',flatten: false},
         ]
       }
     },
@@ -267,8 +280,8 @@ window.L = L;\n\
 
   grunt.registerTask('check', ['eslint:src']);
   grunt.registerTask('prepare', ['browserify:sinon', 'browserify:chai_as_promised']);
-  grunt.registerTask('pack', ['browserify:dist', 'concat:rest', 'uglify:dist', 'copy:dist', 'string-replace', 'compress:dist', 'jsdoc:dist']);
-  grunt.registerTask('dev', ['browserify:dev', 'connect:server']);
+  grunt.registerTask('pack', ['browserify:dist', 'browserify:worker', 'concat:rest', 'uglify:dist', 'copy:dist', 'string-replace', 'compress:dist', 'jsdoc:dist']);
+  grunt.registerTask('dev', ['browserify:dev', 'browserify:worker', 'connect:server']);
   grunt.registerTask('debug', ['browserify:dev']);
   grunt.registerTask('default', ['check', 'pack']);
 };
@@ -23,20 +23,26 @@
 
 <body>
   <h1>Open WebRTC Toolkit</h1>
-  <h2>Sample of QuicTransport</h2>
+  <h2>Sample of WebTransport</h2>
   <div id="description">
-    <p>This sample works with the latest Chrome.</p>
+    <p>This sample works with the Chrome >= 97.</p>
   </div>
   <div class="operations">
     <button id="start-sending">Start sending</button>
     <button id="stop-sending">Stop sending</button>
+    <button id="start-receiving">Start receiving</button>
+    <button id="stop-receiving">Stop receiving</button>
+  </div>
+  <div id="videocontainer">
+    <video id="remote-video" playsinline muted autoplay controls></video>
   </div>
   <div id="conference-status"></div>
   <div class="stats">
   </div>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/3.1.2/socket.io.js" type="text/javascript"></script>
+
+  <script src="scripts/socket.io.js" type="text/javascript"></script>
   <script src="scripts/rest-sample.js" type="text/javascript"></script>
-  <script src="scripts/owt.js" type="text/javascript"></script>
+  <script src="../../../../dist/sdk-debug/owt.js" type="text/javascript"></script>
   <script src="scripts/quic.js" type="text/javascript"></script>
 </body>
 

@@ -2,42 +2,82 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+/* eslint-disable require-jsdoc */
+
 'use strict';
 
 let quicChannel = null;
 let bidirectionalStream = null;
-let writeTask;
-const conference=new Owt.Conference.ConferenceClient();
+let bidiAudioStream = null;
+let writeTask, dataWorker, conferenceId, myId, mixedStream, generatorWriter,
+    mediaPublication;
+const isMedia = true;
+
+window.Module={};
+
+const conference = new Owt.Conference.ConferenceClient(
+    {
+      webTransportConfiguration: {
+        serverCertificateFingerprints: [{
+          value:
+              'FD:CD:87:EB:92:97:84:FD:D9:E9:C1:9F:AF:57:12:0E:32:AF:0D:C0:58:5F:33:BB:59:4A:2E:6E:C3:18:7A:93',
+          algorithm: 'sha-256',
+        }],
+        serverCertificateHashes: [{
+          value: new Uint8Array([
+            0xFD, 0xCD, 0x87, 0xEB, 0x92, 0x97, 0x84, 0xFD, 0xD9, 0xE9, 0xC1,
+            0x9F, 0xAF, 0x57, 0x12, 0x0E, 0x32, 0xAF, 0x0D, 0xC0, 0x58, 0x5F,
+            0x33, 0xBB, 0x59, 0x4A, 0x2E, 0x6E, 0xC3, 0x18, 0x7A, 0x93
+          ]),
+          algorithm: 'sha-256',
+        }],
+      }
+    },
+    '../../../sdk/conference/webtransport');
 conference.addEventListener('streamadded', async (event) => {
   console.log(event.stream);
-  if (event.stream.source.data) {
-    const subscription = await conference.subscribe(event.stream);
-    const reader = subscription.stream.readable.getReader();
-    while (true) {
-      const {value, done} = await reader.read();
-      if (done) {
-        console.log('Subscription ends.');
-        break;
-      }
-      console.log('Received data: '+value);
-    }
+  if (event.stream.origin == myId) {
+    mixStream(
+        conferenceId, event.stream.id, 'common',
+        'http://jianjunz-nuc-ubuntu.sh.intel.com:3001');
   }
+  // if (event.stream.source.data) {
+  //   const subscription = await conference.subscribe(
+  //       event.stream,
+  //       // {transport:{type: 'quic'}});
+  //       {audio: false, video: {codecs: ['h264']}, transport: {type: 'quic'}});
+  //   const reader = subscription.stream.readable.getReader();
+  //   while (true) {
+  //     const {value, done} = await reader.read();
+  //     if (done) {
+  //       console.log('Subscription ends.');
+  //       break;
+  //     }
+  //     //console.log('Received data: ' + value);
+  //   }
+  // }
 });
 
 function updateConferenceStatus(message) {
   document.getElementById('conference-status').innerHTML +=
-    ('<p>' + message + '</p>');
+      ('<p>' + message + '</p>');
 }
 
-
 function joinConference() {
   return new Promise((resolve, reject) => {
-    createToken(undefined, 'user', 'presenter', resp => {
-      conference.join(resp).then(() => {
+    createToken(undefined, 'user', 'presenter', token => {
+      conference.join(token).then((info) => {
+        conferenceId = info.id;
+        myId = info.self.id;
+        for (const stream of info.remoteStreams) {
+          if (stream.source.video === 'mixed') {
+            mixedStream = stream;
+          }
+        }
         updateConferenceStatus('Connected to conference server.');
         resolve();
       });
-    });
+    }, 'http://jianjunz-nuc-ubuntu.sh.intel.com:3001');
   });
 };
 
@@ -55,17 +95,26 @@ function createRandomContentSessionId() {
   return id;
 }
 
+async function attachReader(stream) {
+  const reader = stream.readable.getReader();
+  while (true) {
+    const {value, done} = await reader.read();
+    if (done) {
+      console.log('Ends.');
+      break;
+    }
+    console.log('Received data: ' + value);
+  }
+}
+
 async function createSendChannel() {
   bidirectionalStream = await conference.createSendStream();
-  const localStream=new Owt.Base.LocalStream(bidirectionalStream, new Owt.Base.StreamSourceInfo(undefined, undefined,true));
-  const publication = await conference.publish(localStream);
-  console.log(publication);
   updateConferenceStatus('Created send channel.');
 }
 
 async function windowOnLoad() {
   await joinConference();
-  await createSendChannel();
+  //await createSendChannel();
 }
 
 async function writeUuid() {
@@ -84,7 +133,9 @@ async function writeData() {
   const encoded = encoder.encode('message', {stream: true});
   const writer = bidirectionalStream.writable.getWriter();
   await writer.ready;
-  await writer.write(new ArrayBuffer(2));
+  const ab = new Uint8Array(10000);
+  ab.fill(1, 0);
+  await writer.write(ab);
   writer.releaseLock();
   return;
 }
@@ -94,16 +145,45 @@ window.addEventListener('load', () => {
 });
 
 document.getElementById('start-sending').addEventListener('click', async () => {
-  if (!bidirectionalStream) {
-    updateConferenceStatus('Stream is not created.');
-    return;
+  if (isMedia) {
+    const mediaStream =
+        await navigator.mediaDevices.getUserMedia({audio: true, video: true});
+    const localStream = new Owt.Base.LocalStream(
+        mediaStream, new Owt.Base.StreamSourceInfo('mic', 'camera', undefined));
+    mediaPublication = await conference.publish(localStream, {
+      audio: {codec: 'opus', numberOfChannels: 2, sampleRate: 48000},
+      video: {codec: 'h264'},
+      transport: {type: 'quic'},
+    });
+  } else {
+    if (!bidirectionalStream) {
+      updateConferenceStatus('Stream is not created.');
+      return;
+    }
+    writeTask = setInterval(writeData, 200);
   }
-  await writeUuid();
-  writeTask = setInterval(writeData, 2000);
   updateConferenceStatus('Started sending.');
 });
 
 document.getElementById('stop-sending').addEventListener('click', () => {
-  clearInterval(writeTask);
+  if (isMedia) {
+    if (mediaPublication) {
+      mediaPublication.stop();
+    }
+  } else {
+    clearInterval(writeTask);
+  }
   updateConferenceStatus('Stopped sending.');
 });
+
+document.getElementById('start-receiving')
+    .addEventListener('click', async () => {
+      subscribeMixedStream();
+    });
+
+async function subscribeMixedStream() {
+  const subscription = await conference.subscribe(
+      mixedStream,
+      {audio: false, video: {codecs: ['h264']}, transport: {type: 'quic'}});
+  document.getElementById('remote-video').srcObject = subscription.stream;
+}
@@ -26,7 +26,7 @@
 
 // This file is borrowed from lynckia/licode with some modifications.
 
-/* global window */
+/* global console */
 
 'use strict';
 
@@ -55,13 +55,13 @@ const Logger = (function() {
   };
 
   that.log = (...args) => {
-    window.console.log((new Date()).toISOString(), ...args);
+    console.log((new Date()).toISOString(), ...args);
   };
 
   const bindType = function(type) {
-    if (typeof window.console[type] === 'function') {
+    if (typeof console[type] === 'function') {
       return (...args) => {
-        window.console[type]((new Date()).toISOString(), ...args);
+        console[type]((new Date()).toISOString(), ...args);
       };
     } else {
       return that.log;

@@ -195,17 +195,29 @@ export class PublishOptions {
   // eslint-disable-next-line require-jsdoc
   constructor(audio, video, transport) {
     /**
-     * @member {?Array<Owt.Base.AudioEncodingParameters> | ?Array<RTCRtpEncodingParameters>} audio
+     * @member {?Array<Owt.Base.AudioEncodingParameters> |
+     * ?Array<RTCRtpEncodingParameters> | ?AudioEncoderConfig } audio
      * @instance
      * @memberof Owt.Base.PublishOptions
-     * @desc Parameters for audio RtpSender. Publishing with RTCRtpEncodingParameters is an experimental feature. It is subject to change.
+     * @desc Parameters for audio RtpSender when transport's type is 'webrtc' or
+     * configuration of audio encoder when transport's type is 'quic'.
+     * Publishing with RTCRtpEncodingParameters is an experimental feature. It
+     * is subject to change.
+     * @see {@link https://www.w3.org/TR/webrtc/#rtcrtpencodingparameters|RTCRtpEncodingParameters}
+     * @see {@link https://w3c.github.io/webcodecs/#dictdef-audioencoderconfig|AudioEncoderConfig}
      */
     this.audio = audio;
     /**
-     * @member {?Array<Owt.Base.VideoEncodingParameters> | ?Array<RTCRtpEncodingParameters>} video
+     * @member {?Array<Owt.Base.VideoEncodingParameters> |
+     * ?Array<RTCRtpEncodingParameters> | ?VideoEncoderConfig } video
      * @instance
      * @memberof Owt.Base.PublishOptions
-     * @desc Parameters for video RtpSender. Publishing with RTCRtpEncodingParameters is an experimental feature. It is subject to change.
+     * @desc Parameters for video RtpSender when transport's type is 'webrtc' or
+     * configuration of video encoder when transport's type is 'quic'.
+     * Publishing with RTCRtpEncodingParameters is an experimental feature. It
+     * is subject to change.
+     * @see {@link https://www.w3.org/TR/webrtc/#rtcrtpencodingparameters|RTCRtpEncodingParameters}
+     * @see {@link https://w3c.github.io/webcodecs/#dictdef-videoencoderconfig|VideoEncoderConfig}
      */
     this.video = video;
     /**

@@ -29,7 +29,9 @@ export class TransportConstraints {
      * @member {Array.<Owt.Base.TransportType>} type
      * @instance
      * @memberof Owt.Base.TransportConstraints
-     * @desc Transport type for publication and subscription.
+     * @desc Transport type for publication and subscription. 'quic' is only
+     * supported in conference mode when WebTransport is supported by client and
+     * enabled at server side.
      */
     this.type = type;
     /**

@@ -898,6 +898,7 @@ export class ConferencePeerConnectionChannel extends EventDispatcher {
 
   _createPeerConnection() {
     if (this.pc) {
+      Logger.warning('PeerConnection exists.');
       return;
     }
-Original file line number
+Diff line change
@@ Expand Up @@
       _createPeerConnection() {
         if (this.pc) {
+          Logger.warning('PeerConnection exists.');
           return;
         }
@@ Expand Down @@