From cdc2dae57c7eba9e2fc8ca3fa51e1c08316fbb7c Mon Sep 17 00:00:00 2001 From: Luong Minh Date: Tue, 5 Dec 2023 14:22:57 +0700 Subject: [PATCH 1/2] lazy consumer publisher and added new samples --- coverage/cobertura-coverage.xml | 336 ++++++++++-------- samples/broadcast/publisher.js | 6 +- samples/broadcast/viewer.js | 6 +- samples/conference/app.js | 8 +- samples/consumer-pair/app.js | 2 +- samples/echo-toggle/app.js | 10 +- samples/echo/index.js | 6 +- samples/echo_scroll/app.js | 122 +++++++ samples/echo_scroll/index.html | 5 + samples/echo_toggle_replace_track/app.js | 72 ++++ samples/echo_toggle_replace_track/index.html | 5 + .../app.js | 73 ++++ .../index.html | 5 + samples/echo_toggle_simulcast/app.js | 79 ++++ samples/echo_toggle_simulcast/index.html | 5 + samples/echo_view_unview/app.js | 43 +++ samples/echo_view_unview/index.html | 4 + samples/lazy_consumer/app.js | 27 ++ samples/lazy_consumer/index.html | 3 + samples/lazy_publisher/app.js | 49 +++ samples/lazy_publisher/index.html | 4 + samples/mix_minus/index.html | 3 + samples/mix_minus/index.js | 46 +++ samples/mix_minus/source.html | 27 ++ samples/mix_minus/source.js | 58 +++ samples/mixer/app.js | 122 +++++++ samples/mixer/index.html | 8 + samples/multi_streams_view_unview/app.js | 61 ++++ samples/multi_streams_view_unview/index.html | 6 + samples/simulcast/app.js | 47 +++ samples/simulcast/index.html | 15 + src/lib/core/rpc.ts | 8 + src/lib/core/socket.ts | 32 +- src/lib/core/tracks.ts | 4 +- src/lib/publisher.ts | 8 +- src/lib/receiver-mix-minus.ts | 20 +- src/lib/receiver.test.ts | 1 + src/lib/receiver.ts | 15 +- src/lib/session.ts | 33 +- types/lib/core/rpc.d.ts | 1 + types/lib/core/rpc.d.ts.map | 2 +- types/lib/core/socket.d.ts | 1 + types/lib/core/socket.d.ts.map | 2 +- types/lib/core/tracks.d.ts.map | 2 +- types/lib/publisher.d.ts | 1 + types/lib/publisher.d.ts.map | 2 +- types/lib/receiver-mix-minus.d.ts | 10 +- types/lib/receiver-mix-minus.d.ts.map | 2 +- types/lib/receiver.d.ts | 3 +- types/lib/receiver.d.ts.map | 2 +- types/lib/session.d.ts | 2 +- types/lib/session.d.ts.map | 2 +- 52 files changed, 1182 insertions(+), 234 deletions(-) create mode 100644 samples/echo_scroll/app.js create mode 100644 samples/echo_scroll/index.html create mode 100644 samples/echo_toggle_replace_track/app.js create mode 100644 samples/echo_toggle_replace_track/index.html create mode 100644 samples/echo_toggle_replace_track_simulcast/app.js create mode 100644 samples/echo_toggle_replace_track_simulcast/index.html create mode 100644 samples/echo_toggle_simulcast/app.js create mode 100644 samples/echo_toggle_simulcast/index.html create mode 100644 samples/echo_view_unview/app.js create mode 100644 samples/echo_view_unview/index.html create mode 100644 samples/lazy_consumer/app.js create mode 100644 samples/lazy_consumer/index.html create mode 100644 samples/lazy_publisher/app.js create mode 100644 samples/lazy_publisher/index.html create mode 100644 samples/mix_minus/index.html create mode 100644 samples/mix_minus/index.js create mode 100644 samples/mix_minus/source.html create mode 100644 samples/mix_minus/source.js create mode 100644 samples/mixer/app.js create mode 100644 samples/mixer/index.html create mode 100644 samples/multi_streams_view_unview/app.js create mode 100644 samples/multi_streams_view_unview/index.html create mode 100644 samples/simulcast/app.js create mode 100644 samples/simulcast/index.html diff --git a/coverage/cobertura-coverage.xml b/coverage/cobertura-coverage.xml index 75dc2fb..379aa28 100644 --- a/coverage/cobertura-coverage.xml +++ b/coverage/cobertura-coverage.xml @@ -1,13 +1,13 @@ - + /Users/luongminh/workspace/bluesea/media-sdk-js - + - + @@ -36,52 +36,57 @@ - + - + - + - + - + - + - + + + + + + - + - + - + - + - + - + - + - + - + - + @@ -104,58 +109,61 @@ - - + + - - - - - - - - - - - + + + + + + + + + + + - + - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - + + + + + + + + @@ -322,9 +330,9 @@ - + - + @@ -343,32 +351,32 @@ - + - + - + - + - + - + @@ -376,34 +384,44 @@ - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + @@ -431,54 +449,58 @@ - - + - + - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + + @@ -523,20 +545,22 @@ - + - - + + + + @@ -843,83 +867,83 @@ - + - + - + - + - + - + - + - + - - - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - - - + + + + + + - - - + + + + + diff --git a/samples/broadcast/publisher.js b/samples/broadcast/publisher.js index 46f2baf..2f2f97b 100644 --- a/samples/broadcast/publisher.js +++ b/samples/broadcast/publisher.js @@ -4,7 +4,7 @@ async function onMyStreamAdded(stream) { if (element.consumer) { await element.consumer.unview('main'); } - let consumer = window.bluesea_session.createConsumer(stream); + let consumer = window.atm0sSession.createConsumer(stream); element.srcObject = consumer.view('main'); element.consumer = consumer; } @@ -13,7 +13,7 @@ async function onMyStreamAdded(stream) { // if (element.consumer) { // await element.consumer.unview('main'); // } - // let consumer = window.bluesea_session.createConsumer(stream); + // let consumer = window.atm0sSession.createConsumer(stream); // // element.srcObject = await consumer.view('main'); // // await consumer.view('main'); // element.consumer = consumer; @@ -51,7 +51,7 @@ async function boot() { video: 1, }, }); - window.bluesea_session = session; + window.atm0sSession = session; session.connect(); session.on('mystream_added', onMyStreamAdded); } diff --git a/samples/broadcast/viewer.js b/samples/broadcast/viewer.js index 6652400..9961771 100644 --- a/samples/broadcast/viewer.js +++ b/samples/broadcast/viewer.js @@ -5,7 +5,7 @@ async function onStreamAdded(stream) { if (element.consumer) { element.consumer.unview('main'); } - let consumer = window.bluesea_session.createConsumer(stream); + let consumer = window.atm0sSession.createConsumer(stream); const view = consumer.view('main'); element.srcObject = view; element.consumer = consumer; @@ -16,7 +16,7 @@ async function onStreamAdded(stream) { if (element.consumer) { element.consumer.unview('main'); } - let consumer = window.bluesea_session.createConsumer(stream); + let consumer = window.atm0sSession.createConsumer(stream); const view = consumer.view('main'); element.srcObject = view; element.consumer = consumer; @@ -53,7 +53,7 @@ async function boot() { video: 1, }, }); - window.bluesea_session = session; + window.atm0sSession = session; session.connect(); session.on('stream_added', onStreamAdded); session.on('stream_removed', onStreamRemoved); diff --git a/samples/conference/app.js b/samples/conference/app.js index 3b3542e..48ea186 100644 --- a/samples/conference/app.js +++ b/samples/conference/app.js @@ -2,7 +2,7 @@ async function onMyStreamAdded(stream) { console.log('on my stream added', stream); if (stream.kind == 'audio') { console.log('enable vad and remove noise'); - window.bluesea_session + window.atm0sSession .getSender('audio', 'audio_main') .toggleAudioFeatures(true, false) .then(console.log) @@ -16,7 +16,7 @@ async function onStreamAdded(stream) { console.log('added stream:', stream); if (stream.kind == 'video') { console.log('create video consumer'); - let consumer = window.bluesea_session.createConsumer(stream); + let consumer = window.atm0sSession.createConsumer(stream); let element = document.createElement('video'); element.id = 'video-' + stream.peer_id; element.width = 300; @@ -30,7 +30,7 @@ async function onStreamAdded(stream) { if (stream.kind == 'audio') { console.log('create audio mixer consumer'); - let consumer = await window.bluesea_session.createConsumer(stream); + let consumer = await window.atm0sSession.createConsumer(stream); let element = document.createElement('audio'); element.id = 'audio-' + stream.peer_id; element.hidden = true; @@ -74,7 +74,7 @@ async function boot() { video: 5, }, }); - window.bluesea_session = session; + window.atm0sSession = session; session.connect(); session.on('mystream_added', onMyStreamAdded); session.on('stream_added', onStreamAdded); diff --git a/samples/consumer-pair/app.js b/samples/consumer-pair/app.js index 432e87b..1e19692 100644 --- a/samples/consumer-pair/app.js +++ b/samples/consumer-pair/app.js @@ -13,7 +13,7 @@ async function boot() { video: 1, }, }); - window.bluesea_session = session; + window.atm0sSession = session; let consumer = session.createConsumerPair('rtmp', 'main_audio', 'main_video'); session.connect().then(() => { let element = document.getElementById('my_video'); diff --git a/samples/echo-toggle/app.js b/samples/echo-toggle/app.js index 1539512..ea9057e 100644 --- a/samples/echo-toggle/app.js +++ b/samples/echo-toggle/app.js @@ -1,7 +1,7 @@ async function onMyStreamAdded(stream) { console.log('added mystream:', stream); if (stream.kind == 'video') { - let receiver = await window.bluesea_session.takeReceiver('video'); + let receiver = await window.atm0sSession.takeReceiver('video'); let element = document.getElementById('my_video'); element.srcObject = receiver.stream; element.receiver = receiver; @@ -14,7 +14,7 @@ async function onMyStreamRemoved(stream) { if (stream.kind == 'video') { let element = document.getElementById('my_video'); element.receiver.disconnect(); - window.bluesea_session.backReceiver(element.receiver); + window.atm0sSession.backReceiver(element.receiver); element.receiver = null; element.srcObject = null; } @@ -34,7 +34,7 @@ async function boot() { video: 1, }, }); - window.bluesea_session = session; + window.atm0sSession = session; session.on('mystream_added', onMyStreamAdded); session.on('mystream_removed', onMyStreamRemoved); session.connect(); @@ -42,12 +42,12 @@ async function boot() { window.toggleStream = async function toggleStream() { if (window.video_stream) { - window.bluesea_session.getSender('video', 'video_main').stop(); + window.atm0sSession.getSender('video', 'video_main').stop(); window.video_stream = undefined; } else { console.log('Will toggle stream on'); window.video_stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: true }); - await window.bluesea_session.createSender({ + await window.atm0sSession.createSender({ kind: 'video', name: 'video_main', stream: window.video_stream, diff --git a/samples/echo/index.js b/samples/echo/index.js index fa99e64..0f82168 100644 --- a/samples/echo/index.js +++ b/samples/echo/index.js @@ -1,7 +1,7 @@ async function onMyStreamAdded(stream) { console.log('added mystream:', stream); if (stream.kind == 'video') { - let receiver = window.bluesea_session.takeReceiver('video'); + let receiver = window.atm0sSession.takeReceiver('video'); console.log('take receiver:', receiver); let element = document.getElementById('my_video'); element.srcObject = receiver.stream; @@ -9,7 +9,7 @@ async function onMyStreamAdded(stream) { } if (stream.kind == 'audio') { - let receiver = window.bluesea_session.takeReceiver('audio'); + let receiver = window.atm0sSession.takeReceiver('audio'); let element = document.getElementById('my_audio'); element.srcObject = receiver.stream; receiver.switch(stream); @@ -44,7 +44,7 @@ async function boot() { video: 1, }, }); - window.bluesea_session = session; + window.atm0sSession = session; session.connect(); session.on('mystream_added', onMyStreamAdded); } diff --git a/samples/echo_scroll/app.js b/samples/echo_scroll/app.js new file mode 100644 index 0000000..5b4a632 --- /dev/null +++ b/samples/echo_scroll/app.js @@ -0,0 +1,122 @@ +let videos = {}; +function debounce(func, wait, immediate) { + var timeout; + return function() { + var context = this, args = arguments; + var later = function() { + timeout = null; + if (!immediate) func.apply(context, args); + }; + var callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if (callNow) func.apply(context, args); + }; +}; + +let onApplyDebonde = debounce(function onApplyStream() { + console.log("Will apply now"); + Object.values(videos).map(async (video) => { + if(!video.visible && video.viewing) { + console.log('Switch to hide:', video.id); + video.viewing = false; + video.element.srcObject = null; + video.element.style.backgroundColor = 'gray'; + video.consumer.unview('small_video_element'); + } + }) + + Object.values(videos).map(async (video) => { + if(video.visible && !video.viewing) { + console.log('Get consumer for switch to view:', video.id); + video.viewing = true; + video.consumer = window.atm0sSession.createConsumer(video.stream); + console.log('Switch to view:', video.id, video.consumer); + video.element.srcObject = video.consumer.view('small_video_element'); + video.element.style.backgroundColor = 'black'; + video.consumer.on('state', (state) => { + switch(state) { + case 'live': + if(!video.element.srcObject) + video.element.srcObject = video.stream; + video.element.style.opacity = '1'; + break; + case 'key_only': + if(!video.element.srcObject) + video.element.srcObject = video.stream; + video.element.style.opacity = '0.8'; + break; + case 'disconnected': + video.element.style.opacity = '0.2'; + break; + case 'paused': + video.element.srcObject = null; + video.element.style.opacity = '1'; + break; + + } + }); + } + }) +}, 1000, false); + +function onVisiblityChanged(entries) { + entries.forEach(entry => { + if (entry.isIntersecting) { + videos[entry.target.id].visible = true; + } else { + videos[entry.target.id].visible = false; + } + }); + + onApplyDebonde(); +} + +function onMyStreamAdded(stream) { + console.log('added mystream:', stream); + const options = { threshold: 0.2 }; + const observer = new IntersectionObserver(onVisiblityChanged, options); + if(stream.kind == 'video') { + let panel = document.getElementById('video-panel'); + for(let i = 0; i < 20; i++) { + let element = document.createElement('video'); + element.id = 'video-' + i; + element.stream = stream; + element.style.backgroundColor = 'gray'; + element.style.width = '45%'; + element.style.height = '35%'; + element.style.margin = '2px'; + element.muted = true; + element.autoplay = true; + observer.observe(element); + panel.appendChild(element); + + videos[element.id] = { id: element.id, stream, visible: false, element, consumer: null }; + } + } +} + +async function boot() { + const urlSearchParams = new URLSearchParams(window.location.search); + const params = Object.fromEntries(urlSearchParams.entries()); + + let video_stream = await navigator.mediaDevices.getUserMedia({audio: false, video: true}); + let session = Atm0s.createSession(params['server'], { + roomId: params['room'] || 'demo', + peerId: params['peer'] || 'echo-client-' + new Date().getTime(), + token: params['token'], + senders: [ + { stream: video_stream, name: 'video_main', kind: 'video', simulcast: true } + ], + receivers: { + audio: 1, + video: 8 + } + }); + window.atm0sSession = session; + session.connect(); + session.on('mystream_added', onMyStreamAdded); + session.on('mystream_removed', onMyStreamAdded); +} + +boot(); \ No newline at end of file diff --git a/samples/echo_scroll/index.html b/samples/echo_scroll/index.html new file mode 100644 index 0000000..ed8b865 --- /dev/null +++ b/samples/echo_scroll/index.html @@ -0,0 +1,5 @@ + + +
+ +
\ No newline at end of file diff --git a/samples/echo_toggle_replace_track/app.js b/samples/echo_toggle_replace_track/app.js new file mode 100644 index 0000000..75d2bc9 --- /dev/null +++ b/samples/echo_toggle_replace_track/app.js @@ -0,0 +1,72 @@ +async function onMyStreamAdded(stream) { + console.log('added mystream:', stream); + if (stream.kind == 'video') { + let consumer = await window.atm0sSession.createConsumer(stream); + let element = document.getElementById('my_video'); + element.srcObject = consumer.view('main_video'); + element.consumer = consumer; + } + + if (stream.kind == 'audio') { + let consumer = await window.atm0sSession.createConsumer(stream); + let element = document.getElementById('my_audio'); + element.srcObject = consumer.view('main_audio'); + element.receiver = receiver; + receiver.switch(stream); + } +} + +async function onMyStreamRemoved(stream) { + console.log('removed mystream:', stream); + if (stream.kind == 'video') { + let element = document.getElementById('my_video'); + element.consumer.unview('main_video'); + element.consumer = null; + element.srcObject = null; + } + + if (stream.kind == 'audio') { + let element = document.getElementById('my_audio'); + element.consumer.unview('main_audio'); + element.consumer = null; + element.srcObject = null; + } +} + +async function boot() { + const urlSearchParams = new URLSearchParams(window.location.search); + const params = Object.fromEntries(urlSearchParams.entries()); + + let session = Atm0s.createSession(params['server'], { + roomId: params['room'] || 'demo', + peerId: params['peer'] || 'echo-client-' + new Date().getTime(), + token: params['token'], + senders: [ + // { stream: audio_stream, name: 'audio_main', kind: 'audio' }, + // { stream: video_stream, name: 'video_main', kind: 'video' } + ], + receivers: { + audio: 1, + video: 1, + }, + }); + window.atm0sSession = session; + window.webcam_publisher = session.createPublisher({ kind: 'video', name: 'video_main' }); + session.on('mystream_added', onMyStreamAdded); + session.on('mystream_removed', onMyStreamRemoved); + session.connect(); +} + +window.toggleStream = async function toggleStream() { + if (window.video_stream) { + window.webcam_publisher.switch(null); + window.video_stream.getTracks().forEach((track) => track.stop()); + window.video_stream = undefined; + } else { + console.log('Will toggle stream on'); + window.video_stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: true }); + window.webcam_publisher.switch(window.video_stream); + } +}; + +boot(); diff --git a/samples/echo_toggle_replace_track/index.html b/samples/echo_toggle_replace_track/index.html new file mode 100644 index 0000000..6b27bb2 --- /dev/null +++ b/samples/echo_toggle_replace_track/index.html @@ -0,0 +1,5 @@ + + + + +