From fd9c7a2d920c592cf54a213da7f4ca4944b01625 Mon Sep 17 00:00:00 2001 From: giangndm <45644921+giangndm@users.noreply.github.com> Date: Sat, 8 Jun 2024 15:21:28 +0700 Subject: [PATCH] feat: audio mixer (#306) * refactor: cluster now have features style * feat: room audio-mixer with auto mode * feat: room audio-mixer with manual mode --- Cargo.lock | 9 + Cargo.toml | 2 + bin/gate_z0_n1.sh | 4 +- bin/gate_z256_n1.sh | 4 +- bin/media_z0_n1.sh | 2 +- bin/media_z256_n1.sh | 2 +- bin/src/server/media.rs | 4 +- packages/audio_mixer/Cargo.toml | 7 + packages/audio_mixer/src/lib.rs | 204 ++++++++++ packages/media_core/Cargo.toml | 2 + packages/media_core/src/cluster.rs | 114 +++--- .../media_core/src/cluster/id_generator.rs | 7 + packages/media_core/src/cluster/room.rs | 337 +++++++++++----- .../src/cluster/room/audio_mixer.rs | 293 ++++++++++++++ .../src/cluster/room/audio_mixer/manual.rs | 248 ++++++++++++ .../src/cluster/room/audio_mixer/publisher.rs | 234 +++++++++++ .../cluster/room/audio_mixer/subscriber.rs | 287 +++++++++++++ .../src/cluster/room/media_track.rs | 137 +++++++ .../publisher.rs} | 129 +++--- .../subscriber.rs} | 189 +++++---- .../media_core/src/cluster/room/metadata.rs | 378 +++++++++--------- packages/media_core/src/endpoint.rs | 44 +- packages/media_core/src/endpoint/internal.rs | 94 +++-- .../src/endpoint/internal/local_track.rs | 33 +- .../internal/local_track/voice_activity.rs | 19 + .../src/endpoint/internal/remote_track.rs | 19 +- packages/media_core/src/errors.rs | 1 + packages/media_core/src/transport.rs | 24 +- packages/media_runner/src/worker.rs | 47 ++- packages/protocol/build.rs | 3 +- packages/protocol/proto/features.mixer.proto | 62 +++ packages/protocol/proto/features.proto | 10 +- .../protocol/proto/features_mix_minus.proto | 74 ---- packages/protocol/proto/gateway.proto | 9 +- .../proto/{conn.proto => session.proto} | 36 +- packages/protocol/proto/shared.proto | 19 +- packages/protocol/proto/sync.sh | 1 + packages/protocol/src/endpoint.rs | 132 +++--- packages/protocol/src/endpoint/audio_mixer.rs | 52 +++ packages/protocol/src/endpoint/track.rs | 93 +++++ .../protocol/src/protobuf/features.mixer.rs | 126 ++++++ packages/protocol/src/protobuf/features.rs | 8 +- packages/protocol/src/protobuf/gateway.rs | 6 +- .../src/protobuf/{mix_minus.rs => mixer.rs} | 20 +- packages/protocol/src/protobuf/mod.rs | 10 +- .../src/protobuf/{conn.rs => session.rs} | 46 ++- packages/protocol/src/protobuf/shared.rs | 30 +- packages/protocol/src/transport.rs | 25 +- packages/transport_webrtc/src/lib.rs | 2 +- .../transport_webrtc/src/transport/webrtc.rs | 283 +++++++++---- .../transport_webrtc/src/transport/whep.rs | 11 +- .../transport_webrtc/src/transport/whip.rs | 2 + packages/transport_webrtc/src/worker.rs | 22 +- 53 files changed, 3070 insertions(+), 886 deletions(-) create mode 100644 packages/audio_mixer/Cargo.toml create mode 100644 packages/audio_mixer/src/lib.rs create mode 100644 packages/media_core/src/cluster/room/audio_mixer.rs create mode 100644 packages/media_core/src/cluster/room/audio_mixer/manual.rs create mode 100644 packages/media_core/src/cluster/room/audio_mixer/publisher.rs create mode 100644 packages/media_core/src/cluster/room/audio_mixer/subscriber.rs create mode 100644 packages/media_core/src/cluster/room/media_track.rs rename packages/media_core/src/cluster/room/{channel_pub.rs => media_track/publisher.rs} (54%) rename packages/media_core/src/cluster/room/{channel_sub.rs => media_track/subscriber.rs} (56%) create mode 100644 packages/media_core/src/endpoint/internal/local_track/voice_activity.rs create mode 100644 packages/protocol/proto/features.mixer.proto delete mode 100644 packages/protocol/proto/features_mix_minus.proto rename packages/protocol/proto/{conn.proto => session.proto} (90%) create mode 100644 packages/protocol/proto/sync.sh create mode 100644 packages/protocol/src/endpoint/audio_mixer.rs create mode 100644 packages/protocol/src/endpoint/track.rs create mode 100644 packages/protocol/src/protobuf/features.mixer.rs rename packages/protocol/src/protobuf/{mix_minus.rs => mixer.rs} (88%) rename packages/protocol/src/protobuf/{conn.rs => session.rs} (93%) diff --git a/Cargo.lock b/Cargo.lock index 3431c57d..543b2100 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,6 +257,13 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "audio-mixer" +version = "0.1.0" +dependencies = [ + "log", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -1738,8 +1745,10 @@ name = "media-server-core" version = "0.1.0" dependencies = [ "atm0s-sdn", + "audio-mixer", "derivative", "derive_more", + "indexmap", "log", "media-server-protocol", "media-server-utils", diff --git a/Cargo.toml b/Cargo.toml index 9c4e2f70..e549030c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "packages/transport_webrtc", "packages/media_secure", "packages/media_gateway", + "packages/audio_mixer", ] [workspace.dependencies] @@ -23,3 +24,4 @@ derive_more = "0.99" rand = "0.8" mockall = "0.12" prost = "0.12" +indexmap = "2.2" diff --git a/bin/gate_z0_n1.sh b/bin/gate_z0_n1.sh index 4e3dce24..b2bcbd77 100644 --- a/bin/gate_z0_n1.sh +++ b/bin/gate_z0_n1.sh @@ -1,4 +1,4 @@ -RUST_LOG=atm0s_sdn_network::features::socket=debug,info \ +RUST_LOG=info \ RUST_BACKTRACE=1 \ cargo run -- \ --http-port 3000 \ @@ -8,4 +8,6 @@ cargo run -- \ gateway \ --lat 10 \ --lon 20 \ + --max-memory 100 \ + --max-disk 100 \ --geo-db "../maxminddb-data/GeoLite2-City.mmdb" diff --git a/bin/gate_z256_n1.sh b/bin/gate_z256_n1.sh index 5091df04..f65292a9 100644 --- a/bin/gate_z256_n1.sh +++ b/bin/gate_z256_n1.sh @@ -1,4 +1,4 @@ -RUST_LOG=atm0s_sdn_network::features::socket=debug,info \ +RUST_LOG=info \ RUST_BACKTRACE=1 \ cargo run -- \ --http-port 4000 \ @@ -9,4 +9,6 @@ cargo run -- \ gateway \ --lat 20 \ --lon 30 \ + --max-memory 100 \ + --max-disk 100 \ --geo-db "../maxminddb-data/GeoLite2-City.mmdb" diff --git a/bin/media_z0_n1.sh b/bin/media_z0_n1.sh index 93137d8c..112380d9 100644 --- a/bin/media_z0_n1.sh +++ b/bin/media_z0_n1.sh @@ -1,4 +1,4 @@ -RUST_LOG=atm0s_sdn_network::features::socket=debug,info \ +RUST_LOG=info \ RUST_BACKTRACE=1 \ cargo run -- \ --http-port 3001 \ diff --git a/bin/media_z256_n1.sh b/bin/media_z256_n1.sh index 4aecbff0..e3cc982c 100644 --- a/bin/media_z256_n1.sh +++ b/bin/media_z256_n1.sh @@ -1,4 +1,4 @@ -RUST_LOG=atm0s_sdn_network::features::socket=debug,info \ +RUST_LOG=info \ RUST_BACKTRACE=1 \ cargo run -- \ --http-port 4001 \ diff --git a/bin/src/server/media.rs b/bin/src/server/media.rs index 8bb35e6e..bdc41dca 100644 --- a/bin/src/server/media.rs +++ b/bin/src/server/media.rs @@ -147,13 +147,13 @@ pub async fn run_media_server(workers: usize, http_port: Option, node: Node 0, //because sdn controller allway is run inside worker 0 ExtIn::Sdn(SdnExtIn::ServicesControl( AGENT_SERVICE_ID.into(), - 0.into(), + media_server_runner::UserData::Cluster, media_server_gateway::agent_service::Control::NodeStats(metrics).into(), )), ); } while let Ok(control) = vnet_rx.try_recv() { - controller.send_to_best(ExtIn::Sdn(SdnExtIn::FeaturesControl(0.into(), control.into()))); + controller.send_to_best(ExtIn::Sdn(SdnExtIn::FeaturesControl(media_server_runner::UserData::Cluster, control.into()))); } while let Ok(req) = req_rx.try_recv() { let req_id = req_id_seed; diff --git a/packages/audio_mixer/Cargo.toml b/packages/audio_mixer/Cargo.toml new file mode 100644 index 00000000..b59f3b83 --- /dev/null +++ b/packages/audio_mixer/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "audio-mixer" +version = "0.1.0" +edition = "2021" + +[dependencies] +log.workspace = true diff --git a/packages/audio_mixer/src/lib.rs b/packages/audio_mixer/src/lib.rs new file mode 100644 index 00000000..58ef695a --- /dev/null +++ b/packages/audio_mixer/src/lib.rs @@ -0,0 +1,204 @@ +use std::{ + collections::HashMap, + fmt::Debug, + hash::Hash, + time::{Duration, Instant}, +}; + +const SILENT_LEVEL: i8 = -127; +const SWITCH_AUDIO_THRESHOLD: i16 = 30; +/// if no audio pkt received in AUDIO_SLOT_TIMEOUT, set audio level to SILENT_LEVEL +const AUDIO_SLOT_TIMEOUT: Duration = Duration::from_millis(1000); + +struct SourceState { + last_changed_at: Instant, + slot: Option, +} + +#[allow(unused)] +#[derive(Debug, Clone)] +struct OutputSlotState { + audio_level: i8, + source: Src, +} + +/// Implement lightweight audio mixer with mix-minus feature +/// We will select n highest audio-level tracks +pub struct AudioMixer { + len: usize, + sources: HashMap, + outputs: Vec>>, +} + +impl AudioMixer { + pub fn new(output: usize) -> Self { + log::info!("[AudioMixer] create new with {output} outputs"); + + Self { + len: 0, + sources: HashMap::new(), + outputs: vec![None; output], + } + } + + pub fn on_tick(&mut self, now: Instant) -> Option> { + let mut clear = vec![]; + self.sources.retain(|k, v| { + if v.last_changed_at + AUDIO_SLOT_TIMEOUT <= now { + log::info!("[AudioMixer] del source {:?} after timeout", k); + if let Some(slot) = v.slot { + self.outputs[slot] = None; //clear + self.len -= 1; + clear.push(slot); + } + false + } else { + true + } + }); + if clear.is_empty() { + None + } else { + Some(clear) + } + } + + pub fn on_pkt(&mut self, now: Instant, source: Src, audio_level: Option) -> Option<(usize, bool)> { + let audio_level = audio_level.unwrap_or(SILENT_LEVEL); + if let Some(s) = self.sources.get_mut(&source) { + s.last_changed_at = now; + if let Some(slot) = s.slot { + Some((slot, false)) + } else if self.has_empty_slot() { + let slot = self.find_empty_slot().expect("Should have empty"); + log::info!("[AudioMixer] switch empty slot {} to source {:?}", slot, source); + self.sources.get_mut(&source).expect("Should have source").slot = Some(slot); + self.outputs[slot] = Some(OutputSlotState { audio_level, source }); + self.len += 1; + + Some((slot, true)) + } else { + //We allway have lowest pin_slot here because above check dont have empty_slot + let (lowest_index, lowest_source, lowest_audio_level) = self.lowest_slot().expect("Should have lowest pined"); + if lowest_source != source && audio_level as i16 >= lowest_audio_level as i16 + SWITCH_AUDIO_THRESHOLD { + log::info!( + "[AudioMixer] switch slot {} from source {:?} to source {:?} with higher audio_level", + lowest_index, + lowest_source, + source + ); + self.sources.get_mut(&source).expect("Should have source").slot = Some(lowest_index); + self.sources.get_mut(&lowest_source).expect("Should have lowest_source").slot = None; + self.outputs[lowest_index] = Some(OutputSlotState { audio_level, source: source.clone() }); + Some((lowest_index, true)) + } else { + None + } + } + } else if let Some(slot) = self.find_empty_slot() { + log::info!("[AudioMixer] switch empty slot {} to source {:?}", slot, source); + self.sources.insert( + source.clone(), + SourceState { + last_changed_at: now, + slot: Some(slot), + }, + ); + self.outputs[slot] = Some(OutputSlotState { audio_level, source }); + self.len += 1; + Some((slot, true)) + } else { + log::info!("[AudioMixer] new source {:?}", source); + self.sources.insert(source.clone(), SourceState { last_changed_at: now, slot: None }); + None + } + } + + fn find_empty_slot(&self) -> Option { + for (i, slot) in self.outputs.iter().enumerate() { + if slot.is_none() { + return Some(i); + } + } + None + } + + fn has_empty_slot(&self) -> bool { + self.len < self.outputs.len() + } + + fn lowest_slot(&self) -> Option<(usize, Src, i8)> { + let mut lowest: Option<(usize, Src, i8)> = None; + for (i, slot) in self.outputs.iter().enumerate() { + if let Some(OutputSlotState { audio_level, source }) = slot { + if let Some((_, _, lowest_slot_audio_level)) = &mut lowest { + if *audio_level < *lowest_slot_audio_level || (*audio_level == *lowest_slot_audio_level) { + lowest = Some((i, source.clone(), *audio_level)); + } + } else { + lowest = Some((i, source.clone(), *audio_level)); + } + } + } + lowest + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{AudioMixer, AUDIO_SLOT_TIMEOUT, SWITCH_AUDIO_THRESHOLD}; + + fn ms(m: u64) -> Duration { + Duration::from_millis(m) + } + + #[test] + fn add_remove_correct() { + let mut mixer = AudioMixer::::new(2); + let time_0 = Instant::now(); + + assert_eq!(mixer.on_pkt(time_0, 100, Some(10)), Some((0, true))); + assert_eq!(mixer.on_pkt(time_0, 101, Some(10)), Some((1, true))); + assert_eq!(mixer.on_pkt(time_0, 102, Some(10)), None); + + assert_eq!(mixer.on_pkt(time_0 + ms(10), 100, Some(10)), Some((0, false))); + assert_eq!(mixer.on_pkt(time_0 + ms(10), 101, Some(10)), Some((1, false))); + assert_eq!(mixer.on_pkt(time_0 + ms(10), 102, Some(10)), None); + + assert_eq!(mixer.on_tick(time_0 + AUDIO_SLOT_TIMEOUT), None); + } + + #[test] + fn auto_remove_timeout_source() { + let mut mixer = AudioMixer::::new(1); + let time_0 = Instant::now(); + + assert_eq!(mixer.on_pkt(time_0, 100, Some(10)), Some((0, true))); + assert_eq!(mixer.on_pkt(time_0, 101, Some(10)), None); + + assert_eq!(mixer.on_tick(time_0 + ms(100)), None); + assert_eq!(mixer.on_pkt(time_0 + ms(100), 101, Some(10)), None); + + assert_eq!(mixer.on_tick(time_0 + AUDIO_SLOT_TIMEOUT), Some(vec![0])); //source 100 will be released + assert_eq!(mixer.on_pkt(time_0 + AUDIO_SLOT_TIMEOUT, 101, Some(10)), Some((0, true))); + } + + #[test] + fn auto_switch_higher_source() { + let mut mixer = AudioMixer::::new(1); + let time_0 = Instant::now(); + + assert_eq!(mixer.on_pkt(time_0, 100, Some(10)), Some((0, true))); + assert_eq!(mixer.on_pkt(time_0, 101, Some(10)), None); + + assert_eq!(mixer.on_tick(time_0 + ms(100)), None); + assert_eq!(mixer.on_pkt(time_0 + ms(100), 100, Some(10)), Some((0, false))); + assert_eq!(mixer.on_pkt(time_0 + ms(100), 101, Some(10)), None); + + assert_eq!(mixer.on_tick(time_0 + ms(200)), None); //source 100 will be released + assert_eq!(mixer.on_pkt(time_0 + ms(200), 100, Some(10)), Some((0, false))); + assert_eq!(mixer.on_pkt(time_0 + ms(200), 101, Some(10 + SWITCH_AUDIO_THRESHOLD as i8)), Some((0, true))); + } +} diff --git a/packages/media_core/Cargo.toml b/packages/media_core/Cargo.toml index 65b8fcac..511b798c 100644 --- a/packages/media_core/Cargo.toml +++ b/packages/media_core/Cargo.toml @@ -16,6 +16,8 @@ sans-io-runtime = { workspace = true, default-features = false } atm0s-sdn = { workspace = true } media-server-protocol = { path = "../protocol" } media-server-utils = { path = "../media_utils" } +audio-mixer = { path = "../audio_mixer" } +indexmap = { workspace = true } [dev-dependencies] tracing-subscriber = { workspace = true } diff --git a/packages/media_core/src/cluster.rs b/packages/media_core/src/cluster.rs index 879015f5..6f4a3159 100644 --- a/packages/media_core/src/cluster.rs +++ b/packages/media_core/src/cluster.rs @@ -1,7 +1,8 @@ +//! //! Cluster handle all of logic allow multi node can collaborate to make a giant streaming system. //! //! Cluster is collect of some rooms, each room is independent logic. -//! We use UserData feature from SDN with UserData is ClusterRoomHash to route SDN event to correct room. +//! We use UserData feature from SDN with UserData is RoomUserData to route SDN event to correct room. //! use derive_more::{AsRef, Display, From}; @@ -15,13 +16,14 @@ use std::{ use atm0s_sdn::features::{FeaturesControl, FeaturesEvent}; use media_server_protocol::{ - endpoint::{PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe, TrackMeta, TrackName}, + endpoint::{AudioMixerConfig, PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe, TrackMeta, TrackName, TrackSource}, media::MediaPacket, }; use crate::transport::{LocalTrackId, RemoteTrackId}; use self::room::ClusterRoom; +pub use self::room::RoomUserData; mod id_generator; mod room; @@ -41,7 +43,7 @@ impl From<&RoomId> for ClusterRoomHash { pub enum ClusterRemoteTrackControl { Started(TrackName, TrackMeta), Media(MediaPacket), - Ended, + Ended(TrackName, TrackMeta), } #[derive(Clone, Debug, PartialEq, Eq)] @@ -61,17 +63,31 @@ pub enum ClusterLocalTrackControl { #[derive(Clone, Debug, PartialEq, Eq)] pub enum ClusterLocalTrackEvent { Started, + RelayChanged, SourceChanged, Media(u64, MediaPacket), Ended, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ClusterAudioMixerControl { + Attach(Vec), + Detach(Vec), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ClusterAudioMixerEvent { + SlotSet(u8, PeerId, TrackName), + SlotUnset(u8), +} + #[derive(Debug, PartialEq, Eq)] pub enum ClusterEndpointControl { - Join(PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe), + Join(PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe, Option), Leave, SubscribePeer(PeerId), UnsubscribePeer(PeerId), + AudioMixer(ClusterAudioMixerControl), RemoteTrack(RemoteTrackId, ClusterRemoteTrackControl), LocalTrack(LocalTrackId, ClusterLocalTrackControl), } @@ -82,28 +98,29 @@ pub enum ClusterEndpointEvent { PeerLeaved(PeerId, PeerMeta), TrackStarted(PeerId, TrackName, TrackMeta), TrackStopped(PeerId, TrackName, TrackMeta), + AudioMixer(ClusterAudioMixerEvent), RemoteTrack(RemoteTrackId, ClusterRemoteTrackEvent), LocalTrack(LocalTrackId, ClusterLocalTrackEvent), } -pub enum Input { +pub enum Input { Sdn(ClusterRoomHash, FeaturesEvent), - Endpoint(Owner, ClusterRoomHash, ClusterEndpointControl), + Endpoint(Endpoint, ClusterRoomHash, ClusterEndpointControl), } #[derive(Debug, PartialEq, Eq)] -pub enum Output { - Sdn(ClusterRoomHash, FeaturesControl), - Endpoint(Vec, ClusterEndpointEvent), +pub enum Output { + Sdn(RoomUserData, FeaturesControl), + Endpoint(Vec, ClusterEndpointEvent), Continue, } -pub struct MediaCluster { +pub struct MediaCluster { rooms_map: HashMap, - rooms: TaskGroup, room::Output, ClusterRoom, 16>, + rooms: TaskGroup, room::Output, ClusterRoom, 16>, } -impl Default for MediaCluster { +impl Default for MediaCluster { fn default() -> Self { Self { rooms_map: HashMap::new(), @@ -112,24 +129,24 @@ impl Default for MediaCluster { } } -impl MediaCluster { +impl MediaCluster { pub fn on_tick(&mut self, now: Instant) { self.rooms.on_tick(now); } - pub fn on_sdn_event(&mut self, now: Instant, room: ClusterRoomHash, event: FeaturesEvent) { - let index = return_if_none!(self.rooms_map.get(&room)); - self.rooms.on_event(now, *index, room::Input::Sdn(event)); + pub fn on_sdn_event(&mut self, now: Instant, userdata: RoomUserData, event: FeaturesEvent) { + let index = return_if_none!(self.rooms_map.get(&userdata.0)); + self.rooms.on_event(now, *index, room::Input::Sdn(userdata, event)); } - pub fn on_endpoint_control(&mut self, now: Instant, owner: Owner, room_hash: ClusterRoomHash, control: ClusterEndpointControl) { + pub fn on_endpoint_control(&mut self, now: Instant, endpoint: Endpoint, room_hash: ClusterRoomHash, control: ClusterEndpointControl) { if let Some(index) = self.rooms_map.get(&room_hash) { - self.rooms.on_event(now, *index, room::Input::Endpoint(owner, control)); + self.rooms.on_event(now, *index, room::Input::Endpoint(endpoint, control)); } else { log::info!("[MediaCluster] create room {}", room_hash); let index = self.rooms.add_task(ClusterRoom::new(room_hash)); self.rooms_map.insert(room_hash, index); - self.rooms.on_event(now, index, room::Input::Endpoint(owner, control)); + self.rooms.on_event(now, index, room::Input::Endpoint(endpoint, control)); } } @@ -138,13 +155,13 @@ impl MediaCluster { } } -impl TaskSwitcherChild> for MediaCluster { - type Time = Instant; - fn pop_output(&mut self, now: Instant) -> Option> { - let (index, out) = self.rooms.pop_output(now)?; +impl TaskSwitcherChild> for MediaCluster { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { + let (index, out) = self.rooms.pop_output(())?; match out { room::Output::Sdn(userdata, control) => Some(Output::Sdn(userdata, control)), - room::Output::Endpoint(owners, event) => Some(Output::Endpoint(owners, event)), + room::Output::Endpoint(endpoints, event) => Some(Output::Endpoint(endpoints, event)), room::Output::Destroy(room) => { log::info!("[MediaCluster] remove room index {index}, hash {room}"); self.rooms_map.remove(&room).expect("Should have room with index"); @@ -166,7 +183,11 @@ mod tests { use media_server_protocol::endpoint::{PeerId, PeerInfo, PeerMeta, RoomInfoPublish, RoomInfoSubscribe}; use sans_io_runtime::TaskSwitcherChild; - use crate::cluster::{id_generator, ClusterEndpointEvent}; + use crate::cluster::{ + id_generator, + room::{RoomFeature, RoomUserData}, + ClusterEndpointEvent, + }; use super::{ClusterEndpointControl, ClusterRoomHash, MediaCluster, Output}; @@ -177,9 +198,9 @@ mod tests { fn room_manager_should_work() { let mut cluster = MediaCluster::::default(); - let owner = 1; - let room_hash = ClusterRoomHash(1); - let room_peers_map = id_generator::peers_map(room_hash); + let endpoint = 1; + let userdata = RoomUserData(ClusterRoomHash(1), RoomFeature::MetaData); + let room_peers_map = id_generator::peers_map(userdata.0); let peer = PeerId("peer1".to_string()); let peer_key = id_generator::peers_key(&peer); let peer_info = PeerInfo::new(peer.clone(), PeerMeta { metadata: None }); @@ -188,54 +209,55 @@ mod tests { // Not join room with scope (peer true, track false) should Set and Sub cluster.on_endpoint_control( now, - owner, - room_hash, + endpoint, + userdata.0, ClusterEndpointControl::Join( peer.clone(), peer_info.meta.clone(), RoomInfoPublish { peer: true, tracks: false }, RoomInfoSubscribe { peers: true, tracks: false }, + None, ), ); assert_eq!( - cluster.pop_output(now), + cluster.pop_output(()), Some(Output::Sdn( - room_hash, + userdata, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Set(peer_key, peer_info.serialize()))) )) ); assert_eq!( - cluster.pop_output(now), - Some(Output::Sdn(room_hash, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Sub)))) + cluster.pop_output(()), + Some(Output::Sdn(userdata, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Sub)))) ); - assert_eq!(cluster.pop_output(now), None); + assert_eq!(cluster.pop_output(()), None); assert_eq!(cluster.rooms.tasks(), 1); assert_eq!(cluster.rooms_map.len(), 1); // Correct forward to room cluster.on_sdn_event( now, - room_hash, + userdata, FeaturesEvent::DhtKv(dht_kv::Event::MapEvent(room_peers_map, MapEvent::OnSet(peer_key, 1, peer_info.serialize()))), ); assert_eq!( - cluster.pop_output(now), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::PeerJoined(peer.clone(), peer_info.meta.clone()))) + cluster.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::PeerJoined(peer.clone(), peer_info.meta.clone()))) ); - assert_eq!(cluster.pop_output(now), None); + assert_eq!(cluster.pop_output(()), None); // Now leave room should Del and Unsub - cluster.on_endpoint_control(now, owner, room_hash, ClusterEndpointControl::Leave); + cluster.on_endpoint_control(now, endpoint, userdata.0, ClusterEndpointControl::Leave); assert_eq!( - cluster.pop_output(now), - Some(Output::Sdn(room_hash, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Del(peer_key))))) + cluster.pop_output(()), + Some(Output::Sdn(userdata, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Del(peer_key))))) ); assert_eq!( - cluster.pop_output(now), - Some(Output::Sdn(room_hash, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Unsub)))) + cluster.pop_output(()), + Some(Output::Sdn(userdata, FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, MapControl::Unsub)))) ); - assert_eq!(cluster.pop_output(now), Some(Output::Continue)); //this is for destroy event - assert_eq!(cluster.pop_output(now), None); + assert_eq!(cluster.pop_output(()), Some(Output::Continue)); //this is for destroy event + assert_eq!(cluster.pop_output(()), None); assert_eq!(cluster.rooms.tasks(), 0); assert_eq!(cluster.rooms_map.len(), 0); } diff --git a/packages/media_core/src/cluster/id_generator.rs b/packages/media_core/src/cluster/id_generator.rs index 0d662f4f..4781de78 100644 --- a/packages/media_core/src/cluster/id_generator.rs +++ b/packages/media_core/src/cluster/id_generator.rs @@ -40,3 +40,10 @@ pub fn gen_channel_id>(room: ClusterRoomHash, peer: &PeerId, track: track.as_ref().hash(&mut h); h.finish().into() } + +pub fn gen_mixer_auto_channel_id>(room: ClusterRoomHash) -> T { + let mut h = std::hash::DefaultHasher::new(); + room.as_ref().hash(&mut h); + "mixer_auto".hash(&mut h); + h.finish().into() +} diff --git a/packages/media_core/src/cluster/room.rs b/packages/media_core/src/cluster/room.rs index 84fe7cbf..644c23ca 100644 --- a/packages/media_core/src/cluster/room.rs +++ b/packages/media_core/src/cluster/room.rs @@ -4,196 +4,248 @@ //! Main functions: //! //! - Send/Recv metadata related key-value -//! - Send/Receive pubsub channel +//! - Send/Recv media channel +//! - AudioMixer feature //! use std::{fmt::Debug, hash::Hash, time::Instant}; -use atm0s_sdn::features::{dht_kv, pubsub, FeaturesControl, FeaturesEvent}; -use sans_io_runtime::{collections::DynamicDeque, return_if_none, return_if_some, Task, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; +use atm0s_sdn::features::{dht_kv, FeaturesControl, FeaturesEvent}; +use sans_io_runtime::{return_if_none, Task, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; use crate::transport::{LocalTrackId, RemoteTrackId}; -use self::{channel_pub::RoomChannelPublisher, channel_sub::RoomChannelSubscribe, metadata::RoomMetadata}; +use audio_mixer::AudioMixer; +use media_track::MediaTrack; +use metadata::RoomMetadata; -use super::{ClusterEndpointControl, ClusterEndpointEvent, ClusterLocalTrackControl, ClusterRemoteTrackControl, ClusterRoomHash}; +use super::{id_generator, ClusterEndpointControl, ClusterEndpointEvent, ClusterLocalTrackControl, ClusterRemoteTrackControl, ClusterRoomHash}; -mod channel_pub; -mod channel_sub; +mod audio_mixer; +mod media_track; mod metadata; -pub enum Input { - Sdn(FeaturesEvent), - Endpoint(Owner, ClusterEndpointControl), +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub enum RoomFeature { + MetaData, + MediaTrack, + AudioMixer, } -pub enum Output { - Sdn(ClusterRoomHash, FeaturesControl), - Endpoint(Vec, ClusterEndpointEvent), +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct RoomUserData(pub(crate) ClusterRoomHash, pub(crate) RoomFeature); + +pub enum Input { + Sdn(RoomUserData, FeaturesEvent), + Endpoint(Endpoint, ClusterEndpointControl), +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Output { + Sdn(RoomUserData, FeaturesControl), + Endpoint(Vec, ClusterEndpointEvent), Destroy(ClusterRoomHash), } #[derive(num_enum::TryFromPrimitive, num_enum::IntoPrimitive)] #[repr(usize)] enum TaskType { - Publisher, - Subscriber, Metadata, + MediaTrack, + AudioMixer, } -pub struct ClusterRoom { +pub struct ClusterRoom { room: ClusterRoomHash, - metadata: TaskSwitcherBranch, metadata::Output>, - publisher: TaskSwitcherBranch, channel_pub::Output>, - subscriber: TaskSwitcherBranch, channel_sub::Output>, + metadata: TaskSwitcherBranch, metadata::Output>, + media_track: TaskSwitcherBranch, media_track::Output>, + audio_mixer: TaskSwitcherBranch, audio_mixer::Output>, switcher: TaskSwitcher, - queue: DynamicDeque, 64>, } -impl Task, Output> for ClusterRoom { - fn on_tick(&mut self, _now: Instant) {} +impl Task, Output> for ClusterRoom { + fn on_tick(&mut self, now: Instant) { + self.audio_mixer.input(&mut self.switcher).on_tick(now); + } - fn on_event(&mut self, now: Instant, input: Input) { + fn on_event(&mut self, now: Instant, input: Input) { match input { - Input::Endpoint(owner, control) => self.on_endpoint_control(now, owner, control), - Input::Sdn(event) => self.on_sdn_event(now, event), + Input::Endpoint(endpoint, control) => self.on_endpoint_control(now, endpoint, control), + Input::Sdn(userdata, event) => self.on_sdn_event(now, userdata, event), } } fn on_shutdown(&mut self, _now: Instant) {} } -impl TaskSwitcherChild> for ClusterRoom { - type Time = Instant; - fn pop_output(&mut self, now: Instant) -> Option> { - return_if_some!(self.queue.pop_front()); - +impl TaskSwitcherChild> for ClusterRoom { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { loop { match self.switcher.current()?.try_into().ok()? { - TaskType::Metadata => self.pop_meta_output(now), - TaskType::Publisher => self.pop_publisher_output(now), - TaskType::Subscriber => self.pop_subscriber_output(now), + TaskType::Metadata => { + if let Some(out) = self.metadata.pop_output((), &mut self.switcher) { + match out { + metadata::Output::Kv(control) => break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::MetaData), FeaturesControl::DhtKv(control))), + metadata::Output::Endpoint(endpoints, event) => break Some(Output::Endpoint(endpoints, event)), + metadata::Output::OnResourceEmpty => { + if self.is_empty() { + break Some(Output::Destroy(self.room)); + } + } + } + } + } + TaskType::MediaTrack => { + if let Some(out) = self.media_track.pop_output((), &mut self.switcher) { + match out { + media_track::Output::Endpoint(endpoints, event) => break Some(Output::Endpoint(endpoints, event)), + media_track::Output::Pubsub(control) => break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::MediaTrack), FeaturesControl::PubSub(control))), + media_track::Output::OnResourceEmpty => { + if self.is_empty() { + break Some(Output::Destroy(self.room)); + } + } + } + } + } + TaskType::AudioMixer => { + if let Some(out) = self.audio_mixer.pop_output((), &mut self.switcher) { + match out { + audio_mixer::Output::Endpoint(endpoints, event) => break Some(Output::Endpoint(endpoints, event)), + audio_mixer::Output::Pubsub(control) => break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::AudioMixer), FeaturesControl::PubSub(control))), + audio_mixer::Output::OnResourceEmpty => { + if self.is_empty() { + break Some(Output::Destroy(self.room)); + } + } + } + } + } } - - return_if_some!(self.queue.pop_front()); } } } -impl ClusterRoom { +impl ClusterRoom { pub fn new(room: ClusterRoomHash) -> Self { + let mixer_channel_id = id_generator::gen_mixer_auto_channel_id(room); Self { room, metadata: TaskSwitcherBranch::new(RoomMetadata::new(room), TaskType::Metadata), - publisher: TaskSwitcherBranch::new(RoomChannelPublisher::new(room), TaskType::Publisher), - subscriber: TaskSwitcherBranch::new(RoomChannelSubscribe::new(room), TaskType::Subscriber), + media_track: TaskSwitcherBranch::new(MediaTrack::new(room), TaskType::MediaTrack), + audio_mixer: TaskSwitcherBranch::new(AudioMixer::new(room, mixer_channel_id), TaskType::AudioMixer), switcher: TaskSwitcher::new(3), - queue: DynamicDeque::default(), } } - fn on_sdn_event(&mut self, _now: Instant, event: FeaturesEvent) { - match event { - FeaturesEvent::DhtKv(event) => match event { + pub fn is_empty(&self) -> bool { + self.metadata.is_empty() && self.media_track.is_empty() && self.audio_mixer.is_empty() + } + + fn on_sdn_event(&mut self, now: Instant, userdata: RoomUserData, event: FeaturesEvent) { + match (userdata.1, event) { + (RoomFeature::MetaData, FeaturesEvent::DhtKv(event)) => match event { dht_kv::Event::MapEvent(map, event) => self.metadata.input(&mut self.switcher).on_kv_event(map, event), dht_kv::Event::MapGetRes(_, _) => {} }, - FeaturesEvent::PubSub(pubsub::Event(channel, event)) => match event { - pubsub::ChannelEvent::RouteChanged(next) => { - self.subscriber.input(&mut self.switcher).on_channel_relay_changed(channel, next); - } - pubsub::ChannelEvent::SourceData(_, data) => { - self.subscriber.input(&mut self.switcher).on_channel_data(channel, data); - } - pubsub::ChannelEvent::FeedbackData(fb) => { - self.publisher.input(&mut self.switcher).on_channel_feedback(channel, fb); - } - }, + (RoomFeature::MediaTrack, FeaturesEvent::PubSub(event)) => { + self.media_track.input(&mut self.switcher).on_pubsub_event(event); + } + (RoomFeature::AudioMixer, FeaturesEvent::PubSub(event)) => { + self.audio_mixer.input(&mut self.switcher).on_pubsub_event(now, event); + } _ => {} } } - fn on_endpoint_control(&mut self, now: Instant, owner: Owner, control: ClusterEndpointControl) { + fn on_endpoint_control(&mut self, now: Instant, endpoint: Endpoint, control: ClusterEndpointControl) { match control { - ClusterEndpointControl::Join(peer, meta, publish, subscribe) => { - self.metadata.input(&mut self.switcher).on_join(owner, peer, meta, publish, subscribe); + ClusterEndpointControl::Join(peer, meta, publish, subscribe, mixer) => { + self.audio_mixer.input(&mut self.switcher).on_join(now, endpoint, peer.clone(), mixer); + self.metadata.input(&mut self.switcher).on_join(endpoint, peer, meta, publish, subscribe); } ClusterEndpointControl::Leave => { - self.metadata.input(&mut self.switcher).on_leave(owner); + self.audio_mixer.input(&mut self.switcher).on_leave(now, endpoint); + self.metadata.input(&mut self.switcher).on_leave(endpoint); } ClusterEndpointControl::SubscribePeer(target) => { - self.metadata.input(&mut self.switcher).on_subscribe_peer(owner, target); + self.metadata.input(&mut self.switcher).on_subscribe_peer(endpoint, target); } ClusterEndpointControl::UnsubscribePeer(target) => { - self.metadata.input(&mut self.switcher).on_unsubscribe_peer(owner, target); + self.metadata.input(&mut self.switcher).on_unsubscribe_peer(endpoint, target); + } + ClusterEndpointControl::AudioMixer(control) => { + self.audio_mixer.input(&mut self.switcher).on_control(now, endpoint, control); } - ClusterEndpointControl::RemoteTrack(track, control) => self.on_control_remote_track(now, owner, track, control), - ClusterEndpointControl::LocalTrack(track, control) => self.on_control_local_track(now, owner, track, control), + ClusterEndpointControl::RemoteTrack(track, control) => self.on_control_remote_track(now, endpoint, track, control), + ClusterEndpointControl::LocalTrack(track, control) => self.on_control_local_track(now, endpoint, track, control), } } } -impl ClusterRoom { - fn on_control_remote_track(&mut self, _now: Instant, owner: Owner, track: RemoteTrackId, control: ClusterRemoteTrackControl) { +impl ClusterRoom { + fn on_control_remote_track(&mut self, now: Instant, endpoint: Endpoint, track: RemoteTrackId, control: ClusterRemoteTrackControl) { match control { ClusterRemoteTrackControl::Started(name, meta) => { - let peer = return_if_none!(self.metadata.get_peer_from_owner(owner)); - log::info!("[ClusterRoom {}] started track {:?}/{track} => {peer}/{name}", self.room, owner); + let peer = return_if_none!(self.metadata.get_peer_from_endpoint(endpoint)); + log::info!("[ClusterRoom {}] started track {:?}/{track} => {peer}/{name}", self.room, endpoint); - self.publisher.input(&mut self.switcher).on_track_publish(owner, track, peer, name.clone()); - self.metadata.input(&mut self.switcher).on_track_publish(owner, track, name.clone(), meta.clone()); + if meta.kind.is_audio() { + self.audio_mixer.input(&mut self.switcher).on_track_publish(now, endpoint, track, peer.clone(), name.clone()); + } + self.media_track.input(&mut self.switcher).on_track_publish(endpoint, track, peer, name.clone()); + self.metadata.input(&mut self.switcher).on_track_publish(endpoint, track, name, meta.clone()); } ClusterRemoteTrackControl::Media(media) => { - self.publisher.input(&mut self.switcher).on_track_data(owner, track, media); + if media.meta.is_audio() { + self.audio_mixer.input(&mut self.switcher).on_track_data(now, endpoint, track, &media); + } + self.media_track.input(&mut self.switcher).on_track_data(endpoint, track, media); } - ClusterRemoteTrackControl::Ended => { - log::info!("[ClusterRoom {}] stopped track {:?}/{track}", self.room, owner); - self.publisher.input(&mut self.switcher).on_track_unpublish(owner, track); - self.metadata.input(&mut self.switcher).on_track_unpublish(owner, track); + ClusterRemoteTrackControl::Ended(_name, meta) => { + log::info!("[ClusterRoom {}] stopped track {:?}/{track}", self.room, endpoint); + + if meta.kind.is_audio() { + self.audio_mixer.input(&mut self.switcher).on_track_unpublish(now, endpoint, track); + } + self.media_track.input(&mut self.switcher).on_track_unpublish(endpoint, track); + self.metadata.input(&mut self.switcher).on_track_unpublish(endpoint, track); } } } - fn on_control_local_track(&mut self, now: Instant, owner: Owner, track_id: LocalTrackId, control: ClusterLocalTrackControl) { + fn on_control_local_track(&mut self, now: Instant, endpoint: Endpoint, track_id: LocalTrackId, control: ClusterLocalTrackControl) { match control { - ClusterLocalTrackControl::Subscribe(target_peer, target_track) => self.subscriber.input(&mut self.switcher).on_track_subscribe(owner, track_id, target_peer, target_track), - ClusterLocalTrackControl::RequestKeyFrame => self.subscriber.input(&mut self.switcher).on_track_request_key(owner, track_id), - ClusterLocalTrackControl::DesiredBitrate(bitrate) => self.subscriber.input(&mut self.switcher).on_track_desired_bitrate(now, owner, track_id, bitrate), - ClusterLocalTrackControl::Unsubscribe => self.subscriber.input(&mut self.switcher).on_track_unsubscribe(owner, track_id), + ClusterLocalTrackControl::Subscribe(target_peer, target_track) => self.media_track.input(&mut self.switcher).on_track_subscribe(endpoint, track_id, target_peer, target_track), + ClusterLocalTrackControl::RequestKeyFrame => self.media_track.input(&mut self.switcher).on_track_request_key(endpoint, track_id), + ClusterLocalTrackControl::DesiredBitrate(bitrate) => self.media_track.input(&mut self.switcher).on_track_desired_bitrate(now, endpoint, track_id, bitrate), + ClusterLocalTrackControl::Unsubscribe => self.media_track.input(&mut self.switcher).on_track_unsubscribe(endpoint, track_id), } } +} - fn pop_meta_output(&mut self, now: Instant) { - let out = return_if_none!(self.metadata.pop_output(now, &mut self.switcher)); - let out = match out { - metadata::Output::Kv(control) => Output::Sdn(self.room, FeaturesControl::DhtKv(control)), - metadata::Output::Endpoint(owners, event) => Output::Endpoint(owners, event), - metadata::Output::LastPeerLeaved => Output::Destroy(self.room), - }; - self.queue.push_back(out); - } - - fn pop_publisher_output(&mut self, now: Instant) { - let out = return_if_none!(self.publisher.pop_output(now, &mut self.switcher)); - let out = match out { - channel_pub::Output::Pubsub(control) => Output::Sdn(self.room, FeaturesControl::PubSub(control)), - channel_pub::Output::Endpoint(owners, event) => Output::Endpoint(owners, event), - }; - self.queue.push_back(out); - } - - fn pop_subscriber_output(&mut self, now: Instant) { - let out = return_if_none!(self.subscriber.pop_output(now, &mut self.switcher)); - let out = match out { - channel_sub::Output::Pubsub(control) => Output::Sdn(self.room, FeaturesControl::PubSub(control)), - channel_sub::Output::Endpoint(owners, event) => Output::Endpoint(owners, event), - }; - self.queue.push_back(out); +impl Drop for ClusterRoom { + fn drop(&mut self) { + log::info!("Drop ClusterRoom {}", self.room); + assert!(self.audio_mixer.is_empty(), "Audio mixer not empty"); + assert!(self.media_track.is_empty(), "Media track not empty"); + assert!(self.metadata.is_empty(), "Metadata not empty"); } } #[cfg(test)] mod tests { + use std::time::Instant; + + use atm0s_sdn::features::{dht_kv, pubsub, FeaturesControl}; + use media_server_protocol::endpoint::{AudioMixerConfig, AudioMixerMode, PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe}; + use sans_io_runtime::{Task, TaskSwitcherChild}; + + use crate::cluster::{id_generator, room::RoomFeature, ClusterEndpointControl, RoomUserData}; + + use super::{ClusterRoom, Input, Output}; + //TODO join room should set key-value and SUB to maps //TODO maps event should fire event to endpoint //TODO leave room should del key-value @@ -204,4 +256,83 @@ mod tests { //TODO feddback track should FEEDBACK channel //TODO channel data should fire event to endpoint //TODO unsubscribe track should UNSUB channel + + #[test] + fn cleanup_resource_sub_and_mixer() { + let room_id = 0.into(); + let endpoint = 1; + let peer: PeerId = "peer1".into(); + let t0 = Instant::now(); + let mut room = ClusterRoom::::new(room_id); + room.on_event( + t0, + Input::Endpoint( + endpoint, + ClusterEndpointControl::Join( + peer.clone(), + PeerMeta { metadata: None }, + RoomInfoPublish { peer: false, tracks: false }, + RoomInfoSubscribe { peers: true, tracks: true }, + Some(AudioMixerConfig { + mode: AudioMixerMode::Auto, + outputs: vec![0.into(), 1.into(), 2.into()], + sources: vec![], + }), + ), + ), + ); + + let room_peers_map = id_generator::peers_map(room_id); + let room_tracks_map = id_generator::tracks_map(room_id); + let room_mixer_auto_channel = id_generator::gen_mixer_auto_channel_id(room_id); + + assert_eq!( + room.pop_output(()), + Some(Output::Sdn( + RoomUserData(room_id, RoomFeature::MetaData), + FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, dht_kv::MapControl::Sub)) + )) + ); + assert_eq!( + room.pop_output(()), + Some(Output::Sdn( + RoomUserData(room_id, RoomFeature::MetaData), + FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_tracks_map, dht_kv::MapControl::Sub)) + )) + ); + assert_eq!( + room.pop_output(()), + Some(Output::Sdn( + RoomUserData(room_id, RoomFeature::AudioMixer), + FeaturesControl::PubSub(pubsub::Control(room_mixer_auto_channel, pubsub::ChannelControl::SubAuto)) + )) + ); + assert_eq!(room.pop_output(()), None); + + //after leave we should auto cleanup all resources like kv, pubsub + room.on_event(t0, Input::Endpoint(endpoint, ClusterEndpointControl::Leave)); + assert_eq!( + room.pop_output(()), + Some(Output::Sdn( + RoomUserData(room_id, RoomFeature::MetaData), + FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_peers_map, dht_kv::MapControl::Unsub)) + )) + ); + assert_eq!( + room.pop_output(()), + Some(Output::Sdn( + RoomUserData(room_id, RoomFeature::MetaData), + FeaturesControl::DhtKv(dht_kv::Control::MapCmd(room_tracks_map, dht_kv::MapControl::Unsub)) + )) + ); + assert_eq!( + room.pop_output(()), + Some(Output::Sdn( + RoomUserData(room_id, RoomFeature::AudioMixer), + FeaturesControl::PubSub(pubsub::Control(room_mixer_auto_channel, pubsub::ChannelControl::UnsubAuto)) + )) + ); + assert_eq!(room.pop_output(()), Some(Output::Destroy(room_id))); + assert_eq!(room.pop_output(()), None); + } } diff --git a/packages/media_core/src/cluster/room/audio_mixer.rs b/packages/media_core/src/cluster/room/audio_mixer.rs new file mode 100644 index 00000000..4eb2b1c0 --- /dev/null +++ b/packages/media_core/src/cluster/room/audio_mixer.rs @@ -0,0 +1,293 @@ +//! +//! Audio mixer in room level is split to 2 part: +//! - Publisher: detect top 3 audio and publish to /room_id/audio_mixer channel +//! - Subscriber: subscribe to /room_id/audio_mixer to get all of top-3 audios from other servers +//! calculate top-3 audio for each local endpoint +//! + +//TODO refactor multiple subscriber mode to array instead of manual implement with subscriber1, subscriber2, subscriber3 + +use std::{ + collections::HashMap, + fmt::Debug, + hash::Hash, + time::{Duration, Instant}, +}; + +use atm0s_sdn::features::pubsub::{self, ChannelId}; +use manual::ManualMixer; +use media_server_protocol::{ + endpoint::{AudioMixerConfig, PeerId, TrackName}, + media::MediaPacket, +}; +use sans_io_runtime::{TaskGroup, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; + +use crate::{ + cluster::{ClusterAudioMixerControl, ClusterEndpointEvent, ClusterRoomHash}, + transport::RemoteTrackId, +}; + +use publisher::AudioMixerPublisher; +use subscriber::AudioMixerSubscriber; + +mod manual; +mod publisher; +mod subscriber; + +const TICK_INTERVAL: Duration = Duration::from_millis(1000); + +#[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] +#[repr(usize)] +pub enum TaskType { + Publisher, + Subscriber1, + Subscriber2, + Subscriber3, + Manuals, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Output { + Endpoint(Vec, ClusterEndpointEvent), + Pubsub(pubsub::Control), + OnResourceEmpty, +} + +pub struct AudioMixer { + room: ClusterRoomHash, + mix_channel_id: ChannelId, + //store number of outputs + auto_mode: HashMap, + manual_mode: HashMap, + manual_channels: HashMap>, + publisher: TaskSwitcherBranch, Output>, + subscriber1: TaskSwitcherBranch, Output>, + subscriber2: TaskSwitcherBranch, Output>, + subscriber3: TaskSwitcherBranch, Output>, + manuals: TaskSwitcherBranch, ManualMixer, 4>, (usize, Output)>, + switcher: TaskSwitcher, + last_tick: Instant, +} + +impl AudioMixer { + pub fn new(room: ClusterRoomHash, mix_channel_id: ChannelId) -> Self { + Self { + room, + mix_channel_id, + auto_mode: HashMap::new(), + manual_mode: HashMap::new(), + manual_channels: HashMap::new(), + publisher: TaskSwitcherBranch::new(AudioMixerPublisher::new(mix_channel_id), TaskType::Publisher), + subscriber1: TaskSwitcherBranch::new(AudioMixerSubscriber::new(mix_channel_id), TaskType::Subscriber1), + subscriber2: TaskSwitcherBranch::new(AudioMixerSubscriber::new(mix_channel_id), TaskType::Subscriber2), + subscriber3: TaskSwitcherBranch::new(AudioMixerSubscriber::new(mix_channel_id), TaskType::Subscriber3), + manuals: TaskSwitcherBranch::new(Default::default(), TaskType::Manuals), + switcher: TaskSwitcher::new(5), + last_tick: Instant::now(), + } + } + + /// + /// We need to wait all publisher, subscriber, and manuals ready to remove + /// + pub fn is_empty(&self) -> bool { + self.publisher.is_empty() && self.subscriber1.is_empty() && self.subscriber2.is_empty() && self.subscriber3.is_empty() && self.manuals.tasks() == 0 + } + + pub fn on_tick(&mut self, now: Instant) { + if now >= self.last_tick + TICK_INTERVAL { + self.last_tick = now; + self.publisher.input(&mut self.switcher).on_tick(now); + self.subscriber1.input(&mut self.switcher).on_tick(now); + self.subscriber2.input(&mut self.switcher).on_tick(now); + self.subscriber3.input(&mut self.switcher).on_tick(now); + self.manuals.input(&mut self.switcher).on_tick(now); + } + } + + pub fn on_join(&mut self, now: Instant, endpoint: Endpoint, peer: PeerId, cfg: Option) { + if let Some(cfg) = cfg { + match cfg.mode { + media_server_protocol::endpoint::AudioMixerMode::Auto => { + self.auto_mode.insert(endpoint.clone(), cfg.outputs.len()); + match cfg.outputs.len() { + 1 => self.subscriber1.input(&mut self.switcher).on_endpoint_join(now, endpoint, peer, cfg.outputs), + 2 => self.subscriber2.input(&mut self.switcher).on_endpoint_join(now, endpoint, peer, cfg.outputs), + 3 => self.subscriber3.input(&mut self.switcher).on_endpoint_join(now, endpoint, peer, cfg.outputs), + _ => { + log::warn!("[ClusterRoomAudioMixer] unsupported mixer with {} outputs", cfg.outputs.len()); + } + } + } + media_server_protocol::endpoint::AudioMixerMode::Manual => { + log::info!("[ClusterRoomAudioMixer] add manual mode for {:?} {peer}", endpoint); + let manual_mixer = ManualMixer::new(self.room, endpoint.clone(), cfg.outputs); + let new_index = self.manuals.input(&mut self.switcher).add_task(manual_mixer); + if let Some(_old_index) = self.manual_mode.insert(endpoint, new_index) { + panic!("Manual mixer for endpoint already exist"); + } + } + } + } + } + + pub fn on_control(&mut self, now: Instant, endpoint: Endpoint, control: ClusterAudioMixerControl) { + log::info!("[ClusterRoomAudioMixer] on endpoint {:?} input {:?}", endpoint, control); + let index = *self.manual_mode.get(&endpoint).expect("Manual mixer not found for control"); + let input = match control { + ClusterAudioMixerControl::Attach(sources) => manual::Input::Attach(sources), + ClusterAudioMixerControl::Detach(sources) => manual::Input::Detach(sources), + }; + self.manuals.input(&mut self.switcher).on_event(now, index, input); + } + + pub fn on_leave(&mut self, now: Instant, endpoint: Endpoint) { + if let Some(outputs) = self.auto_mode.remove(&endpoint) { + match outputs { + 1 => self.subscriber1.input(&mut self.switcher).on_endpoint_leave(now, endpoint), + 2 => self.subscriber2.input(&mut self.switcher).on_endpoint_leave(now, endpoint), + 3 => self.subscriber3.input(&mut self.switcher).on_endpoint_leave(now, endpoint), + _ => { + log::warn!("[ClusterRoomAudioMixer] unsupported mixer with {} outputs", outputs); + } + } + } else if let Some(index) = self.manual_mode.remove(&endpoint) { + log::info!("[ClusterRoomAudioMixer] endpoint {:?} leave from manual mode", endpoint); + self.manual_mode.remove(&endpoint); + self.manuals.input(&mut self.switcher).on_event(now, index, manual::Input::LeaveRoom); + } + } + + pub fn on_track_publish(&mut self, now: Instant, endpoint: Endpoint, track: RemoteTrackId, peer: PeerId, name: TrackName) { + if self.auto_mode.contains_key(&endpoint) { + self.publisher.input(&mut self.switcher).on_track_publish(now, endpoint, track, peer, name); + } + } + + pub fn on_track_data(&mut self, now: Instant, endpoint: Endpoint, track: RemoteTrackId, media: &MediaPacket) { + if self.auto_mode.contains_key(&endpoint) { + self.publisher.input(&mut self.switcher).on_track_data(now, endpoint, track, media); + } + } + + pub fn on_track_unpublish(&mut self, now: Instant, endpoint: Endpoint, track: RemoteTrackId) { + if self.auto_mode.contains_key(&endpoint) { + self.publisher.input(&mut self.switcher).on_track_unpublish(now, endpoint, track); + } + } + + pub fn on_pubsub_event(&mut self, now: Instant, event: pubsub::Event) { + match event.1 { + pubsub::ChannelEvent::RouteChanged(_next) => {} + pubsub::ChannelEvent::SourceData(from, data) => { + if event.0 == self.mix_channel_id { + self.subscriber1.input(&mut self.switcher).on_channel_data(now, from, &data); + self.subscriber2.input(&mut self.switcher).on_channel_data(now, from, &data); + self.subscriber3.input(&mut self.switcher).on_channel_data(now, from, &data); + } else if let Some(tasks) = self.manual_channels.get(&event.0) { + for task_index in tasks { + self.manuals.input(&mut self.switcher).on_event(now, *task_index, manual::Input::Pubsub(event.0, from, data.clone())); + } + } + } + pubsub::ChannelEvent::FeedbackData(_fb) => {} + } + } +} + +impl TaskSwitcherChild> for AudioMixer { + type Time = (); + + /// + /// We need to wait all publisher, subscriber, and manuals ready to remove + /// + fn pop_output(&mut self, _now: Self::Time) -> Option> { + loop { + match self.switcher.current()?.try_into().ok()? { + TaskType::Publisher => { + if let Some(out) = self.publisher.pop_output((), &mut self.switcher) { + if let Output::OnResourceEmpty = out { + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } else { + return Some(out); + } + } + } + TaskType::Subscriber1 => { + if let Some(out) = self.subscriber1.pop_output((), &mut self.switcher) { + if let Output::OnResourceEmpty = out { + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } else { + return Some(out); + } + } + } + TaskType::Subscriber2 => { + if let Some(out) = self.subscriber2.pop_output((), &mut self.switcher) { + if let Output::OnResourceEmpty = out { + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } else { + return Some(out); + } + } + } + TaskType::Subscriber3 => { + if let Some(out) = self.subscriber3.pop_output((), &mut self.switcher) { + if let Output::OnResourceEmpty = out { + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } else { + return Some(out); + } + } + } + TaskType::Manuals => { + if let Some((index, out)) = self.manuals.pop_output((), &mut self.switcher) { + match out { + Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::SubAuto)) => { + if let Some(slot) = self.manual_channels.get_mut(&channel_id) { + slot.push(index); + } else { + self.manual_channels.insert(channel_id, vec![index]); + return Some(out); + } + } + Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto)) => { + let slot = self.manual_channels.get_mut(&channel_id).expect("Manual channel map not found"); + let (slot_index, _) = slot.iter().enumerate().find(|(_, task_i)| **task_i == index).expect("Subscribed task not found"); + slot.swap_remove(slot_index); + if slot.is_empty() { + self.manual_channels.remove(&channel_id); + return Some(out); + } + } + Output::OnResourceEmpty => { + self.manuals.input(&mut self.switcher).remove_task(index); + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } + _ => return Some(out), + } + } + } + } + } + } +} + +impl Drop for AudioMixer { + fn drop(&mut self) { + log::info!("[ClusterRoomAudioMixer] Drop {}", self.room); + assert_eq!(self.manual_channels.len(), 0, "Manual channels not empty on drop"); + assert_eq!(self.manual_mode.len(), 0, "Manual modes not empty on drop"); + assert_eq!(self.manuals.tasks(), 0, "Manuals not empty on drop"); + } +} diff --git a/packages/media_core/src/cluster/room/audio_mixer/manual.rs b/packages/media_core/src/cluster/room/audio_mixer/manual.rs new file mode 100644 index 00000000..feeeba19 --- /dev/null +++ b/packages/media_core/src/cluster/room/audio_mixer/manual.rs @@ -0,0 +1,248 @@ +//! +//! This is manual mode of audio mixer. +//! In this mode, each peer has separated mixer logic and subscribe to all audio sources +//! to determine which source is sent to client. +//! + +use std::{collections::HashMap, time::Instant}; + +use atm0s_sdn::{ + features::pubsub::{self, ChannelId}, + NodeId, +}; +use media_server_protocol::{ + endpoint::TrackSource, + media::{MediaMeta, MediaPacket}, + transport::LocalTrackId, +}; +use sans_io_runtime::{collections::DynamicDeque, Task, TaskSwitcherChild}; + +use crate::cluster::{id_generator, ClusterAudioMixerEvent, ClusterEndpointEvent, ClusterLocalTrackEvent, ClusterRoomHash}; + +use super::Output; + +#[derive(Debug)] +pub enum Input { + Attach(Vec), + Detach(Vec), + Pubsub(ChannelId, NodeId, Vec), + LeaveRoom, +} + +pub struct ManualMixer { + endpoint: Endpoint, + room: ClusterRoomHash, + outputs: Vec, + sources: HashMap, + queue: DynamicDeque, 4>, + mixer: audio_mixer::AudioMixer, +} + +impl ManualMixer { + pub fn new(room: ClusterRoomHash, endpoint: Endpoint, outputs: Vec) -> Self { + Self { + endpoint, + room, + mixer: audio_mixer::AudioMixer::new(outputs.len()), + outputs, + sources: HashMap::new(), + queue: Default::default(), + } + } + + fn attach(&mut self, _now: Instant, source: TrackSource) { + let channel_id = id_generator::gen_channel_id(self.room, &source.peer, &source.track); + if !self.sources.contains_key(&channel_id) { + log::info!("[ClusterManualMixer] add source {:?} => sub {channel_id}", source); + self.sources.insert(channel_id, source); + self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::SubAuto))); + } + } + + fn on_source_pkt(&mut self, now: Instant, channel: ChannelId, _from: NodeId, pkt: MediaPacket) { + if let MediaMeta::Opus { audio_level } = pkt.meta { + if let Some((slot, just_set)) = self.mixer.on_pkt(now, channel, audio_level) { + let track_id = self.outputs[slot]; + if just_set { + let source_info = self.sources.get(&channel).expect("Missing source info for channel"); + self.queue.push_back(Output::Endpoint( + vec![self.endpoint.clone()], + ClusterEndpointEvent::LocalTrack(track_id, ClusterLocalTrackEvent::SourceChanged), + )); + self.queue.push_back(Output::Endpoint( + vec![self.endpoint.clone()], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotSet(slot as u8, source_info.peer.clone(), source_info.track.clone())), + )); + } + + self.queue.push_back(Output::Endpoint( + vec![self.endpoint.clone()], + ClusterEndpointEvent::LocalTrack(track_id, ClusterLocalTrackEvent::Media(channel.0, pkt)), + )) + } + } + } + + fn detach(&mut self, _now: Instant, source: TrackSource) { + let channel_id = id_generator::gen_channel_id(self.room, &source.peer, &source.track); + if let Some(_) = self.sources.remove(&channel_id) { + log::info!("[ClusterManualMixer] remove source {:?} => unsub {channel_id}", source); + self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto))); + } + } +} + +impl Task> for ManualMixer { + fn on_tick(&mut self, now: Instant) { + if let Some(removed) = self.mixer.on_tick(now) { + for slot in removed { + self.queue.push_back(Output::Endpoint( + vec![self.endpoint.clone()], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotUnset(slot as u8)), + )); + } + } + } + + fn on_event(&mut self, now: Instant, input: Input) { + match input { + Input::Attach(sources) => { + for source in sources { + self.attach(now, source); + } + } + Input::Detach(sources) => { + for source in sources { + self.detach(now, source); + } + } + Input::Pubsub(channel, from, data) => { + if let Some(pkt) = MediaPacket::deserialize(&data) { + self.on_source_pkt(now, channel, from, pkt); + } + } + Input::LeaveRoom => { + // We need manual release sources because it is from client request, + // we cannot ensure client will release it before it disconnect. + let sources = std::mem::replace(&mut self.sources, Default::default()); + for (channel_id, source) in sources { + log::info!("[ClusterManualMixer] remove source {:?} on queue => unsub {channel_id}", source); + self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto))); + } + self.queue.push_back(Output::OnResourceEmpty); + } + } + } + + fn on_shutdown(&mut self, _now: Instant) {} +} + +impl TaskSwitcherChild> for ManualMixer { + type Time = (); + + fn pop_output(&mut self, _now: Self::Time) -> Option> { + self.queue.pop_front() + } +} + +impl Drop for ManualMixer { + fn drop(&mut self) { + log::info!("[ClusterManualMixer] Drop {}", self.room); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); + assert_eq!(self.sources.len(), 0, "Sources not empty on drop"); + } +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + use atm0s_sdn::features::pubsub; + use media_server_protocol::{ + endpoint::TrackSource, + media::{MediaMeta, MediaPacket}, + }; + use sans_io_runtime::{Task, TaskSwitcherChild}; + + use crate::cluster::{id_generator, ClusterAudioMixerEvent, ClusterEndpointEvent, ClusterLocalTrackEvent}; + + use super::{super::Output, Input, ManualMixer}; + + fn ms(ms: u64) -> Duration { + Duration::from_millis(ms) + } + + #[test] + fn attach_detach() { + let t0 = Instant::now(); + let room = 0.into(); + let endpoint = 1; + let track = 0.into(); + let mut manual = ManualMixer::::new(room, endpoint, vec![track]); + let source = TrackSource { + peer: "peer1".into(), + track: "audio".into(), + }; + let channel_id = id_generator::gen_channel_id(room, &source.peer, &source.track); + + manual.attach(t0, source.clone()); + assert_eq!(manual.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::SubAuto)))); + assert_eq!(manual.pop_output(()), None); + + let pkt = MediaPacket { + ts: 0, + seq: 0, + marker: false, + nackable: false, + layers: None, + meta: MediaMeta::Opus { audio_level: Some(-60) }, + data: vec![1, 2, 3, 4, 5, 6], + }; + manual.on_event(t0, Input::Pubsub(channel_id, 0, pkt.serialize())); + assert_eq!( + manual.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::LocalTrack(track, ClusterLocalTrackEvent::SourceChanged))) + ); + assert_eq!( + manual.pop_output(()), + Some(Output::Endpoint( + vec![1], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotSet(0, source.peer.clone(), source.track.clone())) + )), + ); + assert_eq!( + manual.pop_output(()), + Some(Output::Endpoint( + vec![endpoint], + ClusterEndpointEvent::LocalTrack(track, ClusterLocalTrackEvent::Media(channel_id.0, pkt)) + )), + ); + + manual.detach(t0 + ms(100), source.clone()); + assert_eq!(manual.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto)))); + assert_eq!(manual.pop_output(()), None); + } + + #[test] + fn leave_room() { + let t0 = Instant::now(); + let room = 0.into(); + let endpoint = 1; + let track = 0.into(); + let mut manual = ManualMixer::::new(room, endpoint, vec![track]); + let source = TrackSource { + peer: "peer1".into(), + track: "audio".into(), + }; + let channel_id = id_generator::gen_channel_id(room, &source.peer, &source.track); + + manual.attach(t0, source.clone()); + assert_eq!(manual.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::SubAuto)))); + assert_eq!(manual.pop_output(()), None); + + manual.on_event(t0, Input::LeaveRoom); + assert_eq!(manual.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto)))); + assert_eq!(manual.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(manual.pop_output(()), None); + } +} diff --git a/packages/media_core/src/cluster/room/audio_mixer/publisher.rs b/packages/media_core/src/cluster/room/audio_mixer/publisher.rs new file mode 100644 index 00000000..cc4f9601 --- /dev/null +++ b/packages/media_core/src/cluster/room/audio_mixer/publisher.rs @@ -0,0 +1,234 @@ +use std::{ + collections::HashMap, + fmt::Debug, + hash::Hash, + time::{Duration, Instant}, +}; + +use atm0s_sdn::features::pubsub::{self, ChannelId}; +use media_server_protocol::{ + endpoint::{AudioMixerPkt, PeerHashCode, PeerId, TrackName}, + media::{MediaMeta, MediaPacket}, +}; +use sans_io_runtime::{collections::DynamicDeque, TaskSwitcherChild}; + +use crate::transport::RemoteTrackId; + +use super::Output; + +const FIRE_SOURCE_INTERVAL: Duration = Duration::from_millis(500); + +struct TrackSlot { + peer: PeerId, + name: TrackName, + peer_hash: PeerHashCode, +} + +struct OutputSlot { + last_fired_source: Instant, +} + +pub struct AudioMixerPublisher { + channel_id: pubsub::ChannelId, + tracks: HashMap<(Endpoint, RemoteTrackId), TrackSlot>, + mixer: audio_mixer::AudioMixer<(Endpoint, RemoteTrackId)>, + slots: [Option; 3], + queue: DynamicDeque, 4>, +} + +impl AudioMixerPublisher { + pub fn new(channel_id: ChannelId) -> Self { + Self { + tracks: Default::default(), + channel_id, + mixer: audio_mixer::AudioMixer::new(3), + slots: [None, None, None], + queue: DynamicDeque::default(), + } + } + + pub fn is_empty(&self) -> bool { + self.tracks.is_empty() && self.queue.is_empty() + } + + pub fn on_tick(&mut self, now: Instant) { + if let Some(removed_slots) = self.mixer.on_tick(now) { + for slot in removed_slots { + self.slots[slot] = None; + } + } + } + + pub fn on_track_publish(&mut self, _now: Instant, endpoint: Endpoint, track: RemoteTrackId, peer: PeerId, name: TrackName) { + log::debug!("on track publish {peer}/{name}/{track}"); + let key = (endpoint, track); + assert!(!self.tracks.contains_key(&key)); + if self.tracks.is_empty() { + log::info!("[ClusterAudioMixerPublisher] first track join as Auto mode => publish channel {}", self.channel_id); + self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::PubStart))); + } + self.tracks.insert( + key.clone(), + TrackSlot { + peer_hash: peer.hash_code(), + peer, + name, + }, + ); + } + + pub fn on_track_data(&mut self, now: Instant, endpoint: Endpoint, track: RemoteTrackId, media: &MediaPacket) { + let key = (endpoint, track); + let info = self.tracks.get(&key).expect("Track not found"); + if let MediaMeta::Opus { audio_level } = &media.meta { + if let Some((slot, just_set)) = self.mixer.on_pkt(now, key.clone(), *audio_level) { + let mut source = None; + if just_set { + self.slots[slot] = Some(OutputSlot { last_fired_source: now }); + source = Some((info.peer.clone(), info.name.clone())); + } else { + let slot_info = self.slots[slot].as_mut().expect("Output slot not found"); + if slot_info.last_fired_source + FIRE_SOURCE_INTERVAL <= now { + slot_info.last_fired_source = now; + source = Some((info.peer.clone(), info.name.clone())); + } + }; + let pkt = AudioMixerPkt { + slot: slot as u8, + peer: info.peer_hash, + track, + audio_level: *audio_level, + source, + ts: media.ts, + seq: media.seq, + opus_payload: media.data.clone(), + }; + self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::PubData(pkt.serialize())))) + } + } + } + + pub fn on_track_unpublish(&mut self, _now: Instant, endpoint: Endpoint, track: RemoteTrackId) { + log::debug!("[ClusterAudioMixerPublisher] on track unpublish {track}"); + let key = (endpoint, track); + assert!(self.tracks.contains_key(&key)); + self.tracks.remove(&key); + if self.tracks.is_empty() { + log::info!("[ClusterAudioMixerPublisher] last track leave ind Auto mode => unpublish channel {}", self.channel_id); + self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::PubStop))); + self.queue.push_back(Output::OnResourceEmpty); + } + } +} + +impl TaskSwitcherChild> for AudioMixerPublisher { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { + self.queue.pop_front() + } +} + +impl Drop for AudioMixerPublisher { + fn drop(&mut self) { + log::info!("[ClusterAudioMixerPublisher] Drop {}", self.channel_id); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); + assert_eq!(self.tracks.len(), 0, "Tracks not empty on drop"); + } +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + use atm0s_sdn::features::pubsub; + use media_server_protocol::{ + endpoint::{AudioMixerPkt, PeerId}, + media::{MediaMeta, MediaPacket}, + }; + use sans_io_runtime::TaskSwitcherChild; + + use super::{super::Output, AudioMixerPublisher}; + + fn ms(m: u64) -> Duration { + Duration::from_millis(m) + } + + #[test] + fn track_publish_unpublish() { + let channel = 0.into(); + let peer1: PeerId = "peer1".into(); + let peer2: PeerId = "peer2".into(); + + let mut publisher = AudioMixerPublisher::::new(channel); + + let t0 = Instant::now(); + + publisher.on_track_publish(t0, 1, 0.into(), peer1.clone(), "audio".into()); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::PubStart)))); + assert_eq!(publisher.pop_output(()), None); + + //same endpoint publish more track should not start channel + publisher.on_track_publish(t0, 1, 1.into(), peer1.clone(), "audio2".into()); + assert_eq!(publisher.pop_output(()), None); + + //other endpoint publish more track should not start channel + publisher.on_track_publish(t0, 2, 0.into(), peer2, "audio".into()); + assert_eq!(publisher.pop_output(()), None); + + //when have track data, depend on audio mixer output, it will push to pubsub. in this case we have 3 output then all data is published + let pkt = MediaPacket { + ts: 0, + seq: 1, + marker: false, + nackable: false, + layers: None, + meta: MediaMeta::Opus { audio_level: Some(-60) }, + data: vec![1, 2, 3, 4, 5, 6], + }; + publisher.on_track_data(t0, 1, 0.into(), &pkt); + let expected_pub = AudioMixerPkt { + slot: 0, + peer: peer1.hash_code(), + track: 0.into(), + audio_level: Some(-60), + source: Some((peer1.clone(), "audio".into())), + ts: 0, + seq: 1, + opus_payload: vec![1, 2, 3, 4, 5, 6], + }; + assert_eq!( + publisher.pop_output(()), + Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::PubData(expected_pub.serialize())))) + ); + assert_eq!(publisher.pop_output(()), None); + + //only last track leaved will generate PubStop + publisher.on_track_unpublish(t0 + ms(100), 1, 0.into()); + assert_eq!(publisher.pop_output(()), None); + + publisher.on_track_unpublish(t0 + ms(100), 1, 1.into()); + assert_eq!(publisher.pop_output(()), None); + + publisher.on_track_unpublish(t0 + ms(100), 2, 0.into()); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::PubStop)))); + assert_eq!(publisher.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(publisher.pop_output(()), None); + } + + #[test] + #[should_panic(expected = "Track not found")] + fn invalid_track_data_should_panic() { + let t0 = Instant::now(); + let mut publisher = AudioMixerPublisher::::new(0.into()); + let pkt = MediaPacket { + ts: 0, + seq: 1, + marker: false, + nackable: false, + layers: None, + meta: MediaMeta::Opus { audio_level: Some(-60) }, + data: vec![1, 2, 3, 4, 5, 6], + }; + publisher.on_track_data(t0, 1, 1.into(), &pkt); + } +} diff --git a/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs b/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs new file mode 100644 index 00000000..54dcc38a --- /dev/null +++ b/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs @@ -0,0 +1,287 @@ +use std::{array, fmt::Debug, hash::Hash, time::Instant}; + +use atm0s_sdn::{ + features::pubsub::{self, ChannelId}, + NodeId, +}; +use indexmap::IndexMap; +use media_server_protocol::{ + endpoint::{AudioMixerPkt, PeerHashCode, PeerId, TrackName}, + media::{MediaMeta, MediaPacket}, + transport::LocalTrackId, +}; +use sans_io_runtime::{collections::DynamicDeque, return_if_none, TaskSwitcherChild}; + +use crate::cluster::{ClusterAudioMixerEvent, ClusterEndpointEvent, ClusterLocalTrackEvent}; + +use super::Output; + +struct EndpointSlot { + peer: PeerHashCode, + tracks: Vec, +} + +#[derive(Clone)] +struct OutputSlot { + source: Option<(PeerId, TrackName)>, +} + +pub struct AudioMixerSubscriber { + channel_id: ChannelId, + queue: DynamicDeque, 16>, + endpoints: IndexMap, + outputs: [Option; OUTPUTS], + mixer: audio_mixer::AudioMixer<(NodeId, u8)>, +} + +impl AudioMixerSubscriber { + pub fn new(channel_id: ChannelId) -> Self { + Self { + channel_id, + queue: Default::default(), + endpoints: IndexMap::new(), + outputs: array::from_fn(|_| None), + mixer: audio_mixer::AudioMixer::new(3), //TODO dynamic this + } + } + + pub fn is_empty(&self) -> bool { + self.endpoints.is_empty() && self.queue.is_empty() + } + + pub fn on_tick(&mut self, now: Instant) { + if let Some(removed_slots) = self.mixer.on_tick(now) { + for slot in removed_slots { + self.outputs[slot] = None; + for (endpoint, _) in self.endpoints.iter() { + self.queue.push_back(Output::Endpoint( + vec![endpoint.clone()], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotUnset(slot as u8)), + )); + } + } + } + } + + /// We a endpoint join we need to restore current set slots + pub fn on_endpoint_join(&mut self, _now: Instant, endpoint: Endpoint, peer: PeerId, tracks: Vec) { + assert!(!self.endpoints.contains_key(&endpoint)); + log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] endpoint {:?} peer {peer} join with tracks {:?}", endpoint, tracks); + if self.endpoints.is_empty() { + log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] first endpoint join as Auto mode => subscribe channel {}", self.channel_id); + self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::SubAuto))); + } + + for (index, slot) in self.outputs.iter().enumerate() { + if let Some(slot) = slot { + if let Some((peer, track)) = &slot.source { + self.queue.push_back(Output::Endpoint( + vec![endpoint.clone()], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotSet(index as u8, peer.clone(), track.clone())), + )); + } + } + } + self.endpoints.insert(endpoint, EndpointSlot { peer: peer.hash_code(), tracks }); + } + + /// We we receive audio pkt, we put it into a mixer, it the audio source is selected it will be forwarded to all endpoints except the origin peer. + /// In case output don't have source info and audio pkt has source info, we set it and fire event in to all endpoints + pub fn on_channel_data(&mut self, now: Instant, from: NodeId, pkt: &[u8]) { + if self.endpoints.is_empty() { + return; + } + let audio = return_if_none!(AudioMixerPkt::deserialize(pkt)); + if let Some((slot, just_set)) = self.mixer.on_pkt(now, (from, audio.slot), audio.audio_level) { + // When a source is selected, we just reset the selected slot, + // then wait for next audio pkt which carry source info + if just_set { + self.outputs[slot] = Some(OutputSlot { source: None }); + } + + // If selected slot dont have source info and audio pkt has it, + // we will save it and fire event to all endpoints + if audio.source.is_some() && self.outputs[slot].as_ref().expect("Output slot not found").source.is_none() { + let (peer, track) = audio.source.clone().expect("Audio source not set"); + self.outputs[slot].as_mut().expect("Output slot not found").source = Some((peer.clone(), track.clone())); + for (endpoint, _) in self.endpoints.iter() { + self.queue.push_back(Output::Endpoint( + vec![endpoint.clone()], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotSet(slot as u8, peer.clone(), track.clone())), + )); + } + } + + for (endpoint, endpoint_slot) in self.endpoints.iter() { + if endpoint_slot.peer != audio.peer { + let track_id = endpoint_slot.tracks[slot]; + if just_set { + self.queue.push_back(Output::Endpoint( + vec![endpoint.clone()], + ClusterEndpointEvent::LocalTrack(track_id, ClusterLocalTrackEvent::SourceChanged), + )); + } + self.queue.push_back(Output::Endpoint( + vec![endpoint.clone()], + ClusterEndpointEvent::LocalTrack( + track_id, + ClusterLocalTrackEvent::Media( + (audio.peer.0 << 16) | (audio.track.0 as u64), //TODO better track UUID + MediaPacket { + ts: audio.ts, + seq: audio.seq, + marker: false, + nackable: false, + layers: None, + meta: MediaMeta::Opus { audio_level: audio.audio_level }, + data: audio.opus_payload.clone(), + }, + ), + ), + )) + } + } + } + } + + pub fn on_endpoint_leave(&mut self, _now: Instant, endpoint: Endpoint) { + assert!(self.endpoints.contains_key(&endpoint)); + log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] endpoint {:?} leave", endpoint); + self.endpoints.swap_remove(&endpoint); + if self.endpoints.is_empty() { + log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] last endpoint leave in Auto mode => unsubscribe channel {}", self.channel_id); + self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::UnsubAuto))); + self.queue.push_back(Output::OnResourceEmpty); + } + } +} + +impl TaskSwitcherChild> for AudioMixerSubscriber { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { + self.queue.pop_front() + } +} + +impl Drop for AudioMixerSubscriber { + fn drop(&mut self) { + log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] Drop {}", self.channel_id); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); + assert_eq!(self.endpoints.len(), 0, "Endpoints not empty on drop"); + } +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + use atm0s_sdn::features::pubsub; + use media_server_protocol::{ + endpoint::{AudioMixerPkt, PeerId, TrackName}, + media::{MediaMeta, MediaPacket}, + }; + use sans_io_runtime::TaskSwitcherChild; + + use crate::cluster::{ClusterAudioMixerEvent, ClusterEndpointEvent, ClusterLocalTrackEvent}; + + use super::{super::Output, AudioMixerSubscriber}; + + fn ms(m: u64) -> Duration { + Duration::from_millis(m) + } + + #[test] + fn sub_unsub() { + let t0 = Instant::now(); + let channel = 0.into(); + let endpoint1 = 0; + let peer1: PeerId = "peer1".into(); + let track1: TrackName = "audio".into(); + let endpoint2 = 1; + let mut subscriber = AudioMixerSubscriber::::new(channel); + + //first endpoint should fire Sub + subscriber.on_endpoint_join(t0, endpoint1, peer1.clone(), vec![0.into(), 1.into(), 2.into()]); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::SubAuto)))); + assert_eq!(subscriber.pop_output(()), None); + + //next endpoint should not fire Sub + subscriber.on_endpoint_join(t0, endpoint2, "peer2".into(), vec![0.into(), 1.into(), 2.into()]); + assert_eq!(subscriber.pop_output(()), None); + + //incoming media should rely on audio mixer to forward to endpoints + let node_id = 1; + let pkt = MediaPacket { + ts: 0, + seq: 1, + marker: false, + nackable: false, + layers: None, + meta: MediaMeta::Opus { audio_level: Some(-60) }, + data: vec![1, 2, 3, 4, 5, 6], + }; + let mixer_pkt = AudioMixerPkt { + slot: 0, + peer: peer1.hash_code(), + track: 0.into(), + audio_level: Some(-60), + source: Some((peer1.clone(), track1.clone())), + ts: 0, + seq: 1, + opus_payload: vec![1, 2, 3, 4, 5, 6], + }; + let track_uuid = (mixer_pkt.peer.0 << 16) | (mixer_pkt.track.0 as u64); + subscriber.on_channel_data(t0 + ms(100), node_id, &mixer_pkt.serialize()); + + //sot 0 is set => fire AudioMixer::Set event + assert_eq!( + subscriber.pop_output(()), + Some(Output::Endpoint( + vec![endpoint1], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotSet(0, peer1.clone(), track1.clone())) + )) + ); + assert_eq!( + subscriber.pop_output(()), + Some(Output::Endpoint( + vec![endpoint2], + ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotSet(0, peer1.clone(), track1.clone())) + )) + ); + + //we only forward to peer2 because audio is not forward to same peer + assert_eq!( + subscriber.pop_output(()), + Some(Output::Endpoint(vec![endpoint2], ClusterEndpointEvent::LocalTrack(0.into(), ClusterLocalTrackEvent::SourceChanged))) + ); + assert_eq!( + subscriber.pop_output(()), + Some(Output::Endpoint( + vec![endpoint2], + ClusterEndpointEvent::LocalTrack(0.into(), ClusterLocalTrackEvent::Media(track_uuid, pkt)) + )) + ); + + //after tick timeout should fire unset + subscriber.on_tick(t0 + ms(100 + 2000)); + //sot 0 is unset => fire AudioMixer::UnSet event + assert_eq!( + subscriber.pop_output(()), + Some(Output::Endpoint(vec![endpoint1], ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotUnset(0)))) + ); + assert_eq!( + subscriber.pop_output(()), + Some(Output::Endpoint(vec![endpoint2], ClusterEndpointEvent::AudioMixer(ClusterAudioMixerEvent::SlotUnset(0)))) + ); + + //only last endpoint leave should fire Unsub + subscriber.on_endpoint_leave(t0 + ms(100 + 2000), endpoint1); + assert_eq!(subscriber.pop_output(()), None); + + //now is last endpoint => should fire Unsub + subscriber.on_endpoint_leave(t0 + ms(100 + 2000), endpoint2); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::UnsubAuto)))); + assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(subscriber.pop_output(()), None); + } +} diff --git a/packages/media_core/src/cluster/room/media_track.rs b/packages/media_core/src/cluster/room/media_track.rs new file mode 100644 index 00000000..f115c5dd --- /dev/null +++ b/packages/media_core/src/cluster/room/media_track.rs @@ -0,0 +1,137 @@ +use std::{fmt::Debug, hash::Hash, time::Instant}; + +use atm0s_sdn::features::pubsub; +use media_server_protocol::{ + endpoint::{PeerId, TrackName}, + media::MediaPacket, +}; +use publisher::RoomChannelPublisher; +use sans_io_runtime::{TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; + +use crate::{ + cluster::{ClusterEndpointEvent, ClusterRoomHash}, + transport::{LocalTrackId, RemoteTrackId}, +}; + +use self::subscriber::RoomChannelSubscribe; + +pub mod publisher; +pub mod subscriber; + +#[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] +#[repr(usize)] +pub enum TaskType { + Publisher = 0, + Subscriber = 1, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Output { + Endpoint(Vec, ClusterEndpointEvent), + Pubsub(pubsub::Control), + OnResourceEmpty, +} + +pub struct MediaTrack { + room: ClusterRoomHash, + publisher: TaskSwitcherBranch, Output>, + subscriber: TaskSwitcherBranch, Output>, + switcher: TaskSwitcher, +} + +impl MediaTrack { + pub fn new(room: ClusterRoomHash) -> Self { + Self { + room, + publisher: TaskSwitcherBranch::new(RoomChannelPublisher::new(room), TaskType::Publisher), + subscriber: TaskSwitcherBranch::new(RoomChannelSubscribe::new(room), TaskType::Subscriber), + switcher: TaskSwitcher::new(2), + } + } + + pub fn is_empty(&self) -> bool { + self.publisher.is_empty() && self.subscriber.is_empty() + } + + pub fn on_pubsub_event(&mut self, event: pubsub::Event) { + let channel = event.0; + match event.1 { + pubsub::ChannelEvent::RouteChanged(next) => { + self.subscriber.input(&mut self.switcher).on_track_relay_changed(channel, next); + } + pubsub::ChannelEvent::SourceData(_, data) => { + self.subscriber.input(&mut self.switcher).on_track_data(channel, data); + } + pubsub::ChannelEvent::FeedbackData(fb) => { + self.publisher.input(&mut self.switcher).on_track_feedback(channel, fb); + } + } + } + + pub fn on_track_publish(&mut self, endpoint: Endpoint, track: RemoteTrackId, peer: PeerId, name: TrackName) { + self.publisher.input(&mut self.switcher).on_track_publish(endpoint, track, peer, name); + } + + pub fn on_track_data(&mut self, endpoint: Endpoint, track: RemoteTrackId, media: MediaPacket) { + self.publisher.input(&mut self.switcher).on_track_data(endpoint, track, media); + } + + pub fn on_track_unpublish(&mut self, endpoint: Endpoint, track: RemoteTrackId) { + self.publisher.input(&mut self.switcher).on_track_unpublish(endpoint, track); + } + + pub fn on_track_subscribe(&mut self, endpoint: Endpoint, track: LocalTrackId, target_peer: PeerId, target_track: TrackName) { + self.subscriber.input(&mut self.switcher).on_track_subscribe(endpoint, track, target_peer, target_track); + } + + pub fn on_track_request_key(&mut self, endpoint: Endpoint, track: LocalTrackId) { + self.subscriber.input(&mut self.switcher).on_track_request_key(endpoint, track); + } + + pub fn on_track_desired_bitrate(&mut self, now: Instant, endpoint: Endpoint, track: LocalTrackId, bitrate: u64) { + self.subscriber.input(&mut self.switcher).on_track_desired_bitrate(now, endpoint, track, bitrate); + } + + pub fn on_track_unsubscribe(&mut self, endpoint: Endpoint, track: LocalTrackId) { + self.subscriber.input(&mut self.switcher).on_track_unsubscribe(endpoint, track); + } +} + +impl TaskSwitcherChild> for MediaTrack { + type Time = (); + + fn pop_output(&mut self, _now: Self::Time) -> Option> { + loop { + match self.switcher.current()?.try_into().ok()? { + TaskType::Publisher => { + if let Some(out) = self.publisher.pop_output((), &mut self.switcher) { + if let Output::OnResourceEmpty = out { + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } else { + return Some(out); + } + } + } + TaskType::Subscriber => { + if let Some(out) = self.subscriber.pop_output((), &mut self.switcher) { + if let Output::OnResourceEmpty = out { + if self.is_empty() { + return Some(Output::OnResourceEmpty); + } + } else { + return Some(out); + } + } + } + } + } + } +} + +impl Drop for MediaTrack { + fn drop(&mut self) { + log::info!("[ClusterRoomMediaTrack] Drop {}", self.room); + } +} diff --git a/packages/media_core/src/cluster/room/channel_pub.rs b/packages/media_core/src/cluster/room/media_track/publisher.rs similarity index 54% rename from packages/media_core/src/cluster/room/channel_pub.rs rename to packages/media_core/src/cluster/room/media_track/publisher.rs index 9c2ae02c..4bd5733d 100644 --- a/packages/media_core/src/cluster/room/channel_pub.rs +++ b/packages/media_core/src/cluster/room/media_track/publisher.rs @@ -1,10 +1,11 @@ +//! //! Channel Publisher will takecare of pubsub channel for sending data and handle when received channel feedback +//! use std::{ collections::{HashMap, VecDeque}, fmt::Debug, hash::Hash, - time::Instant, }; use atm0s_sdn::features::pubsub::{self, ChannelControl, ChannelId, Feedback}; @@ -19,6 +20,8 @@ use crate::{ transport::RemoteTrackId, }; +use super::Output; + pub enum FeedbackKind { Bitrate { min: u64, max: u64 }, KeyFrameRequest, @@ -35,20 +38,14 @@ impl TryFrom for FeedbackKind { } } -#[derive(Debug, PartialEq, Eq)] -pub enum Output { - Endpoint(Vec, ClusterEndpointEvent), - Pubsub(pubsub::Control), -} - -pub struct RoomChannelPublisher { +pub struct RoomChannelPublisher { room: ClusterRoomHash, - tracks: HashMap<(Owner, RemoteTrackId), (PeerId, TrackName, ChannelId)>, - tracks_source: HashMap, - queue: VecDeque>, + tracks: HashMap<(Endpoint, RemoteTrackId), (PeerId, TrackName, ChannelId)>, + tracks_source: HashMap, + queue: VecDeque>, } -impl RoomChannelPublisher { +impl RoomChannelPublisher { pub fn new(room: ClusterRoomHash) -> Self { Self { room, @@ -58,66 +55,82 @@ impl RoomChannelPublisher { } } - pub fn on_channel_feedback(&mut self, channel: ChannelId, fb: Feedback) { + pub fn is_empty(&self) -> bool { + self.tracks.is_empty() && self.queue.is_empty() + } + + pub fn on_track_feedback(&mut self, channel: ChannelId, fb: Feedback) { let fb = return_if_err!(FeedbackKind::try_from(fb)); - let (owner, track_id) = return_if_none!(self.tracks_source.get(&channel)); + let (endpoint, track_id) = return_if_none!(self.tracks_source.get(&channel)); match fb { FeedbackKind::Bitrate { min, max } => { log::debug!("[ClusterRoom {}/Publishers] channel {channel} limit bitrate [{min},{max}]", self.room); self.queue.push_back(Output::Endpoint( - vec![*owner], + vec![*endpoint], ClusterEndpointEvent::RemoteTrack(*track_id, ClusterRemoteTrackEvent::LimitBitrate { min, max }), )); } FeedbackKind::KeyFrameRequest => { log::debug!("[ClusterRoom {}/Publishers] channel {channel} request key_frame", self.room); - self.queue - .push_back(Output::Endpoint(vec![*owner], ClusterEndpointEvent::RemoteTrack(*track_id, ClusterRemoteTrackEvent::RequestKeyFrame))); + self.queue.push_back(Output::Endpoint( + vec![*endpoint], + ClusterEndpointEvent::RemoteTrack(*track_id, ClusterRemoteTrackEvent::RequestKeyFrame), + )); } } } - pub fn on_track_publish(&mut self, owner: Owner, track: RemoteTrackId, peer: PeerId, name: TrackName) { + pub fn on_track_publish(&mut self, endpoint: Endpoint, track: RemoteTrackId, peer: PeerId, name: TrackName) { log::info!("[ClusterRoom {}/Publishers] peer ({peer} started track ({name})", self.room); let channel_id = id_generator::gen_channel_id(self.room, &peer, &name); - self.tracks.insert((owner, track), (peer.clone(), name.clone(), channel_id)); - self.tracks_source.insert(channel_id, (owner, track)); + self.tracks.insert((endpoint, track), (peer.clone(), name.clone(), channel_id)); + self.tracks_source.insert(channel_id, (endpoint, track)); self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::PubStart))); } - pub fn on_track_data(&mut self, owner: Owner, track: RemoteTrackId, media: MediaPacket) { + pub fn on_track_data(&mut self, endpoint: Endpoint, track: RemoteTrackId, media: MediaPacket) { log::trace!( "[ClusterRoom {}/Publishers] peer {:?} track {track} publish media meta {:?} seq {}", self.room, - owner, + endpoint, media.meta, media.seq ); - let (_peer, _name, channel_id) = return_if_none!(self.tracks.get(&(owner, track))); + let (_peer, _name, channel_id) = return_if_none!(self.tracks.get(&(endpoint, track))); let data = media.serialize(); self.queue.push_back(Output::Pubsub(pubsub::Control(*channel_id, ChannelControl::PubData(data)))) } - pub fn on_track_unpublish(&mut self, owner: Owner, track: RemoteTrackId) { - let (peer, name, channel_id) = return_if_none!(self.tracks.remove(&(owner, track))); + pub fn on_track_unpublish(&mut self, endpoint: Endpoint, track: RemoteTrackId) { + let (peer, name, channel_id) = return_if_none!(self.tracks.remove(&(endpoint, track))); self.tracks_source.remove(&channel_id).expect("Should have track_source"); log::info!("[ClusterRoom {}/Publishers] peer ({peer} stopped track {name})", self.room); - self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::PubStop))) + self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::PubStop))); + if self.tracks.is_empty() { + self.queue.push_back(Output::OnResourceEmpty); + } } } -impl TaskSwitcherChild> for RoomChannelPublisher { - type Time = Instant; - fn pop_output(&mut self, _now: Instant) -> Option> { +impl TaskSwitcherChild> for RoomChannelPublisher { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } +impl Drop for RoomChannelPublisher { + fn drop(&mut self) { + log::info!("[ClusterRoom {}/Publishers] Drop", self.room); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); + assert_eq!(self.tracks.len(), 0, "Tracks not empty on drop"); + assert_eq!(self.tracks_source.len(), 0, "Tracks source not empty on drop"); + } +} + #[cfg(test)] mod tests { - use std::time::Instant; - use atm0s_sdn::features::pubsub::{ChannelControl, Control, Feedback}; use media_server_protocol::media::{MediaMeta, MediaPacket}; use sans_io_runtime::TaskSwitcherChild; @@ -128,7 +141,7 @@ mod tests { }; use super::id_generator::gen_channel_id; - use super::{Output, RoomChannelPublisher}; + use super::{super::Output, RoomChannelPublisher}; pub fn fake_audio() -> MediaPacket { MediaPacket { @@ -150,26 +163,24 @@ mod tests { let room = 1.into(); let mut publisher = RoomChannelPublisher::::new(room); - let owner = 2; + let endpoint = 2; let track = RemoteTrackId(3); let peer = "peer1".to_string().into(); let name = "audio_main".to_string().into(); let channel_id = gen_channel_id(room, &peer, &name); - publisher.on_track_publish(owner, track, peer, name); - assert_eq!(publisher.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStart)))); - assert_eq!(publisher.pop_output(Instant::now()), None); + publisher.on_track_publish(endpoint, track, peer, name); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStart)))); + assert_eq!(publisher.pop_output(()), None); let media = fake_audio(); - publisher.on_track_data(owner, track, media.clone()); - assert_eq!( - publisher.pop_output(Instant::now()), - Some(Output::Pubsub(Control(channel_id, ChannelControl::PubData(media.serialize())))) - ); - assert_eq!(publisher.pop_output(Instant::now()), None); - - publisher.on_track_unpublish(owner, track); - assert_eq!(publisher.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStop)))); - assert_eq!(publisher.pop_output(Instant::now()), None); + publisher.on_track_data(endpoint, track, media.clone()); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubData(media.serialize()))))); + assert_eq!(publisher.pop_output(()), None); + + publisher.on_track_unpublish(endpoint, track); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStop)))); + assert_eq!(publisher.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(publisher.pop_output(()), None); } //TODO Handle feedback: should handle KeyFrame feedback @@ -179,28 +190,34 @@ mod tests { let room = 1.into(); let mut publisher = RoomChannelPublisher::::new(room); - let owner = 2; + let endpoint = 2; let track = RemoteTrackId(3); let peer = "peer1".to_string().into(); let name = "audio_main".to_string().into(); let channel_id = gen_channel_id(room, &peer, &name); - publisher.on_track_publish(owner, track, peer, name); - assert_eq!(publisher.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStart)))); - assert_eq!(publisher.pop_output(Instant::now()), None); + publisher.on_track_publish(endpoint, track, peer, name); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStart)))); + assert_eq!(publisher.pop_output(()), None); - publisher.on_channel_feedback(channel_id, Feedback::simple(0, 1000, 100, 200)); + publisher.on_track_feedback(channel_id, Feedback::simple(0, 1000, 100, 200)); assert_eq!( - publisher.pop_output(Instant::now()), + publisher.pop_output(()), Some(Output::Endpoint( - vec![owner], + vec![endpoint], ClusterEndpointEvent::RemoteTrack(track, ClusterRemoteTrackEvent::LimitBitrate { min: 1000, max: 1000 }) )) ); - publisher.on_channel_feedback(channel_id, Feedback::simple(1, 1, 100, 200)); + publisher.on_track_feedback(channel_id, Feedback::simple(1, 1, 100, 200)); assert_eq!( - publisher.pop_output(Instant::now()), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::RemoteTrack(track, ClusterRemoteTrackEvent::RequestKeyFrame))) + publisher.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::RemoteTrack(track, ClusterRemoteTrackEvent::RequestKeyFrame))) ); + assert_eq!(publisher.pop_output(()), None); + + publisher.on_track_unpublish(endpoint, track); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStop)))); + assert_eq!(publisher.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(publisher.pop_output(()), None); } } diff --git a/packages/media_core/src/cluster/room/channel_sub.rs b/packages/media_core/src/cluster/room/media_track/subscriber.rs similarity index 56% rename from packages/media_core/src/cluster/room/channel_sub.rs rename to packages/media_core/src/cluster/room/media_track/subscriber.rs index 0ada5077..8daa1959 100644 --- a/packages/media_core/src/cluster/room/channel_sub.rs +++ b/packages/media_core/src/cluster/room/media_track/subscriber.rs @@ -1,3 +1,4 @@ +//! //! Channel Subscriber handle logic for viewer. This module takecare sending Sub or Unsub, and also feedback //! @@ -24,6 +25,8 @@ use crate::{ transport::LocalTrackId, }; +use super::Output; + const BITRATE_FEEDBACK_INTERVAL: u16 = 100; //100 ms const BITRATE_FEEDBACK_TIMEOUT: u16 = 2000; //2 seconds @@ -33,27 +36,21 @@ const KEYFRAME_FEEDBACK_TIMEOUT: u16 = 2000; //2 seconds const BITRATE_FEEDBACK_KIND: u8 = 0; const KEYFRAME_FEEDBACK_KIND: u8 = 1; -#[derive(Debug, PartialEq, Eq)] -pub enum Output { - Endpoint(Vec, ClusterEndpointEvent), - Pubsub(pubsub::Control), -} - #[derive(Derivative)] #[derivative(Default(bound = ""))] -struct ChannelContainer { - owners: Vec<(Owner, LocalTrackId)>, - bitrate_fbs: HashMap, +struct ChannelContainer { + endpoints: Vec<(Endpoint, LocalTrackId)>, + bitrate_fbs: HashMap, } -pub struct RoomChannelSubscribe { +pub struct RoomChannelSubscribe { room: ClusterRoomHash, - channels: HashMap>, - subscribers: HashMap<(Owner, LocalTrackId), (ChannelId, PeerId, TrackName)>, - queue: VecDeque>, + channels: HashMap>, + subscribers: HashMap<(Endpoint, LocalTrackId), (ChannelId, PeerId, TrackName)>, + queue: VecDeque>, } -impl RoomChannelSubscribe { +impl RoomChannelSubscribe { pub fn new(room: ClusterRoomHash) -> Self { Self { room, @@ -63,20 +60,24 @@ impl RoomChannelSubscribe { } } - pub fn on_channel_relay_changed(&mut self, channel: ChannelId, _relay: NodeId) { + pub fn is_empty(&self) -> bool { + self.subscribers.is_empty() && self.queue.is_empty() + } + + pub fn on_track_relay_changed(&mut self, channel: ChannelId, _relay: NodeId) { let channel_container = return_if_none!(self.channels.get(&channel)); log::info!( "[ClusterRoom {}/Subscribers] cluster: channel {channel} source changed => fire event to {:?}", self.room, - channel_container.owners + channel_container.endpoints ); - for (owner, track) in &channel_container.owners { + for (endpoint, track) in &channel_container.endpoints { self.queue - .push_back(Output::Endpoint(vec![*owner], ClusterEndpointEvent::LocalTrack(*track, ClusterLocalTrackEvent::SourceChanged))) + .push_back(Output::Endpoint(vec![*endpoint], ClusterEndpointEvent::LocalTrack(*track, ClusterLocalTrackEvent::RelayChanged))) } } - pub fn on_channel_data(&mut self, channel: ChannelId, data: Vec) { + pub fn on_track_data(&mut self, channel: ChannelId, data: Vec) { let pkt = return_if_none!(MediaPacket::deserialize(&data)); let channel_container = return_if_none!(self.channels.get(&channel)); log::trace!( @@ -84,34 +85,34 @@ impl RoomChannelSubscribe { self.room, pkt.meta, pkt.seq, - channel_container.owners.len() + channel_container.endpoints.len() ); - for (owner, track) in &channel_container.owners { + for (endpoint, track) in &channel_container.endpoints { self.queue.push_back(Output::Endpoint( - vec![*owner], + vec![*endpoint], ClusterEndpointEvent::LocalTrack(*track, ClusterLocalTrackEvent::Media(*channel, pkt.clone())), )) } } - pub fn on_track_subscribe(&mut self, owner: Owner, track: LocalTrackId, target_peer: PeerId, target_track: TrackName) { + pub fn on_track_subscribe(&mut self, endpoint: Endpoint, track: LocalTrackId, target_peer: PeerId, target_track: TrackName) { let channel_id: ChannelId = id_generator::gen_channel_id(self.room, &target_peer, &target_track); log::info!( - "[ClusterRoom {}/Subscribers] owner {:?} track {track} subscribe peer {target_peer} track {target_track}), channel: {channel_id}", + "[ClusterRoom {}/Subscribers] endpoint {:?} track {track} subscribe peer {target_peer} track {target_track}), channel: {channel_id}", self.room, - owner + endpoint ); - self.subscribers.insert((owner, track), (channel_id, target_peer, target_track)); + self.subscribers.insert((endpoint, track), (channel_id, target_peer, target_track)); let channel_container = self.channels.entry(channel_id).or_default(); - channel_container.owners.push((owner, track)); - if channel_container.owners.len() == 1 { + channel_container.endpoints.push((endpoint, track)); + if channel_container.endpoints.len() == 1 { log::info!("[ClusterRoom {}/Subscribers] first subscriber => Sub channel {channel_id}", self.room); self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::SubAuto))); } } - pub fn on_track_request_key(&mut self, owner: Owner, track: LocalTrackId) { - let (channel_id, peer, track) = return_if_none!(self.subscribers.get(&(owner, track))); + pub fn on_track_request_key(&mut self, endpoint: Endpoint, track: LocalTrackId) { + let (channel_id, peer, track) = return_if_none!(self.subscribers.get(&(endpoint, track))); log::info!("[ClusterRoom {}/Subscribers] request key-frame {channel_id} {peer} {track}", self.room); self.queue.push_back(Output::Pubsub(pubsub::Control( *channel_id, @@ -119,11 +120,11 @@ impl RoomChannelSubscribe { ))); } - pub fn on_track_desired_bitrate(&mut self, now: Instant, owner: Owner, track: LocalTrackId, bitrate: u64) { - let (channel_id, _peer, _track) = return_if_none!(self.subscribers.get(&(owner, track))); + pub fn on_track_desired_bitrate(&mut self, now: Instant, endpoint: Endpoint, track: LocalTrackId, bitrate: u64) { + let (channel_id, _peer, _track) = return_if_none!(self.subscribers.get(&(endpoint, track))); let channel_container = return_if_none!(self.channels.get_mut(channel_id)); let fb = Feedback::simple(BITRATE_FEEDBACK_KIND, bitrate, BITRATE_FEEDBACK_INTERVAL, BITRATE_FEEDBACK_TIMEOUT); - channel_container.bitrate_fbs.insert(owner, (now, fb)); + channel_container.bitrate_fbs.insert(endpoint, (now, fb)); //clean if if timeout channel_container @@ -144,32 +145,45 @@ impl RoomChannelSubscribe { .push_back(Output::Pubsub(pubsub::Control(*channel_id, ChannelControl::FeedbackAuto(return_if_none!(sum_fb))))); } - pub fn on_track_unsubscribe(&mut self, owner: Owner, track: LocalTrackId) { - let (channel_id, target_peer, target_track) = return_if_none!(self.subscribers.remove(&(owner, track))); + pub fn on_track_unsubscribe(&mut self, endpoint: Endpoint, track: LocalTrackId) { + let (channel_id, target_peer, target_track) = return_if_none!(self.subscribers.remove(&(endpoint, track))); log::info!( - "[ClusterRoom {}/Subscribers] owner {:?} track {track} unsubscribe from source {target_peer} {target_track}, channel {channel_id}", + "[ClusterRoom {}/Subscribers] endpoint {:?} track {track} unsubscribe from source {target_peer} {target_track}, channel {channel_id}", self.room, - owner + endpoint ); let channel_container = return_if_none!(self.channels.get_mut(&channel_id)); - let (index, _) = return_if_none!(channel_container.owners.iter().enumerate().find(|e| e.1.eq(&(owner, track)))); - channel_container.owners.swap_remove(index); + let (index, _) = return_if_none!(channel_container.endpoints.iter().enumerate().find(|e| e.1.eq(&(endpoint, track)))); + channel_container.endpoints.swap_remove(index); - if channel_container.owners.is_empty() { + if channel_container.endpoints.is_empty() { self.channels.remove(&channel_id); log::info!("[ClusterRoom {}/Subscribers] last unsubscriber => Unsub channel {channel_id}", self.room); self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::UnsubAuto))); } + + if self.subscribers.is_empty() { + self.queue.push_back(Output::OnResourceEmpty); + } } } -impl TaskSwitcherChild> for RoomChannelSubscribe { - type Time = Instant; - fn pop_output(&mut self, _now: Instant) -> Option> { +impl TaskSwitcherChild> for RoomChannelSubscribe { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } +impl Drop for RoomChannelSubscribe { + fn drop(&mut self) { + log::info!("[ClusterRoom {}/Subscriber] Drop", self.room); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); + assert_eq!(self.channels.len(), 0, "Channels not empty on drop"); + assert_eq!(self.subscribers.len(), 0, "Subscribers not empty on drop"); + } +} + #[cfg(test)] mod tests { use std::time::{Duration, Instant}; @@ -182,15 +196,13 @@ mod tests { use sans_io_runtime::TaskSwitcherChild; use crate::{ - cluster::{ - room::channel_sub::{BITRATE_FEEDBACK_INTERVAL, BITRATE_FEEDBACK_KIND, BITRATE_FEEDBACK_TIMEOUT, KEYFRAME_FEEDBACK_INTERVAL, KEYFRAME_FEEDBACK_KIND, KEYFRAME_FEEDBACK_TIMEOUT}, - ClusterEndpointEvent, ClusterLocalTrackEvent, - }, + cluster::{ClusterEndpointEvent, ClusterLocalTrackEvent}, transport::LocalTrackId, }; use super::id_generator::gen_channel_id; use super::{Output, RoomChannelSubscribe}; + use super::{BITRATE_FEEDBACK_INTERVAL, BITRATE_FEEDBACK_KIND, BITRATE_FEEDBACK_TIMEOUT, KEYFRAME_FEEDBACK_INTERVAL, KEYFRAME_FEEDBACK_KIND, KEYFRAME_FEEDBACK_TIMEOUT}; pub fn fake_audio() -> MediaPacket { MediaPacket { @@ -211,26 +223,30 @@ mod tests { let room = 1.into(); let mut subscriber = RoomChannelSubscribe::::new(room); - let owner = 2; + let endpoint = 2; let track = LocalTrackId(3); let target_peer: PeerId = "peer2".to_string().into(); let target_track: TrackName = "audio_main".to_string().into(); let channel_id = gen_channel_id(room, &target_peer, &target_track); - subscriber.on_track_subscribe(owner, track, target_peer.clone(), target_track.clone()); - assert_eq!(subscriber.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::SubAuto)))); - assert_eq!(subscriber.pop_output(Instant::now()), None); + subscriber.on_track_subscribe(endpoint, track, target_peer.clone(), target_track.clone()); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::SubAuto)))); + assert_eq!(subscriber.pop_output(()), None); let pkt = fake_audio(); - subscriber.on_channel_data(channel_id, pkt.serialize()); + subscriber.on_track_data(channel_id, pkt.serialize()); assert_eq!( - subscriber.pop_output(Instant::now()), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::LocalTrack(track, ClusterLocalTrackEvent::Media(*channel_id, pkt)))) + subscriber.pop_output(()), + Some(Output::Endpoint( + vec![endpoint], + ClusterEndpointEvent::LocalTrack(track, ClusterLocalTrackEvent::Media(*channel_id, pkt)) + )) ); - assert_eq!(subscriber.pop_output(Instant::now()), None); + assert_eq!(subscriber.pop_output(()), None); - subscriber.on_track_unsubscribe(owner, track); - assert_eq!(subscriber.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); - assert_eq!(subscriber.pop_output(Instant::now()), None); + subscriber.on_track_unsubscribe(endpoint, track); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); + assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(subscriber.pop_output(()), None); } //TODO Sending key-frame request @@ -239,24 +255,29 @@ mod tests { let room = 1.into(); let mut subscriber = RoomChannelSubscribe::::new(room); - let owner = 2; + let endpoint = 2; let track = LocalTrackId(3); let target_peer: PeerId = "peer2".to_string().into(); let target_track: TrackName = "audio_main".to_string().into(); let channel_id = gen_channel_id(room, &target_peer, &target_track); - subscriber.on_track_subscribe(owner, track, target_peer.clone(), target_track.clone()); - assert_eq!(subscriber.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::SubAuto)))); - assert_eq!(subscriber.pop_output(Instant::now()), None); + subscriber.on_track_subscribe(endpoint, track, target_peer.clone(), target_track.clone()); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::SubAuto)))); + assert_eq!(subscriber.pop_output(()), None); - subscriber.on_track_request_key(owner, track); + subscriber.on_track_request_key(endpoint, track); assert_eq!( - subscriber.pop_output(Instant::now()), + subscriber.pop_output(()), Some(Output::Pubsub(Control( channel_id, ChannelControl::FeedbackAuto(Feedback::simple(KEYFRAME_FEEDBACK_KIND, 1, KEYFRAME_FEEDBACK_INTERVAL, KEYFRAME_FEEDBACK_TIMEOUT)) ))) ); - assert_eq!(subscriber.pop_output(Instant::now()), None); + assert_eq!(subscriber.pop_output(()), None); + + subscriber.on_track_unsubscribe(endpoint, track); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); + assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(subscriber.pop_output(()), None); } //TODO Sending bitrate request single sub @@ -265,38 +286,38 @@ mod tests { let room = 1.into(); let mut subscriber = RoomChannelSubscribe::::new(room); - let owner1 = 2; + let endpoint1 = 2; let track1 = LocalTrackId(3); let target_peer: PeerId = "peer2".to_string().into(); let target_track: TrackName = "audio_main".to_string().into(); let channel_id = gen_channel_id(room, &target_peer, &target_track); - subscriber.on_track_subscribe(owner1, track1, target_peer.clone(), target_track.clone()); - assert_eq!(subscriber.pop_output(Instant::now()), Some(Output::Pubsub(Control(channel_id, ChannelControl::SubAuto)))); - assert_eq!(subscriber.pop_output(Instant::now()), None); + subscriber.on_track_subscribe(endpoint1, track1, target_peer.clone(), target_track.clone()); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::SubAuto)))); + assert_eq!(subscriber.pop_output(()), None); let mut now = Instant::now(); - subscriber.on_track_desired_bitrate(now, owner1, track1, 1000); + subscriber.on_track_desired_bitrate(now, endpoint1, track1, 1000); assert_eq!( - subscriber.pop_output(Instant::now()), + subscriber.pop_output(()), Some(Output::Pubsub(Control( channel_id, ChannelControl::FeedbackAuto(Feedback::simple(BITRATE_FEEDBACK_KIND, 1000, BITRATE_FEEDBACK_INTERVAL, BITRATE_FEEDBACK_TIMEOUT)) ))) ); - assert_eq!(subscriber.pop_output(now), None); + assert_eq!(subscriber.pop_output(()), None); // more local track sub that channel - let owner2 = 3; + let endpoint2 = 3; let track2 = LocalTrackId(4); - subscriber.on_track_subscribe(owner2, track2, target_peer.clone(), target_track.clone()); - assert_eq!(subscriber.pop_output(Instant::now()), None); + subscriber.on_track_subscribe(endpoint2, track2, target_peer.clone(), target_track.clone()); + assert_eq!(subscriber.pop_output(()), None); // more feedback from local track2 now += Duration::from_millis(100); - subscriber.on_track_desired_bitrate(now, owner2, track2, 2000); + subscriber.on_track_desired_bitrate(now, endpoint2, track2, 2000); assert_eq!( - subscriber.pop_output(Instant::now()), + subscriber.pop_output(()), Some(Output::Pubsub(Control( channel_id, ChannelControl::FeedbackAuto(Feedback { @@ -310,13 +331,13 @@ mod tests { }) ))) ); - assert_eq!(subscriber.pop_output(now), None); + assert_eq!(subscriber.pop_output(()), None); //now last update from track2 after long time cause track1 feedback will be timeout now += Duration::from_millis(BITRATE_FEEDBACK_TIMEOUT as u64 - 100); - subscriber.on_track_desired_bitrate(now, owner2, track2, 3000); + subscriber.on_track_desired_bitrate(now, endpoint2, track2, 3000); assert_eq!( - subscriber.pop_output(Instant::now()), + subscriber.pop_output(()), Some(Output::Pubsub(Control( channel_id, ChannelControl::FeedbackAuto(Feedback { @@ -330,6 +351,12 @@ mod tests { }) ))) ); - assert_eq!(subscriber.pop_output(now), None); + assert_eq!(subscriber.pop_output(()), None); + + subscriber.on_track_unsubscribe(endpoint1, track1); + subscriber.on_track_unsubscribe(endpoint2, track2); + assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); + assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(subscriber.pop_output(()), None); } } diff --git a/packages/media_core/src/cluster/room/metadata.rs b/packages/media_core/src/cluster/room/metadata.rs index 5a60cec5..dd7a4895 100644 --- a/packages/media_core/src/cluster/room/metadata.rs +++ b/packages/media_core/src/cluster/room/metadata.rs @@ -7,7 +7,7 @@ //! - Manual: client manual call subscribe on which peer it interested in, this method is useful with some spartial audio application //! -use std::{collections::VecDeque, fmt::Debug, hash::Hash, time::Instant}; +use std::{collections::VecDeque, fmt::Debug, hash::Hash}; use atm0s_sdn::features::dht_kv::{self, Map, MapControl, MapEvent}; use media_server_protocol::endpoint::{PeerId, PeerInfo, PeerMeta, RoomInfoPublish, RoomInfoSubscribe, TrackInfo, TrackMeta, TrackName}; @@ -27,27 +27,27 @@ struct PeerContainer { } #[derive(Debug, PartialEq, Eq)] -pub enum Output { +pub enum Output { Kv(dht_kv::Control), - Endpoint(Vec, ClusterEndpointEvent), - LastPeerLeaved, + Endpoint(Vec, ClusterEndpointEvent), + OnResourceEmpty, } -pub struct RoomMetadata { +pub struct RoomMetadata { room: ClusterRoomHash, peers_map: Map, tracks_map: Map, - peers: SmallMap, - peers_map_subscribers: SmallSet, - tracks_map_subscribers: SmallSet, - //This is for storing list of owners subscribe manual a target track - peers_tracks_subs: SmallMap>, + peers: SmallMap, + peers_map_subscribers: SmallSet, + tracks_map_subscribers: SmallSet, + //This is for storing list of endpoints subscribe manual a target track + peers_tracks_subs: SmallMap>, cluster_peers: SmallMap, cluster_tracks: SmallMap, - queue: VecDeque>, + queue: VecDeque>, } -impl RoomMetadata { +impl RoomMetadata { pub fn new(room: ClusterRoomHash) -> Self { Self { room, @@ -63,16 +63,20 @@ impl RoomMetadata { } } - pub fn get_peer_from_owner(&self, owner: Owner) -> Option { - Some(self.peers.get(&owner)?.peer.clone()) + pub fn is_empty(&self) -> bool { + self.peers.is_empty() && self.queue.is_empty() } - /// We put peer to list and register owner to peers and tracks list subscriber based on level - pub fn on_join(&mut self, owner: Owner, peer: PeerId, meta: PeerMeta, publish: RoomInfoPublish, subscribe: RoomInfoSubscribe) { + pub fn get_peer_from_endpoint(&self, endpoint: Endpoint) -> Option { + Some(self.peers.get(&endpoint)?.peer.clone()) + } + + /// We put peer to list and register endpoint to peers and tracks list subscriber based on level + pub fn on_join(&mut self, endpoint: Endpoint, peer: PeerId, meta: PeerMeta, publish: RoomInfoPublish, subscribe: RoomInfoSubscribe) { log::info!("[ClusterRoom {}] join peer ({peer})", self.room); - // First let insert to peers cache for reuse when we need information of owner + // First let insert to peers cache for reuse when we need information of endpoint self.peers.insert( - owner, + endpoint, PeerContainer { peer: peer.clone(), publish: publish.clone(), @@ -89,14 +93,14 @@ impl RoomMetadata { } // Let Sub to peers_map if need need subscribe.peers if subscribe.peers { - self.peers_map_subscribers.insert(owner, ()); + self.peers_map_subscribers.insert(endpoint, ()); log::info!("[ClusterRoom {}] next peer sub peers => restore {} remote peers", self.room, self.cluster_peers.len()); // Restore already added peers for (_track_key, info) in self.cluster_peers.iter() { //TODO avoiding duplicate same peer self.queue - .push_back(Output::Endpoint(vec![owner], ClusterEndpointEvent::PeerJoined(info.peer.clone(), info.meta.clone()))); + .push_back(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::PeerJoined(info.peer.clone(), info.meta.clone()))); } // If this is first peer which subscribed to peers_map, the should send Sub @@ -107,14 +111,14 @@ impl RoomMetadata { } // Let Sub to tracks_map if need need subscribe.tracks if subscribe.tracks { - self.tracks_map_subscribers.insert(owner, ()); + self.tracks_map_subscribers.insert(endpoint, ()); log::info!("[ClusterRoom {}] next peer sub tracks => restore {} remote tracks", self.room, self.cluster_tracks.len()); // Restore already added tracks for (_track_key, info) in self.cluster_tracks.iter() { //TODO avoiding duplicate same peer self.queue.push_back(Output::Endpoint( - vec![owner], + vec![endpoint], ClusterEndpointEvent::TrackStarted(info.peer.clone(), info.track.clone(), info.meta.clone()), )); } @@ -127,8 +131,8 @@ impl RoomMetadata { }; } - pub fn on_leave(&mut self, owner: Owner) { - let peer = return_if_none!(self.peers.remove(&owner)); + pub fn on_leave(&mut self, endpoint: Endpoint) { + let peer = return_if_none!(self.peers.remove(&endpoint)); log::info!("[ClusterRoom {}] leave peer {}", self.room, peer.peer); let peer_key = id_generator::peers_key(&peer.peer); // If remain remote tracks, must to delete from list. @@ -144,12 +148,12 @@ impl RoomMetadata { self.queue.push_back(Output::Kv(dht_kv::Control::MapCmd(peer_map, MapControl::Del(track_key)))); } - if self.peers_map_subscribers.remove(&owner).is_some() && self.peers_map_subscribers.is_empty() { + if self.peers_map_subscribers.remove(&endpoint).is_some() && self.peers_map_subscribers.is_empty() { log::info!("[ClusterRoom {}] last peer unsub peers map => unsubscribe", self.room); self.queue.push_back(Output::Kv(dht_kv::Control::MapCmd(self.peers_map, MapControl::Unsub))); } - if self.tracks_map_subscribers.remove(&owner).is_some() && self.tracks_map_subscribers.is_empty() { + if self.tracks_map_subscribers.remove(&endpoint).is_some() && self.tracks_map_subscribers.is_empty() { log::info!("[ClusterRoom {}] last peer unsub tracks map => unsubscribe", self.room); self.queue.push_back(Output::Kv(dht_kv::Control::MapCmd(self.tracks_map, MapControl::Unsub))); } @@ -158,7 +162,7 @@ impl RoomMetadata { for (target, _) in peer.sub_peers.into_iter() { let target_peer_map = id_generator::peer_map(self.room, &target); let subs = self.peers_tracks_subs.get_mut(&target_peer_map).expect("Should have private peer_map"); - subs.remove(&owner); + subs.remove(&endpoint); if subs.is_empty() { self.peers_tracks_subs.remove(&target_peer_map); self.queue.push_back(Output::Kv(dht_kv::Control::MapCmd(target_peer_map, MapControl::Unsub))); @@ -167,16 +171,16 @@ impl RoomMetadata { if self.peers.is_empty() { log::info!("[ClusterRoom {}] last peer leaed => destroy metadata", self.room); - self.queue.push_back(Output::LastPeerLeaved); + self.queue.push_back(Output::OnResourceEmpty); } } - pub fn on_subscribe_peer(&mut self, owner: Owner, target: PeerId) { - let peer = self.peers.get_mut(&owner).expect("Should have peer"); + pub fn on_subscribe_peer(&mut self, endpoint: Endpoint, target: PeerId) { + let peer = self.peers.get_mut(&endpoint).expect("Should have peer"); let target_peer_map = id_generator::peer_map(self.room, &target); let subs = self.peers_tracks_subs.entry(target_peer_map).or_default(); let need_sub = subs.is_empty(); - subs.insert(owner, ()); + subs.insert(endpoint, ()); peer.sub_peers.insert(target, ()); if need_sub { @@ -184,11 +188,11 @@ impl RoomMetadata { } } - pub fn on_unsubscribe_peer(&mut self, owner: Owner, target: PeerId) { - let peer = self.peers.get_mut(&owner).expect("Should have peer"); + pub fn on_unsubscribe_peer(&mut self, endpoint: Endpoint, target: PeerId) { + let peer = self.peers.get_mut(&endpoint).expect("Should have peer"); let target_peer_map = id_generator::peer_map(self.room, &target); let subs = self.peers_tracks_subs.entry(target_peer_map).or_default(); - subs.remove(&owner); + subs.remove(&endpoint); peer.sub_peers.remove(&target); if subs.is_empty() { self.peers_tracks_subs.remove(&target_peer_map); @@ -196,8 +200,8 @@ impl RoomMetadata { } } - pub fn on_track_publish(&mut self, owner: Owner, track_id: RemoteTrackId, track: TrackName, meta: TrackMeta) { - let peer = return_if_none!(self.peers.get_mut(&owner)); + pub fn on_track_publish(&mut self, endpoint: Endpoint, track_id: RemoteTrackId, track: TrackName, meta: TrackMeta) { + let peer = return_if_none!(self.peers.get_mut(&endpoint)); if peer.publish.tracks { let info = TrackInfo { peer: peer.peer.clone(), @@ -213,8 +217,8 @@ impl RoomMetadata { } } - pub fn on_track_unpublish(&mut self, owner: Owner, track_id: RemoteTrackId) { - let peer = return_if_none!(self.peers.get_mut(&owner)); + pub fn on_track_unpublish(&mut self, endpoint: Endpoint, track_id: RemoteTrackId) { + let peer = return_if_none!(self.peers.get_mut(&endpoint)); let track = return_if_none!(peer.pub_tracks.remove(&track_id)); let track_key = id_generator::tracks_key(&peer.peer, &track); @@ -340,17 +344,26 @@ impl RoomMetadata { } } -impl TaskSwitcherChild> for RoomMetadata { - type Time = Instant; - fn pop_output(&mut self, _now: Instant) -> Option> { +impl TaskSwitcherChild> for RoomMetadata { + type Time = (); + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } +impl Drop for RoomMetadata { + fn drop(&mut self) { + log::info!("[ClusterRoomMetadata] Drop {}", self.room); + assert_eq!(self.queue.len(), 0, "Queue not empty"); + assert_eq!(self.peers.len(), 0, "Peers not empty"); + assert_eq!(self.peers_map_subscribers.len(), 0, "Peers subscriber not empty"); + assert_eq!(self.tracks_map_subscribers.len(), 0, "Tracks subscriber not empty"); + assert_eq!(self.peers_tracks_subs.len(), 0, "Peers tracks subs not empty"); + } +} + #[cfg(test)] mod tests { - use std::time::Instant; - use atm0s_sdn::features::dht_kv::{Control, MapControl, MapEvent}; use media_server_protocol::endpoint::{PeerId, PeerInfo, PeerMeta, RoomInfoPublish, RoomInfoSubscribe, TrackInfo, TrackName}; use sans_io_runtime::TaskSwitcherChild; @@ -369,17 +382,21 @@ mod tests { let mut room_meta: RoomMetadata = RoomMetadata::::new(room); let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; - let owner = 1; + let endpoint = 1; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.get_peer_from_owner(1), Some(peer_id)); - assert_eq!(room_meta.get_peer_from_owner(2), None); + assert_eq!(room_meta.get_peer_from_endpoint(1), Some(peer_id)); + assert_eq!(room_meta.get_peer_from_endpoint(2), None); + + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } /// Test join as peer only => should subscribe peers, fire only peer @@ -394,52 +411,49 @@ mod tests { let peer_meta = PeerMeta { metadata: None }; let peer_info = PeerInfo::new(peer_id.clone(), peer_meta.clone()); let peer_key = id_generator::peers_key(&peer_id); - let owner = 1; + let endpoint = 1; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: true, tracks: false }, RoomInfoSubscribe { peers: true, tracks: false }, ); - assert_eq!( - room_meta.pop_output(Instant::now()), - Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Set(peer_key, peer_info.serialize())))) - ); - assert_eq!(room_meta.pop_output(Instant::now()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Sub)))); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Set(peer_key, peer_info.serialize()))))); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Sub)))); + assert_eq!(room_meta.pop_output(()), None); // should handle incoming event with only peer and reject track room_meta.on_kv_event(peers_map, MapEvent::OnSet(peer_key, 0, peer_info.serialize())); assert_eq!( - room_meta.pop_output(Instant::now()), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::PeerJoined(peer_id.clone(), peer_meta.clone()))) + room_meta.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::PeerJoined(peer_id.clone(), peer_meta.clone()))) ); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), None); let track_name: TrackName = "audio_main".to_string().into(); let track_info = TrackInfo::simple_audio(peer_id.clone()); let track_key = id_generator::tracks_key(&peer_id, &track_name); room_meta.on_kv_event(tracks_map, MapEvent::OnSet(track_key, 0, track_info.serialize())); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), None); // should only handle remove peer event, reject track room_meta.on_kv_event(tracks_map, MapEvent::OnDel(track_key, 0)); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), None); room_meta.on_kv_event(peers_map, MapEvent::OnDel(peer_key, 0)); assert_eq!( - room_meta.pop_output(Instant::now()), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::PeerLeaved(peer_id.clone(), peer_info.meta))) + room_meta.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::PeerLeaved(peer_id.clone(), peer_info.meta))) ); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), None); // peer leave should send unsub and del - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(Instant::now()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Del(peer_key))))); - assert_eq!(room_meta.pop_output(Instant::now()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(Instant::now()), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(Instant::now()), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Del(peer_key))))); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Unsub)))); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } #[test] @@ -453,24 +467,29 @@ mod tests { let peer2_info = PeerInfo::new(peer2, PeerMeta { metadata: None }); room_meta.on_kv_event(peers_map, MapEvent::OnSet(peer2_key, 0, peer2_info.serialize())); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), None); - let owner = 1; + let endpoint = 1; let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: true, tracks: false }, ); assert_eq!( - room_meta.pop_output(Instant::now()), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::PeerJoined(peer2_info.peer.clone(), peer2_info.meta.clone()))) + room_meta.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::PeerJoined(peer2_info.peer.clone(), peer2_info.meta.clone()))) ); - assert_eq!(room_meta.pop_output(Instant::now()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Sub)))); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Sub)))); + assert_eq!(room_meta.pop_output(()), None); + + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Unsub)))); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } //TODO Test join as track only => should subscribe only tracks, fire only track events @@ -484,51 +503,53 @@ mod tests { let peer_meta = PeerMeta { metadata: None }; let peer_info = PeerInfo::new(peer_id.clone(), peer_meta.clone()); let peer_key = id_generator::peers_key(&peer_id); - let owner = 1; - let now = Instant::now(); + let endpoint = 1; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: true }, RoomInfoSubscribe { peers: false, tracks: true }, ); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Sub)))); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Sub)))); + assert_eq!(room_meta.pop_output(()), None); // should handle incoming event with only track and reject peer room_meta.on_kv_event(peers_map, MapEvent::OnSet(peer_key, 0, peer_info.serialize())); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let track_name: TrackName = "audio_main".to_string().into(); let track_info = TrackInfo::simple_audio(peer_id.clone()); let track_key = id_generator::tracks_key(&peer_id, &track_name); room_meta.on_kv_event(tracks_map, MapEvent::OnSet(track_key, 0, track_info.serialize())); assert_eq!( - room_meta.pop_output(now), + room_meta.pop_output(()), Some(Output::Endpoint( - vec![owner], + vec![endpoint], ClusterEndpointEvent::TrackStarted(peer_id.clone(), track_name.clone(), track_info.meta.clone()) )) ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // should only handle remove track event, reject peer room_meta.on_kv_event(tracks_map, MapEvent::OnDel(track_key, 0)); assert_eq!( - room_meta.pop_output(now), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::TrackStopped(peer_id.clone(), track_name.clone(), track_info.meta))) + room_meta.pop_output(()), + Some(Output::Endpoint( + vec![endpoint], + ClusterEndpointEvent::TrackStopped(peer_id.clone(), track_name.clone(), track_info.meta) + )) ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); room_meta.on_kv_event(peers_map, MapEvent::OnDel(peer_key, 0)); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // peer leave should send unsub - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Unsub)))); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } //join track only should restore old tracks @@ -544,27 +565,32 @@ mod tests { let track_info = TrackInfo::simple_audio(peer2); room_meta.on_kv_event(tracks_map, MapEvent::OnSet(track_key, 0, track_info.serialize())); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), None); - let owner = 1; + let endpoint = 1; let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: true }, ); assert_eq!( - room_meta.pop_output(Instant::now()), + room_meta.pop_output(()), Some(Output::Endpoint( - vec![owner], + vec![endpoint], ClusterEndpointEvent::TrackStarted(track_info.peer.clone(), track_info.track.clone(), track_info.meta.clone()) )) ); - assert_eq!(room_meta.pop_output(Instant::now()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Sub)))); - assert_eq!(room_meta.pop_output(Instant::now()), None); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Sub)))); + assert_eq!(room_meta.pop_output(()), None); + + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Unsub)))); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } //Test manual no subscribe peer => dont fire any event @@ -578,38 +604,37 @@ mod tests { let peer_meta = PeerMeta { metadata: None }; let peer_info = PeerInfo::new(peer_id.clone(), peer_meta.clone()); let peer_key = id_generator::peers_key(&peer_id); - let owner = 1; - let now = Instant::now(); + let endpoint = 1; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // should handle incoming event with only track and reject peer room_meta.on_kv_event(peers_map, MapEvent::OnSet(peer_key, 0, peer_info.serialize())); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let track_name: TrackName = "audio_main".to_string().into(); let track_info = TrackInfo::simple_audio(peer_id.clone()); let track_key = id_generator::tracks_key(&peer_id, &track_name); room_meta.on_kv_event(tracks_map, MapEvent::OnSet(track_key, 0, track_info.serialize())); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // should only handle remove track event, reject peer room_meta.on_kv_event(tracks_map, MapEvent::OnDel(track_key, 0)); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); room_meta.on_kv_event(peers_map, MapEvent::OnDel(peer_key, 0)); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // peer leave should send unsub - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } //TODO Test manual and subscribe peer => should fire event @@ -619,22 +644,21 @@ mod tests { let mut room_meta: RoomMetadata = RoomMetadata::::new(room); let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; - let owner = 1; - let now = Instant::now(); + let endpoint = 1; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let peer2: PeerId = "peer1".to_string().into(); let peer2_map = id_generator::peer_map(room, &peer2); - room_meta.on_subscribe_peer(owner, peer2.clone()); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Sub)))); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_subscribe_peer(endpoint, peer2.clone()); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Sub)))); + assert_eq!(room_meta.pop_output(()), None); // should handle incoming event with only track and reject peer let track_name: TrackName = "audio_main".to_string().into(); @@ -642,31 +666,31 @@ mod tests { let track_key = id_generator::tracks_key(&peer2, &track_name); room_meta.on_kv_event(peer2_map, MapEvent::OnSet(track_key, 0, track_info.serialize())); assert_eq!( - room_meta.pop_output(now), + room_meta.pop_output(()), Some(Output::Endpoint( - vec![owner], + vec![endpoint], ClusterEndpointEvent::TrackStarted(peer2.clone(), track_name.clone(), track_info.meta.clone()) )) ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // should only handle remove track event, reject peer room_meta.on_kv_event(peer2_map, MapEvent::OnDel(track_key, 0)); assert_eq!( - room_meta.pop_output(now), - Some(Output::Endpoint(vec![owner], ClusterEndpointEvent::TrackStopped(peer2.clone(), track_name.clone(), track_info.meta))) + room_meta.pop_output(()), + Some(Output::Endpoint(vec![endpoint], ClusterEndpointEvent::TrackStopped(peer2.clone(), track_name.clone(), track_info.meta))) ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); // should send unsub when unsubscribe peer - room_meta.on_unsubscribe_peer(owner, peer2.clone()); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_unsubscribe_peer(endpoint, peer2.clone()); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Unsub)))); + assert_eq!(room_meta.pop_output(()), None); // peer leave should not send unsub - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } //TODO Test track publish => should set key to both single peer map and tracks map @@ -676,45 +700,44 @@ mod tests { let tracks_map = id_generator::tracks_map(room); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); - let owner = 1; + let endpoint = 1; let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; - let now = Instant::now(); room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: true }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let track_id: RemoteTrackId = RemoteTrackId(1); let track_name: TrackName = "audio_main".to_string().into(); let track_info = TrackInfo::simple_audio(peer_id.clone()); let peer_map = id_generator::peer_map(room, &peer_id); let track_key = id_generator::tracks_key(&peer_id, &track_name); - room_meta.on_track_publish(owner, track_id, track_name, track_info.meta.clone()); + room_meta.on_track_publish(endpoint, track_id, track_name, track_info.meta.clone()); assert_eq!( - room_meta.pop_output(now), + room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Set(track_key, track_info.serialize())))) ); assert_eq!( - room_meta.pop_output(now), + room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Set(track_key, track_info.serialize())))) ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); //after unpublish should delete all tracks - room_meta.on_track_unpublish(owner, track_id); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Del(track_key))))); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Del(track_key))))); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_track_unpublish(endpoint, track_id); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Del(track_key))))); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Del(track_key))))); + assert_eq!(room_meta.pop_output(()), None); //should not pop anything after leave - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } //TODO Test track publish in disable mode => should not set key to both single peer map and tracks map @@ -723,33 +746,32 @@ mod tests { let room: ClusterRoomHash = 1.into(); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); - let now = Instant::now(); - let owner = 1; + let endpoint = 1; let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let track_id: RemoteTrackId = RemoteTrackId(1); let track_name: TrackName = "audio_main".to_string().into(); let track_info = TrackInfo::simple_audio(peer_id.clone()); - room_meta.on_track_publish(owner, track_id, track_name, track_info.meta.clone()); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_track_publish(endpoint, track_id, track_name, track_info.meta.clone()); + assert_eq!(room_meta.pop_output(()), None); //after unpublish should delete all tracks - room_meta.on_track_unpublish(owner, track_id); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_track_unpublish(endpoint, track_id); + assert_eq!(room_meta.pop_output(()), None); //should not pop anything after leave - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } /// Test leave room auto del remain remote tracks @@ -759,41 +781,40 @@ mod tests { let tracks_map = id_generator::tracks_map(room); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); - let now = Instant::now(); - let owner = 1; + let endpoint = 1; let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: true }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let track_id: RemoteTrackId = RemoteTrackId(1); let track_name: TrackName = "audio_main".to_string().into(); let track_info = TrackInfo::simple_audio(peer_id.clone()); let peer_map = id_generator::peer_map(room, &peer_id); let track_key = id_generator::tracks_key(&peer_id, &track_name); - room_meta.on_track_publish(owner, track_id, track_name, track_info.meta.clone()); + room_meta.on_track_publish(endpoint, track_id, track_name, track_info.meta.clone()); assert_eq!( - room_meta.pop_output(now), + room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Set(track_key, track_info.serialize())))) ); assert_eq!( - room_meta.pop_output(now), + room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Set(track_key, track_info.serialize())))) ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); //after leave should auto delete all tracks - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Del(track_key))))); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Del(track_key))))); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Del(track_key))))); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Del(track_key))))); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } // Leave room auto unsub private peer maps @@ -803,27 +824,26 @@ mod tests { let mut room_meta: RoomMetadata = RoomMetadata::::new(room); let peer_id: PeerId = "peer1".to_string().into(); let peer_meta = PeerMeta { metadata: None }; - let owner = 1; - let now = Instant::now(); + let endpoint = 1; room_meta.on_join( - owner, + endpoint, peer_id.clone(), peer_meta.clone(), RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: false }, ); - assert_eq!(room_meta.pop_output(now), None); + assert_eq!(room_meta.pop_output(()), None); let peer2: PeerId = "peer1".to_string().into(); let peer2_map = id_generator::peer_map(room, &peer2); - room_meta.on_subscribe_peer(owner, peer2.clone()); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Sub)))); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_subscribe_peer(endpoint, peer2.clone()); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Sub)))); + assert_eq!(room_meta.pop_output(()), None); // peer leave should send unsub of peer2_map - room_meta.on_leave(owner); - assert_eq!(room_meta.pop_output(now), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(now), Some(Output::LastPeerLeaved)); - assert_eq!(room_meta.pop_output(now), None); + room_meta.on_leave(endpoint); + assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Unsub)))); + assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); + assert_eq!(room_meta.pop_output(()), None); } } diff --git a/packages/media_core/src/endpoint.rs b/packages/media_core/src/endpoint.rs index b7157060..845396c1 100644 --- a/packages/media_core/src/endpoint.rs +++ b/packages/media_core/src/endpoint.rs @@ -3,7 +3,7 @@ use std::{marker::PhantomData, time::Instant}; use media_server_protocol::{ - endpoint::{BitrateControlMode, PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe, TrackMeta, TrackName, TrackPriority}, + endpoint::{AudioMixerConfig, BitrateControlMode, PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe, TrackMeta, TrackName, TrackPriority, TrackSource}, media::MediaPacket, protobuf, transport::RpcResult, @@ -52,21 +52,6 @@ pub enum EndpointRemoteTrackRes { Config(RpcResult<()>), } -#[derive(Debug, PartialEq, Eq)] -pub struct EndpointLocalTrackSource { - pub peer: PeerId, - pub track: TrackName, -} - -impl From for EndpointLocalTrackSource { - fn from(value: protobuf::shared::receiver::Source) -> Self { - Self { - peer: value.peer.into(), - track: value.track.into(), - } - } -} - #[derive(Debug, PartialEq, Eq)] pub struct EndpointLocalTrackConfig { pub priority: TrackPriority, @@ -90,7 +75,7 @@ impl From for EndpointLocalTrackConfig { #[derive(Debug, PartialEq, Eq)] pub enum EndpointLocalTrackReq { - Attach(EndpointLocalTrackSource, EndpointLocalTrackConfig), + Attach(TrackSource, EndpointLocalTrackConfig), Detach(), Config(EndpointLocalTrackConfig), } @@ -102,16 +87,29 @@ pub enum EndpointLocalTrackRes { Config(RpcResult<()>), } +#[derive(Debug, PartialEq, Eq)] +pub enum EndpointAudioMixerReq { + Attach(Vec), + Detach(Vec), +} + +#[derive(Debug, PartialEq, Eq)] +pub enum EndpointAudioMixerRes { + Attach(RpcResult<()>), + Detach(RpcResult<()>), +} + #[derive(Debug, PartialEq, Eq, derive_more::From)] pub struct EndpointReqId(pub u32); /// This is control APIs, which is used to control server from Endpoint SDK #[derive(Debug, PartialEq, Eq)] pub enum EndpointReq { - JoinRoom(RoomId, PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe), + JoinRoom(RoomId, PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe, Option), LeaveRoom, SubscribePeer(PeerId), UnsubscribePeer(PeerId), + AudioMixer(EndpointAudioMixerReq), RemoteTrack(RemoteTrackId, EndpointRemoteTrackReq), LocalTrack(LocalTrackId, EndpointLocalTrackReq), } @@ -123,6 +121,7 @@ pub enum EndpointRes { LeaveRoom(RpcResult<()>), SubscribePeer(RpcResult<()>), UnsubscribePeer(RpcResult<()>), + AudioMixer(EndpointAudioMixerRes), RemoteTrack(RemoteTrackId, EndpointRemoteTrackRes), LocalTrack(LocalTrackId, EndpointLocalTrackRes), } @@ -132,6 +131,7 @@ pub enum EndpointRes { pub enum EndpointLocalTrackEvent { Media(MediaPacket), Status(protobuf::shared::receiver::Status), + VoiceActivity(i8), } /// This is used for controlling the remote track, which is sent from endpoint @@ -141,12 +141,20 @@ pub enum EndpointRemoteTrackEvent { LimitBitrateBps { min: u64, max: u64 }, } +/// This is used for controlling audio mixer feature +#[derive(Debug, PartialEq, Eq)] +pub enum EndpointAudioMixerEvent { + SlotSet(u8, PeerId, TrackName), + SlotUnset(u8), +} + #[derive(Debug, PartialEq, Eq)] pub enum EndpointEvent { PeerJoined(PeerId, PeerMeta), PeerLeaved(PeerId, PeerMeta), PeerTrackStarted(PeerId, TrackName, TrackMeta), PeerTrackStopped(PeerId, TrackName, TrackMeta), + AudioMixer(EndpointAudioMixerEvent), RemoteMediaTrack(RemoteTrackId, EndpointRemoteTrackEvent), LocalMediaTrack(LocalTrackId, EndpointLocalTrackEvent), /// Egress est params diff --git a/packages/media_core/src/endpoint/internal.rs b/packages/media_core/src/endpoint/internal.rs index b20d80ca..42d18666 100644 --- a/packages/media_core/src/endpoint/internal.rs +++ b/packages/media_core/src/endpoint/internal.rs @@ -3,21 +3,21 @@ use std::{collections::VecDeque, time::Instant}; use media_server_protocol::{ - endpoint::{PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe}, + endpoint::{AudioMixerConfig, AudioMixerMode, PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe}, transport::RpcError, }; use media_server_utils::Small2dMap; use sans_io_runtime::{return_if_none, return_if_some, TaskGroup, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; use crate::{ - cluster::{ClusterEndpointControl, ClusterEndpointEvent, ClusterLocalTrackEvent, ClusterRemoteTrackEvent, ClusterRoomHash}, + cluster::{ClusterAudioMixerControl, ClusterAudioMixerEvent, ClusterEndpointControl, ClusterEndpointEvent, ClusterLocalTrackEvent, ClusterRemoteTrackEvent, ClusterRoomHash}, errors::EndpointErrors, transport::{LocalTrackEvent, LocalTrackId, RemoteTrackEvent, RemoteTrackId, TransportEvent, TransportState, TransportStats}, }; use self::{bitrate_allocator::BitrateAllocator, local_track::EndpointLocalTrack, remote_track::EndpointRemoteTrack}; -use super::{middleware::EndpointMiddleware, EndpointCfg, EndpointEvent, EndpointReq, EndpointReqId, EndpointRes}; +use super::{middleware::EndpointMiddleware, EndpointAudioMixerEvent, EndpointAudioMixerReq, EndpointAudioMixerRes, EndpointCfg, EndpointEvent, EndpointReq, EndpointReqId, EndpointRes}; mod bitrate_allocator; mod local_track; @@ -42,8 +42,8 @@ pub enum InternalOutput { pub struct EndpointInternal { cfg: EndpointCfg, state: TransportState, - wait_join: Option<(EndpointReqId, RoomId, PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe)>, - joined: Option<(ClusterRoomHash, RoomId, PeerId)>, + wait_join: Option<(EndpointReqId, RoomId, PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe, Option)>, + joined: Option<(ClusterRoomHash, RoomId, PeerId, Option)>, local_tracks_id: Small2dMap, remote_tracks_id: Small2dMap, local_tracks: TaskSwitcherBranch, (usize, local_track::Output)>, @@ -113,16 +113,16 @@ impl EndpointInternal { pub fn on_transport_rpc(&mut self, now: Instant, req_id: EndpointReqId, req: EndpointReq) { match req { - EndpointReq::JoinRoom(room, peer, meta, publish, subscribe) => { + EndpointReq::JoinRoom(room, peer, meta, publish, subscribe, mixer) => { if matches!(self.state, TransportState::Connecting) { log::info!("[EndpointInternal] join_room({room}, {peer}) but in Connecting state => wait"); - self.wait_join = Some((req_id, room, peer, meta, publish, subscribe)); + self.wait_join = Some((req_id, room, peer, meta, publish, subscribe, mixer)); } else { - self.join_room(now, req_id, room, peer, meta, publish, subscribe); + self.join_room(now, req_id, room, peer, meta, publish, subscribe, mixer); } } EndpointReq::LeaveRoom => { - if let Some((_req_id, room, peer, _meta, _publish, _subscribe)) = self.wait_join.take() { + if let Some((_req_id, room, peer, _meta, _publish, _subscribe, _mixer)) = self.wait_join.take() { log::info!("[EndpointInternal] leave_room({room}, {peer}) but in Connecting state => only clear local"); self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::LeaveRoom(Ok(())))); } else { @@ -131,7 +131,7 @@ impl EndpointInternal { } } EndpointReq::SubscribePeer(peer) => { - if let Some((room, _, _)) = &self.joined { + if let Some((room, _, _, _)) = &self.joined { self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::SubscribePeer(Ok(())))); self.queue.push_back(InternalOutput::Cluster(*room, ClusterEndpointControl::SubscribePeer(peer))); } else { @@ -140,7 +140,7 @@ impl EndpointInternal { } } EndpointReq::UnsubscribePeer(peer) => { - if let Some((room, _, _)) = &self.joined { + if let Some((room, _, _, _)) = &self.joined { self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::UnsubscribePeer(Ok(())))); self.queue.push_back(InternalOutput::Cluster(*room, ClusterEndpointControl::UnsubscribePeer(peer))); } else { @@ -156,6 +156,32 @@ impl EndpointInternal { let index = return_if_none!(self.local_tracks_id.get1(&track_id)); self.local_tracks.input(&mut self.switcher).on_event(now, *index, local_track::Input::RpcReq(req_id, req)); } + EndpointReq::AudioMixer(req) => match req { + EndpointAudioMixerReq::Attach(sources) => { + if let Some((room, _, _, Some(AudioMixerMode::Manual))) = &self.joined { + self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::AudioMixer(EndpointAudioMixerRes::Attach(Ok(()))))); + self.queue + .push_back(InternalOutput::Cluster(*room, ClusterEndpointControl::AudioMixer(ClusterAudioMixerControl::Attach(sources)))); + } else { + self.queue.push_back(InternalOutput::RpcRes( + req_id, + EndpointRes::AudioMixer(EndpointAudioMixerRes::Attach(Err(RpcError::new2(EndpointErrors::AudioMixerWrongMode)))), + )); + } + } + EndpointAudioMixerReq::Detach(sources) => { + if let Some((room, _, _, Some(AudioMixerMode::Manual))) = &self.joined { + self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::AudioMixer(EndpointAudioMixerRes::Detach(Ok(()))))); + self.queue + .push_back(InternalOutput::Cluster(*room, ClusterEndpointControl::AudioMixer(ClusterAudioMixerControl::Detach(sources)))); + } else { + self.queue.push_back(InternalOutput::RpcRes( + req_id, + EndpointRes::AudioMixer(EndpointAudioMixerRes::Detach(Err(RpcError::new2(EndpointErrors::AudioMixerWrongMode)))), + )); + } + } + }, } } @@ -171,9 +197,9 @@ impl EndpointInternal { } TransportState::Connected => { log::info!("[EndpointInternal] connected"); - let (req_id, room, peer, meta, publish, subscribe) = return_if_none!(self.wait_join.take()); + let (req_id, room, peer, meta, publish, subscribe, mixer) = return_if_none!(self.wait_join.take()); log::info!("[EndpointInternal] join_room({room}, {peer}) after connected"); - self.join_room(now, req_id, room, peer, meta, publish, subscribe); + self.join_room(now, req_id, room, peer, meta, publish, subscribe, mixer); } TransportState::Reconnecting => { log::info!("[EndpointInternal] reconnecting"); @@ -211,16 +237,16 @@ impl EndpointInternal { fn on_transport_stats(&mut self, _now: Instant, _stats: TransportStats) {} #[allow(clippy::too_many_arguments)] - fn join_room(&mut self, now: Instant, req_id: EndpointReqId, room: RoomId, peer: PeerId, meta: PeerMeta, publish: RoomInfoPublish, subscribe: RoomInfoSubscribe) { + fn join_room(&mut self, now: Instant, req_id: EndpointReqId, room: RoomId, peer: PeerId, meta: PeerMeta, publish: RoomInfoPublish, subscribe: RoomInfoSubscribe, mixer: Option) { let room_hash: ClusterRoomHash = (&room).into(); log::info!("[EndpointInternal] join_room({room}, {peer}), room_hash {room_hash}"); self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::JoinRoom(Ok(())))); self.leave_room(now); - self.joined = Some(((&room).into(), room.clone(), peer.clone())); + self.joined = Some(((&room).into(), room.clone(), peer.clone(), mixer.as_ref().map(|m| m.mode))); self.queue - .push_back(InternalOutput::Cluster((&room).into(), ClusterEndpointControl::Join(peer, meta, publish, subscribe))); + .push_back(InternalOutput::Cluster((&room).into(), ClusterEndpointControl::Join(peer, meta, publish, subscribe, mixer))); for (_track_id, index) in self.local_tracks_id.pairs() { self.local_tracks.input(&mut self.switcher).on_event(now, index, local_track::Input::JoinRoom(room_hash)); @@ -232,7 +258,7 @@ impl EndpointInternal { } fn leave_room(&mut self, now: Instant) { - let (hash, room, peer) = return_if_none!(self.joined.take()); + let (hash, room, peer, _) = return_if_none!(self.joined.take()); log::info!("[EndpointInternal] leave_room({room}, {peer})"); for (_track_id, index) in self.local_tracks_id.pairs() { @@ -243,6 +269,14 @@ impl EndpointInternal { self.remote_tracks.input(&mut self.switcher).on_event(now, index, remote_track::Input::LeaveRoom); } + while let Some(task) = self.switcher.current() { + match task.try_into().expect("Should valid task type") { + TaskType::BitrateAllocator => self.pop_bitrate_allocator(now), + TaskType::LocalTracks => self.pop_local_tracks(now), + TaskType::RemoteTracks => self.pop_remote_tracks(now), + } + } + self.queue.push_back(InternalOutput::Cluster(hash, ClusterEndpointControl::Leave)); } } @@ -255,6 +289,12 @@ impl EndpointInternal { ClusterEndpointEvent::PeerLeaved(peer, meta) => self.queue.push_back(InternalOutput::Event(EndpointEvent::PeerLeaved(peer, meta))), ClusterEndpointEvent::TrackStarted(peer, track, meta) => self.queue.push_back(InternalOutput::Event(EndpointEvent::PeerTrackStarted(peer, track, meta))), ClusterEndpointEvent::TrackStopped(peer, track, meta) => self.queue.push_back(InternalOutput::Event(EndpointEvent::PeerTrackStopped(peer, track, meta))), + ClusterEndpointEvent::AudioMixer(event) => match event { + ClusterAudioMixerEvent::SlotSet(slot, peer, track) => self + .queue + .push_back(InternalOutput::Event(EndpointEvent::AudioMixer(EndpointAudioMixerEvent::SlotSet(slot, peer, track)))), + ClusterAudioMixerEvent::SlotUnset(slot) => self.queue.push_back(InternalOutput::Event(EndpointEvent::AudioMixer(EndpointAudioMixerEvent::SlotUnset(slot)))), + }, ClusterEndpointEvent::RemoteTrack(track, event) => self.on_cluster_remote_track(now, track, event), ClusterEndpointEvent::LocalTrack(track, event) => self.on_cluster_local_track(now, track, event), } @@ -391,12 +431,12 @@ mod tests { let meta = PeerMeta { metadata: None }; let publish = RoomInfoPublish { peer: true, tracks: true }; let subscribe = RoomInfoSubscribe { peers: true, tracks: true }; - internal.on_transport_rpc(now, 0.into(), EndpointReq::JoinRoom(room.clone(), peer.clone(), meta.clone(), publish.clone(), subscribe.clone())); + internal.on_transport_rpc(now, 0.into(), EndpointReq::JoinRoom(room.clone(), peer.clone(), meta.clone(), publish.clone(), subscribe.clone(), None)); assert_eq!(internal.pop_output(now), Some(InternalOutput::RpcRes(0.into(), EndpointRes::JoinRoom(Ok(()))))); let room_hash = ClusterRoomHash::from(&room); assert_eq!( internal.pop_output(now), - Some(InternalOutput::Cluster(room_hash, ClusterEndpointControl::Join(peer, meta, publish, subscribe))) + Some(InternalOutput::Cluster(room_hash, ClusterEndpointControl::Join(peer, meta, publish, subscribe, None))) ); assert_eq!(internal.pop_output(now), None); @@ -424,14 +464,18 @@ mod tests { let meta = PeerMeta { metadata: None }; let publish = RoomInfoPublish { peer: true, tracks: true }; let subscribe = RoomInfoSubscribe { peers: true, tracks: true }; - internal.on_transport_rpc(now, 0.into(), EndpointReq::JoinRoom(room1.clone(), peer.clone(), meta.clone(), publish.clone(), subscribe.clone())); + internal.on_transport_rpc( + now, + 0.into(), + EndpointReq::JoinRoom(room1.clone(), peer.clone(), meta.clone(), publish.clone(), subscribe.clone(), None), + ); assert_eq!(internal.pop_output(now), Some(InternalOutput::RpcRes(0.into(), EndpointRes::JoinRoom(Ok(()))))); assert_eq!( internal.pop_output(now), Some(InternalOutput::Cluster( room1_hash, - ClusterEndpointControl::Join(peer.clone(), meta.clone(), publish.clone(), subscribe.clone()) + ClusterEndpointControl::Join(peer.clone(), meta.clone(), publish.clone(), subscribe.clone(), None), )) ); assert_eq!(internal.pop_output(now), None); @@ -440,7 +484,11 @@ mod tests { let room2: RoomId = "room2".into(); let room2_hash = ClusterRoomHash::from(&room2); - internal.on_transport_rpc(now, 1.into(), EndpointReq::JoinRoom(room2.clone(), peer.clone(), meta.clone(), publish.clone(), subscribe.clone())); + internal.on_transport_rpc( + now, + 1.into(), + EndpointReq::JoinRoom(room2.clone(), peer.clone(), meta.clone(), publish.clone(), subscribe.clone(), None), + ); assert_eq!(internal.pop_output(now), Some(InternalOutput::RpcRes(1.into(), EndpointRes::JoinRoom(Ok(()))))); //it will auto leave room1 assert_eq!(internal.pop_output(now), Some(InternalOutput::Cluster(room1_hash, ClusterEndpointControl::Leave))); @@ -450,7 +498,7 @@ mod tests { internal.pop_output(now), Some(InternalOutput::Cluster( room2_hash, - ClusterEndpointControl::Join(peer.clone(), meta.clone(), publish.clone(), subscribe.clone()) + ClusterEndpointControl::Join(peer.clone(), meta.clone(), publish.clone(), subscribe.clone(), None), )) ); assert_eq!(internal.pop_output(now), None); diff --git a/packages/media_core/src/endpoint/internal/local_track.rs b/packages/media_core/src/endpoint/internal/local_track.rs index 98f0bc11..c0541a8b 100644 --- a/packages/media_core/src/endpoint/internal/local_track.rs +++ b/packages/media_core/src/endpoint/internal/local_track.rs @@ -7,7 +7,7 @@ use std::{collections::VecDeque, time::Instant}; use atm0s_sdn::TimePivot; use media_server_protocol::{ endpoint::{PeerId, TrackName, TrackPriority}, - media::MediaKind, + media::{MediaKind, MediaMeta}, protobuf::shared::receiver::Status as ProtoStatus, transport::RpcError, }; @@ -20,11 +20,13 @@ use crate::{ transport::LocalTrackEvent, }; -use self::packet_selector::PacketSelector; +use packet_selector::PacketSelector; +use voice_activity::VoiceActivityDetector; use super::bitrate_allocator::EgressAction; mod packet_selector; +mod voice_activity; const MEDIA_TIMEOUT_MS: u64 = 2_000; //after 2s not receive media, the track will become inactive @@ -60,6 +62,7 @@ pub struct EndpointLocalTrack { queue: VecDeque, selector: PacketSelector, timer: TimePivot, + voice_activity: VoiceActivityDetector, } impl EndpointLocalTrack { @@ -72,6 +75,7 @@ impl EndpointLocalTrack { queue: VecDeque::new(), selector: PacketSelector::new(kind, 2, 2), timer: TimePivot::build(), + voice_activity: VoiceActivityDetector::default(), } } @@ -94,10 +98,17 @@ impl EndpointLocalTrack { fn on_cluster_event(&mut self, now: Instant, event: ClusterLocalTrackEvent) { match event { ClusterLocalTrackEvent::Started => todo!(), + ClusterLocalTrackEvent::RelayChanged => { + if self.kind.is_video() { + let room = return_if_none!(self.room.as_ref()); + log::info!("[EndpointLocalTrack] relay changed => request key-frame"); + self.queue.push_back(Output::Cluster(*room, ClusterLocalTrackControl::RequestKeyFrame)); + } + } ClusterLocalTrackEvent::SourceChanged => { - let room = return_if_none!(self.room.as_ref()); - log::info!("[EndpointLocalTrack] source changed => request key-frame and reset seq, ts rewrite"); - self.queue.push_back(Output::Cluster(*room, ClusterLocalTrackControl::RequestKeyFrame)); + //currently for audio_mixer + log::info!("[EndpointLocalTrack] source changed => reset seq, ts rewrite"); + self.selector.reset(); } ClusterLocalTrackEvent::Media(channel, mut pkt) => { log::trace!("[EndpointLocalTrack] on media payload {:?} seq {}", pkt.meta, pkt.seq); @@ -117,6 +128,12 @@ impl EndpointLocalTrack { } } + if let MediaMeta::Opus { audio_level } = &pkt.meta { + if let Some(level) = self.voice_activity.on_audio(now_ms, *audio_level) { + self.queue.push_back(Output::Event(EndpointLocalTrackEvent::VoiceActivity(level))); + } + } + self.queue.push_back(Output::Event(EndpointLocalTrackEvent::Media(pkt))); } } @@ -263,6 +280,12 @@ impl TaskSwitcherChild for EndpointLocalTrack { } } +impl Drop for EndpointLocalTrack { + fn drop(&mut self) { + assert_eq!(self.queue.len(), 0); + } +} + #[cfg(test)] mod tests { //TODO view not in room diff --git a/packages/media_core/src/endpoint/internal/local_track/voice_activity.rs b/packages/media_core/src/endpoint/internal/local_track/voice_activity.rs new file mode 100644 index 00000000..e38d631d --- /dev/null +++ b/packages/media_core/src/endpoint/internal/local_track/voice_activity.rs @@ -0,0 +1,19 @@ +const AUDIO_LEVEL_THRESHOLD: i8 = -40; +const VOICE_ACTIVITY_INTERVAL: u64 = 500; + +#[derive(Default)] +pub struct VoiceActivityDetector { + last_activity: u64, +} + +impl VoiceActivityDetector { + pub fn on_audio(&mut self, now: u64, audio_level: Option) -> Option { + let audio_level = audio_level?; + if audio_level >= AUDIO_LEVEL_THRESHOLD && self.last_activity + VOICE_ACTIVITY_INTERVAL <= now { + self.last_activity = now; + Some(audio_level) + } else { + None + } + } +} diff --git a/packages/media_core/src/endpoint/internal/remote_track.rs b/packages/media_core/src/endpoint/internal/remote_track.rs index ddb96ced..823405c4 100644 --- a/packages/media_core/src/endpoint/internal/remote_track.rs +++ b/packages/media_core/src/endpoint/internal/remote_track.rs @@ -40,7 +40,7 @@ pub enum Output { pub struct EndpointRemoteTrack { meta: TrackMeta, room: Option, - name: Option, + name: Option, queue: VecDeque, allocate_bitrate: Option, /// This is for storing current stream layers, everytime key-frame arrived we will set this if it not set @@ -68,14 +68,15 @@ impl EndpointRemoteTrack { log::info!("[EndpointRemoteTrack] join room {room}"); let name = return_if_none!(self.name.clone()); log::info!("[EndpointRemoteTrack] started as name {name} after join room"); - self.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Started(TrackName(name), self.meta.clone()))); + self.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Started(name, self.meta.clone()))); } + fn on_leave_room(&mut self, _now: Instant) { let room = self.room.take().expect("Must have room here"); log::info!("[EndpointRemoteTrack] leave room {room}"); - let name = return_if_none!(self.name.as_ref()); + let name = return_if_none!(self.name.clone()); log::info!("[EndpointRemoteTrack] stopped as name {name} after leave room"); - self.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Ended)); + self.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Ended(name, self.meta.clone()))); } fn on_cluster_event(&mut self, _now: Instant, event: ClusterRemoteTrackEvent) { @@ -95,7 +96,7 @@ impl EndpointRemoteTrack { fn on_transport_event(&mut self, _now: Instant, event: RemoteTrackEvent) { match event { RemoteTrackEvent::Started { name, priority, meta: _ } => { - self.name = Some(name.clone()); + self.name = Some(name.clone().into()); let room = return_if_none!(self.room.as_ref()); log::info!("[EndpointRemoteTrack] started as name {name} in room {room}"); self.queue.push_back(Output::Cluster(*room, ClusterRemoteTrackControl::Started(TrackName(name), self.meta.clone()))); @@ -123,7 +124,7 @@ impl EndpointRemoteTrack { let name = return_if_none!(self.name.take()); let room = return_if_none!(self.room.as_ref()); log::info!("[EndpointRemoteTrack] stopped with name {name} in room {room}"); - self.queue.push_back(Output::Cluster(*room, ClusterRemoteTrackControl::Ended)); + self.queue.push_back(Output::Cluster(*room, ClusterRemoteTrackControl::Ended(name, self.meta.clone()))); self.queue.push_back(Output::Stopped(self.meta.kind)); } } @@ -192,6 +193,12 @@ impl TaskSwitcherChild for EndpointRemoteTrack { } } +impl Drop for EndpointRemoteTrack { + fn drop(&mut self) { + assert_eq!(self.queue.len(), 0); + } +} + #[cfg(test)] mod tests { //TODO start in room diff --git a/packages/media_core/src/errors.rs b/packages/media_core/src/errors.rs index f8a91912..0c6cfbfc 100644 --- a/packages/media_core/src/errors.rs +++ b/packages/media_core/src/errors.rs @@ -5,4 +5,5 @@ pub enum EndpointErrors { LocalTrackNotPinSource = 0x1001, LocalTrackInvalidPriority = 0x1002, RemoteTrackInvalidPriority = 0x2001, + AudioMixerWrongMode = 0x3001, } diff --git a/packages/media_core/src/transport.rs b/packages/media_core/src/transport.rs index 4f6e78a3..5d63a3cf 100644 --- a/packages/media_core/src/transport.rs +++ b/packages/media_core/src/transport.rs @@ -1,5 +1,5 @@ use derive_more::{Display, From}; -use std::{hash::Hash, time::Instant}; +use std::time::Instant; use media_server_protocol::{ endpoint::{TrackMeta, TrackPriority}, @@ -13,28 +13,10 @@ use sans_io_runtime::{ use crate::endpoint::{EndpointEvent, EndpointReq, EndpointReqId, EndpointRes}; -#[derive(From, Debug, Clone, Copy, PartialEq, Eq, Display)] -pub struct TransportId(pub u64); +pub use media_server_protocol::transport::{LocalTrackId, RemoteTrackId}; -/// RemoteTrackId is used for track which received media from client #[derive(From, Debug, Clone, Copy, PartialEq, Eq, Display)] -pub struct RemoteTrackId(pub u16); - -impl Hash for RemoteTrackId { - fn hash(&self, state: &mut H) { - self.0.hash(state); - } -} - -/// LocalTrackId is used for track which send media to client -#[derive(From, Debug, Clone, Copy, PartialEq, Eq, Display)] -pub struct LocalTrackId(pub u16); - -impl Hash for LocalTrackId { - fn hash(&self, state: &mut H) { - self.0.hash(state); - } -} +pub struct TransportId(pub u64); #[derive(Debug, PartialEq, Eq)] pub enum TransportError { diff --git a/packages/media_runner/src/worker.rs b/packages/media_runner/src/worker.rs index b52b4503..110e3994 100644 --- a/packages/media_runner/src/worker.rs +++ b/packages/media_runner/src/worker.rs @@ -25,7 +25,7 @@ use sans_io_runtime::{ collections::DynamicDeque, TaskSwitcher, TaskSwitcherBranch, }; -use transport_webrtc::{GroupInput, MediaWorkerWebrtc, VariantParams, WebrtcOwner}; +use transport_webrtc::{GroupInput, MediaWorkerWebrtc, VariantParams, WebrtcSession}; const FEEDBACK_GATEWAY_AGENT_INTERVAL: u64 = 1000; //only feedback every second @@ -45,7 +45,11 @@ pub enum Owner { } //for sdn -pub type UserData = cluster::ClusterRoomHash; +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub enum UserData { + Cluster, + Room(cluster::RoomUserData), +} #[derive(Clone, Debug, convert_enum::From, convert_enum::TryInto)] pub enum SC { Visual(visualization::Control), @@ -84,8 +88,8 @@ enum TaskType { } #[derive(convert_enum::From, Debug, Clone, Copy, Hash, PartialEq, Eq)] -enum MediaClusterOwner { - Webrtc(WebrtcOwner), +enum MediaClusterEndpoint { + Webrtc(WebrtcSession), } #[allow(clippy::type_complexity)] @@ -93,7 +97,7 @@ pub struct MediaServerWorker { worker: u16, sdn_slot: usize, sdn_worker: TaskSwitcherBranch, SdnWorkerOutput>, - media_cluster: TaskSwitcherBranch, cluster::Output>, + media_cluster: TaskSwitcherBranch, cluster::Output>, media_webrtc: TaskSwitcherBranch, transport_webrtc::GroupOutput>, switcher: TaskSwitcher, queue: DynamicDeque, @@ -168,7 +172,7 @@ impl MediaServerWorker { now_ms, SdnWorkerInput::Ext(SdnExtIn::ServicesControl( AGENT_SERVICE_ID.into(), - 0.into(), + UserData::Cluster, media_server_gateway::agent_service::Control::WorkerUsage(ServiceKind::Webrtc, self.worker, webrtc_live).into(), )), ); @@ -220,7 +224,7 @@ impl MediaServerWorker { } } TaskType::MediaCluster => { - if let Some(out) = self.media_cluster.pop_output(now, &mut self.switcher) { + if let Some(out) = self.media_cluster.pop_output((), &mut self.switcher) { return Some(self.output_cluster(now, out)); } } @@ -247,7 +251,8 @@ impl MediaServerWorker { match out { SdnWorkerOutput::Ext(out) => Output::ExtSdn(out), SdnWorkerOutput::ExtWorker(out) => match out { - SdnExtOut::FeaturesEvent(room, event) => { + SdnExtOut::FeaturesEvent(UserData::Cluster, _event) => Output::Continue, + SdnExtOut::FeaturesEvent(UserData::Room(room), event) => { self.media_cluster.input(&mut self.switcher).on_sdn_event(now, room, event); Output::Continue } @@ -263,20 +268,20 @@ impl MediaServerWorker { } } - fn output_cluster(&mut self, now: Instant, out: cluster::Output) -> Output { + fn output_cluster(&mut self, now: Instant, out: cluster::Output) -> Output { match out { - cluster::Output::Sdn(userdata, control) => { + cluster::Output::Sdn(room, control) => { let now_ms = self.timer.timestamp_ms(now); self.sdn_worker .input(&mut self.switcher) - .on_event(now_ms, SdnWorkerInput::ExtWorker(SdnExtIn::FeaturesControl(userdata, control))); + .on_event(now_ms, SdnWorkerInput::ExtWorker(SdnExtIn::FeaturesControl(UserData::Room(room), control))); Output::Continue } - cluster::Output::Endpoint(owners, event) => { - for owner in owners { - match owner { - MediaClusterOwner::Webrtc(owner) => { - self.media_webrtc.input(&mut self.switcher).on_event(now, transport_webrtc::GroupInput::Cluster(owner, event.clone())); + cluster::Output::Endpoint(endpoints, event) => { + for endpoint in endpoints { + match endpoint { + MediaClusterEndpoint::Webrtc(session) => { + self.media_webrtc.input(&mut self.switcher).on_event(now, transport_webrtc::GroupInput::Cluster(session, event.clone())); } } } @@ -289,11 +294,11 @@ impl MediaServerWorker { fn output_webrtc(&mut self, now: Instant, out: transport_webrtc::GroupOutput) -> Output { match out { transport_webrtc::GroupOutput::Net(out) => Output::Net(Owner::MediaWebrtc, out), - transport_webrtc::GroupOutput::Cluster(owner, room, control) => { - self.media_cluster.input(&mut self.switcher).on_endpoint_control(now, owner.into(), room, control); + transport_webrtc::GroupOutput::Cluster(session, room, control) => { + self.media_cluster.input(&mut self.switcher).on_endpoint_control(now, session.into(), room, control); Output::Continue } - transport_webrtc::GroupOutput::Ext(owner, ext) => match ext { + transport_webrtc::GroupOutput::Ext(session, ext) => match ext { transport_webrtc::ExtOut::RemoteIce(req_id, variant, res) => match variant { transport_webrtc::Variant::Whip => Output::ExtRpc(req_id, RpcRes::Whip(whip::RpcRes::RemoteIce(res.map(|_| WhipRemoteIceRes {})))), transport_webrtc::Variant::Whep => Output::ExtRpc(req_id, RpcRes::Whep(whep::RpcRes::RemoteIce(res.map(|_| WhepRemoteIceRes {})))), @@ -303,7 +308,7 @@ impl MediaServerWorker { req_id, RpcRes::Webrtc(webrtc::RpcRes::RestartIce(res.map(|(ice_lite, sdp)| { ( - owner.index(), + session.index(), ConnectResponse { conn_id: "".to_string(), sdp, @@ -313,7 +318,7 @@ impl MediaServerWorker { }))), ), }, - transport_webrtc::GroupOutput::Shutdown(_owner) => Output::Continue, + transport_webrtc::GroupOutput::Shutdown(_session) => Output::Continue, transport_webrtc::GroupOutput::Continue => Output::Continue, } } diff --git a/packages/protocol/build.rs b/packages/protocol/build.rs index 382dc667..abc9d2eb 100644 --- a/packages/protocol/build.rs +++ b/packages/protocol/build.rs @@ -15,8 +15,9 @@ fn main() -> Result<()> { .compile_protos( &[ "./proto/shared.proto", - "./proto/conn.proto", + "./proto/session.proto", "./proto/features.proto", + "./proto/features.mixer.proto", "./proto/gateway.proto", "./proto/cluster_gateway.proto", ], diff --git a/packages/protocol/proto/features.mixer.proto b/packages/protocol/proto/features.mixer.proto new file mode 100644 index 00000000..6208be06 --- /dev/null +++ b/packages/protocol/proto/features.mixer.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; + +import "shared.proto"; + +package features.mixer; + +enum Mode { + AUTO = 0; + MANUAL = 1; +} + +message Config { + Mode mode = 1; + repeated string outputs = 2; + repeated shared.Receiver.Source sources = 3; +} + +message Request { + message Attach { + repeated shared.Receiver.Source sources = 1; + } + + message Detach { + repeated shared.Receiver.Source sources = 1; + } + + oneof request { + Attach attach = 1; + Detach detach = 2; + } +} + +message Response { + message Attach { + + } + + message Detach { + + } + + oneof response { + Attach attach = 1; + Detach detach = 2; + } +} + +message ServerEvent { + message SlotSet { + uint32 slot = 1; + shared.Receiver.Source source = 2; + } + + message SlotUnset { + uint32 slot = 1; + } + + oneof event { + SlotSet slot_set = 1; + SlotUnset slot_unset = 2; + } +} diff --git a/packages/protocol/proto/features.proto b/packages/protocol/proto/features.proto index d0504618..73b03463 100644 --- a/packages/protocol/proto/features.proto +++ b/packages/protocol/proto/features.proto @@ -1,27 +1,27 @@ syntax = "proto3"; -import "features_mix_minus.proto"; +import "features.mixer.proto"; package features; message Config { - optional mix_minus.Config mix_minus = 1; + optional features.mixer.Config mixer = 1; } message Request { oneof request { - mix_minus.Request mix_minus = 1; + features.mixer.Request mixer = 1; } } message Response { oneof response { - mix_minus.Response mix_minus = 1; + features.mixer.Response mixer = 1; } } message ServerEvent { oneof event { - mix_minus.ServerEvent mix_minus = 1; + features.mixer.ServerEvent mixer = 1; } } diff --git a/packages/protocol/proto/features_mix_minus.proto b/packages/protocol/proto/features_mix_minus.proto deleted file mode 100644 index c447e686..00000000 --- a/packages/protocol/proto/features_mix_minus.proto +++ /dev/null @@ -1,74 +0,0 @@ -syntax = "proto3"; - -package mix_minus; - -enum Mode { - AUTO = 0; - MANUAL = 1; -} - -message Source { - string peer = 1; - string track = 2; -} - -message Config { - Mode mode = 1; - repeated Source sources = 2; -} - -message Request { - message Attach { - repeated Source sources = 1; - } - - message Detach { - repeated Source sources = 1; - } - - oneof request { - Attach attach = 1; - Detach detach = 2; - } -} - -message Response { - message Attach { - - } - - message Detach { - - } - - oneof response { - Attach attach = 1; - Detach detach = 2; - } -} - -message ServerEvent { - message MappingSlotSet { - uint32 slot = 1; - Source source = 2; - } - - message MappingSlotDel { - uint32 slot = 1; - } - - message SlotAudioLevel { - uint32 slot = 1; - int32 audio_level = 2; - } - - message MappingSlotsAudioLevel { - repeated SlotAudioLevel slots = 1; - } - - oneof event { - MappingSlotSet slot_set = 1; - MappingSlotDel slot_del = 2; - MappingSlotsAudioLevel slots_audio_level = 3; - } -} diff --git a/packages/protocol/proto/gateway.proto b/packages/protocol/proto/gateway.proto index 8efb92d0..1f01f109 100644 --- a/packages/protocol/proto/gateway.proto +++ b/packages/protocol/proto/gateway.proto @@ -1,16 +1,15 @@ syntax = "proto3"; import "shared.proto"; -import "features.proto"; +import "session.proto"; package gateway; message ConnectRequest { string version = 2; - optional shared.RoomJoin join = 3; - features.Config features = 4; - shared.Tracks tracks = 5; - string sdp = 6; + optional session.RoomJoin join = 3; + shared.Tracks tracks = 4; + string sdp = 5; } message ConnectResponse { diff --git a/packages/protocol/proto/conn.proto b/packages/protocol/proto/session.proto similarity index 90% rename from packages/protocol/proto/conn.proto rename to packages/protocol/proto/session.proto index b49e2193..d99e57c3 100644 --- a/packages/protocol/proto/conn.proto +++ b/packages/protocol/proto/session.proto @@ -3,16 +3,25 @@ syntax = "proto3"; import "shared.proto"; import "features.proto"; -package conn; +package session; + +message RoomJoin { + string room = 1; + string peer = 2; + shared.RoomInfoPublish publish = 3; + shared.RoomInfoSubscribe subscribe = 4; + features.Config features = 5; + optional string metadata = 6; +} message Request { message Session { - message RoomJoin { - shared.RoomJoin info = 1; + message Join { + RoomJoin info = 1; string token = 2; } - message RoomLeave { + message Leave { } @@ -26,8 +35,8 @@ message Request { } oneof request { - RoomJoin join = 1; - RoomLeave leave = 2; + Join join = 1; + Leave leave = 2; UpdateSdp sdp = 3; Disconnect disconnect = 4; } @@ -96,11 +105,11 @@ message Request { message Response { message Session { - message RoomJoin { + message Join { } - message RoomLeave { + message Leave { } @@ -113,8 +122,8 @@ message Response { } oneof response { - RoomJoin join = 1; - RoomLeave leave = 2; + Join join = 1; + Leave leave = 2; UpdateSdp sdp = 3; Disconnect disconnect = 4; } @@ -182,7 +191,7 @@ message Response { Room room = 4; Sender sender = 5; Receiver receiver = 6; - features.Request features = 7; + features.Response features = 7; } } @@ -299,10 +308,15 @@ message ServerEvent { optional Transmit transmit = 2; } + message VoiceActivity { + int32 audio_level = 1; + } + string name = 1; oneof event { State state = 2; Stats stats = 3; + VoiceActivity voice_activity = 4; } } diff --git a/packages/protocol/proto/shared.proto b/packages/protocol/proto/shared.proto index edb70f70..57bb18ad 100644 --- a/packages/protocol/proto/shared.proto +++ b/packages/protocol/proto/shared.proto @@ -2,6 +2,11 @@ syntax = "proto3"; package shared; +message Error { + uint32 code = 1; + string message = 2; +} + enum Kind { AUDIO = 0; VIDEO = 1; @@ -64,7 +69,6 @@ message Sender { State state = 3; } - message Tracks { repeated Receiver receivers = 1; repeated Sender senders = 2; @@ -80,20 +84,7 @@ message RoomInfoSubscribe { bool tracks = 2; } -message RoomJoin { - string room = 1; - string peer = 2; - RoomInfoPublish publish = 3; - RoomInfoSubscribe subscribe = 4; - optional string metadata = 5; -} - enum BitrateControlMode { DYNAMIC_CONSUMERS = 0; MAX_BITRATE = 1; } - -message Error { - uint32 code = 1; - string message = 2; -} diff --git a/packages/protocol/proto/sync.sh b/packages/protocol/proto/sync.sh new file mode 100644 index 00000000..231c149b --- /dev/null +++ b/packages/protocol/proto/sync.sh @@ -0,0 +1 @@ +cp /Users/giangminh/Workspace/8xff/dev/atm0s-media-sdk-ts/lib/protobuf/* ./ diff --git a/packages/protocol/src/endpoint.rs b/packages/protocol/src/endpoint.rs index 5e76f875..69767435 100644 --- a/packages/protocol/src/endpoint.rs +++ b/packages/protocol/src/endpoint.rs @@ -1,13 +1,23 @@ use derive_more::{AsRef, From}; use serde::{Deserialize, Serialize}; -use std::{fmt::Display, str::FromStr}; - -use crate::{ - media::{MediaKind, MediaScaling}, - protobuf, - transport::ConnLayer, +use std::{ + fmt::Display, + hash::{DefaultHasher, Hash, Hasher}, + str::FromStr, }; +use crate::{protobuf, transport::ConnLayer}; + +mod audio_mixer; +mod track; + +pub use audio_mixer::*; +pub use track::*; + +/// +/// ClusterConnId is used for re-router request from gateway to correct node +/// This is a pair of node info and node inner worker and index +/// #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ClusterConnId { pub node: u32, @@ -51,6 +61,10 @@ impl ConnLayer for ClusterConnId { } } +/// +/// ServerConnId is for routing inside a node, which is a pair of worker index and task index +/// Note that task index maybe a pair of task type and index +/// #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ServerConnId { pub worker: u16, @@ -113,6 +127,14 @@ impl ConnLayer for usize { fn get_down_part(&self) -> Self::DownRes {} } +/// +/// This is config for endpoint publish level +/// +/// - peer: it will publish peer info to cluster +/// - tracks: it will publish all tracks info to cluster +/// +/// We can combine with RoomInfoSubscribe for adapting with difference kind of applications +/// #[derive(Debug, Clone, PartialEq, Eq)] pub struct RoomInfoPublish { pub peer: bool, @@ -128,6 +150,14 @@ impl From for RoomInfoPublish { } } +/// +/// This is config for endpoint subscribe level, this defined which kind of data the endpoint interted to +/// +/// - peers: interested in all published peer info +/// - tracks: interested in all published track info +/// +/// We can combine with RoomInfoPublish for adapting with difference kind of applications +/// #[derive(Debug, Clone, PartialEq, Eq)] pub struct RoomInfoSubscribe { pub peers: bool, @@ -143,6 +173,12 @@ impl From for RoomInfoSubscribe { } } +/// +/// RoomId type, we should use this type instead of direct String +/// This is useful when we can validate +/// +/// TODO: validate with uuid type (maybe max 32 bytes + [a-z]_- ) +/// #[derive(From, AsRef, Debug, derive_more::Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct RoomId(pub String); @@ -152,6 +188,12 @@ impl From<&str> for RoomId { } } +/// +/// PeerId type, we should use this type instead of direct String +/// This is useful when we can validate +/// +/// TODO: validate with uuid type (maybe max 32 bytes + [a-z]_- ) +/// #[derive(From, AsRef, Debug, derive_more::Display, derive_more::FromStr, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct PeerId(pub String); @@ -161,11 +203,30 @@ impl From<&str> for PeerId { } } +impl PeerId { + pub fn hash_code(&self) -> PeerHashCode { + let mut hash = DefaultHasher::new(); + self.0.hash(&mut hash); + PeerHashCode(hash.finish()) + } +} + +#[derive(From, AsRef, Debug, derive_more::Display, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PeerHashCode(pub u64); + +/// +/// PeerMeta will store custom information +/// +/// TODO: implement it +/// #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PeerMeta { pub metadata: Option, } +/// +/// PeerInfo will be used for broadcast to cluster +/// #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub peer: PeerId, @@ -188,62 +249,9 @@ impl PeerInfo { } } -#[derive(From, AsRef, Debug, derive_more::Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct TrackName(pub String); - -impl From<&str> for TrackName { - fn from(value: &str) -> Self { - Self(value.to_string()) - } -} - -#[derive(From, AsRef, Debug, derive_more::Display, derive_more::Add, derive_more::AddAssign, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct TrackPriority(pub u32); - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TrackMeta { - pub kind: MediaKind, - pub scaling: MediaScaling, - pub control: BitrateControlMode, - pub metadata: Option, -} - -impl TrackMeta { - pub fn default_audio() -> Self { - Self { - kind: MediaKind::Audio, - scaling: MediaScaling::None, - control: BitrateControlMode::MaxBitrate, - metadata: None, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrackInfo { - pub peer: PeerId, - pub track: TrackName, - pub meta: TrackMeta, -} - -impl TrackInfo { - pub fn simple_audio(peer: PeerId) -> Self { - Self { - peer, - track: "audio_main".to_string().into(), - meta: TrackMeta::default_audio(), - } - } - - pub fn serialize(&self) -> Vec { - bincode::serialize(self).expect("should ok") - } - - pub fn deserialize(data: &[u8]) -> Option { - bincode::deserialize::(data).ok() - } -} - +/// +/// We useBitrateControlMode for controlling how server adapt with consumer bitrates +/// #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum BitrateControlMode { /// Only limit with sender network and CAP with fixed MAX_BITRATE diff --git a/packages/protocol/src/endpoint/audio_mixer.rs b/packages/protocol/src/endpoint/audio_mixer.rs new file mode 100644 index 00000000..9fde0b5f --- /dev/null +++ b/packages/protocol/src/endpoint/audio_mixer.rs @@ -0,0 +1,52 @@ +use serde::{Deserialize, Serialize}; + +use crate::{ + protobuf::features::mixer::Mode, + transport::{LocalTrackId, RemoteTrackId}, +}; + +use super::{PeerHashCode, PeerId, TrackName, TrackSource}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum AudioMixerMode { + Auto, + Manual, +} + +impl From for AudioMixerMode { + fn from(value: Mode) -> Self { + match value { + Mode::Auto => AudioMixerMode::Auto, + Mode::Manual => AudioMixerMode::Manual, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct AudioMixerConfig { + pub mode: AudioMixerMode, + pub outputs: Vec, + pub sources: Vec, +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AudioMixerPkt { + pub slot: u8, + pub peer: PeerHashCode, + pub track: RemoteTrackId, + pub audio_level: Option, + pub source: Option<(PeerId, TrackName)>, + pub ts: u32, + pub seq: u16, + pub opus_payload: Vec, +} + +impl AudioMixerPkt { + pub fn serialize(&self) -> Vec { + bincode::serialize(self).expect("should ok") + } + + pub fn deserialize(data: &[u8]) -> Option { + bincode::deserialize::(data).ok() + } +} diff --git a/packages/protocol/src/endpoint/track.rs b/packages/protocol/src/endpoint/track.rs new file mode 100644 index 00000000..b01566c2 --- /dev/null +++ b/packages/protocol/src/endpoint/track.rs @@ -0,0 +1,93 @@ +use derive_more::{AsRef, From}; +use serde::{Deserialize, Serialize}; + +use crate::{ + media::{MediaKind, MediaScaling}, + protobuf, +}; + +use super::{BitrateControlMode, PeerId}; + +/// +/// TrackName type, we should use this type instead of direct String +/// This is useful when we can validate +/// +/// TODO: validate with uuid type (maybe max 32 bytes + [a-z]_- ) +/// +#[derive(From, AsRef, Debug, derive_more::Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct TrackName(pub String); + +impl From<&str> for TrackName { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +#[derive(From, AsRef, Debug, derive_more::Display, derive_more::Add, derive_more::AddAssign, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct TrackPriority(pub u32); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrackMeta { + pub kind: MediaKind, + pub scaling: MediaScaling, + pub control: BitrateControlMode, + pub metadata: Option, +} + +impl TrackMeta { + pub fn default_audio() -> Self { + Self { + kind: MediaKind::Audio, + scaling: MediaScaling::None, + control: BitrateControlMode::MaxBitrate, + metadata: None, + } + } +} + +/// +/// TrackInfo will be used for broadcast to cluster +/// +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrackInfo { + pub peer: PeerId, + pub track: TrackName, + pub meta: TrackMeta, +} + +impl TrackInfo { + pub fn simple_audio(peer: PeerId) -> Self { + Self { + peer, + track: "audio_main".to_string().into(), + meta: TrackMeta::default_audio(), + } + } + + pub fn serialize(&self) -> Vec { + bincode::serialize(self).expect("should ok") + } + + pub fn deserialize(data: &[u8]) -> Option { + bincode::deserialize::(data).ok() + } +} + +/// +/// TrackSource is identify of a track in a room, this is used for attaching a source into a consumer. +/// A consumer can be: local track, audio_mixer ... +/// +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +pub struct TrackSource { + pub peer: PeerId, + pub track: TrackName, +} + +impl From for TrackSource { + fn from(value: protobuf::shared::receiver::Source) -> Self { + Self { + peer: value.peer.into(), + track: value.track.into(), + } + } +} diff --git a/packages/protocol/src/protobuf/features.mixer.rs b/packages/protocol/src/protobuf/features.mixer.rs new file mode 100644 index 00000000..7548c370 --- /dev/null +++ b/packages/protocol/src/protobuf/features.mixer.rs @@ -0,0 +1,126 @@ +// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Config { + #[prost(enumeration = "Mode", tag = "1")] + pub mode: i32, + #[prost(string, repeated, tag = "2")] + pub outputs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "3")] + pub sources: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Request { + #[prost(oneof = "request::Request", tags = "1, 2")] + pub request: ::core::option::Option, +} +/// Nested message and enum types in `Request`. +pub mod request { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Attach { + #[prost(message, repeated, tag = "1")] + pub sources: ::prost::alloc::vec::Vec< + super::super::super::shared::receiver::Source, + >, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Detach { + #[prost(message, repeated, tag = "1")] + pub sources: ::prost::alloc::vec::Vec< + super::super::super::shared::receiver::Source, + >, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Request { + #[prost(message, tag = "1")] + Attach(Attach), + #[prost(message, tag = "2")] + Detach(Detach), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Response { + #[prost(oneof = "response::Response", tags = "1, 2")] + pub response: ::core::option::Option, +} +/// Nested message and enum types in `Response`. +pub mod response { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Attach {} + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Detach {} + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Response { + #[prost(message, tag = "1")] + Attach(Attach), + #[prost(message, tag = "2")] + Detach(Detach), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ServerEvent { + #[prost(oneof = "server_event::Event", tags = "1, 2")] + pub event: ::core::option::Option, +} +/// Nested message and enum types in `ServerEvent`. +pub mod server_event { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct SlotSet { + #[prost(uint32, tag = "1")] + pub slot: u32, + #[prost(message, optional, tag = "2")] + pub source: ::core::option::Option< + super::super::super::shared::receiver::Source, + >, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct SlotUnset { + #[prost(uint32, tag = "1")] + pub slot: u32, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Event { + #[prost(message, tag = "1")] + SlotSet(SlotSet), + #[prost(message, tag = "2")] + SlotUnset(SlotUnset), + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Mode { + Auto = 0, + Manual = 1, +} +impl Mode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Mode::Auto => "AUTO", + Mode::Manual => "MANUAL", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "AUTO" => Some(Self::Auto), + "MANUAL" => Some(Self::Manual), + _ => None, + } + } +} diff --git a/packages/protocol/src/protobuf/features.rs b/packages/protocol/src/protobuf/features.rs index 32bb2e92..5211a477 100644 --- a/packages/protocol/src/protobuf/features.rs +++ b/packages/protocol/src/protobuf/features.rs @@ -3,7 +3,7 @@ #[derive(Clone, PartialEq, ::prost::Message)] pub struct Config { #[prost(message, optional, tag = "1")] - pub mix_minus: ::core::option::Option, + pub mixer: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -17,7 +17,7 @@ pub mod request { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Request { #[prost(message, tag = "1")] - MixMinus(super::super::mix_minus::Request), + Mixer(super::mixer::Request), } } #[allow(clippy::derive_partial_eq_without_eq)] @@ -32,7 +32,7 @@ pub mod response { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Response { #[prost(message, tag = "1")] - MixMinus(super::super::mix_minus::Response), + Mixer(super::mixer::Response), } } #[allow(clippy::derive_partial_eq_without_eq)] @@ -47,6 +47,6 @@ pub mod server_event { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Event { #[prost(message, tag = "1")] - MixMinus(super::super::mix_minus::ServerEvent), + Mixer(super::mixer::ServerEvent), } } diff --git a/packages/protocol/src/protobuf/gateway.rs b/packages/protocol/src/protobuf/gateway.rs index f0139805..a70aa941 100644 --- a/packages/protocol/src/protobuf/gateway.rs +++ b/packages/protocol/src/protobuf/gateway.rs @@ -5,12 +5,10 @@ pub struct ConnectRequest { #[prost(string, tag = "2")] pub version: ::prost::alloc::string::String, #[prost(message, optional, tag = "3")] - pub join: ::core::option::Option, + pub join: ::core::option::Option, #[prost(message, optional, tag = "4")] - pub features: ::core::option::Option, - #[prost(message, optional, tag = "5")] pub tracks: ::core::option::Option, - #[prost(string, tag = "6")] + #[prost(string, tag = "5")] pub sdp: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/packages/protocol/src/protobuf/mix_minus.rs b/packages/protocol/src/protobuf/mixer.rs similarity index 88% rename from packages/protocol/src/protobuf/mix_minus.rs rename to packages/protocol/src/protobuf/mixer.rs index e88e62ea..6fe2ed72 100644 --- a/packages/protocol/src/protobuf/mix_minus.rs +++ b/packages/protocol/src/protobuf/mixer.rs @@ -1,19 +1,13 @@ // This file is @generated by prost-build. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct Source { - #[prost(string, tag = "1")] - pub peer: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub track: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] pub struct Config { #[prost(enumeration = "Mode", tag = "1")] pub mode: i32, - #[prost(message, repeated, tag = "2")] - pub sources: ::prost::alloc::vec::Vec, + #[prost(string, repeated, tag = "2")] + pub outputs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "3")] + pub sources: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -27,13 +21,13 @@ pub mod request { #[derive(Clone, PartialEq, ::prost::Message)] pub struct Attach { #[prost(message, repeated, tag = "1")] - pub sources: ::prost::alloc::vec::Vec, + pub sources: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Detach { #[prost(message, repeated, tag = "1")] - pub sources: ::prost::alloc::vec::Vec, + pub sources: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] @@ -81,7 +75,7 @@ pub mod server_event { #[prost(uint32, tag = "1")] pub slot: u32, #[prost(message, optional, tag = "2")] - pub source: ::core::option::Option, + pub source: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/packages/protocol/src/protobuf/mod.rs b/packages/protocol/src/protobuf/mod.rs index de7f7f0d..53962482 100644 --- a/packages/protocol/src/protobuf/mod.rs +++ b/packages/protocol/src/protobuf/mod.rs @@ -2,17 +2,17 @@ pub mod cluster_gateway { include!("cluster_gateway.rs"); } -pub mod conn { - include!("conn.rs"); -} pub mod features { include!("features.rs"); + pub mod mixer { + include!("features.mixer.rs"); + } } pub mod gateway { include!("gateway.rs"); } -pub mod mix_minus { - include!("mix_minus.rs"); +pub mod session { + include!("session.rs"); } pub mod shared { include!("shared.rs"); diff --git a/packages/protocol/src/protobuf/conn.rs b/packages/protocol/src/protobuf/session.rs similarity index 93% rename from packages/protocol/src/protobuf/conn.rs rename to packages/protocol/src/protobuf/session.rs index f1013dac..67de64ad 100644 --- a/packages/protocol/src/protobuf/conn.rs +++ b/packages/protocol/src/protobuf/session.rs @@ -1,6 +1,22 @@ // This file is @generated by prost-build. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct RoomJoin { + #[prost(string, tag = "1")] + pub room: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub peer: ::prost::alloc::string::String, + #[prost(message, optional, tag = "3")] + pub publish: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub subscribe: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub features: ::core::option::Option, + #[prost(string, optional, tag = "6")] + pub metadata: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { #[prost(uint32, tag = "1")] pub req_id: u32, @@ -19,15 +35,15 @@ pub mod request { pub mod session { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] - pub struct RoomJoin { + pub struct Join { #[prost(message, optional, tag = "1")] - pub info: ::core::option::Option, + pub info: ::core::option::Option, #[prost(string, tag = "2")] pub token: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] - pub struct RoomLeave {} + pub struct Leave {} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateSdp { @@ -43,9 +59,9 @@ pub mod request { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Request { #[prost(message, tag = "1")] - Join(RoomJoin), + Join(Join), #[prost(message, tag = "2")] - Leave(RoomLeave), + Leave(Leave), #[prost(message, tag = "3")] Sdp(UpdateSdp), #[prost(message, tag = "4")] @@ -188,10 +204,10 @@ pub mod response { pub mod session { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] - pub struct RoomJoin {} + pub struct Join {} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] - pub struct RoomLeave {} + pub struct Leave {} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateSdp { @@ -205,9 +221,9 @@ pub mod response { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Response { #[prost(message, tag = "1")] - Join(RoomJoin), + Join(Join), #[prost(message, tag = "2")] - Leave(RoomLeave), + Leave(Leave), #[prost(message, tag = "3")] Sdp(UpdateSdp), #[prost(message, tag = "4")] @@ -307,7 +323,7 @@ pub mod response { #[prost(message, tag = "6")] Receiver(Receiver), #[prost(message, tag = "7")] - Features(super::super::features::Request), + Features(super::super::features::Response), } } #[allow(clippy::derive_partial_eq_without_eq)] @@ -488,7 +504,7 @@ pub mod server_event { pub struct Receiver { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, - #[prost(oneof = "receiver::Event", tags = "2, 3")] + #[prost(oneof = "receiver::Event", tags = "2, 3, 4")] pub event: ::core::option::Option, } /// Nested message and enum types in `Receiver`. @@ -536,12 +552,20 @@ pub mod server_event { } } #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct VoiceActivity { + #[prost(int32, tag = "1")] + pub audio_level: i32, + } + #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Event { #[prost(message, tag = "2")] State(State), #[prost(message, tag = "3")] Stats(Stats), + #[prost(message, tag = "4")] + VoiceActivity(VoiceActivity), } } #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/packages/protocol/src/protobuf/shared.rs b/packages/protocol/src/protobuf/shared.rs index 42a05fc6..15ca1c9f 100644 --- a/packages/protocol/src/protobuf/shared.rs +++ b/packages/protocol/src/protobuf/shared.rs @@ -1,6 +1,14 @@ // This file is @generated by prost-build. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct Error { + #[prost(uint32, tag = "1")] + pub code: u32, + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Receiver { #[prost(enumeration = "Kind", tag = "1")] pub kind: i32, @@ -180,28 +188,6 @@ pub struct RoomInfoSubscribe { #[prost(bool, tag = "2")] pub tracks: bool, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct RoomJoin { - #[prost(string, tag = "1")] - pub room: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub peer: ::prost::alloc::string::String, - #[prost(message, optional, tag = "3")] - pub publish: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub subscribe: ::core::option::Option, - #[prost(string, optional, tag = "5")] - pub metadata: ::core::option::Option<::prost::alloc::string::String>, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Error { - #[prost(uint32, tag = "1")] - pub code: u32, - #[prost(string, tag = "2")] - pub message: ::prost::alloc::string::String, -} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum Kind { diff --git a/packages/protocol/src/transport.rs b/packages/protocol/src/transport.rs index 447a7f8e..45cfdc89 100644 --- a/packages/protocol/src/transport.rs +++ b/packages/protocol/src/transport.rs @@ -1,4 +1,7 @@ -use std::fmt::Display; +use std::{fmt::Display, hash::Hash}; + +use derive_more::{Display, From}; +use serde::{Deserialize, Serialize}; use crate::protobuf; @@ -6,6 +9,26 @@ pub mod webrtc; pub mod whep; pub mod whip; +/// RemoteTrackId is used for track which received media from client +#[derive(From, Debug, Clone, Copy, PartialEq, Eq, Display, Serialize, Deserialize)] +pub struct RemoteTrackId(pub u16); + +impl Hash for RemoteTrackId { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +/// LocalTrackId is used for track which send media to client +#[derive(From, Debug, Clone, Copy, PartialEq, Eq, Display, Serialize, Deserialize)] +pub struct LocalTrackId(pub u16); + +impl Hash for LocalTrackId { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + pub trait ConnLayer { type Up; type UpParam; diff --git a/packages/transport_webrtc/src/lib.rs b/packages/transport_webrtc/src/lib.rs index be422f0d..75e4b176 100644 --- a/packages/transport_webrtc/src/lib.rs +++ b/packages/transport_webrtc/src/lib.rs @@ -4,7 +4,7 @@ mod transport; mod worker; pub use transport::{ExtIn, ExtOut, Variant, VariantParams}; -pub use worker::{GroupInput, GroupOutput, MediaWorkerWebrtc, WebrtcOwner}; +pub use worker::{GroupInput, GroupOutput, MediaWorkerWebrtc, WebrtcSession}; #[derive(num_enum::TryFromPrimitive, num_enum::IntoPrimitive, derive_more::Display)] #[repr(u32)] diff --git a/packages/transport_webrtc/src/transport/webrtc.rs b/packages/transport_webrtc/src/transport/webrtc.rs index 88ad2efa..36366f0d 100644 --- a/packages/transport_webrtc/src/transport/webrtc.rs +++ b/packages/transport_webrtc/src/transport/webrtc.rs @@ -4,24 +4,32 @@ use std::{ }; use media_server_core::{ - endpoint::{EndpointEvent, EndpointLocalTrackEvent, EndpointLocalTrackReq, EndpointRemoteTrackReq, EndpointReq, EndpointReqId, EndpointRes}, + endpoint::{EndpointAudioMixerReq, EndpointEvent, EndpointLocalTrackEvent, EndpointLocalTrackReq, EndpointRemoteTrackReq, EndpointReq, EndpointReqId, EndpointRes}, transport::{LocalTrackEvent, LocalTrackId, RemoteTrackEvent, RemoteTrackId, TransportError, TransportEvent, TransportOutput, TransportState}, }; use media_server_protocol::{ - endpoint::{PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe}, + endpoint::{AudioMixerConfig, PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe}, protobuf::{ self, - conn::{ + features::{ + mixer::{ + server_event::{Event as ProtoFeatureMixerEvent2, SlotSet, SlotUnset}, + ServerEvent as ProtoFeatureMixerEvent, + }, + server_event::Event as ProtoFeaturesEvent2, + ServerEvent as ProtoFeaturesEvent, + }, + gateway::ConnectRequest, + session::{ server_event::{ - receiver::{Event as ProtoReceiverEvent, State as ProtoReceiverState}, + receiver::{Event as ProtoReceiverEvent, State as ProtoReceiverState, VoiceActivity as ProtoReceiverVoiceActivity}, room::{Event as ProtoRoomEvent2, PeerJoined, PeerLeaved, TrackStarted, TrackStopped}, sender::{Event as ProtoSenderEvent, State as ProtoSenderState}, Event as ProtoServerEvent, Receiver as ProtoReceiverEventContainer, Room as ProtoRoomEvent, Sender as ProtoSenderEventContainer, }, ClientEvent, }, - gateway::ConnectRequest, - shared::{sender::Status as ProtoSenderStatus, Kind}, + shared::{receiver::Source as ProtoReceiverSource, sender::Status as ProtoSenderStatus, Kind}, }, tokens::WebrtcToken, transport::{RpcError, RpcResult}, @@ -71,6 +79,7 @@ pub struct TransportWebrtcSdk { event_seq: u32, local_tracks: Vec, remote_tracks: Vec, + audio_mixer: Option, media_convert: RemoteMediaConvert, bwe_state: BweState, secure: Arc, @@ -79,19 +88,49 @@ pub struct TransportWebrtcSdk { impl TransportWebrtcSdk { pub fn new(req: ConnectRequest, secure: Arc) -> Self { let tracks = req.tracks.unwrap_or_default(); - Self { - join: req - .join - .map(|j| (j.room.into(), j.peer.into(), j.metadata, j.publish.unwrap_or_default().into(), j.subscribe.unwrap_or_default().into())), - state: State::New, - local_tracks: tracks.receivers.into_iter().enumerate().map(|(index, r)| LocalTrack::new((index as u16).into(), r)).collect(), - remote_tracks: tracks.senders.into_iter().enumerate().map(|(index, s)| RemoteTrack::new((index as u16).into(), s)).collect(), - queue: Default::default(), - channel: None, - event_seq: 0, - media_convert: RemoteMediaConvert::default(), - bwe_state: BweState::default(), - secure, + let local_tracks: Vec = tracks.receivers.into_iter().enumerate().map(|(index, r)| LocalTrack::new((index as u16).into(), r)).collect(); + let remote_tracks: Vec = tracks.senders.into_iter().enumerate().map(|(index, s)| RemoteTrack::new((index as u16).into(), s)).collect(); + if let Some(j) = req.join { + Self { + join: Some((j.room.into(), j.peer.into(), j.metadata, j.publish.unwrap_or_default().into(), j.subscribe.unwrap_or_default().into())), + state: State::New, + audio_mixer: j.features.and_then(|f| { + f.mixer.and_then(|m| { + Some(AudioMixerConfig { + mode: m.mode().into(), + outputs: m + .outputs + .iter() + .map(|r| local_tracks.iter().find(|l| l.name() == r.as_str()).map(|l| l.id())) + .flatten() + .collect::>(), + sources: m.sources.into_iter().map(|s| s.into()).collect::>(), + }) + }) + }), + local_tracks, + remote_tracks, + queue: Default::default(), + channel: None, + event_seq: 0, + media_convert: RemoteMediaConvert::default(), + bwe_state: BweState::default(), + secure, + } + } else { + Self { + join: None, + state: State::New, + local_tracks, + remote_tracks, + audio_mixer: None, + queue: Default::default(), + channel: None, + event_seq: 0, + media_convert: RemoteMediaConvert::default(), + bwe_state: BweState::default(), + secure, + } } } @@ -119,21 +158,21 @@ impl TransportWebrtcSdk { self.local_tracks.iter_mut().find(|t| t.name() == name) } - fn send_event(&mut self, event: protobuf::conn::server_event::Event) { + fn send_event(&mut self, event: protobuf::session::server_event::Event) { let channel = return_if_none!(self.channel); let seq = self.event_seq; self.event_seq += 1; - let event = protobuf::conn::ServerEvent { seq, event: Some(event) }; + let event = protobuf::session::ServerEvent { seq, event: Some(event) }; self.queue.push_back(InternalOutput::Str0mSendData(channel, event.encode_to_vec())); } - fn send_rpc_res(&mut self, req_id: u32, res: protobuf::conn::response::Response) { - self.send_event(protobuf::conn::server_event::Event::Response(protobuf::conn::Response { req_id, response: Some(res) })); + fn send_rpc_res(&mut self, req_id: u32, res: protobuf::session::response::Response) { + self.send_event(protobuf::session::server_event::Event::Response(protobuf::session::Response { req_id, response: Some(res) })); } fn send_rpc_res_err(&mut self, req_id: u32, err: RpcError) { - let response = protobuf::conn::response::Response::Error(err.into()); - self.send_event(protobuf::conn::server_event::Event::Response(protobuf::conn::Response { req_id, response: Some(response) })) + let response = protobuf::session::response::Response::Error(err.into()); + self.send_event(protobuf::session::server_event::Event::Response(protobuf::session::Response { req_id, response: Some(response) })) } } @@ -182,8 +221,8 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { Ok(res) => match res { InternalRpcRes::SetRemoteSdp(answer) => self.send_rpc_res( req_id, - protobuf::conn::response::Response::Session(protobuf::conn::response::Session { - response: Some(protobuf::conn::response::session::Response::Sdp(protobuf::conn::response::session::UpdateSdp { sdp: answer })), + protobuf::session::response::Response::Session(protobuf::session::response::Session { + response: Some(protobuf::session::response::session::Response::Sdp(protobuf::session::response::session::UpdateSdp { sdp: answer })), }), ), }, @@ -231,6 +270,27 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { })), })); } + EndpointEvent::AudioMixer(event) => match event { + media_server_core::endpoint::EndpointAudioMixerEvent::SlotSet(slot, peer, track) => { + log::info!("[TransportWebrtcSdk] audio mixer slot {slot} set to {peer}/{track}"); + self.send_event(ProtoServerEvent::Features(ProtoFeaturesEvent { + event: Some(ProtoFeaturesEvent2::Mixer(ProtoFeatureMixerEvent { + event: Some(ProtoFeatureMixerEvent2::SlotSet(SlotSet { + slot: slot as u32, + source: Some(ProtoReceiverSource { peer: peer.0, track: track.0 }), + })), + })), + })) + } + media_server_core::endpoint::EndpointAudioMixerEvent::SlotUnset(slot) => { + log::info!("[TransportWebrtcSdk] audio mixer slot {slot} unset"); + self.send_event(ProtoServerEvent::Features(ProtoFeaturesEvent { + event: Some(ProtoFeaturesEvent2::Mixer(ProtoFeatureMixerEvent { + event: Some(ProtoFeatureMixerEvent2::SlotUnset(SlotUnset { slot: slot as u32 })), + })), + })) + } + }, EndpointEvent::RemoteMediaTrack(track_id, event) => match event { media_server_core::endpoint::EndpointRemoteTrackEvent::RequestKeyFrame => { let track = return_if_none!(self.remote_track(track_id)); @@ -264,6 +324,14 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { event: Some(ProtoReceiverEvent::State(ProtoReceiverState { status: status as i32 })), })); } + EndpointLocalTrackEvent::VoiceActivity(level) => { + let track = return_if_none!(self.local_track(track_id)).name().to_string(); + log::info!("[TransportWebrtcSdk] track {track} set audio_level {:?}", level); + self.send_event(ProtoServerEvent::Receiver(ProtoReceiverEventContainer { + name: track, + event: Some(ProtoReceiverEvent::VoiceActivity(ProtoReceiverVoiceActivity { audio_level: level as i32 })), + })); + } }, EndpointEvent::BweConfig { current, desired } => { let (current, desired) = self.bwe_state.filter_bwe_config(current, desired); @@ -278,15 +346,15 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { match res { EndpointRes::JoinRoom(Ok(_)) => self.send_rpc_res( req_id.0, - protobuf::conn::response::Response::Session(protobuf::conn::response::Session { - response: Some(protobuf::conn::response::session::Response::Join(protobuf::conn::response::session::RoomJoin {})), + protobuf::session::response::Response::Session(protobuf::session::response::Session { + response: Some(protobuf::session::response::session::Response::Join(protobuf::session::response::session::Join {})), }), ), EndpointRes::JoinRoom(Err(err)) => self.send_rpc_res_err(req_id.0, err), EndpointRes::LeaveRoom(Ok(_)) => self.send_rpc_res( req_id.0, - protobuf::conn::response::Response::Session(protobuf::conn::response::Session { - response: Some(protobuf::conn::response::session::Response::Leave(protobuf::conn::response::session::RoomLeave {})), + protobuf::session::response::Response::Session(protobuf::session::response::Session { + response: Some(protobuf::session::response::session::Response::Leave(protobuf::session::response::session::Leave {})), }), ), EndpointRes::LeaveRoom(Err(err)) => self.send_rpc_res_err(req_id.0, err), @@ -295,8 +363,8 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { EndpointRes::RemoteTrack(_track_id, res) => match res { media_server_core::endpoint::EndpointRemoteTrackRes::Config(Ok(_)) => self.send_rpc_res( req_id.0, - protobuf::conn::response::Response::Sender(protobuf::conn::response::Sender { - response: Some(protobuf::conn::response::sender::Response::Config(protobuf::conn::response::sender::Config {})), + protobuf::session::response::Response::Sender(protobuf::session::response::Sender { + response: Some(protobuf::session::response::sender::Response::Config(protobuf::session::response::sender::Config {})), }), ), media_server_core::endpoint::EndpointRemoteTrackRes::Config(Err(err)) => self.send_rpc_res_err(req_id.0, err), @@ -304,26 +372,46 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { EndpointRes::LocalTrack(_track_id, res) => match res { media_server_core::endpoint::EndpointLocalTrackRes::Attach(Ok(_)) => self.send_rpc_res( req_id.0, - protobuf::conn::response::Response::Receiver(protobuf::conn::response::Receiver { - response: Some(protobuf::conn::response::receiver::Response::Attach(protobuf::conn::response::receiver::Attach {})), + protobuf::session::response::Response::Receiver(protobuf::session::response::Receiver { + response: Some(protobuf::session::response::receiver::Response::Attach(protobuf::session::response::receiver::Attach {})), }), ), media_server_core::endpoint::EndpointLocalTrackRes::Detach(Ok(_)) => self.send_rpc_res( req_id.0, - protobuf::conn::response::Response::Receiver(protobuf::conn::response::Receiver { - response: Some(protobuf::conn::response::receiver::Response::Detach(protobuf::conn::response::receiver::Detach {})), + protobuf::session::response::Response::Receiver(protobuf::session::response::Receiver { + response: Some(protobuf::session::response::receiver::Response::Detach(protobuf::session::response::receiver::Detach {})), }), ), media_server_core::endpoint::EndpointLocalTrackRes::Config(Ok(_)) => self.send_rpc_res( req_id.0, - protobuf::conn::response::Response::Receiver(protobuf::conn::response::Receiver { - response: Some(protobuf::conn::response::receiver::Response::Config(protobuf::conn::response::receiver::Config {})), + protobuf::session::response::Response::Receiver(protobuf::session::response::Receiver { + response: Some(protobuf::session::response::receiver::Response::Config(protobuf::session::response::receiver::Config {})), }), ), media_server_core::endpoint::EndpointLocalTrackRes::Attach(Err(err)) => self.send_rpc_res_err(req_id.0, err), media_server_core::endpoint::EndpointLocalTrackRes::Detach(Err(err)) => self.send_rpc_res_err(req_id.0, err), media_server_core::endpoint::EndpointLocalTrackRes::Config(Err(err)) => self.send_rpc_res_err(req_id.0, err), }, + EndpointRes::AudioMixer(res) => match res { + media_server_core::endpoint::EndpointAudioMixerRes::Attach(Ok(_)) => self.send_rpc_res( + req_id.0, + protobuf::session::response::Response::Features(protobuf::features::Response { + response: Some(protobuf::features::response::Response::Mixer(protobuf::features::mixer::Response { + response: Some(protobuf::features::mixer::response::Response::Attach(protobuf::features::mixer::response::Attach {})), + })), + }), + ), + media_server_core::endpoint::EndpointAudioMixerRes::Detach(Ok(_)) => self.send_rpc_res( + req_id.0, + protobuf::session::response::Response::Features(protobuf::features::Response { + response: Some(protobuf::features::response::Response::Mixer(protobuf::features::mixer::Response { + response: Some(protobuf::features::mixer::response::Response::Detach(protobuf::features::mixer::response::Detach {})), + })), + }), + ), + media_server_core::endpoint::EndpointAudioMixerRes::Attach(Err(err)) => self.send_rpc_res_err(req_id.0, err), + media_server_core::endpoint::EndpointAudioMixerRes::Detach(Err(err)) => self.send_rpc_res_err(req_id.0, err), + }, } } @@ -338,7 +426,14 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { if let Some((room, peer, metadata, publish, subscribe)) = &self.join { self.queue.push_back(InternalOutput::TransportOutput(TransportOutput::RpcReq( 0.into(), - EndpointReq::JoinRoom(room.clone(), peer.clone(), PeerMeta { metadata: metadata.clone() }, publish.clone(), subscribe.clone()), + EndpointReq::JoinRoom( + room.clone(), + peer.clone(), + PeerMeta { metadata: metadata.clone() }, + publish.clone(), + subscribe.clone(), + self.audio_mixer.take(), + ), ))); } } @@ -498,25 +593,30 @@ impl TransportWebrtcSdk { fn on_str0m_channel_event(&mut self, event: ClientEvent) { log::info!("[TransportWebrtcSdk] on client event {:?}", event); match return_if_none!(event.event) { - protobuf::conn::client_event::Event::Request(req) => match req.request { - Some(protobuf::conn::request::Request::Session(session)) => match session.request { + protobuf::session::client_event::Event::Request(req) => match req.request { + Some(protobuf::session::request::Request::Session(session)) => match session.request { Some(session_req) => self.on_session_req(req.req_id, session_req), None => self.send_rpc_res_err(req.req_id, RpcError::new2(WebrtcError::RpcInvalidRequest)), }, - Some(protobuf::conn::request::Request::Sender(sender)) => match sender.request { + Some(protobuf::session::request::Request::Sender(sender)) => match sender.request { Some(sender_req) => self.on_sender_req(req.req_id, &sender.name, sender_req), None => self.send_rpc_res_err(req.req_id, RpcError::new2(WebrtcError::RpcInvalidRequest)), }, - Some(protobuf::conn::request::Request::Receiver(receiver)) => match receiver.request { + Some(protobuf::session::request::Request::Receiver(receiver)) => match receiver.request { Some(receiver_req) => self.on_recever_req(req.req_id, &receiver.name, receiver_req), None => self.send_rpc_res_err(req.req_id, RpcError::new2(WebrtcError::RpcInvalidRequest)), }, - Some(protobuf::conn::request::Request::Room(_room)) => { - todo!() - } - Some(protobuf::conn::request::Request::Features(_features)) => { + Some(protobuf::session::request::Request::Room(_room)) => { todo!() } + Some(protobuf::session::request::Request::Features(features_req)) => match features_req.request { + Some(protobuf::features::request::Request::Mixer(mixer_req)) => { + if let Some(mixer_req) = mixer_req.request { + self.on_mixer_req(req.req_id, mixer_req); + } + } + None => {} + }, None => self.send_rpc_res_err(req.req_id, RpcError::new2(WebrtcError::RpcInvalidRequest)), }, } @@ -525,20 +625,30 @@ impl TransportWebrtcSdk { ///This is for handling rpc from client impl TransportWebrtcSdk { - fn on_session_req(&mut self, req_id: u32, req: protobuf::conn::request::session::Request) { + fn on_session_req(&mut self, req_id: u32, req: protobuf::session::request::session::Request) { let build_req = |req: EndpointReq| InternalOutput::TransportOutput(TransportOutput::RpcReq(req_id.into(), req)); match req { - protobuf::conn::request::session::Request::Join(req) => { + protobuf::session::request::session::Request::Join(req) => { let info = req.info.unwrap_or_default(); let meta = PeerMeta { metadata: info.metadata }; if let Some(token) = self.secure.decode_obj::("webrtc", &req.token) { if token.room == Some(info.room.clone()) && token.peer == Some(info.peer.clone()) { + let mixer_cfg = info.features.and_then(|f| { + f.mixer.and_then(|m| { + Some(AudioMixerConfig { + mode: m.mode().into(), + outputs: m.outputs.iter().map(|r| self.local_track_by_name(r.as_str()).map(|l| l.id())).flatten().collect::>(), + sources: m.sources.into_iter().map(|s| s.into()).collect::>(), + }) + }) + }); self.queue.push_back(build_req(EndpointReq::JoinRoom( info.room.into(), info.peer.into(), meta, info.publish.unwrap_or_default().into(), info.subscribe.unwrap_or_default().into(), + mixer_cfg, ))); } else { self.send_rpc_res_err(req_id, RpcError::new2(WebrtcError::RpcTokenRoomPeerNotMatch)); @@ -547,8 +657,8 @@ impl TransportWebrtcSdk { self.send_rpc_res_err(req_id, RpcError::new2(WebrtcError::RpcTokenInvalid)); } } - protobuf::conn::request::session::Request::Leave(_req) => self.queue.push_back(build_req(EndpointReq::LeaveRoom)), - protobuf::conn::request::session::Request::Sdp(req) => { + protobuf::session::request::session::Request::Leave(_req) => self.queue.push_back(build_req(EndpointReq::LeaveRoom)), + protobuf::session::request::session::Request::Sdp(req) => { let tracks = req.tracks.unwrap_or_default(); for (index, s) in tracks.senders.into_iter().enumerate() { if self.remote_track_by_name(&s.name).is_none() { @@ -565,7 +675,7 @@ impl TransportWebrtcSdk { } self.queue.push_back(InternalOutput::RpcReq(req_id, InternalRpcReq::SetRemoteSdp(req.sdp))); } - protobuf::conn::request::session::Request::Disconnect(_) => { + protobuf::session::request::session::Request::Disconnect(_) => { log::info!("[TransportWebrtcSdk] switched to disconnected with close action from client"); self.state = State::Disconnected; self.queue @@ -574,7 +684,7 @@ impl TransportWebrtcSdk { } } - fn on_sender_req(&mut self, req_id: u32, name: &str, req: protobuf::conn::request::sender::Request) { + fn on_sender_req(&mut self, req_id: u32, name: &str, req: protobuf::session::request::sender::Request) { let track = if let Some(track) = self.remote_track_by_name(name) { track } else { @@ -585,7 +695,7 @@ impl TransportWebrtcSdk { let build_req = |req: EndpointReq| InternalOutput::TransportOutput(TransportOutput::RpcReq(req_id.into(), req)); match req { - protobuf::conn::request::sender::Request::Attach(attach) => { + protobuf::session::request::sender::Request::Attach(attach) => { if !track.has_source() { track.set_source(attach.source.unwrap_or_default()); let event = InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::RemoteTrack( @@ -598,8 +708,8 @@ impl TransportWebrtcSdk { ))); self.send_rpc_res( req_id, - protobuf::conn::response::Response::Sender(protobuf::conn::response::Sender { - response: Some(protobuf::conn::response::sender::Response::Attach(protobuf::conn::response::sender::Attach {})), + protobuf::session::response::Response::Sender(protobuf::session::response::Sender { + response: Some(protobuf::session::response::sender::Response::Attach(protobuf::session::response::sender::Attach {})), }), ); self.queue.push_back(event); @@ -607,14 +717,14 @@ impl TransportWebrtcSdk { self.send_rpc_res_err(req_id, RpcError::new2(WebrtcError::RpcTrackAlreadyAttached)); } } - protobuf::conn::request::sender::Request::Detach(_) => { + protobuf::session::request::sender::Request::Detach(_) => { if track.has_source() { track.del_source(); let event = InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::RemoteTrack(track_id, RemoteTrackEvent::Ended))); self.send_rpc_res( req_id, - protobuf::conn::response::Response::Sender(protobuf::conn::response::Sender { - response: Some(protobuf::conn::response::sender::Response::Detach(protobuf::conn::response::sender::Detach {})), + protobuf::session::response::Response::Sender(protobuf::session::response::Sender { + response: Some(protobuf::session::response::sender::Response::Detach(protobuf::session::response::sender::Detach {})), }), ); self.queue.push_back(event); @@ -622,11 +732,11 @@ impl TransportWebrtcSdk { self.send_rpc_res_err(req_id, RpcError::new2(WebrtcError::RpcTrackNotAttached)); } } - protobuf::conn::request::sender::Request::Config(config) => self.queue.push_back(build_req(EndpointReq::RemoteTrack(track_id, EndpointRemoteTrackReq::Config(config.into())))), + protobuf::session::request::sender::Request::Config(config) => self.queue.push_back(build_req(EndpointReq::RemoteTrack(track_id, EndpointRemoteTrackReq::Config(config.into())))), } } - fn on_recever_req(&mut self, req_id: u32, name: &str, req: protobuf::conn::request::receiver::Request) { + fn on_recever_req(&mut self, req_id: u32, name: &str, req: protobuf::session::request::receiver::Request) { let track = if let Some(track) = self.local_track_by_name(name) { track } else { @@ -637,20 +747,39 @@ impl TransportWebrtcSdk { let build_req = |req: EndpointLocalTrackReq| InternalOutput::TransportOutput(TransportOutput::RpcReq(req_id.into(), EndpointReq::LocalTrack(track_id, req))); match req { - protobuf::conn::request::receiver::Request::Attach(attach) => { + protobuf::session::request::receiver::Request::Attach(attach) => { self.queue.push_back(build_req(EndpointLocalTrackReq::Attach( attach.source.unwrap_or_default().into(), attach.config.unwrap_or_default().into(), ))); } - protobuf::conn::request::receiver::Request::Detach(_) => { + protobuf::session::request::receiver::Request::Detach(_) => { self.queue.push_back(build_req(EndpointLocalTrackReq::Detach())); } - protobuf::conn::request::receiver::Request::Config(config) => { + protobuf::session::request::receiver::Request::Config(config) => { self.queue.push_back(build_req(EndpointLocalTrackReq::Config(config.into()))); } } } + + fn on_mixer_req(&mut self, req_id: u32, req: protobuf::features::mixer::request::Request) { + match req { + protobuf::features::mixer::request::Request::Attach(req) => { + let sources = req.sources.into_iter().map(|s| s.into()).collect::>(); + self.queue.push_back(InternalOutput::TransportOutput(TransportOutput::RpcReq( + req_id.into(), + EndpointReq::AudioMixer(EndpointAudioMixerReq::Attach(sources)), + ))); + } + protobuf::features::mixer::request::Request::Detach(req) => { + let sources = req.sources.into_iter().map(|s| s.into()).collect::>(); + self.queue.push_back(InternalOutput::TransportOutput(TransportOutput::RpcReq( + req_id.into(), + EndpointReq::AudioMixer(EndpointAudioMixerReq::Detach(sources)), + ))); + } + } + } } #[cfg(test)] @@ -664,8 +793,9 @@ mod tests { use media_server_protocol::{ endpoint::{PeerMeta, RoomInfoPublish, RoomInfoSubscribe}, protobuf::{ - conn::{self, client_event, ClientEvent}, - gateway, shared, + gateway, + session::{self, client_event, ClientEvent}, + shared, }, tokens::WebrtcToken, }; @@ -687,12 +817,13 @@ mod tests { #[test] fn join_room_first() { let req = gateway::ConnectRequest { - join: Some(shared::RoomJoin { + join: Some(session::RoomJoin { room: "room".to_string(), peer: "peer".to_string(), publish: Some(shared::RoomInfoPublish { peer: true, tracks: true }), subscribe: Some(shared::RoomInfoSubscribe { peers: true, tracks: true }), metadata: Some("metadata".to_string()), + features: None, }), ..Default::default() }; @@ -720,7 +851,8 @@ mod tests { metadata: Some("metadata".to_string()) }, RoomInfoPublish { peer: true, tracks: true }, - RoomInfoSubscribe { peers: true, tracks: true } + RoomInfoSubscribe { peers: true, tracks: true }, + None, ) ))) ); @@ -756,16 +888,17 @@ mod tests { ); transport.on_str0m_channel_event(ClientEvent { seq: 0, - event: Some(client_event::Event::Request(conn::Request { + event: Some(client_event::Event::Request(session::Request { req_id: 1, - request: Some(conn::request::Request::Session(conn::request::Session { - request: Some(conn::request::session::Request::Join(conn::request::session::RoomJoin { - info: Some(shared::RoomJoin { + request: Some(session::request::Request::Session(session::request::Session { + request: Some(session::request::session::Request::Join(session::request::session::Join { + info: Some(session::RoomJoin { room: "demo".to_string(), peer: "peer1".to_string(), metadata: None, publish: None, subscribe: None, + features: None, }), token: token.clone(), })), @@ -782,7 +915,8 @@ mod tests { "peer1".to_string().into(), PeerMeta { metadata: None }, RoomInfoPublish { peer: false, tracks: false }, - RoomInfoSubscribe { peers: false, tracks: false } + RoomInfoSubscribe { peers: false, tracks: false }, + None, ) ))) ); @@ -796,4 +930,5 @@ mod tests { //TODO test local track //TODO test local track lazy //TODO test local track attach, detach + //TODO test audio mixer event } diff --git a/packages/transport_webrtc/src/transport/whep.rs b/packages/transport_webrtc/src/transport/whep.rs index 79b21427..efa229ca 100644 --- a/packages/transport_webrtc/src/transport/whep.rs +++ b/packages/transport_webrtc/src/transport/whep.rs @@ -4,11 +4,11 @@ use std::{ }; use media_server_core::{ - endpoint::{EndpointEvent, EndpointLocalTrackConfig, EndpointLocalTrackEvent, EndpointLocalTrackReq, EndpointLocalTrackSource, EndpointReq}, + endpoint::{EndpointEvent, EndpointLocalTrackConfig, EndpointLocalTrackEvent, EndpointLocalTrackReq, EndpointReq}, transport::{LocalTrackEvent, LocalTrackId, TransportError, TransportEvent, TransportOutput, TransportState}, }; use media_server_protocol::{ - endpoint::{PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe, TrackMeta, TrackName, TrackPriority}, + endpoint::{PeerId, PeerMeta, RoomId, RoomInfoPublish, RoomInfoSubscribe, TrackMeta, TrackName, TrackPriority, TrackSource}, media::MediaKind, }; use sans_io_runtime::{collections::DynamicDeque, return_if_none}; @@ -149,6 +149,7 @@ impl TransportWebrtcInternal for TransportWebrtcWhep { self.queue.push_back(InternalOutput::Str0mSendMedia(mid, pkt)); } EndpointLocalTrackEvent::Status(_) => {} + EndpointLocalTrackEvent::VoiceActivity(_) => {} }, EndpointEvent::RemoteMediaTrack(_track, _event) => {} EndpointEvent::BweConfig { current, desired } => { @@ -156,6 +157,7 @@ impl TransportWebrtcInternal for TransportWebrtcWhep { self.queue.push_back(InternalOutput::Str0mBwe(current, desired)); } EndpointEvent::GoAway(_seconds, _reason) => {} + EndpointEvent::AudioMixer(_) => {} } } @@ -172,6 +174,7 @@ impl TransportWebrtcInternal for TransportWebrtcWhep { PeerMeta { metadata: None }, RoomInfoPublish { peer: false, tracks: false }, RoomInfoSubscribe { peers: false, tracks: true }, + None, ), ))); self.queue @@ -289,7 +292,7 @@ impl TransportWebrtcWhep { EndpointReq::LocalTrack( AUDIO_TRACK, EndpointLocalTrackReq::Attach( - EndpointLocalTrackSource { peer, track }, + TrackSource { peer, track }, EndpointLocalTrackConfig { priority: DEFAULT_PRIORITY, max_spatial: 2, @@ -312,7 +315,7 @@ impl TransportWebrtcWhep { EndpointReq::LocalTrack( VIDEO_TRACK, EndpointLocalTrackReq::Attach( - EndpointLocalTrackSource { peer, track }, + TrackSource { peer, track }, EndpointLocalTrackConfig { priority: DEFAULT_PRIORITY, max_spatial: 2, diff --git a/packages/transport_webrtc/src/transport/whip.rs b/packages/transport_webrtc/src/transport/whip.rs index ce761586..807d9f6a 100644 --- a/packages/transport_webrtc/src/transport/whip.rs +++ b/packages/transport_webrtc/src/transport/whip.rs @@ -134,6 +134,7 @@ impl TransportWebrtcInternal for TransportWebrtcWhip { EndpointEvent::LocalMediaTrack(_, _) => {} EndpointEvent::BweConfig { .. } => {} EndpointEvent::GoAway(_, _) => {} + EndpointEvent::AudioMixer(_) => {} } } @@ -152,6 +153,7 @@ impl TransportWebrtcInternal for TransportWebrtcWhip { PeerMeta { metadata: None }, RoomInfoPublish { peer: true, tracks: true }, RoomInfoSubscribe { peers: false, tracks: false }, + None, ), ))); } diff --git a/packages/transport_webrtc/src/worker.rs b/packages/transport_webrtc/src/worker.rs index 9adcb791..8b0052ae 100644 --- a/packages/transport_webrtc/src/worker.rs +++ b/packages/transport_webrtc/src/worker.rs @@ -18,21 +18,21 @@ use crate::{ WebrtcError, }; -group_owner_type!(WebrtcOwner); +group_owner_type!(WebrtcSession); pub enum GroupInput { Net(BackendIncoming), - Cluster(WebrtcOwner, ClusterEndpointEvent), - Ext(WebrtcOwner, ExtIn), - Close(WebrtcOwner), + Cluster(WebrtcSession, ClusterEndpointEvent), + Ext(WebrtcSession, ExtIn), + Close(WebrtcSession), } #[derive(Debug)] pub enum GroupOutput { Net(BackendOutgoing), - Cluster(WebrtcOwner, ClusterRoomHash, ClusterEndpointControl), - Ext(WebrtcOwner, ExtOut), - Shutdown(WebrtcOwner), + Cluster(WebrtcSession, ClusterRoomHash, ClusterEndpointControl), + Ext(WebrtcSession, ExtOut), + Shutdown(WebrtcSession), Continue, } @@ -78,6 +78,7 @@ impl MediaWorkerWebrtc { let (tran, ufrag, sdp) = TransportWebrtc::new(variant, offer, self.dtls_cert.clone(), self.addrs.clone(), self.ice_lite)?; let endpoint = Endpoint::new(cfg, tran); let index = self.endpoints.add_task(endpoint); + log::info!("[TransportWebrtc] create endpoint {index}"); self.shared_port.add_ufrag(ufrag, index); Ok((self.ice_lite, sdp, index)) } @@ -85,13 +86,14 @@ impl MediaWorkerWebrtc { fn process_output(&mut self, index: usize, out: EndpointOutput) -> GroupOutput { match out { EndpointOutput::Net(net) => GroupOutput::Net(net), - EndpointOutput::Cluster(room, control) => GroupOutput::Cluster(WebrtcOwner(index), room, control), + EndpointOutput::Cluster(room, control) => GroupOutput::Cluster(WebrtcSession(index), room, control), EndpointOutput::Destroy => { + log::info!("[TransportWebrtc] destroy endpoint {index}"); self.endpoints.remove_task(index); self.shared_port.remove_task(index); - GroupOutput::Shutdown(WebrtcOwner(index)) + GroupOutput::Shutdown(WebrtcSession(index)) } - EndpointOutput::Ext(ext) => GroupOutput::Ext(WebrtcOwner(index), ext), + EndpointOutput::Ext(ext) => GroupOutput::Ext(WebrtcSession(index), ext), EndpointOutput::Continue => GroupOutput::Continue, } }