diff --git a/Cargo.lock b/Cargo.lock index b875e5af..aab95f4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,7 +468,7 @@ dependencies = [ "atm0s-sdn", "clap", "convert-enum", - "derive_more 1.0.0", + "derive_more", "hex", "local-ip-address", "log", @@ -502,9 +502,9 @@ dependencies = [ [[package]] name = "atm0s-sdn" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b95dae8a1c71d28f90b1c891eddf9d80a84249ae16f557a36e1e690c2d83ab6" +checksum = "9ae4bafdf42069f4419ca9b361a3a31d59d8bbdbf87b14a0eee7fdcd744d30db" dependencies = [ "atm0s-sdn-identity", "atm0s-sdn-network", @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "atm0s-sdn-network" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f890567c3ee41e7748ddee4ab6c8edd7c1fbdc93fde5d322187eeb00673dfe0b" +checksum = "68bb1728d2a0f5f517c6be60a6e32bd73767b8bf70bb9df5f006d188def58210" dependencies = [ "aes-gcm 0.10.3", "atm0s-sdn-identity", @@ -546,7 +546,7 @@ dependencies = [ "convert-enum", "derivative", "log", - "mockall 0.12.1", + "mockall", "num", "num_enum", "parking_lot", @@ -561,14 +561,14 @@ dependencies = [ [[package]] name = "atm0s-sdn-router" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88e27523f92b50bd5c827281b52ef73cd8497c5df6c7bc95a167e503f28d60" +checksum = "6679fb3c1797ae958c247a0549c3e33224c7c0352e5b28bb984f8a7e90bd7bf7" dependencies = [ "atm0s-sdn-identity", "atm0s-sdn-utils", "log", - "mockall 0.12.1", + "mockall", "serde", ] @@ -1142,12 +1142,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -1519,19 +1513,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_more" -version = "0.99.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version 0.4.1", - "syn 2.0.79", -] - [[package]] name = "derive_more" version = "1.0.0" @@ -1547,7 +1528,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ - "convert_case 0.6.0", + "convert_case", "proc-macro2", "quote", "syn 2.0.79", @@ -1698,6 +1679,27 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -3233,15 +3235,16 @@ dependencies = [ "atm0s-sdn", "audio-mixer", "derivative", - "derive_more 1.0.0", + "derive_more", "indexmap", "log", "media-server-protocol", "media-server-utils", - "mockall 0.13.0", + "mockall", "num_enum", "sans-io-runtime", "smallmap", + "test-log", "tracing-subscriber", ] @@ -3250,7 +3253,7 @@ name = "media-server-gateway" version = "0.1.0" dependencies = [ "atm0s-sdn", - "derive_more 1.0.0", + "derive_more", "httpmock", "log", "media-server-protocol", @@ -3283,7 +3286,7 @@ dependencies = [ "bincode", "convert-enum", "derivative", - "derive_more 1.0.0", + "derive_more", "log", "prost", "prost-build", @@ -3358,6 +3361,7 @@ dependencies = [ name = "media-server-utils" version = "0.1.0" dependencies = [ + "derive_more", "log", "once_cell", "pin-project-lite", @@ -3433,21 +3437,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" -[[package]] -name = "mockall" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "lazy_static", - "mockall_derive 0.12.1", - "predicates", - "predicates-tree", -] - [[package]] name = "mockall" version = "0.13.0" @@ -3457,23 +3446,11 @@ dependencies = [ "cfg-if", "downcast", "fragile", - "mockall_derive 0.13.0", + "mockall_derive", "predicates", "predicates-tree", ] -[[package]] -name = "mockall_derive" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 2.0.79", -] - [[package]] name = "mockall_derive" version = "0.13.0" @@ -4240,7 +4217,7 @@ checksum = "5c354a706a81a17657da8441869c6ce3d486ef5e72705919d6035fb038a6f7b5" dependencies = [ "base64 0.22.1", "bytes 1.7.2", - "derive_more 1.0.0", + "derive_more", "futures-util", "indexmap", "mime", @@ -5213,12 +5190,12 @@ dependencies = [ [[package]] name = "sans-io-runtime" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecb86ad7989fbacc137308565897a1e401a101043c1ccfbe5bd8a9fea79675" +checksum = "00a1be6434e4714a0e0d75634aca19691a5ea6bee5163391b3b82260565bbb68" dependencies = [ "convert-enum", - "derive_more 0.99.18", + "derive_more", "heapless", "log", "num_enum", @@ -6442,6 +6419,28 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "test-log" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dffced63c2b5c7be278154d76b479f9f9920ed34e7574201407f0b14e2bbb93" +dependencies = [ + "env_logger", + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-macros" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5999e24eaa32083191ba4e425deb75cdf25efefabe5aaccb7446dd0d4122a3f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "thiserror" version = "1.0.64" @@ -6738,7 +6737,7 @@ dependencies = [ name = "transport-rtpengine" version = "0.1.0" dependencies = [ - "derive_more 1.0.0", + "derive_more", "log", "media-server-codecs", "media-server-core", @@ -6756,7 +6755,7 @@ dependencies = [ name = "transport-webrtc" version = "0.1.0" dependencies = [ - "derive_more 1.0.0", + "derive_more", "log", "media-server-core", "media-server-protocol", diff --git a/Cargo.toml b/Cargo.toml index 8759e812..1486341a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,9 @@ members = [ ] [workspace.dependencies] -sans-io-runtime = { version = "0.2", default-features = false } -atm0s-sdn = { version = "0.2.2", default-features = false } +sans-io-runtime = { version = "0.3", default-features = false } +atm0s-sdn = { version = "0.2", default-features = false } +atm0s-sdn-network = { version = "0.6", default-features = false } tokio = "1.37" tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } convert-enum = "0.1" @@ -37,3 +38,4 @@ prost = "0.13" indexmap = "2.2" spin = "0.9" httpmock = "0.7" +test-log = "0.2" diff --git a/bin/src/server/media/runtime_worker.rs b/bin/src/server/media/runtime_worker.rs index 0aa282f7..25a4dd8f 100644 --- a/bin/src/server/media/runtime_worker.rs +++ b/bin/src/server/media/runtime_worker.rs @@ -50,6 +50,7 @@ pub struct MediaRuntimeWorker { index: u16, worker: MediaServerWorker, queue: VecDeque, + shutdown: bool, } impl WorkerInner, SCfg> for MediaRuntimeWorker { @@ -72,7 +73,12 @@ impl WorkerInner u16 { @@ -83,6 +89,10 @@ impl WorkerInner bool { + self.shutdown && self.queue.is_empty() && self.worker.is_empty() + } + fn spawn(&mut self, _now: Instant, _cfg: SCfg) { panic!("Not supported") } @@ -104,7 +114,11 @@ impl WorkerInner, @@ -24,6 +25,7 @@ struct OutputSlotState { /// Implement lightweight audio mixer with mix-minus feature /// We will select n highest audio-level tracks +#[derive(Debug)] pub struct AudioMixer { len: usize, sources: HashMap, diff --git a/packages/media_connector/src/agent_service.rs b/packages/media_connector/src/agent_service.rs index 4d09621f..c4741fc8 100644 --- a/packages/media_connector/src/agent_service.rs +++ b/packages/media_connector/src/agent_service.rs @@ -35,6 +35,7 @@ pub struct ConnectorAgentService { subscriber: Option>, msg_queue: MessageQueue, queue: VecDeque>, + shutdown: bool, _tmp: std::marker::PhantomData<(UserData, SC, SE, TC, TW)>, } @@ -52,6 +53,7 @@ impl ConnectorAgentService { subscriber: None, queue: VecDeque::from([ServiceOutput::FeatureControl(data::Control::DataListen(DATA_PORT).into())]), msg_queue: MessageQueue::default(), + shutdown: false, _tmp: std::marker::PhantomData, } } @@ -62,6 +64,10 @@ where SC: From + TryInto + Debug, SE: From + TryInto, { + fn is_service_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + fn service_id(&self) -> u8 { AGENT_SERVICE_ID } @@ -134,6 +140,10 @@ where } } + fn on_shutdown(&mut self, _ctx: &ServiceCtx, _now: u64) { + self.shutdown = true; + } + fn pop_output2(&mut self, now: u64) -> Option> { if let Some(out) = self.queue.pop_front() { return Some(out); @@ -151,9 +161,14 @@ where pub struct ConnectorAgentServiceWorker { queue: VecDeque>, + shutdown: bool, } impl ServiceWorker for ConnectorAgentServiceWorker { + fn is_service_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + fn service_id(&self) -> u8 { AGENT_SERVICE_ID } @@ -172,6 +187,10 @@ impl ServiceWorker Option> { self.queue.pop_front() } @@ -218,7 +237,10 @@ where } fn create_worker(&self) -> Box> { - Box::new(ConnectorAgentServiceWorker { queue: Default::default() }) + Box::new(ConnectorAgentServiceWorker { + queue: Default::default(), + shutdown: false, + }) } } diff --git a/packages/media_connector/src/handler_service.rs b/packages/media_connector/src/handler_service.rs index 536e7769..b22fed9d 100644 --- a/packages/media_connector/src/handler_service.rs +++ b/packages/media_connector/src/handler_service.rs @@ -31,6 +31,7 @@ pub struct ConnectorHandlerService { lru: LruCache, subscriber: Option>, queue: VecDeque>, + shutdown: bool, _tmp: std::marker::PhantomData<(UserData, SC, SE, TC, TW)>, } @@ -46,6 +47,7 @@ impl ConnectorHandlerService subscriber: None, lru: LruCache::new(NonZeroUsize::new(10000).expect("should be non-zero")), queue: VecDeque::from([ServiceOutput::FeatureControl(data::Control::DataListen(DATA_PORT).into())]), + shutdown: false, _tmp: std::marker::PhantomData, } } @@ -64,6 +66,10 @@ where HANDLER_SERVICE_NAME } + fn is_service_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + fn on_shared_input<'a>(&mut self, _ctx: &ServiceCtx, _now: u64, _input: ServiceSharedInput) {} fn on_input(&mut self, _ctx: &ServiceCtx, _now: u64, input: ServiceInput) { @@ -116,6 +122,10 @@ where } } + fn on_shutdown(&mut self, _ctx: &ServiceCtx, _now: u64) { + self.shutdown = true; + } + fn pop_output2(&mut self, _now: u64) -> Option> { self.queue.pop_front() } @@ -123,6 +133,7 @@ where pub struct ConnectorHandlerServiceWorker { queue: VecDeque>, + shutdown: bool, } impl ServiceWorker for ConnectorHandlerServiceWorker { @@ -134,6 +145,10 @@ impl ServiceWorker bool { + self.shutdown && self.queue.is_empty() + } + fn on_tick(&mut self, _ctx: &ServiceWorkerCtx, _now: u64, _tick_count: u64) {} fn on_input(&mut self, _ctx: &ServiceWorkerCtx, _now: u64, input: ServiceWorkerInput) { @@ -144,6 +159,10 @@ impl ServiceWorker Option> { self.queue.pop_front() } @@ -190,6 +209,9 @@ where } fn create_worker(&self) -> Box> { - Box::new(ConnectorHandlerServiceWorker { queue: Default::default() }) + Box::new(ConnectorHandlerServiceWorker { + queue: Default::default(), + shutdown: false, + }) } } diff --git a/packages/media_core/Cargo.toml b/packages/media_core/Cargo.toml index f0a11138..b73c983d 100644 --- a/packages/media_core/Cargo.toml +++ b/packages/media_core/Cargo.toml @@ -21,3 +21,4 @@ indexmap = { workspace = true } [dev-dependencies] tracing-subscriber = { workspace = true } +test-log = { workspace = true } \ No newline at end of file diff --git a/packages/media_core/src/cluster.rs b/packages/media_core/src/cluster.rs index 6e71e0f0..194922cc 100644 --- a/packages/media_core/src/cluster.rs +++ b/packages/media_core/src/cluster.rs @@ -6,7 +6,7 @@ //! use derive_more::{AsRef, Display, From}; -use sans_io_runtime::{return_if_none, TaskGroup, TaskSwitcherChild}; +use sans_io_runtime::{return_if_none, TaskGroup, TaskGroupOutput, TaskSwitcherChild}; use std::{ collections::HashMap, fmt::Debug, @@ -126,12 +126,14 @@ pub enum Input { pub enum Output { Sdn(RoomUserData, FeaturesControl), Endpoint(Vec, ClusterEndpointEvent), + OnResourceEmpty, Continue, } pub struct MediaCluster { rooms_map: HashMap, rooms: TaskGroup, room::Output, ClusterRoom, 16>, + shutdown: bool, } impl Default for MediaCluster { @@ -139,6 +141,7 @@ impl Default for MediaCluster MediaCluster } pub fn shutdown(&mut self, now: Instant) { + if self.shutdown { + return; + } self.rooms.on_shutdown(now); + self.shutdown = true; } } impl TaskSwitcherChild> for MediaCluster { type Time = (); + + fn is_empty(&self) -> bool { + self.shutdown && self.rooms.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { - let (index, out) = self.rooms.pop_output(())?; + let (index, out) = match self.rooms.pop_output(())? { + TaskGroupOutput::TaskOutput(index, out) => (index, out), + TaskGroupOutput::OnResourceEmpty => return Some(Output::Continue), + }; match out { room::Output::Sdn(userdata, control) => Some(Output::Sdn(userdata, control)), room::Output::Endpoint(endpoints, event) => Some(Output::Endpoint(endpoints, event)), - room::Output::Destroy(room) => { + room::Output::OnResourceEmpty(room) => { log::info!("[MediaCluster] remove room index {index}, hash {room}"); self.rooms_map.remove(&room).expect("Should have room with index"); self.rooms.remove_task(index); @@ -208,7 +227,7 @@ mod tests { use super::{ClusterEndpointControl, ClusterRoomHash, MediaCluster, Output}; - #[test] + #[test_log::test] fn multi_tenancy_room() { let app_root = AppContext { app: AppId::root_app() }; let app1 = AppContext { app: AppId::from("app1") }; @@ -228,7 +247,7 @@ mod tests { //TODO should create room when new room event arrived //TODO should route to correct room //TODO should remove room after all peers leaved - #[test] + #[test_log::test] fn room_manager_should_work() { let mut cluster = MediaCluster::::default(); diff --git a/packages/media_core/src/cluster/room.rs b/packages/media_core/src/cluster/room.rs index be13cc8e..92eea02a 100644 --- a/packages/media_core/src/cluster/room.rs +++ b/packages/media_core/src/cluster/room.rs @@ -52,7 +52,7 @@ pub enum Input { pub enum Output { Sdn(RoomUserData, FeaturesControl), Endpoint(Vec, ClusterEndpointEvent), - Destroy(ClusterRoomHash), + OnResourceEmpty(ClusterRoomHash), } #[derive(num_enum::TryFromPrimitive, num_enum::IntoPrimitive)] @@ -86,11 +86,22 @@ impl Task, Output TaskSwitcherChild> for ClusterRoom { type Time = (); + + fn is_empty(&self) -> bool { + self.metadata.is_empty() && self.media_track.is_empty() && self.audio_mixer.is_empty() && self.message_channel.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty(self.room) + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { loop { match self.switcher.current()?.try_into().ok()? { @@ -100,9 +111,7 @@ impl TaskSwitcherChild break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::MetaData), FeaturesControl::DhtKv(control))), metadata::Output::Endpoint(endpoints, event) => break Some(Output::Endpoint(endpoints, event)), metadata::Output::OnResourceEmpty => { - if self.is_empty() { - break Some(Output::Destroy(self.room)); - } + log::info!("[ClusterRoom] on metadata empty"); } } } @@ -113,9 +122,7 @@ impl TaskSwitcherChild break Some(Output::Endpoint(endpoints, event)), media_track::Output::Pubsub(control) => break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::MediaTrack), FeaturesControl::PubSub(control))), media_track::Output::OnResourceEmpty => { - if self.is_empty() { - break Some(Output::Destroy(self.room)); - } + log::info!("[ClusterRoom] on media track empty"); } } } @@ -126,9 +133,7 @@ impl TaskSwitcherChild break Some(Output::Endpoint(endpoints, event)), audio_mixer::Output::Pubsub(control) => break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::AudioMixer), FeaturesControl::PubSub(control))), audio_mixer::Output::OnResourceEmpty => { - if self.is_empty() { - break Some(Output::Destroy(self.room)); - } + log::info!("[ClusterRoom] on audio mixer empty"); } } } @@ -139,9 +144,7 @@ impl TaskSwitcherChild break Some(Output::Endpoint(endpoints, event)), message_channel::Output::Pubsub(control) => break Some(Output::Sdn(RoomUserData(self.room, RoomFeature::MessageChannel), FeaturesControl::PubSub(control))), message_channel::Output::OnResourceEmpty => { - if self.is_empty() { - break Some(Output::Destroy(self.room)); - } + log::info!("[ClusterRoom] on message channel empty"); } } } @@ -165,10 +168,6 @@ impl ClusterRoom { } } - pub fn is_empty(&self) -> bool { - self.metadata.is_empty() && self.media_track.is_empty() && self.audio_mixer.is_empty() && self.message_channel.is_empty() - } - fn on_sdn_event(&mut self, now: Instant, userdata: RoomUserData, event: FeaturesEvent) { match (userdata.1, event) { (RoomFeature::MetaData, FeaturesEvent::DhtKv(event)) => match event { @@ -272,10 +271,10 @@ impl ClusterRoom { impl Drop for ClusterRoom { fn drop(&mut self) { log::info!("Drop ClusterRoom {}", self.room); - assert!(self.audio_mixer.is_empty(), "Audio mixer not empty"); - assert!(self.media_track.is_empty(), "Media track not empty"); - assert!(self.metadata.is_empty(), "Metadata not empty"); - assert!(self.message_channel.is_empty(), "Data channel not empty"); + assert!(self.audio_mixer.is_empty(), "Audio mixer not empty, {:?}", self.audio_mixer); + assert!(self.media_track.is_empty(), "Media track not empty, {:?}", self.media_track); + assert!(self.metadata.is_empty(), "Metadata not empty, {:?}", self.metadata); + assert!(self.message_channel.is_empty(), "Data channel not empty, {:?}", self.message_channel); } } @@ -302,7 +301,7 @@ mod tests { //TODO channel data should fire event to endpoint //TODO unsubscribe track should UNSUB channel - #[test] + #[test_log::test] fn cleanup_resource_sub_and_mixer() { let room_id = 0.into(); let endpoint = 1; @@ -377,7 +376,7 @@ mod tests { FeaturesControl::PubSub(pubsub::Control(room_mixer_auto_channel, pubsub::ChannelControl::UnsubAuto)) )) ); - assert_eq!(room.pop_output(()), Some(Output::Destroy(room_id))); assert_eq!(room.pop_output(()), None); + assert!(room.is_empty()); } } diff --git a/packages/media_core/src/cluster/room/audio_mixer.rs b/packages/media_core/src/cluster/room/audio_mixer.rs index 7bfce0a2..a232f075 100644 --- a/packages/media_core/src/cluster/room/audio_mixer.rs +++ b/packages/media_core/src/cluster/room/audio_mixer.rs @@ -20,7 +20,7 @@ use media_server_protocol::{ endpoint::{AudioMixerConfig, PeerId, TrackName}, media::MediaPacket, }; -use sans_io_runtime::{TaskGroup, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; +use sans_io_runtime::{TaskGroup, TaskGroupOutput, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; use crate::{ cluster::{ClusterAudioMixerControl, ClusterEndpointEvent, ClusterRoomHash}, @@ -53,9 +53,10 @@ pub enum Output { OnResourceEmpty, } -type AudioMixerManuals = TaskSwitcherBranch, ManualMixer, 4>, (usize, Output)>; +type AudioMixerManuals = TaskSwitcherBranch, ManualMixer, 4>, TaskGroupOutput>>; -pub struct AudioMixer { +#[derive(Debug)] +pub struct AudioMixer { room: ClusterRoomHash, mix_channel_id: ChannelId, //store number of outputs @@ -89,13 +90,6 @@ impl AudioMixer { } } - /// - /// We need to wait all publisher, subscriber, and manuals ready to remove - /// - pub fn is_empty(&self) -> bool { - self.publisher.is_empty() && self.subscriber1.is_empty() && self.subscriber2.is_empty() && self.subscriber3.is_empty() && self.manuals.tasks() == 0 - } - pub fn on_tick(&mut self, now: Instant) { if now >= self.last_tick + TICK_INTERVAL { self.last_tick = now; @@ -194,6 +188,20 @@ impl AudioMixer { impl TaskSwitcherChild> for AudioMixer { type Time = (); + fn is_empty(&self) -> bool { + self.manual_channels.is_empty() + && self.manual_mode.is_empty() + && self.publisher.is_empty() + && self.subscriber1.is_empty() + && self.subscriber2.is_empty() + && self.subscriber3.is_empty() + && self.manuals.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + /// /// We need to wait all publisher, subscriber, and manuals ready to remove /// @@ -203,9 +211,7 @@ impl TaskSwitcherChild> fo TaskType::Publisher => { if let Some(out) = self.publisher.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -214,9 +220,7 @@ impl TaskSwitcherChild> fo TaskType::Subscriber1 => { if let Some(out) = self.subscriber1.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -225,9 +229,7 @@ impl TaskSwitcherChild> fo TaskType::Subscriber2 => { if let Some(out) = self.subscriber2.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -236,42 +238,49 @@ impl TaskSwitcherChild> fo TaskType::Subscriber3 => { if let Some(out) = self.subscriber3.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } } } TaskType::Manuals => { - if let Some((index, out)) = self.manuals.pop_output((), &mut self.switcher) { - match out { - Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::SubAuto)) => { - if let Some(slot) = self.manual_channels.get_mut(&channel_id) { - slot.push(index); - } else { - self.manual_channels.insert(channel_id, vec![index]); - return Some(out); - } + let (index, out) = match self.manuals.pop_output((), &mut self.switcher) { + Some(TaskGroupOutput::TaskOutput(index, out)) => (index, out), + Some(TaskGroupOutput::OnResourceEmpty) => { + // we dont need to forward OnResourceEmpty to parent + continue; + } + None => { + continue; + } + }; + + match out { + Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::SubAuto)) => { + if let Some(slot) = self.manual_channels.get_mut(&channel_id) { + slot.push(index); + } else { + self.manual_channels.insert(channel_id, vec![index]); + return Some(out); } - Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto)) => { - let slot = self.manual_channels.get_mut(&channel_id).expect("Manual channel map not found"); - let (slot_index, _) = slot.iter().enumerate().find(|(_, task_i)| **task_i == index).expect("Subscribed task not found"); - slot.swap_remove(slot_index); - if slot.is_empty() { - self.manual_channels.remove(&channel_id); - return Some(out); - } + } + Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto)) => { + let slot = self.manual_channels.get_mut(&channel_id).expect("Manual channel map not found"); + let (slot_index, _) = slot.iter().enumerate().find(|(_, task_i)| **task_i == index).expect("Subscribed task not found"); + slot.swap_remove(slot_index); + if slot.is_empty() { + self.manual_channels.remove(&channel_id); + return Some(out); } - Output::OnResourceEmpty => { - self.manuals.input(&mut self.switcher).remove_task(index); - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + } + Output::OnResourceEmpty => { + self.manuals.input(&mut self.switcher).remove_task(index); + if self.is_empty() { + return Some(Output::OnResourceEmpty); } - _ => return Some(out), } + _ => return Some(out), } } } @@ -279,11 +288,15 @@ impl TaskSwitcherChild> fo } } -impl Drop for AudioMixer { +impl Drop for AudioMixer { fn drop(&mut self) { log::info!("[ClusterRoomAudioMixer] Drop {}", self.room); - assert_eq!(self.manual_channels.len(), 0, "Manual channels not empty on drop"); - assert_eq!(self.manual_mode.len(), 0, "Manual modes not empty on drop"); - assert_eq!(self.manuals.tasks(), 0, "Manuals not empty on drop"); + assert_eq!(self.manual_channels.len(), 0, "Manual channels not empty on drop {:?}", self.manual_channels); + assert_eq!(self.manual_mode.len(), 0, "Manual modes not empty on drop {:?}", self.manual_mode); + assert!(self.manuals.is_empty(), "AudioMixerManuals not empty on drop {:?}", self.manuals); + assert!(self.publisher.is_empty(), "AudioMixerPublisher not empty on drop {:?}", self.publisher); + assert!(self.subscriber1.is_empty(), "AudioMixerSubscriber1 not empty on drop {:?}", self.subscriber1); + assert!(self.subscriber2.is_empty(), "AudioMixerSubscriber2 not empty on drop {:?}", self.subscriber2); + assert!(self.subscriber3.is_empty(), "AudioMixerSubscriber3 not empty on drop {:?}", self.subscriber3); } } diff --git a/packages/media_core/src/cluster/room/audio_mixer/manual.rs b/packages/media_core/src/cluster/room/audio_mixer/manual.rs index 9abf79f4..2e5f1a86 100644 --- a/packages/media_core/src/cluster/room/audio_mixer/manual.rs +++ b/packages/media_core/src/cluster/room/audio_mixer/manual.rs @@ -4,7 +4,7 @@ //! to determine which source is sent to client. //! -use std::{collections::HashMap, time::Instant}; +use std::{collections::HashMap, fmt::Debug, time::Instant}; use atm0s_sdn::{ features::pubsub::{self, ChannelId}, @@ -30,7 +30,8 @@ pub enum Input { LeaveRoom, } -pub struct ManualMixer { +#[derive(Debug)] +pub struct ManualMixer { _c: Count, endpoint: Endpoint, room: ClusterRoomHash, @@ -40,7 +41,7 @@ pub struct ManualMixer { mixer: audio_mixer::AudioMixer, } -impl ManualMixer { +impl ManualMixer { pub fn new(room: ClusterRoomHash, endpoint: Endpoint, outputs: Vec) -> Self { Self { _c: Default::default(), @@ -95,7 +96,7 @@ impl ManualMixer { } } -impl Task> for ManualMixer { +impl Task> for ManualMixer { fn on_tick(&mut self, now: Instant) { if let Some(removed) = self.mixer.on_tick(now) { for slot in removed { @@ -132,27 +133,36 @@ impl Task> for ManualMixer { log::info!("[ClusterManualMixer] remove source {:?} on queue => unsub {channel_id}", source); self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto))); } - self.queue.push_back(Output::OnResourceEmpty); } } } - fn on_shutdown(&mut self, _now: Instant) {} + fn on_shutdown(&mut self, _now: Instant) { + // this is depend on endpoint, so we cannot shutdown until endpoint is empty + } } -impl TaskSwitcherChild> for ManualMixer { +impl TaskSwitcherChild> for ManualMixer { type Time = (); + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.sources.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for ManualMixer { +impl Drop for ManualMixer { fn drop(&mut self) { log::info!("[ClusterManualMixer] Drop {}", self.room); - assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.sources.len(), 0, "Sources not empty on drop"); + assert_eq!(self.queue.len(), 0, "AudioMixerManual Queue not empty on drop {:?}", self.queue); + assert_eq!(self.sources.len(), 0, "AudioMixerManual Sources not empty on drop {:?}", self.sources); } } @@ -175,7 +185,7 @@ mod test { Duration::from_millis(ms) } - #[test] + #[test_log::test] fn attach_detach() { let t0 = Instant::now(); let room = 0.into(); @@ -226,7 +236,7 @@ mod test { assert_eq!(manual.pop_output(()), None); } - #[test] + #[test_log::test] fn leave_room() { let t0 = Instant::now(); let room = 0.into(); @@ -245,7 +255,7 @@ mod test { manual.on_event(t0, Input::LeaveRoom); assert_eq!(manual.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel_id, pubsub::ChannelControl::UnsubAuto)))); - assert_eq!(manual.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(manual.pop_output(()), None); + assert!(manual.is_empty()); } } diff --git a/packages/media_core/src/cluster/room/audio_mixer/publisher.rs b/packages/media_core/src/cluster/room/audio_mixer/publisher.rs index 713566d1..e72f3325 100644 --- a/packages/media_core/src/cluster/room/audio_mixer/publisher.rs +++ b/packages/media_core/src/cluster/room/audio_mixer/publisher.rs @@ -19,17 +19,20 @@ use super::Output; const FIRE_SOURCE_INTERVAL: Duration = Duration::from_millis(500); +#[derive(Debug)] struct TrackSlot { peer: PeerId, name: TrackName, peer_hash: PeerHashCode, } +#[derive(Debug)] struct OutputSlot { last_fired_source: Instant, } -pub struct AudioMixerPublisher { +#[derive(Debug)] +pub struct AudioMixerPublisher { _c: Count, channel_id: pubsub::ChannelId, tracks: HashMap<(Endpoint, RemoteTrackId), TrackSlot>, @@ -50,10 +53,6 @@ impl AudioMixerPublisher { } } - pub fn is_empty(&self) -> bool { - self.tracks.is_empty() && self.queue.is_empty() - } - pub fn on_tick(&mut self, now: Instant) { if let Some(removed_slots) = self.mixer.on_tick(now) { for slot in removed_slots { @@ -119,23 +118,31 @@ impl AudioMixerPublisher { if self.tracks.is_empty() { log::info!("[ClusterAudioMixerPublisher] last track leave ind Auto mode => unpublish channel {}", self.channel_id); self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::PubStop))); - self.queue.push_back(Output::OnResourceEmpty); } } } -impl TaskSwitcherChild> for AudioMixerPublisher { +impl TaskSwitcherChild> for AudioMixerPublisher { type Time = (); + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.tracks.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for AudioMixerPublisher { +impl Drop for AudioMixerPublisher { fn drop(&mut self) { log::info!("[ClusterAudioMixerPublisher] Drop {}", self.channel_id); - assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.tracks.len(), 0, "Tracks not empty on drop"); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop {:?}", self.queue); + assert_eq!(self.tracks.len(), 0, "Tracks not empty on drop {:?}", self.tracks); } } @@ -156,7 +163,7 @@ mod test { Duration::from_millis(m) } - #[test] + #[test_log::test] fn track_publish_unpublish() { let channel = 0.into(); let peer1: PeerId = "peer1".into(); @@ -214,11 +221,11 @@ mod test { publisher.on_track_unpublish(t0 + ms(100), 2, 0.into()); assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::PubStop)))); - assert_eq!(publisher.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(publisher.pop_output(()), None); + assert!(publisher.is_empty()); } - #[test] + #[test_log::test] #[should_panic(expected = "Track not found")] fn invalid_track_data_should_panic() { let t0 = Instant::now(); diff --git a/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs b/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs index aa297a8a..6dbb12d7 100644 --- a/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs +++ b/packages/media_core/src/cluster/room/audio_mixer/subscriber.rs @@ -17,17 +17,19 @@ use crate::cluster::{ClusterAudioMixerEvent, ClusterEndpointEvent, ClusterLocalT use super::Output; +#[derive(Debug)] struct EndpointSlot { peer: PeerHashCode, tracks: Vec, } -#[derive(Clone)] +#[derive(Clone, Debug)] struct OutputSlot { source: Option<(PeerId, TrackName)>, } -pub struct AudioMixerSubscriber { +#[derive(Debug)] +pub struct AudioMixerSubscriber { _c: Count, channel_id: ChannelId, queue: DynamicDeque, 16>, @@ -48,10 +50,6 @@ impl AudioMixerSubscr } } - pub fn is_empty(&self) -> bool { - self.endpoints.is_empty() && self.queue.is_empty() - } - pub fn on_tick(&mut self, now: Instant) { if let Some(removed_slots) = self.mixer.on_tick(now) { for slot in removed_slots { @@ -154,23 +152,31 @@ impl AudioMixerSubscr if self.endpoints.is_empty() { log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] last endpoint leave in Auto mode => unsubscribe channel {}", self.channel_id); self.queue.push_back(Output::Pubsub(pubsub::Control(self.channel_id, pubsub::ChannelControl::UnsubAuto))); - self.queue.push_back(Output::OnResourceEmpty); } } } -impl TaskSwitcherChild> for AudioMixerSubscriber { +impl TaskSwitcherChild> for AudioMixerSubscriber { type Time = (); + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.endpoints.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for AudioMixerSubscriber { +impl Drop for AudioMixerSubscriber { fn drop(&mut self) { log::info!("[ClusterAudioMixerSubscriber {OUTPUTS}] Drop {}", self.channel_id); - assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.endpoints.len(), 0, "Endpoints not empty on drop"); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop {:?}", self.queue); + assert_eq!(self.endpoints.len(), 0, "Endpoints not empty on drop {:?}", self.endpoints); } } @@ -193,7 +199,7 @@ mod test { Duration::from_millis(m) } - #[test] + #[test_log::test] fn sub_unsub() { let t0 = Instant::now(); let channel = 0.into(); @@ -284,7 +290,7 @@ mod test { //now is last endpoint => should fire Unsub subscriber.on_endpoint_leave(t0 + ms(100 + 2000), endpoint2); assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(pubsub::Control(channel, pubsub::ChannelControl::UnsubAuto)))); - assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(subscriber.pop_output(()), None); + assert!(subscriber.is_empty()); } } diff --git a/packages/media_core/src/cluster/room/media_track.rs b/packages/media_core/src/cluster/room/media_track.rs index f115c5dd..fce2874b 100644 --- a/packages/media_core/src/cluster/room/media_track.rs +++ b/packages/media_core/src/cluster/room/media_track.rs @@ -32,7 +32,8 @@ pub enum Output { OnResourceEmpty, } -pub struct MediaTrack { +#[derive(Debug)] +pub struct MediaTrack { room: ClusterRoomHash, publisher: TaskSwitcherBranch, Output>, subscriber: TaskSwitcherBranch, Output>, @@ -49,10 +50,6 @@ impl MediaTrack { } } - pub fn is_empty(&self) -> bool { - self.publisher.is_empty() && self.subscriber.is_empty() - } - pub fn on_pubsub_event(&mut self, event: pubsub::Event) { let channel = event.0; match event.1 { @@ -100,15 +97,21 @@ impl MediaTrack { impl TaskSwitcherChild> for MediaTrack { type Time = (); + fn is_empty(&self) -> bool { + self.publisher.is_empty() && self.subscriber.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { loop { match self.switcher.current()?.try_into().ok()? { TaskType::Publisher => { if let Some(out) = self.publisher.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -117,9 +120,7 @@ impl TaskSwitcherChild> for TaskType::Subscriber => { if let Some(out) = self.subscriber.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -130,8 +131,10 @@ impl TaskSwitcherChild> for } } -impl Drop for MediaTrack { +impl Drop for MediaTrack { fn drop(&mut self) { log::info!("[ClusterRoomMediaTrack] Drop {}", self.room); + assert!(self.publisher.is_empty(), "MediaTrackPublisher not empty on drop {:?}", self.publisher); + assert!(self.subscriber.is_empty(), "MediaTrackSubscriber not empty on drop {:?}", self.subscriber); } } diff --git a/packages/media_core/src/cluster/room/media_track/publisher.rs b/packages/media_core/src/cluster/room/media_track/publisher.rs index 5cdbdadb..600d9772 100644 --- a/packages/media_core/src/cluster/room/media_track/publisher.rs +++ b/packages/media_core/src/cluster/room/media_track/publisher.rs @@ -39,7 +39,8 @@ impl TryFrom for FeedbackKind { } } -pub struct RoomChannelPublisher { +#[derive(Debug)] +pub struct RoomChannelPublisher { _c: Count, room: ClusterRoomHash, tracks: HashMap<(Endpoint, RemoteTrackId), (PeerId, TrackName, ChannelId)>, @@ -58,10 +59,6 @@ impl RoomChannelPublisher { } } - pub fn is_empty(&self) -> bool { - self.tracks.is_empty() && self.queue.is_empty() - } - pub fn on_track_feedback(&mut self, channel: ChannelId, fb: Feedback) { let fb = return_if_err!(FeedbackKind::try_from(fb)); let sources = return_if_none!(self.tracks_source.get(&channel)); @@ -118,27 +115,32 @@ impl RoomChannelPublisher { self.tracks_source.remove(&channel_id).expect("Should remove source channel on unpublish"); self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::PubStop))); } - log::info!("[ClusterRoom {}/Publishers] peer ({peer} stopped track {name})", self.room); - if self.tracks.is_empty() { - self.queue.push_back(Output::OnResourceEmpty); - } } } impl TaskSwitcherChild> for RoomChannelPublisher { type Time = (); + + fn is_empty(&self) -> bool { + self.tracks.is_empty() && self.tracks_source.is_empty() && self.queue.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for RoomChannelPublisher { +impl Drop for RoomChannelPublisher { fn drop(&mut self) { log::info!("[ClusterRoom {}/Publishers] Drop", self.room); - assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.tracks.len(), 0, "Tracks not empty on drop"); - assert_eq!(self.tracks_source.len(), 0, "Tracks source not empty on drop"); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop {:?}", self.queue); + assert_eq!(self.tracks.len(), 0, "Tracks not empty on drop {:?}", self.tracks); + assert_eq!(self.tracks_source.len(), 0, "Tracks source not empty on drop {:?}", self.tracks_source); } } @@ -174,7 +176,7 @@ mod tests { //Track start => should register with SDN //Track stop => should unregister with SDN //Track media => should send data over SDN - #[test] + #[test_log::test] fn channel_publish_data() { let room = 1.into(); let mut publisher = RoomChannelPublisher::::new(room); @@ -195,13 +197,13 @@ mod tests { publisher.on_track_unpublish(endpoint, track); assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStop)))); - assert_eq!(publisher.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(publisher.pop_output(()), None); + assert!(publisher.is_empty()); } //TODO Handle feedback: should handle KeyFrame feedback //TODO Handle feedback: should handle Bitrate feedback - #[test] + #[test_log::test] fn channel_feedback() { let room = 1.into(); let mut publisher = RoomChannelPublisher::::new(room); @@ -233,11 +235,11 @@ mod tests { publisher.on_track_unpublish(endpoint, track); assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStop)))); - assert_eq!(publisher.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(publisher.pop_output(()), None); + assert!(publisher.is_empty()); } - #[test] + #[test_log::test] fn two_sessions_same_room_peer_should_not_crash() { let room = 1.into(); let mut publisher = RoomChannelPublisher::::new(room); @@ -247,6 +249,7 @@ mod tests { let track = RemoteTrackId::from(3); let peer: PeerId = "peer1".to_string().into(); let name: TrackName = "audio_main".to_string().into(); + let channel_id = gen_track_channel_id(room, &peer, &name); publisher.on_track_publish(endpoint1, track, peer.clone(), name.clone()); publisher.on_track_publish(endpoint2, track, peer, name); @@ -257,8 +260,8 @@ mod tests { publisher.on_track_unpublish(endpoint1, track); publisher.on_track_unpublish(endpoint2, track); - assert!(publisher.pop_output(()).is_some()); // PubStop - assert!(publisher.pop_output(()).is_some()); // OnResourceEmpty - assert!(publisher.pop_output(()).is_none()); + assert_eq!(publisher.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::PubStop)))); // PubStop + assert_eq!(publisher.pop_output(()), None); + assert!(publisher.is_empty()); } } diff --git a/packages/media_core/src/cluster/room/media_track/subscriber.rs b/packages/media_core/src/cluster/room/media_track/subscriber.rs index c2a3dc56..67776329 100644 --- a/packages/media_core/src/cluster/room/media_track/subscriber.rs +++ b/packages/media_core/src/cluster/room/media_track/subscriber.rs @@ -37,14 +37,15 @@ const KEYFRAME_FEEDBACK_TIMEOUT: u16 = 2000; //2 seconds const BITRATE_FEEDBACK_KIND: u8 = 0; const KEYFRAME_FEEDBACK_KIND: u8 = 1; -#[derive(Derivative)] +#[derive(Derivative, Debug)] #[derivative(Default(bound = ""))] -struct ChannelContainer { +struct ChannelContainer { endpoints: Vec<(Endpoint, LocalTrackId)>, bitrate_fbs: HashMap, } -pub struct RoomChannelSubscribe { +#[derive(Debug)] +pub struct RoomChannelSubscribe { _c: Count, room: ClusterRoomHash, channels: HashMap>, @@ -52,7 +53,7 @@ pub struct RoomChannelSubscribe { queue: VecDeque>, } -impl RoomChannelSubscribe { +impl RoomChannelSubscribe { pub fn new(room: ClusterRoomHash) -> Self { Self { _c: Default::default(), @@ -63,10 +64,6 @@ impl RoomChannelSubscribe { } } - pub fn is_empty(&self) -> bool { - self.subscribers.is_empty() && self.queue.is_empty() - } - pub fn on_track_relay_changed(&mut self, channel: ChannelId, _relay: NodeId) { let channel_container = return_if_none!(self.channels.get(&channel)); log::info!( @@ -164,26 +161,31 @@ impl RoomChannelSubscribe { log::info!("[ClusterRoom {}/Subscribers] last unsubscriber => Unsub channel {channel_id}", self.room); self.queue.push_back(Output::Pubsub(pubsub::Control(channel_id, ChannelControl::UnsubAuto))); } - - if self.subscribers.is_empty() { - self.queue.push_back(Output::OnResourceEmpty); - } } } impl TaskSwitcherChild> for RoomChannelSubscribe { type Time = (); + + fn is_empty(&self) -> bool { + self.subscribers.is_empty() && self.channels.is_empty() && self.queue.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for RoomChannelSubscribe { +impl Drop for RoomChannelSubscribe { fn drop(&mut self) { log::info!("[ClusterRoom {}/Subscriber] Drop", self.room); - assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.channels.len(), 0, "Channels not empty on drop"); - assert_eq!(self.subscribers.len(), 0, "Subscribers not empty on drop"); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop {:?}", self.queue); + assert_eq!(self.channels.len(), 0, "Channels not empty on drop {:?}", self.channels); + assert_eq!(self.subscribers.len(), 0, "Subscribers not empty on drop {:?}", self.subscribers); } } @@ -221,7 +223,7 @@ mod tests { //TODO First Subscribe channel should sending Sub //TODO Last Unsubscribe channel should sending Unsub - #[test] + #[test_log::test] fn normal_sub_ubsub() { let room = 1.into(); let mut subscriber = RoomChannelSubscribe::::new(room); @@ -248,12 +250,12 @@ mod tests { subscriber.on_track_unsubscribe(endpoint, track); assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); - assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(subscriber.pop_output(()), None); + assert!(subscriber.is_empty()); } //TODO Sending key-frame request - #[test] + #[test_log::test] fn send_key_frame() { let room = 1.into(); let mut subscriber = RoomChannelSubscribe::::new(room); @@ -279,12 +281,12 @@ mod tests { subscriber.on_track_unsubscribe(endpoint, track); assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); - assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(subscriber.pop_output(()), None); + assert!(subscriber.is_empty()); } //TODO Sending bitrate request single sub - #[test] + #[test_log::test] fn send_bitrate_limit_speed() { let room = 1.into(); let mut subscriber = RoomChannelSubscribe::::new(room); @@ -359,7 +361,7 @@ mod tests { subscriber.on_track_unsubscribe(endpoint1, track1); subscriber.on_track_unsubscribe(endpoint2, track2); assert_eq!(subscriber.pop_output(()), Some(Output::Pubsub(Control(channel_id, ChannelControl::UnsubAuto)))); - assert_eq!(subscriber.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(subscriber.pop_output(()), None); + assert!(subscriber.is_empty()); } } diff --git a/packages/media_core/src/cluster/room/message_channel.rs b/packages/media_core/src/cluster/room/message_channel.rs index 10d0d132..27f49c4c 100644 --- a/packages/media_core/src/cluster/room/message_channel.rs +++ b/packages/media_core/src/cluster/room/message_channel.rs @@ -27,7 +27,8 @@ pub enum Output { OnResourceEmpty, } -pub struct RoomMessageChannel { +#[derive(Debug)] +pub struct RoomMessageChannel { room: ClusterRoomHash, publisher: TaskSwitcherBranch, Output>, subscriber: TaskSwitcherBranch, Output>, @@ -45,10 +46,6 @@ impl RoomMessageChannel { } } - pub fn is_empty(&self) -> bool { - self.publisher.is_empty() && self.subscriber.is_empty() - } - pub fn on_pubsub_event(&mut self, event: pubsub::Event) { let channel_id = event.0; if let pubsub::ChannelEvent::SourceData(_, data) = event.1 { @@ -85,15 +82,21 @@ impl RoomMessageChannel { impl TaskSwitcherChild> for RoomMessageChannel { type Time = (); + fn is_empty(&self) -> bool { + self.publisher.is_empty() && self.subscriber.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { loop { match self.switcher.current()?.try_into().ok()? { TaskType::Publisher => { if let Some(out) = self.publisher.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -102,9 +105,7 @@ impl TaskSwitcherChild> for TaskType::Subscriber => { if let Some(out) = self.subscriber.pop_output((), &mut self.switcher) { if let Output::OnResourceEmpty = out { - if self.is_empty() { - return Some(Output::OnResourceEmpty); - } + // we dont need to forward OnResourceEmpty to parent } else { return Some(out); } @@ -115,9 +116,11 @@ impl TaskSwitcherChild> for } } -impl Drop for RoomMessageChannel { +impl Drop for RoomMessageChannel { fn drop(&mut self) { log::info!("[ClusterRoomDataChannel] Drop {}", self.room); + assert!(self.publisher.is_empty(), "MessageChannelPublisher not empty on drop {:?}", self.publisher); + assert!(self.subscriber.is_empty(), "MessageChannelSubscriber not empty on drop {:?}", self.subscriber); } } @@ -138,7 +141,7 @@ mod tests { endpoint::MessageChannelLabel, }; - #[test] + #[test_log::test] fn start_stop_publish() { let now = (); let room_id = 1.into(); @@ -165,7 +168,7 @@ mod tests { assert_eq!(room.pop_output(now), None); } - #[test] + #[test_log::test] fn sub_unsub() { let now = (); let room_id = 1.into(); @@ -208,7 +211,7 @@ mod tests { assert!(room.subscriber.is_empty()); } - #[test] + #[test_log::test] fn receive_data() { let now = (); let room_id = 1.into(); @@ -259,7 +262,7 @@ mod tests { assert_eq!(room.pop_output(now), None); } - #[test] + #[test_log::test] fn publish_data() { let now = (); let room_id = 1.into(); @@ -298,7 +301,7 @@ mod tests { assert_eq!(room.pop_output(now), None); } - #[test] + #[test_log::test] fn leave_room() { let now = (); let room_id = 1.into(); diff --git a/packages/media_core/src/cluster/room/message_channel/publisher.rs b/packages/media_core/src/cluster/room/message_channel/publisher.rs index e8ed4d02..7b571b00 100644 --- a/packages/media_core/src/cluster/room/message_channel/publisher.rs +++ b/packages/media_core/src/cluster/room/message_channel/publisher.rs @@ -16,11 +16,13 @@ use crate::{ use super::Output; -struct ChannelContainer { +#[derive(Debug)] +struct ChannelContainer { publishers: HashSet, } -pub struct MessageChannelPublisher { +#[derive(Debug)] +pub struct MessageChannelPublisher { _c: Count, room: ClusterRoomHash, channels: HashMap>, @@ -28,7 +30,7 @@ pub struct MessageChannelPublisher { queue: VecDeque>, } -impl MessageChannelPublisher { +impl MessageChannelPublisher { pub fn new(room: ClusterRoomHash) -> Self { Self { _c: Default::default(), @@ -39,10 +41,6 @@ impl MessageChannelPublisher { } } - pub fn is_empty(&self) -> bool { - self.queue.is_empty() && self.channels.is_empty() && self.publishers.is_empty() - } - pub fn on_channel_pub_start(&mut self, endpoint: Endpoint, label: &MessageChannelLabel) { log::info!("[ClusterRoomDataChannel {}/Publishers] publish start message channel", self.room); @@ -115,16 +113,25 @@ impl MessageChannelPublisher { impl TaskSwitcherChild> for MessageChannelPublisher { type Time = (); + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.channels.is_empty() && self.publishers.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for MessageChannelPublisher { +impl Drop for MessageChannelPublisher { fn drop(&mut self) { log::info!("[ClusterRoomDataChannel {}/Publishers] Drop", self.room); - assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.publishers.len(), 0, "Publishers not empty on drop"); - assert_eq!(self.channels.len(), 0, "Channels not empty on drop"); + assert_eq!(self.queue.len(), 0, "Queue not empty on drop {:?}", self.queue); + assert_eq!(self.publishers.len(), 0, "Publishers not empty on drop {:?}", self.publishers); + assert_eq!(self.channels.len(), 0, "Channels not empty on drop {:?}", self.channels); } } diff --git a/packages/media_core/src/cluster/room/message_channel/subscriber.rs b/packages/media_core/src/cluster/room/message_channel/subscriber.rs index ba630961..659146ed 100644 --- a/packages/media_core/src/cluster/room/message_channel/subscriber.rs +++ b/packages/media_core/src/cluster/room/message_channel/subscriber.rs @@ -15,12 +15,14 @@ use crate::{ endpoint::MessageChannelLabel, }; -struct ChannelContainer { +#[derive(Debug)] +struct ChannelContainer { subscribers: HashSet, label: MessageChannelLabel, } -pub struct MessageChannelSubscriber { +#[derive(Debug)] +pub struct MessageChannelSubscriber { _c: Count, room: ClusterRoomHash, channels: HashMap>, @@ -28,7 +30,7 @@ pub struct MessageChannelSubscriber { queue: VecDeque>, } -impl MessageChannelSubscriber { +impl MessageChannelSubscriber { pub fn new(room: ClusterRoomHash) -> Self { Self { _c: Default::default(), @@ -39,10 +41,6 @@ impl MessageChannelSubscriber { } } - pub fn is_empty(&self) -> bool { - self.queue.is_empty() && self.subscriptions.is_empty() && self.channels.is_empty() - } - pub fn on_channel_subscribe(&mut self, endpoint: Endpoint, label: &MessageChannelLabel) { log::info!("[ClusterRoomDataChannel {}/Subscribers] Subscribe channel", self.room); @@ -117,16 +115,25 @@ impl MessageChannelSubscriber { impl TaskSwitcherChild> for MessageChannelSubscriber { type Time = (); + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.subscriptions.is_empty() && self.channels.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for MessageChannelSubscriber { +impl Drop for MessageChannelSubscriber { fn drop(&mut self) { log::info!("[ClusterRoomDataChannel {}/Subscriber] Drop", self.room); assert_eq!(self.queue.len(), 0, "Queue not empty on drop"); - assert_eq!(self.subscriptions.len(), 0, "Subscribers not empty on drop"); - assert_eq!(self.channels.len(), 0, "Channels not not empty on drop"); + assert_eq!(self.subscriptions.len(), 0, "Subscribers not empty on drop {:?}", self.subscriptions); + assert_eq!(self.channels.len(), 0, "Channels not not empty on drop {:?}", self.channels); } } diff --git a/packages/media_core/src/cluster/room/metadata.rs b/packages/media_core/src/cluster/room/metadata.rs index 707c635e..82bde410 100644 --- a/packages/media_core/src/cluster/room/metadata.rs +++ b/packages/media_core/src/cluster/room/metadata.rs @@ -19,6 +19,7 @@ use crate::{ transport::RemoteTrackId, }; +#[derive(Debug)] struct PeerContainer { peer: PeerId, publish: RoomInfoPublish, @@ -33,7 +34,8 @@ pub enum Output { OnResourceEmpty, } -pub struct RoomMetadata { +#[derive(Debug)] +pub struct RoomMetadata { room: ClusterRoomHash, peers_map: Map, tracks_map: Map, @@ -47,7 +49,7 @@ pub struct RoomMetadata { queue: VecDeque>, } -impl RoomMetadata { +impl RoomMetadata { pub fn new(room: ClusterRoomHash) -> Self { Self { room, @@ -63,10 +65,6 @@ impl RoomMetadata { } } - pub fn is_empty(&self) -> bool { - self.peers.is_empty() && self.queue.is_empty() - } - pub fn get_peer_from_endpoint(&self, endpoint: Endpoint) -> Option { Some(self.peers.get(&endpoint)?.peer.clone()) } @@ -168,11 +166,6 @@ impl RoomMetadata { self.queue.push_back(Output::Kv(dht_kv::Control::MapCmd(target_peer_map, MapControl::Unsub))); } } - - if self.peers.is_empty() { - log::info!("[ClusterRoom {}] last peer leaed => destroy metadata", self.room); - self.queue.push_back(Output::OnResourceEmpty); - } } pub fn on_subscribe_peer(&mut self, endpoint: Endpoint, target: PeerId) { @@ -344,21 +337,30 @@ impl RoomMetadata { } } -impl TaskSwitcherChild> for RoomMetadata { +impl TaskSwitcherChild> for RoomMetadata { type Time = (); + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.peers.is_empty() && self.peers_map_subscribers.is_empty() && self.tracks_map_subscribers.is_empty() && self.peers_tracks_subs.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Self::Time) -> Option> { self.queue.pop_front() } } -impl Drop for RoomMetadata { +impl Drop for RoomMetadata { fn drop(&mut self) { log::info!("[ClusterRoomMetadata] Drop {}", self.room); - assert_eq!(self.queue.len(), 0, "Queue not empty"); - assert_eq!(self.peers.len(), 0, "Peers not empty"); - assert_eq!(self.peers_map_subscribers.len(), 0, "Peers subscriber not empty"); - assert_eq!(self.tracks_map_subscribers.len(), 0, "Tracks subscriber not empty"); - assert_eq!(self.peers_tracks_subs.len(), 0, "Peers tracks subs not empty"); + assert_eq!(self.queue.len(), 0, "Metadata Queue not empty {:?}", self.queue); + assert_eq!(self.peers.len(), 0, "Metadata Peers not empty {:?}", self.peers); + assert_eq!(self.peers_map_subscribers.len(), 0, "Metadata Peers subscriber not empty {:?}", self.peers_map_subscribers); + assert_eq!(self.tracks_map_subscribers.len(), 0, "Metadata Tracks subscriber not empty {:?}", self.tracks_map_subscribers); + assert_eq!(self.peers_tracks_subs.len(), 0, "Metadata Peers tracks subs not empty {:?}", self.peers_tracks_subs); } } @@ -376,7 +378,7 @@ mod tests { use super::{Output, RoomMetadata}; /// Test correct get peer info - #[test] + #[test_log::test] fn correct_get_peer() { let room: ClusterRoomHash = 1.into(); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); @@ -395,13 +397,13 @@ mod tests { assert_eq!(room_meta.get_peer_from_endpoint(2), None); room_meta.on_leave(endpoint); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } /// Test join as peer only => should subscribe peers, fire only peer /// After leave should unsubscribe only peers, and del - #[test] + #[test_log::test] fn join_peer_only() { let room: ClusterRoomHash = 1.into(); let peers_map = id_generator::peers_map(room); @@ -452,11 +454,11 @@ mod tests { room_meta.on_leave(endpoint); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Del(peer_key))))); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } - #[test] + #[test_log::test] fn join_sub_peer_only_should_restore_old_peers() { let room: ClusterRoomHash = 1.into(); let peers_map = id_generator::peers_map(room); @@ -488,12 +490,12 @@ mod tests { room_meta.on_leave(endpoint); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peers_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } //TODO Test join as track only => should subscribe only tracks, fire only track events - #[test] + #[test_log::test] fn join_track_only() { let room: ClusterRoomHash = 1.into(); let peers_map = id_generator::peers_map(room); @@ -548,12 +550,12 @@ mod tests { // peer leave should send unsub room_meta.on_leave(endpoint); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } //join track only should restore old tracks - #[test] + #[test_log::test] fn join_sub_track_only_should_restore_old_tracks() { let room: ClusterRoomHash = 1.into(); let tracks_map = id_generator::tracks_map(room); @@ -589,12 +591,12 @@ mod tests { room_meta.on_leave(endpoint); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } //Test manual no subscribe peer => dont fire any event - #[test] + #[test_log::test] fn join_manual_no_subscribe_peer() { let room: ClusterRoomHash = 1.into(); let peers_map = id_generator::peers_map(room); @@ -633,12 +635,12 @@ mod tests { // peer leave should send unsub room_meta.on_leave(endpoint); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } //TODO Test manual and subscribe peer => should fire event - #[test] + #[test_log::test] fn join_manual_with_subscribe() { let room: ClusterRoomHash = 1.into(); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); @@ -689,12 +691,12 @@ mod tests { // peer leave should not send unsub room_meta.on_leave(endpoint); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } //TODO Test track publish => should set key to both single peer map and tracks map - #[test] + #[test_log::test] fn track_publish_enable() { let room: ClusterRoomHash = 1.into(); let tracks_map = id_generator::tracks_map(room); @@ -736,12 +738,12 @@ mod tests { //should not pop anything after leave room_meta.on_leave(endpoint); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } //TODO Test track publish in disable mode => should not set key to both single peer map and tracks map - #[test] + #[test_log::test] fn track_publish_disable() { let room: ClusterRoomHash = 1.into(); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); @@ -770,12 +772,12 @@ mod tests { //should not pop anything after leave room_meta.on_leave(endpoint); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } /// Test leave room auto del remain remote tracks - #[test] + #[test_log::test] fn leave_room_auto_del_remote_tracks() { let room: ClusterRoomHash = 1.into(); let tracks_map = id_generator::tracks_map(room); @@ -813,12 +815,12 @@ mod tests { room_meta.on_leave(endpoint); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(tracks_map, MapControl::Del(track_key))))); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer_map, MapControl::Del(track_key))))); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } // Leave room auto unsub private peer maps - #[test] + #[test_log::test] fn leave_room_auto_unsub_private_peer_maps() { let room: ClusterRoomHash = 1.into(); let mut room_meta: RoomMetadata = RoomMetadata::::new(room); @@ -843,7 +845,7 @@ mod tests { // peer leave should send unsub of peer2_map room_meta.on_leave(endpoint); assert_eq!(room_meta.pop_output(()), Some(Output::Kv(Control::MapCmd(peer2_map, MapControl::Unsub)))); - assert_eq!(room_meta.pop_output(()), Some(Output::OnResourceEmpty)); assert_eq!(room_meta.pop_output(()), None); + assert!(room_meta.is_empty()); } } diff --git a/packages/media_core/src/endpoint.rs b/packages/media_core/src/endpoint.rs index 450bfde6..f0554b42 100644 --- a/packages/media_core/src/endpoint.rs +++ b/packages/media_core/src/endpoint.rs @@ -206,8 +206,8 @@ pub enum EndpointOutput { PeerEvent(AppId, u64, Instant, peer_event::Event), RecordEvent(u64, Instant, SessionRecordEvent), Ext(Ext), + OnResourceEmpty, Continue, - Destroy, } #[derive(num_enum::TryFromPrimitive, num_enum::IntoPrimitive)] @@ -232,6 +232,7 @@ pub struct Endpoint, ExtIn, ExtOut> { transport: TaskSwitcherBranch>, internal: TaskSwitcherBranch, switcher: TaskSwitcher, + shutdown: bool, _tmp: PhantomData<(ExtIn, ExtOut)>, } @@ -244,6 +245,7 @@ impl, ExtIn, ExtOut> Endpoint { transport: TaskSwitcherBranch::new(transport, TaskType::Transport), internal: TaskSwitcherBranch::new(EndpointInternal::new(cfg), TaskType::Internal), switcher: TaskSwitcher::new(2), + shutdown: false, _tmp: PhantomData, } } @@ -273,7 +275,12 @@ where } fn on_shutdown(&mut self, now: Instant) { - self.transport.input(&mut self.switcher).on_input(now, TransportInput::SystemClose); + if self.shutdown { + return; + } + self.shutdown = true; + self.internal.input(&mut self.switcher).on_shutdown(now); + self.transport.input(&mut self.switcher).on_shutdown(now); } } @@ -282,6 +289,15 @@ where T::Time: From, { type Time = Instant; + + fn is_empty(&self) -> bool { + self.internal.is_empty() && self.transport.is_empty() + } + + fn empty_event(&self) -> EndpointOutput { + EndpointOutput::OnResourceEmpty + } + fn pop_output(&mut self, now: Instant) -> Option> { loop { match self.switcher.current()?.try_into().ok()? { @@ -313,6 +329,10 @@ impl, ExtIn, ExtOut> Endpoint { self.internal.input(&mut self.switcher).on_transport_rpc(now, req_id, req); None } + TransportOutput::OnResourceEmpty => { + // we don't need to forward this event to parent, itself will fire OnResourceEmpty + Some(EndpointOutput::Continue) + } } } @@ -327,7 +347,10 @@ impl, ExtIn, ExtOut> Endpoint { None } InternalOutput::Cluster(room, control) => Some(EndpointOutput::Cluster(room, control)), - InternalOutput::Destroy => Some(EndpointOutput::Destroy), + InternalOutput::OnResourceEmpty => { + // we don't need to forward this event to parent, itself will fire OnResourceEmpty + Some(EndpointOutput::Continue) + } InternalOutput::PeerEvent(ts, event) => Some(EndpointOutput::PeerEvent(self.app.clone(), self.session_id, ts, event)), InternalOutput::RecordEvent(ts, event) => Some(EndpointOutput::RecordEvent(self.session_id, ts, event)), } diff --git a/packages/media_core/src/endpoint/internal.rs b/packages/media_core/src/endpoint/internal.rs index e55ad88e..68422878 100644 --- a/packages/media_core/src/endpoint/internal.rs +++ b/packages/media_core/src/endpoint/internal.rs @@ -9,7 +9,7 @@ use media_server_protocol::{ transport::RpcError, }; use media_server_utils::Small2dMap; -use sans_io_runtime::{return_if_none, return_if_some, TaskGroup, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; +use sans_io_runtime::{return_if_none, return_if_some, TaskGroup, TaskGroupOutput, TaskSwitcher, TaskSwitcherBranch, TaskSwitcherChild}; use crate::{ cluster::{ @@ -44,7 +44,7 @@ pub enum InternalOutput { RecordEvent(Instant, SessionRecordEvent), RpcRes(EndpointReqId, EndpointRes), Cluster(ClusterRoomHash, ClusterEndpointControl), - Destroy, + OnResourceEmpty, } type EndpointInternalWaitJoin = Option<(EndpointReqId, RoomId, PeerId, PeerMeta, RoomInfoPublish, RoomInfoSubscribe, Option)>; @@ -56,10 +56,11 @@ pub struct EndpointInternal { joined: Option<(ClusterRoomHash, RoomId, PeerId, Option)>, local_tracks_id: Small2dMap, remote_tracks_id: Small2dMap, - local_tracks: TaskSwitcherBranch, (usize, local_track::Output)>, - remote_tracks: TaskSwitcherBranch, (usize, remote_track::Output)>, + local_tracks: TaskSwitcherBranch, TaskGroupOutput>, + remote_tracks: TaskSwitcherBranch, TaskGroupOutput>, bitrate_allocator: TaskSwitcherBranch, queue: VecDeque, + shutdown: bool, switcher: TaskSwitcher, } @@ -75,6 +76,7 @@ impl EndpointInternal { remote_tracks: TaskSwitcherBranch::default(TaskType::RemoteTracks), bitrate_allocator: TaskSwitcherBranch::new(BitrateAllocator::new(cfg.max_ingress_bitrate, cfg.max_ingress_bitrate), TaskType::BitrateAllocator), queue: Default::default(), + shutdown: false, switcher: TaskSwitcher::new(3), cfg, } @@ -85,10 +87,46 @@ impl EndpointInternal { self.local_tracks.input(&mut self.switcher).on_tick(now); self.remote_tracks.input(&mut self.switcher).on_tick(now); } + + pub fn on_shutdown(&mut self, now: Instant) { + if self.shutdown { + return; + } + self.shutdown = true; + self.local_tracks.input(&mut self.switcher).on_shutdown(now); + self.remote_tracks.input(&mut self.switcher).on_shutdown(now); + + // after shutdown, we need to pop all the remaining tasks + while let Some(task) = self.switcher.current() { + match task.try_into().expect("Should valid task type") { + TaskType::BitrateAllocator => self.pop_bitrate_allocator(now), + TaskType::LocalTracks => self.pop_local_tracks(now), + TaskType::RemoteTracks => self.pop_remote_tracks(now), + } + } + + // if joined, send leave event + let (hash, room, peer, _) = return_if_none!(self.joined.take()); + self.queue.push_back(InternalOutput::Cluster(hash, ClusterEndpointControl::Leave)); + if self.cfg.record { + self.queue.push_back(InternalOutput::RecordEvent(now, SessionRecordEvent::LeaveRoom)); + } + self.queue + .push_back(InternalOutput::PeerEvent(now, peer_event::Event::Leave(peer_event::Leave { room: room.into(), peer: peer.into() }))); + } } impl TaskSwitcherChild for EndpointInternal { type Time = Instant; + + fn is_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() && self.local_tracks.is_empty() && self.remote_tracks.is_empty() + } + + fn empty_event(&self) -> InternalOutput { + InternalOutput::OnResourceEmpty + } + fn pop_output(&mut self, now: Instant) -> Option { return_if_some!(self.queue.pop_front()); @@ -301,7 +339,7 @@ impl EndpointInternal { error: 0, }), )); - self.queue.push_back(InternalOutput::Destroy); + self.on_shutdown(now); } TransportState::Connected(ip) => { log::info!("[EndpointInternal] connected"); @@ -334,8 +372,6 @@ impl EndpointInternal { } TransportState::Disconnected(err) => { log::info!("[EndpointInternal] disconnected {:?}", err); - self.leave_room(now); - self.clear_tracks(); self.queue.push_back(InternalOutput::PeerEvent( now, peer_event::Event::Disconnected(peer_event::Disconnected { duration_ms: 0, reason: 0 }), //TODO provide correct reason @@ -343,19 +379,19 @@ impl EndpointInternal { if self.cfg.record { self.queue.push_back(InternalOutput::RecordEvent(now, SessionRecordEvent::Disconnected)); } - self.queue.push_back(InternalOutput::Destroy); + self.on_shutdown(now); } } } fn on_transport_remote_track(&mut self, now: Instant, track: RemoteTrackId, event: RemoteTrackEvent) { - if let Some((name, priority, meta)) = event.need_create() { + if let Some((name, _priority, meta)) = event.need_create() { log::info!("[EndpointInternal] create remote track {:?}", track); let room = self.joined.as_ref().map(|j| j.0); let index = self .remote_tracks .input(&mut self.switcher) - .add_task(EndpointRemoteTrack::new(room, track, name, priority, meta, self.cfg.record)); + .add_task(EndpointRemoteTrack::new(room, track, name, meta, self.cfg.record)); self.remote_tracks_id.insert(track, index); } let index = return_if_none!(self.remote_tracks_id.get1(&track)); @@ -439,19 +475,6 @@ impl EndpointInternal { self.queue .push_back(InternalOutput::PeerEvent(now, peer_event::Event::Leave(peer_event::Leave { room: room.into(), peer: peer.into() }))); } - - /// only call when endpoint shutdown or disconnect - fn clear_tracks(&mut self) { - for (id, index) in self.local_tracks_id.pairs() { - self.local_tracks.input(&mut self.switcher).remove_task(index); - self.local_tracks_id.remove1(&id); - } - - for (id, index) in self.remote_tracks_id.pairs() { - self.remote_tracks.input(&mut self.switcher).remove_task(index); - self.remote_tracks_id.remove1(&id); - } - } } /// This block is for cluster related events @@ -488,7 +511,10 @@ impl EndpointInternal { /// This block for internal local and remote track impl EndpointInternal { fn pop_remote_tracks(&mut self, now: Instant) { - let (index, out) = return_if_none!(self.remote_tracks.pop_output(now, &mut self.switcher)); + let (index, out) = match return_if_none!(self.remote_tracks.pop_output(now, &mut self.switcher)) { + TaskGroupOutput::TaskOutput(index, out) => (index, out), + TaskGroupOutput::OnResourceEmpty => return, + }; let id = *self.remote_tracks_id.get2(&index).expect("Should have remote_track_id"); match out { @@ -516,7 +542,6 @@ impl EndpointInternal { self.bitrate_allocator.input(&mut self.switcher).del_ingress_video_track(id); } self.remote_tracks.input(&mut self.switcher).remove_task(index); - self.remote_tracks_id.remove1(&id); } remote_track::Output::PeerEvent(ts, event) => { self.queue.push_back(InternalOutput::PeerEvent(ts, event)); @@ -528,7 +553,10 @@ impl EndpointInternal { } fn pop_local_tracks(&mut self, now: Instant) { - let (index, out) = return_if_none!(self.local_tracks.pop_output(now, &mut self.switcher)); + let (index, out) = match return_if_none!(self.local_tracks.pop_output(now, &mut self.switcher)) { + TaskGroupOutput::TaskOutput(index, out) => (index, out), + TaskGroupOutput::OnResourceEmpty => return, + }; let id = *self.local_tracks_id.get2(&index).expect("Should have local_track_id"); match out { local_track::Output::Event(event) => { @@ -540,8 +568,8 @@ impl EndpointInternal { local_track::Output::RpcRes(req_id, res) => { self.queue.push_back(InternalOutput::RpcRes(req_id, EndpointRes::LocalTrack(id, res))); } - local_track::Output::Started(kind, priority) => { - log::info!("[EndpointInternal] local track started {kind} priority {priority}"); + local_track::Output::Bind(kind, priority) => { + log::info!("[EndpointInternal] local track bind {kind} priority {priority}"); if kind.is_video() { self.bitrate_allocator.input(&mut self.switcher).set_egress_video_track(id, priority); } @@ -551,8 +579,8 @@ impl EndpointInternal { self.bitrate_allocator.input(&mut self.switcher).set_egress_video_track(id, priority); } } - local_track::Output::Stopped(kind) => { - log::info!("[EndpointInternal] local track stopped {kind}"); + local_track::Output::Unbind(kind) => { + log::info!("[EndpointInternal] local track unbind {kind}"); if kind.is_video() { self.bitrate_allocator.input(&mut self.switcher).del_egress_video_track(id); } @@ -560,6 +588,10 @@ impl EndpointInternal { local_track::Output::PeerEvent(ts, event) => { self.queue.push_back(InternalOutput::PeerEvent(ts, event)); } + local_track::Output::OnResourceEmpty => { + self.local_tracks.input(&mut self.switcher).remove_task(index); + self.local_tracks_id.remove1(&id); + } } } @@ -612,7 +644,7 @@ mod tests { use super::EndpointInternal; - #[test] + #[test_log::test] fn test_join_leave_room_success() { let app = AppContext::root_app(); let mut internal = EndpointInternal::new(EndpointCfg { @@ -732,7 +764,7 @@ mod tests { assert_eq!(internal.pop_output(now), None); } - #[test] + #[test_log::test] fn test_join_overwrite_auto_leave() { let app = AppContext::root_app(); let mut internal = EndpointInternal::new(EndpointCfg { diff --git a/packages/media_core/src/endpoint/internal/bitrate_allocator.rs b/packages/media_core/src/endpoint/internal/bitrate_allocator.rs index 50d87c03..d51d47f0 100644 --- a/packages/media_core/src/endpoint/internal/bitrate_allocator.rs +++ b/packages/media_core/src/endpoint/internal/bitrate_allocator.rs @@ -60,6 +60,15 @@ impl BitrateAllocator { impl TaskSwitcherChild for BitrateAllocator { type Time = Instant; + fn is_empty(&self) -> bool { + // we never send empty event + false + } + + fn empty_event(&self) -> Output { + unreachable!() + } + fn pop_output(&mut self, _now: Instant) -> Option { if let Some(out) = self.egress.pop_output() { let out = match out { diff --git a/packages/media_core/src/endpoint/internal/bitrate_allocator/egress.rs b/packages/media_core/src/endpoint/internal/bitrate_allocator/egress.rs index 918ff956..4b883a9b 100644 --- a/packages/media_core/src/endpoint/internal/bitrate_allocator/egress.rs +++ b/packages/media_core/src/endpoint/internal/bitrate_allocator/egress.rs @@ -108,7 +108,7 @@ mod test { const MAX_BW: u64 = 2_500_000; - #[test] + #[test_log::test] fn no_source() { let mut allocator = EgressBitrateAllocator::new(MAX_BW); allocator.set_egress_estimate(200_000); @@ -118,7 +118,7 @@ mod test { assert_eq!(allocator.pop_output(), None); } - #[test] + #[test_log::test] fn single_source() { let mut allocator = EgressBitrateAllocator::new(MAX_BW); allocator.set_video_track(0.into(), 1.into()); @@ -136,7 +136,7 @@ mod test { assert_eq!(allocator.pop_output(), None); } - #[test] + #[test_log::test] fn multi_source() { let mut allocator = EgressBitrateAllocator::new(MAX_BW); allocator.set_video_track(0.into(), 1.into()); diff --git a/packages/media_core/src/endpoint/internal/bitrate_allocator/ingress.rs b/packages/media_core/src/endpoint/internal/bitrate_allocator/ingress.rs index e07adeb6..a4b4395a 100644 --- a/packages/media_core/src/endpoint/internal/bitrate_allocator/ingress.rs +++ b/packages/media_core/src/endpoint/internal/bitrate_allocator/ingress.rs @@ -74,7 +74,7 @@ mod test { const TEST_BITRATE: u64 = 2_000_000; - #[test] + #[test_log::test] fn single_source() { let mut allocator = IngressBitrateAllocator::new(TEST_BITRATE); allocator.set_video_track(0.into(), 1.into()); @@ -83,7 +83,7 @@ mod test { assert_eq!(allocator.pop_output(), Some((0.into(), Action::SetBitrate(TEST_BITRATE)))); } - #[test] + #[test_log::test] fn multi_source() { let mut allocator = IngressBitrateAllocator::new(TEST_BITRATE); allocator.set_video_track(0.into(), 1.into()); diff --git a/packages/media_core/src/endpoint/internal/local_track.rs b/packages/media_core/src/endpoint/internal/local_track.rs index 737bfd4a..edf31b97 100644 --- a/packages/media_core/src/endpoint/internal/local_track.rs +++ b/packages/media_core/src/endpoint/internal/local_track.rs @@ -11,7 +11,6 @@ use media_server_protocol::{ protobuf::{cluster_connector::peer_event, shared::receiver::Status as ProtoStatus}, transport::{LocalTrackId, RpcError}, }; -use media_server_utils::Count; use sans_io_runtime::{return_if_none, Task, TaskSwitcherChild}; use crate::{ @@ -45,9 +44,10 @@ pub enum Output { Cluster(ClusterRoomHash, ClusterLocalTrackControl), PeerEvent(Instant, peer_event::Event), RpcRes(EndpointReqId, EndpointLocalTrackRes), - Started(MediaKind, TrackPriority), + Bind(MediaKind, TrackPriority), Updated(MediaKind, TrackPriority), - Stopped(MediaKind), + Unbind(MediaKind), + OnResourceEmpty, } #[derive(Debug, PartialEq, Eq)] @@ -58,7 +58,6 @@ enum Status { } pub struct EndpointLocalTrack { - _c: Count, track: LocalTrackId, kind: MediaKind, room: Option, @@ -67,13 +66,13 @@ pub struct EndpointLocalTrack { selector: PacketSelector, timer: TimePivot, voice_activity: VoiceActivityDetector, + shutdown: bool, } impl EndpointLocalTrack { pub fn new(track: LocalTrackId, kind: MediaKind, room: Option) -> Self { log::info!("[EndpointLocalTrack] track {kind}, room {:?}", room); Self { - _c: Default::default(), track, kind, room, @@ -82,6 +81,7 @@ impl EndpointLocalTrack { selector: PacketSelector::new(kind, 2, 2), timer: TimePivot::build(), voice_activity: VoiceActivityDetector::default(), + shutdown: false, } } @@ -184,11 +184,11 @@ impl EndpointLocalTrack { self.kind ); self.queue.push_back(Output::Cluster(*room, ClusterLocalTrackControl::Unsubscribe)); - self.queue.push_back(Output::Stopped(self.kind)); + self.queue.push_back(Output::Unbind(self.kind)); } self.bind = Some((peer.clone(), track.clone(), Status::Waiting)); self.selector.set_limit_layer(now_ms, config.max_spatial, config.max_temporal); - self.queue.push_back(Output::Started(self.kind, config.priority)); + self.queue.push_back(Output::Bind(self.kind, config.priority)); self.queue.push_back(Output::Cluster(*room, ClusterLocalTrackControl::Subscribe(peer.clone(), track.clone()))); self.queue.push_back(Output::PeerEvent( now, @@ -211,7 +211,7 @@ impl EndpointLocalTrack { if let Some((peer, track, _)) = self.bind.take() { log::info!("[EndpointLocalTrack] unview room {room} peer {peer} track {track}"); self.queue.push_back(Output::RpcRes(req_id, EndpointLocalTrackRes::Detach(Ok(())))); - self.queue.push_back(Output::Stopped(self.kind)); + self.queue.push_back(Output::Unbind(self.kind)); self.queue.push_back(Output::Cluster(*room, ClusterLocalTrackControl::Unsubscribe)); self.queue.push_back(Output::PeerEvent( now, @@ -298,11 +298,28 @@ impl Task for EndpointLocalTrack { } } - fn on_shutdown(&mut self, _now: Instant) {} + fn on_shutdown(&mut self, now: Instant) { + if self.shutdown { + return; + } + self.shutdown = true; + if self.room.is_some() { + self.on_leave_room(now); + } + } } impl TaskSwitcherChild for EndpointLocalTrack { type Time = Instant; + + fn is_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + + fn empty_event(&self) -> Output { + Output::OnResourceEmpty + } + fn pop_output(&mut self, _now: Instant) -> Option { self.queue.pop_front() } diff --git a/packages/media_core/src/endpoint/internal/local_track/packet_selector.rs b/packages/media_core/src/endpoint/internal/local_track/packet_selector.rs index 5991baae..8cefabd9 100644 --- a/packages/media_core/src/endpoint/internal/local_track/packet_selector.rs +++ b/packages/media_core/src/endpoint/internal/local_track/packet_selector.rs @@ -277,7 +277,7 @@ mod tests { } } - #[test] + #[test_log::test] fn audio_should_not_request_key_frame() { let mut selector = PacketSelector::new(MediaKind::Audio, 2, 2); @@ -286,7 +286,7 @@ mod tests { assert_eq!(selector.pop_output(0), None); } - #[test] + #[test_log::test] fn video_should_not_request_key_frame_with_first_is_key() { let mut selector = PacketSelector::new(MediaKind::Video, 2, 2); @@ -297,7 +297,7 @@ mod tests { assert_eq!(selector.pop_output(0), None); } - #[test] + #[test_log::test] fn video_should_request_key_frame_with_first_is_not_key() { let mut selector = PacketSelector::new(MediaKind::Video, 2, 2); @@ -325,6 +325,6 @@ mod tests { assert_eq!(selector.pop_output(0), None); } - #[test] + #[test_log::test] fn pkt_rewrite_after_switch_channel() {} } diff --git a/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_h264_sim.rs b/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_h264_sim.rs index 14aedba7..debe5bb6 100644 --- a/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_h264_sim.rs +++ b/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_h264_sim.rs @@ -222,7 +222,7 @@ mod tests { } } - #[test] + #[test_log::test] fn up_spatial() { test( 200, @@ -247,7 +247,7 @@ mod tests { ) } - #[test] + #[test_log::test] fn down_spatial() { test( 800, diff --git a/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp8_sim.rs b/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp8_sim.rs index a9cba716..833e0317 100644 --- a/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp8_sim.rs +++ b/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp8_sim.rs @@ -324,7 +324,7 @@ mod tests { } /// Test if first arrived pkt match filter - #[test] + #[test_log::test] fn start_low_layer() { let channel = 0; test( @@ -335,7 +335,7 @@ mod tests { } /// Test if first arrived pkt not match filter - #[test] + #[test_log::test] fn start_high_layer() { let channel = 0; test( @@ -363,7 +363,7 @@ mod tests { } /// Test with up temporal, need to wait pkt with layer_sync flag - #[test] + #[test_log::test] fn up_temporal_wait_layer_sync() { let channel = 0; test( @@ -387,7 +387,7 @@ mod tests { } /// Test with down temporal - #[test] + #[test_log::test] fn down_temporal() { let channel = 0; test( @@ -410,7 +410,7 @@ mod tests { } /// Test with up temporal, need to wait pkt with layer_sync flag - #[test] + #[test_log::test] fn up_spatial_wait_key_frame() { let channel = 0; test( @@ -449,7 +449,7 @@ mod tests { } /// Test with up temporal, need to wait pkt with layer_sync flag - #[test] + #[test_log::test] fn down_spatial_wait_key_frame() { let channel = 0; test( diff --git a/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp9_svc.rs b/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp9_svc.rs index d6365a96..fb8ac6e6 100644 --- a/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp9_svc.rs +++ b/packages/media_core/src/endpoint/internal/local_track/packet_selector/video_vp9_svc.rs @@ -342,7 +342,7 @@ mod tests { } /// Test if first arrived pkt match filter - #[test] + #[test_log::test] fn start_low_layer() { let channel = 0; test( @@ -353,7 +353,7 @@ mod tests { } /// Test if first arrived pkt not match filter - #[test] + #[test_log::test] fn start_high_layer() { let channel = 0; test( @@ -383,7 +383,7 @@ mod tests { } /// Test with up temporal, need to wait current frame end and switching point - #[test] + #[test_log::test] fn up_temporal_wait_layer_sync() { let channel = 0; test( @@ -407,7 +407,7 @@ mod tests { } /// Test with down temporal - #[test] + #[test_log::test] fn down_temporal() { let channel = 0; test( @@ -430,7 +430,7 @@ mod tests { } /// Test with up temporal, need to wait pkt with layer_sync flag - #[test] + #[test_log::test] fn up_spatial_wait_key_frame() { let c = 0; test( @@ -469,7 +469,7 @@ mod tests { } /// Test with down spatial, need to wait key-frame for non k-svc type => dont need wait key-frame, only endframe - #[test] + #[test_log::test] fn down_spatial_wait_end_frame() { let c = 0; test( diff --git a/packages/media_core/src/endpoint/internal/remote_track.rs b/packages/media_core/src/endpoint/internal/remote_track.rs index 1e01774b..aff1971c 100644 --- a/packages/media_core/src/endpoint/internal/remote_track.rs +++ b/packages/media_core/src/endpoint/internal/remote_track.rs @@ -1,43 +1,15 @@ //! RemoteTrack take care about publish local media to sdn, and react with feedback from consumers -//! -//! State Machine Diagram: -//! ```ascii -//! JoinRoom -//! ┌─────────────────────────────────┐ -//! │ ▼ -//! ┌───────────┐ ┌─────────┐ -//! │ Waiting │ │ │ -//! │ JoinRoom │◄───────────────────│ InRoom │ -//! └───────────┘ LeaveRoom │ │ -//! │ └─────────┘ -//! │ │ -//! │ TrackEnded │ -//! │ │ -//! │ ┌─────────┐ │ -//! └──────────►│ │◄───────┘ -//! │ Stopped │ -//! │ │ -//! └─────────┘ -//! ``` -//! -//! State Transitions: -//! - WaitingJoinRoom -> InRoom: via JoinRoom event -//! - InRoom -> WaitingJoinRoom: via LeaveRoom event -//! - WaitingJoinRoom/InRoom -> Stopped: via TrackEnded event -//! - Stopped: Terminal state, no transitions out -//! use std::{collections::VecDeque, time::Instant}; use media_server_protocol::{ endpoint::{BitrateControlMode, TrackMeta, TrackName, TrackPriority}, - media::{MediaKind, MediaLayersBitrate, MediaPacket}, + media::{MediaKind, MediaLayersBitrate}, protobuf::{cluster_connector::peer_event, shared::Kind}, record::SessionRecordEvent, transport::{RemoteTrackId, RpcError}, }; -use media_server_utils::Count; -use sans_io_runtime::{Task, TaskSwitcherChild}; +use sans_io_runtime::{return_if_none, Task, TaskSwitcherChild}; use crate::{ cluster::{ClusterRemoteTrackControl, ClusterRemoteTrackEvent, ClusterRoomHash}, @@ -69,427 +41,185 @@ pub enum Output { Stopped(MediaKind), } -struct StateContext { +pub struct EndpointRemoteTrack { id: RemoteTrackId, - name: TrackName, meta: TrackMeta, - priority: TrackPriority, + room: Option, + name: TrackName, queue: VecDeque, + allocate_bitrate: Option, /// This is for storing current stream layers, everytime key-frame arrived we will set this if it not set last_layers: Option, cluster_bitrate_limit: Option<(u64, u64)>, record: bool, - allocate_bitrate: Option, - next_state: Option, -} - -impl StateContext { - fn calc_limit_bitrate(&self) -> Option<(u64, u64)> { - let cluster_limit = self.meta.control.eq(&BitrateControlMode::DynamicConsumers).then_some(self.cluster_bitrate_limit).flatten(); - match (self.allocate_bitrate, cluster_limit) { - (Some(b1), Some((min, max))) => Some((min.min(b1), max.min(b1))), - (Some(b1), None) => Some((b1, b1)), - (None, Some((min, max))) => Some((min, max)), - (None, None) => None, - } - } - - fn limit_bitrate(&mut self, action: IngressAction) { - match action { - IngressAction::SetBitrate(bitrate) => { - log::info!("[EndpointRemoteTrack] on allocation bitrate {bitrate}"); - self.allocate_bitrate = Some(bitrate); - if let Some((min, max)) = self.calc_limit_bitrate() { - self.queue.push_back(Output::Event(EndpointRemoteTrackEvent::LimitBitrateBps { min, max })) - } - } - } - } -} - -trait StateLogic { - fn on_join_room(&mut self, ctx: &mut StateContext, now: Instant, room: ClusterRoomHash); - fn on_leave_room(&mut self, ctx: &mut StateContext, now: Instant); - fn on_track_started(&mut self, ctx: &mut StateContext, now: Instant); - fn on_track_media(&mut self, ctx: &mut StateContext, now: Instant, media: MediaPacket); - fn on_track_ended(&mut self, ctx: &mut StateContext, now: Instant); - fn on_rpc_req(&mut self, ctx: &mut StateContext, now: Instant, req_id: EndpointReqId, req: EndpointRemoteTrackReq); - fn on_cluster_event(&mut self, ctx: &mut StateContext, now: Instant, event: ClusterRemoteTrackEvent); - fn on_bitrate_allocation(&mut self, ctx: &mut StateContext, now: Instant, action: IngressAction); - fn pop_output(&mut self, ctx: &mut StateContext, now: Instant) -> Option; -} - -struct WaitingJoinRoom; -struct InRoom { - room: ClusterRoomHash, -} -struct Stopped { - waiting: bool, -} - -enum State { - WaitingJoinRoom(WaitingJoinRoom), - InRoom(InRoom), - Stopped(Stopped), -} -impl StateLogic for State { - fn on_join_room(&mut self, ctx: &mut StateContext, now: Instant, room: ClusterRoomHash) { - match self { - State::WaitingJoinRoom(state) => state.on_join_room(ctx, now, room), - State::InRoom(state) => state.on_join_room(ctx, now, room), - State::Stopped(state) => state.on_join_room(ctx, now, room), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_leave_room(&mut self, ctx: &mut StateContext, now: Instant) { - match self { - State::WaitingJoinRoom(state) => state.on_leave_room(ctx, now), - State::InRoom(state) => state.on_leave_room(ctx, now), - State::Stopped(state) => state.on_leave_room(ctx, now), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_track_started(&mut self, ctx: &mut StateContext, now: Instant) { - match self { - State::WaitingJoinRoom(state) => state.on_track_started(ctx, now), - State::InRoom(state) => state.on_track_started(ctx, now), - State::Stopped(state) => state.on_track_started(ctx, now), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_track_media(&mut self, ctx: &mut StateContext, now: Instant, media: MediaPacket) { - match self { - State::WaitingJoinRoom(state) => state.on_track_media(ctx, now, media), - State::InRoom(state) => state.on_track_media(ctx, now, media), - State::Stopped(state) => state.on_track_media(ctx, now, media), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_track_ended(&mut self, ctx: &mut StateContext, now: Instant) { - match self { - State::WaitingJoinRoom(state) => state.on_track_ended(ctx, now), - State::InRoom(state) => state.on_track_ended(ctx, now), - State::Stopped(state) => state.on_track_ended(ctx, now), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_rpc_req(&mut self, ctx: &mut StateContext, now: Instant, req_id: EndpointReqId, req: EndpointRemoteTrackReq) { - match self { - State::WaitingJoinRoom(state) => state.on_rpc_req(ctx, now, req_id, req), - State::InRoom(state) => state.on_rpc_req(ctx, now, req_id, req), - State::Stopped(state) => state.on_rpc_req(ctx, now, req_id, req), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_cluster_event(&mut self, ctx: &mut StateContext, now: Instant, event: ClusterRemoteTrackEvent) { - match self { - State::WaitingJoinRoom(state) => state.on_cluster_event(ctx, now, event), - State::InRoom(state) => state.on_cluster_event(ctx, now, event), - State::Stopped(state) => state.on_cluster_event(ctx, now, event), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn on_bitrate_allocation(&mut self, ctx: &mut StateContext, now: Instant, action: IngressAction) { - match self { - State::WaitingJoinRoom(state) => state.on_bitrate_allocation(ctx, now, action), - State::InRoom(state) => state.on_bitrate_allocation(ctx, now, action), - State::Stopped(state) => state.on_bitrate_allocation(ctx, now, action), - } - - if let Some(next_state) = ctx.next_state.take() { - *self = next_state; - } - } - - fn pop_output(&mut self, ctx: &mut StateContext, now: Instant) -> Option { - match self { - State::WaitingJoinRoom(state) => state.pop_output(ctx, now), - State::InRoom(state) => state.pop_output(ctx, now), - State::Stopped(state) => state.pop_output(ctx, now), - } - } + shutdown: bool, } -impl StateLogic for WaitingJoinRoom { - fn on_join_room(&mut self, ctx: &mut StateContext, now: Instant, room: ClusterRoomHash) { - log::info!("[EndpointRemoteTrack] join room {room}"); - let name = ctx.name.clone(); +impl EndpointRemoteTrack { + pub fn new(room: Option, id: RemoteTrackId, name: TrackName, meta: TrackMeta, record: bool) -> Self { + log::info!("[EndpointRemoteTrack] created with room {:?} meta {:?}", room, meta); + Self { + id, + meta, + room, + name, + queue: VecDeque::new(), + allocate_bitrate: None, + last_layers: None, + cluster_bitrate_limit: None, + record, + shutdown: false, + } + } + + fn on_join_room(&mut self, now: Instant, room: ClusterRoomHash) { + assert_eq!(self.room, None); + let name = self.name.clone(); + self.room = Some(room); + log::info!("[EndpointRemoteTrack] join room {room} as name {name}"); log::info!("[EndpointRemoteTrack] started as name {name} after join room"); - ctx.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Started(name.clone(), ctx.meta.clone()))); - if ctx.record { - ctx.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStarted(ctx.id, name.clone(), ctx.meta.clone()))); + self.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Started(name.clone(), self.meta.clone()))); + if self.record { + self.queue + .push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStarted(self.id, name.clone(), self.meta.clone()))); } - ctx.queue.push_back(Output::PeerEvent( + self.queue.push_back(Output::PeerEvent( now, peer_event::Event::RemoteTrackStarted(peer_event::RemoteTrackStarted { track: name.into(), - kind: Kind::from(ctx.meta.kind) as i32, + kind: Kind::from(self.meta.kind) as i32, }), )); - ctx.next_state = Some(State::InRoom(InRoom { room })); } - fn on_leave_room(&mut self, _ctx: &mut StateContext, _now: Instant) { - log::warn!("[EndpointRemoteTrack] leave room but not in room"); - } - - fn on_track_started(&mut self, ctx: &mut StateContext, _now: Instant) { - ctx.queue.push_back(Output::Started(ctx.meta.kind, ctx.priority)); - } - - fn on_track_media(&mut self, _ctx: &mut StateContext, _now: Instant, _media: MediaPacket) {} - - fn on_track_ended(&mut self, ctx: &mut StateContext, _now: Instant) { - ctx.next_state = Some(State::Stopped(Stopped { waiting: true })); - } - - fn on_rpc_req(&mut self, ctx: &mut StateContext, _now: Instant, req_id: EndpointReqId, req: EndpointRemoteTrackReq) { - match req { - EndpointRemoteTrackReq::Config(config) => { - if *config.priority == 0 { - log::warn!("[EndpointRemoteTrack] view with invalid priority"); - ctx.queue - .push_back(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Err(RpcError::new2(EndpointErrors::RemoteTrackInvalidPriority))))); - } else { - ctx.meta.control = config.control; - ctx.queue.push_back(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Ok(())))); - ctx.queue.push_back(Output::Update(ctx.meta.kind, config.priority)); - } - } - } - } - - fn on_cluster_event(&mut self, _ctx: &mut StateContext, _now: Instant, _event: ClusterRemoteTrackEvent) { - log::warn!("[EndpointRemoteTrack] on cluster event but not in room"); - } - - fn on_bitrate_allocation(&mut self, ctx: &mut StateContext, _now: Instant, action: IngressAction) { - ctx.limit_bitrate(action); - } - - fn pop_output(&mut self, ctx: &mut StateContext, _now: Instant) -> Option { - ctx.queue.pop_front() - } -} - -impl StateLogic for InRoom { - fn on_join_room(&mut self, _ctx: &mut StateContext, _now: Instant, _room: ClusterRoomHash) { - log::warn!("[EndpointRemoteTrack] join room but already in room"); - } - - fn on_leave_room(&mut self, ctx: &mut StateContext, now: Instant) { - let room = self.room; - log::info!("[EndpointRemoteTrack] leave room {room}"); - let name = ctx.name.clone(); + fn on_leave_room(&mut self, now: Instant) { + let room = self.room.take().expect("Must have room here"); + let name = self.name.clone(); + log::info!("[EndpointRemoteTrack] leave room {room} as name {name}"); log::info!("[EndpointRemoteTrack] stopped as name {name} after leave room"); - ctx.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Ended(name.clone(), ctx.meta.clone()))); - if ctx.record { - ctx.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStopped(ctx.id))); + self.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Ended(name.clone(), self.meta.clone()))); + if self.record { + self.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStopped(self.id))); } - ctx.queue.push_back(Output::PeerEvent( + self.queue.push_back(Output::PeerEvent( now, peer_event::Event::RemoteTrackEnded(peer_event::RemoteTrackEnded { track: name.into(), - kind: Kind::from(ctx.meta.kind) as i32, + kind: Kind::from(self.meta.kind) as i32, }), )); - ctx.next_state = Some(State::WaitingJoinRoom(WaitingJoinRoom)); } - fn on_track_started(&mut self, ctx: &mut StateContext, now: Instant) { - let room = self.room; - let name = ctx.name.clone(); - log::info!("[EndpointRemoteTrack] started as name {name} with room {room}"); - ctx.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Started(name.clone(), ctx.meta.clone()))); - ctx.queue.push_back(Output::Started(ctx.meta.kind, ctx.priority)); - if ctx.record { - ctx.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStarted(ctx.id, name.clone(), ctx.meta.clone()))); + fn on_cluster_event(&mut self, _now: Instant, event: ClusterRemoteTrackEvent) { + match event { + ClusterRemoteTrackEvent::RequestKeyFrame => self.queue.push_back(Output::Event(EndpointRemoteTrackEvent::RequestKeyFrame)), + ClusterRemoteTrackEvent::LimitBitrate { min, max } => { + self.cluster_bitrate_limit = Some((min, max)); + if self.meta.control.eq(&BitrateControlMode::DynamicConsumers) { + if let Some((min, max)) = self.calc_limit_bitrate() { + self.queue.push_back(Output::Event(EndpointRemoteTrackEvent::LimitBitrateBps { min, max })); + } + } + } } - ctx.queue.push_back(Output::PeerEvent( - now, - peer_event::Event::RemoteTrackStarted(peer_event::RemoteTrackStarted { - track: name.into(), - kind: Kind::from(ctx.meta.kind) as i32, - }), - )); } - fn on_track_media(&mut self, ctx: &mut StateContext, now: Instant, mut media: MediaPacket) { - // We restore last_layer if key frame not contain for allow consumers fast switching - if media.meta.is_video_key() && media.layers.is_none() && ctx.last_layers.is_some() { - log::debug!("[EndpointRemoteTrack] set layers info to key-frame {:?}", media.layers); - media.layers.clone_from(&ctx.last_layers); - } + fn on_transport_event(&mut self, now: Instant, event: RemoteTrackEvent) { + match event { + RemoteTrackEvent::Started { name, priority, meta } => { + let room = return_if_none!(self.room.as_ref()); + log::info!("[EndpointRemoteTrack] started as name {name} in room {room}"); + self.queue.push_back(Output::Cluster(*room, ClusterRemoteTrackControl::Started(name.clone().into(), self.meta.clone()))); + self.queue.push_back(Output::Started(self.meta.kind, priority)); + if self.record { + self.queue + .push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStarted(self.id, name.clone().into(), self.meta.clone()))); + } + self.queue.push_back(Output::PeerEvent( + now, + peer_event::Event::RemoteTrackStarted(peer_event::RemoteTrackStarted { + track: name, + kind: Kind::from(meta.kind) as i32, + }), + )); + } + RemoteTrackEvent::Paused => {} + RemoteTrackEvent::Resumed => {} + RemoteTrackEvent::Media(mut media) => { + //TODO clear self.last_layer if switched to new track + if media.layers.is_some() { + log::debug!("[EndpointRemoteTrack] on layers info {:?}", media.layers); + self.last_layers.clone_from(&media.layers); + } - if ctx.record { - ctx.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackMedia(ctx.id, media.clone()))); - } + // We restore last_layer if key frame not contain for allow consumers fast switching + if media.meta.is_video_key() && media.layers.is_none() && self.last_layers.is_some() { + log::debug!("[EndpointRemoteTrack] set layers info to key-frame {:?}", media.layers); + media.layers.clone_from(&self.last_layers); + } - ctx.queue.push_back(Output::Cluster(self.room, ClusterRemoteTrackControl::Media(media))); - } + if self.record { + self.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackMedia(self.id, media.clone()))); + } - fn on_track_ended(&mut self, ctx: &mut StateContext, now: Instant) { - let room = self.room; - let name = ctx.name.clone(); - log::info!("[EndpointRemoteTrack] stopped with name {name} in room {room}"); - ctx.queue.push_back(Output::Cluster(room, ClusterRemoteTrackControl::Ended(name.clone(), ctx.meta.clone()))); - if ctx.record { - ctx.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStopped(ctx.id))); + let room = return_if_none!(self.room.as_ref()); + self.queue.push_back(Output::Cluster(*room, ClusterRemoteTrackControl::Media(media))); + } + RemoteTrackEvent::Ended => { + let name = self.name.clone(); + let room = return_if_none!(self.room.as_ref()); + log::info!("[EndpointRemoteTrack] stopped with name {name} in room {room}"); + self.queue.push_back(Output::Cluster(*room, ClusterRemoteTrackControl::Ended(name.clone(), self.meta.clone()))); + if self.record { + self.queue.push_back(Output::RecordEvent(now, SessionRecordEvent::TrackStopped(self.id))); + } + self.queue.push_back(Output::PeerEvent( + now, + peer_event::Event::RemoteTrackEnded(peer_event::RemoteTrackEnded { + track: name.into(), + kind: Kind::from(self.meta.kind) as i32, + }), + )); + self.shutdown = true; + } } - ctx.queue.push_back(Output::PeerEvent( - now, - peer_event::Event::RemoteTrackEnded(peer_event::RemoteTrackEnded { - track: name.into(), - kind: Kind::from(ctx.meta.kind) as i32, - }), - )); - ctx.next_state = Some(State::Stopped(Stopped { waiting: true })); } - fn on_rpc_req(&mut self, ctx: &mut StateContext, _now: Instant, req_id: EndpointReqId, req: EndpointRemoteTrackReq) { + fn on_rpc_req(&mut self, _now: Instant, req_id: EndpointReqId, req: EndpointRemoteTrackReq) { match req { EndpointRemoteTrackReq::Config(config) => { if *config.priority == 0 { log::warn!("[EndpointRemoteTrack] view with invalid priority"); - ctx.queue + self.queue .push_back(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Err(RpcError::new2(EndpointErrors::RemoteTrackInvalidPriority))))); } else { - ctx.meta.control = config.control; - ctx.queue.push_back(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Ok(())))); - ctx.queue.push_back(Output::Update(ctx.meta.kind, config.priority)); + self.meta.control = config.control; + self.queue.push_back(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Ok(())))); + self.queue.push_back(Output::Update(self.meta.kind, config.priority)); } } } } - fn on_cluster_event(&mut self, ctx: &mut StateContext, _now: Instant, event: ClusterRemoteTrackEvent) { - match event { - ClusterRemoteTrackEvent::RequestKeyFrame => ctx.queue.push_back(Output::Event(EndpointRemoteTrackEvent::RequestKeyFrame)), - ClusterRemoteTrackEvent::LimitBitrate { min, max } => { - ctx.cluster_bitrate_limit = Some((min, max)); - if ctx.meta.control.eq(&BitrateControlMode::DynamicConsumers) { - if let Some((min, max)) = ctx.calc_limit_bitrate() { - ctx.queue.push_back(Output::Event(EndpointRemoteTrackEvent::LimitBitrateBps { min, max })); - } + fn on_bitrate_allocation_action(&mut self, _now: Instant, action: IngressAction) { + match action { + IngressAction::SetBitrate(bitrate) => { + log::info!("[EndpointRemoteTrack] on allocation bitrate {bitrate}"); + self.allocate_bitrate = Some(bitrate); + if let Some((min, max)) = self.calc_limit_bitrate() { + self.queue.push_back(Output::Event(EndpointRemoteTrackEvent::LimitBitrateBps { min, max })) } } } } - fn on_bitrate_allocation(&mut self, ctx: &mut StateContext, _now: Instant, action: IngressAction) { - ctx.limit_bitrate(action); - } - - fn pop_output(&mut self, ctx: &mut StateContext, _now: Instant) -> Option { - ctx.queue.pop_front() - } -} - -impl StateLogic for Stopped { - fn on_join_room(&mut self, _ctx: &mut StateContext, _now: Instant, _room: ClusterRoomHash) { - log::warn!("[EndpointRemoteTrack] join room but stopped"); - } - - fn on_leave_room(&mut self, _ctx: &mut StateContext, _now: Instant) { - log::warn!("[EndpointRemoteTrack] leave room but stopped"); - } - - fn on_track_started(&mut self, _ctx: &mut StateContext, _now: Instant) { - log::warn!("[EndpointRemoteTrack] track started but stopped"); - } - - fn on_track_media(&mut self, _ctx: &mut StateContext, _now: Instant, _media: MediaPacket) { - log::warn!("[EndpointRemoteTrack] track media but stopped"); - } - - fn on_track_ended(&mut self, _ctx: &mut StateContext, _now: Instant) { - log::warn!("[EndpointRemoteTrack] track ended but stopped"); - } - - fn on_rpc_req(&mut self, ctx: &mut StateContext, _now: Instant, req_id: EndpointReqId, req: EndpointRemoteTrackReq) { - match req { - EndpointRemoteTrackReq::Config(_config) => { - ctx.queue - .push_back(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Err(RpcError::new2(EndpointErrors::RemoteTrackStopped))))); - } - } - } - - fn on_cluster_event(&mut self, _ctx: &mut StateContext, _now: Instant, _event: ClusterRemoteTrackEvent) { - log::warn!("[EndpointRemoteTrack] on cluster event but stopped"); - } - - fn on_bitrate_allocation(&mut self, _ctx: &mut StateContext, _now: Instant, _action: IngressAction) { - log::warn!("[EndpointRemoteTrack] on bitrate allocation but stopped"); - } - - fn pop_output(&mut self, ctx: &mut StateContext, _now: Instant) -> Option { - if ctx.queue.is_empty() && self.waiting { - self.waiting = false; - // We must send Stopped at last, if not we missed some event - Some(Output::Stopped(ctx.meta.kind)) - } else { - ctx.queue.pop_front() - } - } -} - -pub struct EndpointRemoteTrack { - _c: Count, - ctx: StateContext, - state: State, -} - -impl EndpointRemoteTrack { - pub fn new(room: Option, id: RemoteTrackId, name: TrackName, priority: TrackPriority, meta: TrackMeta, record: bool) -> Self { - log::info!("[EndpointRemoteTrack] created with room {:?} meta {:?}", room, meta); - Self { - _c: Default::default(), - ctx: StateContext { - id, - name, - meta, - priority, - queue: Default::default(), - last_layers: None, - cluster_bitrate_limit: None, - record, - next_state: None, - allocate_bitrate: None, - }, - state: if let Some(room) = room { - State::InRoom(InRoom { room }) - } else { - State::WaitingJoinRoom(WaitingJoinRoom) - }, + fn calc_limit_bitrate(&self) -> Option<(u64, u64)> { + let cluster_limit = self.meta.control.eq(&BitrateControlMode::DynamicConsumers).then_some(self.cluster_bitrate_limit).flatten(); + match (self.allocate_bitrate, cluster_limit) { + (Some(b1), Some((min, max))) => Some((min.min(b1), max.min(b1))), + (Some(b1), None) => Some((b1, b1)), + (None, Some((min, max))) => Some((min, max)), + (None, None) => None, } } } @@ -499,46 +229,45 @@ impl Task for EndpointRemoteTrack { fn on_event(&mut self, now: Instant, input: Input) { match input { - Input::JoinRoom(room) => self.state.on_join_room(&mut self.ctx, now, room), - Input::LeaveRoom => self.state.on_leave_room(&mut self.ctx, now), - Input::Cluster(event) => self.state.on_cluster_event(&mut self.ctx, now, event), - Input::Event(event) => match event { - RemoteTrackEvent::Started { .. } => { - self.state.on_track_started(&mut self.ctx, now); - } - RemoteTrackEvent::Paused => {} - RemoteTrackEvent::Resumed => {} - RemoteTrackEvent::Media(media) => { - //TODO clear self.last_layer if switched to new track - if media.layers.is_some() { - log::debug!("[EndpointRemoteTrack] on layers info {:?}", media.layers); - self.ctx.last_layers.clone_from(&media.layers); - } - - self.state.on_track_media(&mut self.ctx, now, media); - } - RemoteTrackEvent::Ended => { - self.state.on_track_ended(&mut self.ctx, now); - } - }, - Input::RpcReq(req_id, req) => self.state.on_rpc_req(&mut self.ctx, now, req_id, req), - Input::BitrateAllocation(action) => self.state.on_bitrate_allocation(&mut self.ctx, now, action), + Input::JoinRoom(room) => self.on_join_room(now, room), + Input::LeaveRoom => self.on_leave_room(now), + Input::Cluster(event) => self.on_cluster_event(now, event), + Input::Event(event) => self.on_transport_event(now, event), + Input::RpcReq(req_id, req) => self.on_rpc_req(now, req_id, req), + Input::BitrateAllocation(action) => self.on_bitrate_allocation_action(now, action), } } - fn on_shutdown(&mut self, _now: Instant) {} + fn on_shutdown(&mut self, now: Instant) { + if self.shutdown { + return; + } + self.shutdown = true; + if self.room.is_some() { + self.on_leave_room(now); + } + } } impl TaskSwitcherChild for EndpointRemoteTrack { type Time = Instant; - fn pop_output(&mut self, now: Instant) -> Option { - self.state.pop_output(&mut self.ctx, now) + + fn is_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + + fn empty_event(&self) -> Output { + Output::Stopped(self.meta.kind) + } + + fn pop_output(&mut self, _now: Instant) -> Option { + self.queue.pop_front() } } impl Drop for EndpointRemoteTrack { fn drop(&mut self) { - assert_eq!(self.ctx.queue.len(), 0, "remote track queue should empty on drop"); + assert_eq!(self.queue.len(), 0, "remote track queue should empty on drop"); } } @@ -547,23 +276,16 @@ mod tests { use std::time::{Duration, Instant}; use media_server_protocol::{ - endpoint::{BitrateControlMode, TrackMeta, TrackName}, - media::MediaKind, + endpoint::{TrackMeta, TrackName}, protobuf::{cluster_connector::peer_event, shared::Kind}, - transport::RpcError, }; use sans_io_runtime::{Task, TaskSwitcherChild}; - use crate::{ - cluster::ClusterRemoteTrackControl, - endpoint::{EndpointRemoteTrackConfig, EndpointRemoteTrackReq, EndpointRemoteTrackRes, EndpointReqId}, - errors::EndpointErrors, - transport::RemoteTrackEvent, - }; + use crate::{cluster::ClusterRemoteTrackControl, transport::RemoteTrackEvent}; use super::{EndpointRemoteTrack, Input, Output}; - #[test] + #[test_log::test] fn start_in_room() { let room = 0.into(); let track_name = TrackName::from("audio_main"); @@ -571,7 +293,7 @@ mod tests { let track_priority = 2.into(); let meta = TrackMeta::default_audio(); let now = Instant::now(); - let mut track = EndpointRemoteTrack::new(Some(room), track_id, track_name.clone(), track_priority, meta.clone(), false); + let mut track = EndpointRemoteTrack::new(Some(room), track_id, track_name.clone(), meta.clone(), false); assert_eq!(track.pop_output(now), None); track.on_event( @@ -612,50 +334,9 @@ mod tests { }), )) ); - //we need Output::Stopped at last - assert_eq!(track.pop_output(now), Some(Output::Stopped(meta.kind))); - assert_eq!(track.pop_output(now), None); - } - - #[test] - fn should_wait_for_stopped() { - let name = TrackName::from("audio_main"); - let id = 1.into(); - let priority = 2.into(); - let meta = TrackMeta::default_audio(); - let now = Instant::now(); - let mut track = EndpointRemoteTrack::new(None, id, name.clone(), priority, meta.clone(), false); - - track.on_event( - now, - Input::Event(RemoteTrackEvent::Started { - name: name.clone().into(), - priority, - meta, - }), - ); - assert_eq!(track.pop_output(now), Some(Output::Started(MediaKind::Audio, priority))); - assert_eq!(track.pop_output(now), None); - track.on_event(now, Input::Event(RemoteTrackEvent::Ended)); - - let req_id = EndpointReqId(0); - track.on_event( - now, - Input::RpcReq( - req_id, - EndpointRemoteTrackReq::Config(EndpointRemoteTrackConfig { - priority, - control: BitrateControlMode::DynamicConsumers, - }), - ), - ); - assert_eq!( - track.pop_output(now), - Some(Output::RpcRes(req_id, EndpointRemoteTrackRes::Config(Err(RpcError::new2(EndpointErrors::RemoteTrackStopped))))) - ); - //we need Output::Stopped at last - assert_eq!(track.pop_output(now), Some(Output::Stopped(MediaKind::Audio))); assert_eq!(track.pop_output(now), None); + //we dont need Output::Stopped here, it will be fired with TaskSwitcherChild::pop_output with is_empty true + assert!(track.is_empty()); } //TODO start not in room diff --git a/packages/media_core/src/errors.rs b/packages/media_core/src/errors.rs index 94774ab3..e56e23bd 100644 --- a/packages/media_core/src/errors.rs +++ b/packages/media_core/src/errors.rs @@ -7,4 +7,5 @@ pub enum EndpointErrors { RemoteTrackInvalidPriority = 0x2001, RemoteTrackStopped = 0x2002, AudioMixerWrongMode = 0x3001, + Destroying = 0x4001, } diff --git a/packages/media_core/src/transport.rs b/packages/media_core/src/transport.rs index bdbc35ab..b912de7e 100644 --- a/packages/media_core/src/transport.rs +++ b/packages/media_core/src/transport.rs @@ -18,7 +18,7 @@ pub use media_server_protocol::transport::{LocalTrackId, RemoteTrackId}; #[derive(From, Debug, Clone, Copy, PartialEq, Eq, Display)] pub struct TransportId(pub u64); -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TransportError { Timeout, } @@ -94,7 +94,6 @@ pub enum TransportInput { Endpoint(EndpointEvent), RpcRes(EndpointReqId, EndpointRes), Ext(Ext), - SystemClose, } /// This is event from transport, in general is is result of transport protocol @@ -104,9 +103,11 @@ pub enum TransportOutput { Event(TransportEvent), RpcReq(EndpointReqId, EndpointReq), Ext(Ext), + OnResourceEmpty, } pub trait Transport: TaskSwitcherChild> { fn on_tick(&mut self, now: Instant); fn on_input(&mut self, now: Instant, input: TransportInput); + fn on_shutdown(&mut self, now: Instant); } diff --git a/packages/media_gateway/src/agent_service.rs b/packages/media_gateway/src/agent_service.rs index 34259e48..240c6055 100644 --- a/packages/media_gateway/src/agent_service.rs +++ b/packages/media_gateway/src/agent_service.rs @@ -49,6 +49,7 @@ pub struct GatewayAgentService { seq: u16, node: NodeMetrics, services: HashMap, + shutdown: bool, _tmp: std::marker::PhantomData<(UserData, SC, SE, TC, TW)>, } @@ -59,6 +60,7 @@ impl GatewayAgentService { seq: 0, node: Default::default(), services: HashMap::from_iter(max.into_iter().map(|(k, v)| (k, ServiceWorkersStats { max: v, workers: HashMap::new() }))), + shutdown: false, _tmp: std::marker::PhantomData, } } @@ -69,6 +71,10 @@ where SC: From + TryInto + Debug, SE: From + TryInto, { + fn is_service_empty(&self) -> bool { + self.shutdown && self.output.is_none() + } + fn service_id(&self) -> u8 { AGENT_SERVICE_ID } @@ -121,6 +127,10 @@ where } } + fn on_shutdown(&mut self, _ctx: &ServiceCtx, _now: u64) { + self.shutdown = true; + } + fn pop_output2(&mut self, _now: u64) -> Option> { self.output.take() } @@ -128,6 +138,7 @@ where pub struct GatewayAgentServiceWorker { queue: VecDeque>, + shutdown: bool, } impl ServiceWorker for GatewayAgentServiceWorker { @@ -139,6 +150,10 @@ impl ServiceWorker bool { + self.shutdown && self.queue.is_empty() + } + fn on_tick(&mut self, _ctx: &ServiceWorkerCtx, _now: u64, _tick_count: u64) {} fn on_input(&mut self, _ctx: &ServiceWorkerCtx, _now: u64, input: ServiceWorkerInput) { @@ -152,6 +167,10 @@ impl ServiceWorker Option> { self.queue.pop_front() } + + fn on_shutdown(&mut self, _ctx: &ServiceWorkerCtx, _now: u64) { + self.shutdown = true; + } } pub struct GatewayAgentServiceBuilder { @@ -190,6 +209,9 @@ where } fn create_worker(&self) -> Box> { - Box::new(GatewayAgentServiceWorker { queue: Default::default() }) + Box::new(GatewayAgentServiceWorker { + queue: Default::default(), + shutdown: false, + }) } } diff --git a/packages/media_gateway/src/store_service.rs b/packages/media_gateway/src/store_service.rs index cb0a84d1..cd471e86 100644 --- a/packages/media_gateway/src/store_service.rs +++ b/packages/media_gateway/src/store_service.rs @@ -41,6 +41,7 @@ pub struct GatewayStoreService { queue: VecDeque>, store: GatewayStore, seq: u16, + shutdown: bool, _tmp: std::marker::PhantomData<(UserData, SC, SE, TC, TW)>, } @@ -54,6 +55,7 @@ where store: GatewayStore::new(zone, Location { lat, lon }, max_cpu, max_memory, max_disk), queue: VecDeque::from([ServiceOutput::FeatureControl(data::Control::DataListen(DATA_PORT).into())]), seq: 0, + shutdown: false, _tmp: std::marker::PhantomData, } } @@ -98,6 +100,10 @@ where STORE_SERVICE_NAME } + fn is_service_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + fn on_shared_input<'a>(&mut self, _ctx: &ServiceCtx, now: u64, input: ServiceSharedInput) { match input { ServiceSharedInput::Tick(_) => { @@ -157,6 +163,10 @@ where } } + fn on_shutdown(&mut self, _ctx: &ServiceCtx, _now: u64) { + self.shutdown = true; + } + fn pop_output2(&mut self, _now: u64) -> Option> { self.queue.pop_front() } @@ -164,6 +174,7 @@ where pub struct GatewayStoreServiceWorker { queue: VecDeque>, + shutdown: bool, } impl ServiceWorker for GatewayStoreServiceWorker { @@ -175,6 +186,10 @@ impl ServiceWorker bool { + self.shutdown && self.queue.is_empty() + } + fn on_tick(&mut self, _ctx: &ServiceWorkerCtx, _now: u64, _tick_count: u64) {} fn on_input(&mut self, _ctx: &ServiceWorkerCtx, _now: u64, input: ServiceWorkerInput) { @@ -188,6 +203,10 @@ impl ServiceWorker Option> { self.queue.pop_front() } @@ -242,6 +261,9 @@ where } fn create_worker(&self) -> Box> { - Box::new(GatewayStoreServiceWorker { queue: Default::default() }) + Box::new(GatewayStoreServiceWorker { + queue: Default::default(), + shutdown: false, + }) } } diff --git a/packages/media_record/bin/run_worker.sh b/packages/media_record/bin/run_worker.sh new file mode 100644 index 00000000..f714bb03 --- /dev/null +++ b/packages/media_record/bin/run_worker.sh @@ -0,0 +1,4 @@ +cargo run --bin convert_record_worker -- \ + --input-s3-uri http://ows-storage:tzraxj7wptoh@storage.dev.owslab.io/atm0s-record \ + --multi-tenancy-sync https://atm0s.wiremockapi.cloud/cp/apps \ + --secret zsz94nsrj3xvmbu555nmu25hwqo6shiq \ No newline at end of file diff --git a/packages/media_runner/Cargo.toml b/packages/media_runner/Cargo.toml index bb6ac6b9..5ceb841b 100644 --- a/packages/media_runner/Cargo.toml +++ b/packages/media_runner/Cargo.toml @@ -18,7 +18,7 @@ media-server-core = { path = "../media_core" } sans-io-runtime = { workspace = true, default-features = false } atm0s-sdn = { workspace = true } -atm0s-sdn-network = "0.5" +atm0s-sdn-network = { workspace = true } transport-webrtc = { path = "../transport_webrtc", optional = true } transport-rtpengine = { path = "../transport_rtpengine", optional = true } diff --git a/packages/media_runner/src/worker.rs b/packages/media_runner/src/worker.rs index 2a372a7d..9ad5259c 100644 --- a/packages/media_runner/src/worker.rs +++ b/packages/media_runner/src/worker.rs @@ -132,6 +132,7 @@ pub struct MediaServerWorker { timer: TimePivot, last_feedback_gateway_agent: u64, secure: Arc, + shutdown: bool, } impl MediaServerWorker { @@ -219,6 +220,7 @@ impl MediaServerWorker { secure, sdn_backend_addrs: Default::default(), sdn_backend_slots: Default::default(), + shutdown: false, } } @@ -226,6 +228,10 @@ impl MediaServerWorker { self.sdn_worker.tasks() + self.sdn_worker.tasks() } + pub fn is_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() && self.sdn_worker.is_empty() && self.media_cluster.is_empty() && self.media_webrtc.is_empty() && self.media_rtpengine.is_empty() + } + pub fn on_tick(&mut self, now: Instant) { let s = &mut self.switcher; let now_ms = self.timer.timestamp_ms(now); @@ -369,9 +375,9 @@ impl MediaServerWorker { None } - pub fn shutdown(&mut self, now: Instant) { + pub fn on_shutdown(&mut self, now: Instant) { let now_ms = self.timer.timestamp_ms(now); - self.sdn_worker.input(&mut self.switcher).on_event(now_ms, SdnWorkerInput::ShutdownRequest); + self.sdn_worker.input(&mut self.switcher).on_shutdown(now_ms); self.media_cluster.input(&mut self.switcher).shutdown(now); self.media_webrtc.input(&mut self.switcher).shutdown(now); self.media_rtpengine.input(&mut self.switcher).shutdown(now); @@ -406,7 +412,7 @@ impl MediaServerWorker { } }, SdnWorkerOutput::Bus(event) => Output::Bus(event), - SdnWorkerOutput::ShutdownResponse => Output::Continue, + SdnWorkerOutput::OnResourceEmpty => Output::Continue, SdnWorkerOutput::Continue => Output::Continue, } } @@ -435,6 +441,7 @@ impl MediaServerWorker { } Output::Continue } + cluster::Output::OnResourceEmpty => Output::Continue, cluster::Output::Continue => Output::Continue, } } @@ -492,7 +499,7 @@ impl MediaServerWorker { transport_webrtc::Variant::Webrtc => Output::ExtRpc(req_id, RpcRes::Webrtc(webrtc::RpcRes::Delete(res))), }, }, - transport_webrtc::GroupOutput::Shutdown(_session) => Output::Continue, + transport_webrtc::GroupOutput::OnResourceEmpty => Output::Continue, transport_webrtc::GroupOutput::Continue => Output::Continue, } } @@ -529,7 +536,7 @@ impl MediaServerWorker { Output::Continue } transport_rtpengine::GroupOutput::RecordEvent(_, session_id, ts, event) => Output::Record(session_id, ts, event), - transport_rtpengine::GroupOutput::Shutdown(_) => Output::Continue, + transport_rtpengine::GroupOutput::OnResourceEmpty => Output::Continue, transport_rtpengine::GroupOutput::Continue => Output::Continue, } } diff --git a/packages/media_utils/Cargo.toml b/packages/media_utils/Cargo.toml index 60af3231..74ba7783 100644 --- a/packages/media_utils/Cargo.toml +++ b/packages/media_utils/Cargo.toml @@ -14,4 +14,5 @@ uriparse = "0.6" serde-querystring = "0.2" pin-project-lite = "0.2" spin = { workspace = true } -once_cell = "1.20" \ No newline at end of file +once_cell = "1.20" +derive_more = "1.0" diff --git a/packages/media_utils/src/lib.rs b/packages/media_utils/src/lib.rs index d67d6a31..6f823579 100644 --- a/packages/media_utils/src/lib.rs +++ b/packages/media_utils/src/lib.rs @@ -4,6 +4,7 @@ mod select; mod seq_extend; mod seq_rewrite; mod small_2dmap; +mod state; mod time; mod ts_rewrite; mod uri; @@ -14,6 +15,7 @@ pub use select::*; pub use seq_extend::RtpSeqExtend; pub use seq_rewrite::SeqRewrite; pub use small_2dmap::Small2dMap; +pub use state::*; pub use time::now_ms; pub use ts_rewrite::TsRewrite; pub use uri::CustomUri; diff --git a/packages/media_utils/src/state.rs b/packages/media_utils/src/state.rs new file mode 100644 index 00000000..d808a3da --- /dev/null +++ b/packages/media_utils/src/state.rs @@ -0,0 +1,182 @@ +//! This module implements the state machine pattern for easy state transition +//! + +use derive_more::From; +use std::{ + collections::VecDeque, + ops::{Deref, DerefMut}, +}; + +pub trait StateQueue { + fn push(&mut self, event: T); + fn pop(&mut self) -> Option; +} + +pub struct StateDestroyingQueue { + data: VecDeque, + destroy_event: Option, +} + +impl Deref for StateDestroyingQueue { + type Target = VecDeque; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl StateDestroyingQueue { + pub fn build(data: &mut VecDeque, destroy_event: T) -> Self { + let data = std::mem::take(data); + Self { + data, + destroy_event: Some(destroy_event), + } + } +} + +impl StateQueue for StateDestroyingQueue { + fn push(&mut self, event: T) { + self.data.push_back(event); + } + + fn pop(&mut self) -> Option { + if self.destroy_event.is_none() { + assert!(self.data.is_empty()); + return None; + } + if self.data.is_empty() { + self.destroy_event.take() + } else { + self.data.pop_front() + } + } +} + +pub struct McContext { + ctx: C, + next: Option, +} + +impl McContext { + pub fn switch(&mut self, next: S) { + self.next = Some(next); + } + + pub fn next_state(&mut self) -> Option { + self.next.take() + } +} + +impl From for McContext { + fn from(ctx: C) -> Self { + Self { ctx, next: None } + } +} + +impl Deref for McContext { + type Target = C; + + fn deref(&self) -> &Self::Target { + &self.ctx + } +} + +impl DerefMut for McContext { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.ctx + } +} + +impl StateQueue for VecDeque { + fn push(&mut self, event: T) { + self.push_back(event); + } + + fn pop(&mut self) -> Option { + self.pop_front() + } +} + +#[allow(unused)] +#[cfg(test)] +mod tests { + use super::*; + + enum Input { + Req, + Destroy, + } + + enum Output { + Res(Result<(), ()>), + Destroy, + } + + struct Context {} + + trait State { + fn on_event(&mut self, ctx: &mut McContext>, event: Input); + fn pop_event(&mut self, ctx: &mut McContext>) -> Option; + } + + struct Running(VecDeque); + struct Destroying(StateDestroyingQueue); + + struct Object { + ctx: McContext>, + states: Box, + } + + impl Object { + pub fn new() -> Self { + Self { + ctx: Context {}.into(), + states: Box::new(Running(VecDeque::new())), + } + } + + pub fn on_event(&mut self, event: Input) { + self.states.on_event(&mut self.ctx, event); + if let Some(next) = self.ctx.next_state() { + self.states = next; + } + } + + pub fn pop_event(&mut self, ctx: &mut McContext>) -> Option { + self.states.pop_event(ctx) + } + } + + impl State for Running { + fn on_event(&mut self, ctx: &mut McContext>, event: Input) { + match event { + Input::Req => { + self.0.push_back(Output::Res(Ok(()))); + } + Input::Destroy => ctx.switch(Box::new(Destroying(StateDestroyingQueue::build(&mut self.0, Output::Destroy)))), + } + } + + fn pop_event(&mut self, _ctx: &mut McContext>) -> Option { + self.0.pop_front() + } + } + + impl State for Destroying { + fn on_event(&mut self, _ctx: &mut McContext>, event: Input) { + match event { + Input::Req => { + self.0.push(Output::Res(Err(()))); + } + Input::Destroy => { + //do nothing + } + } + } + + fn pop_event(&mut self, _ctx: &mut McContext>) -> Option { + self.0.pop() + } + } +} diff --git a/packages/transport_rtpengine/src/transport.rs b/packages/transport_rtpengine/src/transport.rs index 5ec065ef..2eff1fc8 100644 --- a/packages/transport_rtpengine/src/transport.rs +++ b/packages/transport_rtpengine/src/transport.rs @@ -62,6 +62,7 @@ pub struct TransportRtpEngine { pcma_to_opus: AudioTranscoder, opus_to_pcma: AudioTranscoder, tmp_buf: [u8; 1500], + shutdown: bool, } impl TransportRtpEngine { @@ -92,6 +93,7 @@ impl TransportRtpEngine { pcma_to_opus: AudioTranscoder::new(PcmaDecoder::default(), OpusEncoder::default()), opus_to_pcma: AudioTranscoder::new(OpusDecoder::default(), PcmaEncoder::default()), tmp_buf: [0; 1500], + shutdown: false, }, answer, )) @@ -135,6 +137,7 @@ impl TransportRtpEngine { pcma_to_opus: AudioTranscoder::new(PcmaDecoder::default(), OpusEncoder::default()), opus_to_pcma: AudioTranscoder::new(OpusDecoder::default(), PcmaEncoder::default()), tmp_buf: [0; 1500], + shutdown: false, }, answer, )) @@ -160,17 +163,20 @@ impl TransportRtpEngine { impl Transport for TransportRtpEngine { fn on_tick(&mut self, _now: Instant) { - let last_activity = match (self.last_recv_rtp, self.last_send_rtp) { - (None, None) => self.created, - (Some(_time), None) => self.created, //we need two way, if only one-way => disconnect - (None, Some(_time)) => self.created, //we need two way, if only one-way => disconnect - (Some(time1), Some(time2)) => time1.max(time2), - }; + if !self.shutdown { + let last_activity = match (self.last_recv_rtp, self.last_send_rtp) { + (None, None) => self.created, + (Some(_time), None) => self.created, //we need two way, if only one-way => disconnect + (None, Some(_time)) => self.created, //we need two way, if only one-way => disconnect + (Some(time1), Some(time2)) => time1.max(time2), + }; - if last_activity.elapsed() >= Duration::from_millis(TIMEOUT_DURATION_MS) { - log::warn!("[TransportRtpEngine] timeout after {TIMEOUT_DURATION_MS} ms don't has activity"); - self.queue - .push_back(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(Some(TransportError::Timeout))))); + if last_activity.elapsed() >= Duration::from_millis(TIMEOUT_DURATION_MS) { + log::warn!("[TransportRtpEngine] timeout after {TIMEOUT_DURATION_MS} ms don't has activity"); + self.queue + .push_back(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(Some(TransportError::Timeout))))); + self.shutdown = true; + } } } @@ -196,9 +202,13 @@ impl Transport for TransportRtpEngine { self.queue.push_back(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None)))); } }, - TransportInput::SystemClose => { - self.queue.push_back(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None)))); - } + } + } + + fn on_shutdown(&mut self, _now: Instant) { + if !self.shutdown { + log::info!("[TransportRtpEngine] shutdown request"); + self.shutdown = true; } } } @@ -353,6 +363,14 @@ impl TransportRtpEngine { impl TaskSwitcherChild> for TransportRtpEngine { type Time = Instant; + fn is_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() + } + + fn empty_event(&self) -> TransportOutput { + TransportOutput::OnResourceEmpty + } + fn pop_output(&mut self, _now: Instant) -> Option> { self.queue.pop_front() } diff --git a/packages/transport_rtpengine/src/worker.rs b/packages/transport_rtpengine/src/worker.rs index c9a6fb14..53ab5d1f 100644 --- a/packages/transport_rtpengine/src/worker.rs +++ b/packages/transport_rtpengine/src/worker.rs @@ -13,7 +13,7 @@ use media_server_protocol::{ }; use sans_io_runtime::{ backend::{BackendIncoming, BackendOutgoing}, - group_owner_type, return_if_some, TaskGroup, TaskSwitcherChild, + group_owner_type, return_if_some, TaskGroup, TaskGroupOutput, TaskSwitcherChild, }; use crate::transport::{ExtIn, ExtOut, TransportRtpEngine}; @@ -34,7 +34,7 @@ pub enum GroupOutput { PeerEvent(RtpEngineSession, AppId, u64, Instant, peer_event::Event), RecordEvent(RtpEngineSession, u64, Instant, SessionRecordEvent), Ext(RtpEngineSession, ExtOut), - Shutdown(RtpEngineSession), + OnResourceEmpty, Continue, } @@ -43,6 +43,7 @@ pub struct MediaWorkerRtpEngine { ip: IpAddr, endpoints: TaskGroup, EndpointOutput, Endpoint, 16>, queue: VecDeque, + shutdown: bool, } impl MediaWorkerRtpEngine { @@ -51,6 +52,7 @@ impl MediaWorkerRtpEngine { ip, endpoints: TaskGroup::default(), queue: VecDeque::new(), + shutdown: false, } } @@ -77,10 +79,10 @@ impl MediaWorkerRtpEngine { EndpointOutput::Cluster(room, control) => GroupOutput::Cluster(RtpEngineSession(index), room, control), EndpointOutput::PeerEvent(app, session_id, ts, event) => GroupOutput::PeerEvent(RtpEngineSession(index), app, session_id, ts, event), EndpointOutput::RecordEvent(session_id, ts, event) => GroupOutput::RecordEvent(RtpEngineSession(index), session_id, ts, event), - EndpointOutput::Destroy => { + EndpointOutput::OnResourceEmpty => { log::info!("[TransportRtpEngine] destroy endpoint {index}"); self.endpoints.remove_task(index); - GroupOutput::Shutdown(RtpEngineSession(index)) + GroupOutput::Continue } EndpointOutput::Ext(ext) => GroupOutput::Ext(RtpEngineSession(index), ext), EndpointOutput::Continue => GroupOutput::Continue, @@ -120,15 +122,30 @@ impl MediaWorkerRtpEngine { } pub fn shutdown(&mut self, now: Instant) { - self.endpoints.on_shutdown(now); + if !self.shutdown { + self.shutdown = true; + self.endpoints.on_shutdown(now); + } } } impl TaskSwitcherChild for MediaWorkerRtpEngine { type Time = Instant; + + fn empty_event(&self) -> GroupOutput { + GroupOutput::OnResourceEmpty + } + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.endpoints.tasks() == 0 && self.shutdown + } + fn pop_output(&mut self, now: Instant) -> Option { return_if_some!(self.queue.pop_front()); - let (index, out) = self.endpoints.pop_output(now)?; + let (index, out) = match self.endpoints.pop_output(now)? { + TaskGroupOutput::TaskOutput(index, out) => (index, out), + TaskGroupOutput::OnResourceEmpty => return Some(GroupOutput::Continue), + }; Some(self.process_output(index, out)) } } diff --git a/packages/transport_webrtc/src/lib.rs b/packages/transport_webrtc/src/lib.rs index e00a6457..9d23155f 100644 --- a/packages/transport_webrtc/src/lib.rs +++ b/packages/transport_webrtc/src/lib.rs @@ -19,4 +19,5 @@ pub enum WebrtcError { RpcTokenInvalid = 0x2007, RpcTokenRoomPeerNotMatch = 0x2008, RpcTokenAppNotMatch = 0x2009, + RpcAlreadyDisconnected = 0x2010, } diff --git a/packages/transport_webrtc/src/transport.rs b/packages/transport_webrtc/src/transport.rs index f7987a35..ff3f2980 100644 --- a/packages/transport_webrtc/src/transport.rs +++ b/packages/transport_webrtc/src/transport.rs @@ -101,7 +101,8 @@ trait TransportWebrtcInternal { fn on_transport_rpc_res(&mut self, now: Instant, req_id: EndpointReqId, res: EndpointRes); fn on_endpoint_event(&mut self, now: Instant, input: EndpointEvent); fn on_str0m_event(&mut self, now: Instant, event: str0m::Event); - fn close(&mut self, now: Instant); + fn is_empty(&self) -> bool; + fn on_shutdown(&mut self, now: Instant); fn pop_output(&mut self, now: Instant) -> Option; } @@ -337,21 +338,31 @@ impl Transport for TransportWebrtc } } ExtIn::Disconnect(req_id, variant) => { - self.internal.close(now); + self.internal.on_shutdown(now); self.queue.push_back(TransportOutput::Ext(ExtOut::Disconnect(req_id, variant, Ok(())))); } }, - TransportInput::SystemClose => { - log::info!("[TransportWebrtc] system close request"); - self.internal.close(now); - } } } + + fn on_shutdown(&mut self, now: Instant) { + log::info!("[TransportWebrtc] shutdown request"); + self.internal.on_shutdown(now); + self.rtc.disconnect(); + } } impl TaskSwitcherChild> for TransportWebrtc { type Time = Instant; + fn empty_event(&self) -> TransportOutput { + TransportOutput::OnResourceEmpty + } + + fn is_empty(&self) -> bool { + self.queue.is_empty() && self.internal.is_empty() + } + fn pop_output(&mut self, now: Instant) -> Option> { return_if_some!(self.queue.pop_front()); while let Some(out) = self.internal.pop_output(now) { diff --git a/packages/transport_webrtc/src/transport/webrtc.rs b/packages/transport_webrtc/src/transport/webrtc.rs index 1848c7a7..041e5168 100644 --- a/packages/transport_webrtc/src/transport/webrtc.rs +++ b/packages/transport_webrtc/src/transport/webrtc.rs @@ -198,6 +198,10 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { self.media_convert.set_config(cfg); } + fn is_empty(&self) -> bool { + matches!(self.state, State::Disconnected) && self.queue.is_empty() + } + fn on_tick(&mut self, now: Instant) { if let Some(init_bitrate) = self.bwe_state.on_tick(now) { self.queue.push_back(InternalOutput::Str0mResetBwe(init_bitrate)); @@ -517,9 +521,12 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { } Str0mEvent::ChannelClose(_channel) => { log::info!("[TransportWebrtcSdk] channel closed, leave room {:?}", self.join); - self.state = State::Disconnected; - self.queue - .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + if !matches!(self.state, State::Disconnected) { + log::info!("[TransportWebrtcSdk] switched to disconnected"); + self.state = State::Disconnected; + self.queue + .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + } } Str0mEvent::IceConnectionStateChange(state) => self.on_str0m_state(now, state), Str0mEvent::MediaAdded(media) => self.on_str0m_media_added(now, media), @@ -583,11 +590,15 @@ impl TransportWebrtcInternal for TransportWebrtcSdk { } } - fn close(&mut self, _now: Instant) { - log::info!("[TransportWebrtcSdk] switched to disconnected with close action"); - self.state = State::Disconnected; - self.queue - .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))) + fn on_shutdown(&mut self, _now: Instant) { + if !matches!(self.state, State::Disconnected) { + log::info!("[TransportWebrtcSdk] switched to disconnected with close action"); + self.state = State::Disconnected; + self.queue + .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + } else { + log::warn!("[TransportWebrtcSdk] already disconnected, ignore close action"); + } } fn pop_output(&mut self, _now: Instant) -> Option { @@ -758,15 +769,21 @@ impl TransportWebrtcSdk { self.queue.push_back(InternalOutput::RpcReq(req_id, InternalRpcReq::SetRemoteSdp(req.sdp))); } protobuf::session::request::session::Request::Disconnect(_req) => { - log::info!("[TransportWebrtcSdk] switched to disconnected with close action from client"); - self.queue - .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); - self.send_rpc_res( - req_id, - protobuf::session::response::Response::Session(protobuf::session::response::Session { - response: Some(protobuf::session::response::session::Response::Disconnect(protobuf::session::response::session::Disconnect {})), - }), - ); + if !matches!(self.state, State::Disconnected) { + log::info!("[TransportWebrtcSdk] switched to disconnected with close action from client"); + self.state = State::Disconnected; + self.queue + .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + self.send_rpc_res( + req_id, + protobuf::session::response::Response::Session(protobuf::session::response::Session { + response: Some(protobuf::session::response::session::Response::Disconnect(protobuf::session::response::session::Disconnect {})), + }), + ); + } else { + log::warn!("[TransportWebrtcSdk] already disconnected, ignore close action from client"); + self.send_rpc_res_err(req_id, RpcError::new2(WebrtcError::RpcAlreadyDisconnected)); + } } } } diff --git a/packages/transport_webrtc/src/transport/whep.rs b/packages/transport_webrtc/src/transport/whep.rs index 37ccceec..ac6e1535 100644 --- a/packages/transport_webrtc/src/transport/whep.rs +++ b/packages/transport_webrtc/src/transport/whep.rs @@ -85,6 +85,10 @@ impl TransportWebrtcWhep { impl TransportWebrtcInternal for TransportWebrtcWhep { fn on_codec_config(&mut self, _cfg: &str0m::format::CodecConfig) {} + fn is_empty(&self) -> bool { + matches!(self.state, State::Disconnected) && self.queue.is_empty() + } + fn on_tick(&mut self, now: Instant) { if let Some(init_bitrate) = self.bwe_state.on_tick(now) { self.queue.push_back(InternalOutput::Str0mResetBwe(init_bitrate)); @@ -217,11 +221,15 @@ impl TransportWebrtcInternal for TransportWebrtcWhep { } } - fn close(&mut self, _now: Instant) { - log::info!("[TransportWebrtcWhep] switched to disconnected with close action"); - self.state = State::Disconnected; - self.queue - .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + fn on_shutdown(&mut self, _now: Instant) { + if !matches!(self.state, State::Disconnected) { + log::info!("[TransportWebrtcWhep] switched to disconnected with close action"); + self.state = State::Disconnected; + self.queue + .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + } else { + log::warn!("[TransportWebrtcWhep] already disconnected, ignore close action"); + } } fn pop_output(&mut self, _now: Instant) -> Option { diff --git a/packages/transport_webrtc/src/transport/whip.rs b/packages/transport_webrtc/src/transport/whip.rs index 737a9706..d6ed8956 100644 --- a/packages/transport_webrtc/src/transport/whip.rs +++ b/packages/transport_webrtc/src/transport/whip.rs @@ -78,6 +78,10 @@ impl TransportWebrtcInternal for TransportWebrtcWhip { self.media_convert.set_config(cfg); } + fn is_empty(&self) -> bool { + matches!(self.state, State::Disconnected) && self.queue.is_empty() + } + fn on_tick(&mut self, now: Instant) { match &self.state { State::New => { @@ -199,11 +203,15 @@ impl TransportWebrtcInternal for TransportWebrtcWhip { } } - fn close(&mut self, _now: Instant) { - log::info!("[TransportWebrtcWhip] switched to disconnected with close action"); - self.state = State::Disconnected; - self.queue - .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + fn on_shutdown(&mut self, _now: Instant) { + if !matches!(self.state, State::Disconnected) { + log::info!("[TransportWebrtcWhip] switched to disconnected with close action"); + self.state = State::Disconnected; + self.queue + .push_back(InternalOutput::TransportOutput(TransportOutput::Event(TransportEvent::State(TransportState::Disconnected(None))))); + } else { + log::warn!("[TransportWebrtcWhip] already disconnected, ignore close action"); + } } fn pop_output(&mut self, _now: Instant) -> Option { diff --git a/packages/transport_webrtc/src/worker.rs b/packages/transport_webrtc/src/worker.rs index f832973a..7bb9a686 100644 --- a/packages/transport_webrtc/src/worker.rs +++ b/packages/transport_webrtc/src/worker.rs @@ -19,7 +19,7 @@ use media_server_protocol::{ use media_server_secure::MediaEdgeSecure; use sans_io_runtime::{ backend::{BackendIncoming, BackendOutgoing}, - group_owner_type, return_if_none, return_if_some, TaskGroup, TaskSwitcherChild, + group_owner_type, return_if_none, return_if_some, TaskGroup, TaskGroupOutput, TaskSwitcherChild, }; use str0m::change::DtlsCert; @@ -45,7 +45,7 @@ pub enum GroupOutput { PeerEvent(WebrtcSession, AppId, u64, Instant, peer_event::Event), RecordEvent(WebrtcSession, u64, Instant, SessionRecordEvent), Ext(WebrtcSession, ExtOut), - Shutdown(WebrtcSession), + OnResourceEmpty, Continue, } @@ -59,6 +59,7 @@ pub struct MediaWorkerWebrtc { addrs: Vec<(SocketAddr, usize)>, queue: VecDeque, secure: Arc, + shutdown: bool, } impl MediaWorkerWebrtc { @@ -72,6 +73,7 @@ impl MediaWorkerWebrtc { addrs: vec![], queue: VecDeque::from(addrs.iter().map(|addr| GroupOutput::Net(BackendOutgoing::UdpListen { addr: *addr, reuse: false })).collect::>()), secure, + shutdown: false, } } @@ -110,11 +112,11 @@ impl MediaWorkerWebrtc { EndpointOutput::Cluster(room, control) => GroupOutput::Cluster(WebrtcSession(index), room, control), EndpointOutput::PeerEvent(app, session_id, ts, event) => GroupOutput::PeerEvent(WebrtcSession(index), app, session_id, ts, event), EndpointOutput::RecordEvent(session_id, ts, event) => GroupOutput::RecordEvent(WebrtcSession(index), session_id, ts, event), - EndpointOutput::Destroy => { + EndpointOutput::OnResourceEmpty => { log::info!("[TransportWebrtc] destroy endpoint {index}"); self.endpoints.remove_task(index); self.shared_port.remove_task(index); - GroupOutput::Shutdown(WebrtcSession(index)) + GroupOutput::Continue } EndpointOutput::Ext(ext) => GroupOutput::Ext(WebrtcSession(index), ext), EndpointOutput::Continue => GroupOutput::Continue, @@ -179,15 +181,31 @@ impl MediaWorkerWebrtc { } pub fn shutdown(&mut self, now: Instant) { - self.endpoints.on_shutdown(now); + if !self.shutdown { + log::info!("[MediaWorkerWebrtc] shutdown request"); + self.shutdown = true; + self.endpoints.on_shutdown(now); + } } } impl TaskSwitcherChild for MediaWorkerWebrtc { type Time = Instant; + + fn empty_event(&self) -> GroupOutput { + GroupOutput::OnResourceEmpty + } + + fn is_empty(&self) -> bool { + self.shutdown && self.queue.is_empty() && self.endpoints.is_empty() + } + fn pop_output(&mut self, now: Instant) -> Option { return_if_some!(self.queue.pop_front()); - let (index, out) = self.endpoints.pop_output(now)?; + let (index, out) = match self.endpoints.pop_output(now)? { + TaskGroupOutput::TaskOutput(index, out) => (index, out), + TaskGroupOutput::OnResourceEmpty => return Some(GroupOutput::Continue), + }; Some(self.process_output(index, out)) } }