diff --git a/zenoh-ext/Cargo.toml b/zenoh-ext/Cargo.toml index 47204a2d66..4f2613cb70 100644 --- a/zenoh-ext/Cargo.toml +++ b/zenoh-ext/Cargo.toml @@ -29,9 +29,7 @@ maintenance = { status = "actively-developed" } [features] unstable = [] default = [] -shared-memory = [ - "zenoh/shared-memory", -] +shared-memory = ["zenoh/shared-memory"] [dependencies] tokio = { workspace = true, features = [ @@ -53,5 +51,8 @@ serde_json = { workspace = true } zenoh = { workspace = true, features = ["unstable", "internal"], default-features = false } zenoh-macros = { workspace = true } +[dev-dependencies] +zenoh = { workspace = true, features = ["unstable"], default-features = true } + [package.metadata.docs.rs] features = ["unstable"] diff --git a/zenoh-ext/tests/liveliness.rs b/zenoh-ext/tests/liveliness.rs new file mode 100644 index 0000000000..23e901d458 --- /dev/null +++ b/zenoh-ext/tests/liveliness.rs @@ -0,0 +1,354 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +use zenoh::{ + config::{self, EndPoint, WhatAmI}, + sample::SampleKind, +}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_querying_subscriber_clique() { + use std::time::Duration; + + use zenoh::{internal::ztimeout, prelude::*}; + use zenoh_ext::SubscriberBuilderExt; + + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const PEER1_ENDPOINT: &str = "udp/localhost:47447"; + + const LIVELINESS_KEYEXPR_1: &str = "test/liveliness/querying-subscriber/brokered/1"; + const LIVELINESS_KEYEXPR_2: &str = "test/liveliness/querying-subscriber/brokered/2"; + const LIVELINESS_KEYEXPR_ALL: &str = "test/liveliness/querying-subscriber/brokered/*"; + + zenoh_util::try_init_log_from_env(); + + let peer1 = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (1) ZID: {}", s.zid()); + s + }; + + let peer2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (2) ZID: {}", s.zid()); + s + }; + + let token1 = ztimeout!(peer2.liveliness().declare_token(LIVELINESS_KEYEXPR_1)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sub = ztimeout!(peer1 + .liveliness() + .declare_subscriber(LIVELINESS_KEYEXPR_ALL) + .querying()) + .unwrap(); + tokio::time::sleep(SLEEP).await; + + let _token2 = ztimeout!(peer2.liveliness().declare_token(LIVELINESS_KEYEXPR_2)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_2); + + drop(token1); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Delete); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_querying_subscriber_brokered() { + use std::time::Duration; + + use zenoh::{internal::ztimeout, prelude::*}; + use zenoh_ext::SubscriberBuilderExt; + + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const ROUTER_ENDPOINT: &str = "tcp/localhost:47448"; + + const LIVELINESS_KEYEXPR_1: &str = "test/liveliness/querying-subscriber/brokered/1"; + const LIVELINESS_KEYEXPR_2: &str = "test/liveliness/querying-subscriber/brokered/2"; + const LIVELINESS_KEYEXPR_ALL: &str = "test/liveliness/querying-subscriber/brokered/*"; + + zenoh_util::try_init_log_from_env(); + + let _router = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Router)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Router ZID: {}", s.zid()); + s + }; + + let client1 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (1) ZID: {}", s.zid()); + s + }; + + let client2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (2) ZID: {}", s.zid()); + s + }; + + let client3 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (3) ZID: {}", s.zid()); + s + }; + + let token1 = ztimeout!(client2.liveliness().declare_token(LIVELINESS_KEYEXPR_1)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sub = ztimeout!(client1 + .liveliness() + .declare_subscriber(LIVELINESS_KEYEXPR_ALL) + .querying()) + .unwrap(); + tokio::time::sleep(SLEEP).await; + + let _token2 = ztimeout!(client3.liveliness().declare_token(LIVELINESS_KEYEXPR_2)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_2); + + drop(token1); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Delete); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_fetching_subscriber_clique() { + use std::time::Duration; + + use zenoh::{internal::ztimeout, prelude::*}; + use zenoh_ext::SubscriberBuilderExt; + + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const PEER1_ENDPOINT: &str = "udp/localhost:47449"; + + const LIVELINESS_KEYEXPR_1: &str = "test/liveliness/querying-subscriber/brokered/1"; + const LIVELINESS_KEYEXPR_2: &str = "test/liveliness/querying-subscriber/brokered/2"; + const LIVELINESS_KEYEXPR_ALL: &str = "test/liveliness/querying-subscriber/brokered/*"; + + zenoh_util::try_init_log_from_env(); + + let peer1 = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (1) ZID: {}", s.zid()); + s + }; + + let peer2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (2) ZID: {}", s.zid()); + s + }; + + let token1 = ztimeout!(peer2.liveliness().declare_token(LIVELINESS_KEYEXPR_1)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sub = ztimeout!(peer1 + .liveliness() + .declare_subscriber(LIVELINESS_KEYEXPR_ALL) + .fetching(|cb| peer1 + .liveliness() + .get(LIVELINESS_KEYEXPR_ALL) + .callback(cb) + .wait())) + .unwrap(); + tokio::time::sleep(SLEEP).await; + + let _token2 = ztimeout!(peer2.liveliness().declare_token(LIVELINESS_KEYEXPR_2)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_2); + + drop(token1); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Delete); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_fetching_subscriber_brokered() { + use std::time::Duration; + + use zenoh::{internal::ztimeout, prelude::*}; + use zenoh_ext::SubscriberBuilderExt; + + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const ROUTER_ENDPOINT: &str = "tcp/localhost:47450"; + + const LIVELINESS_KEYEXPR_1: &str = "test/liveliness/querying-subscriber/brokered/1"; + const LIVELINESS_KEYEXPR_2: &str = "test/liveliness/querying-subscriber/brokered/2"; + const LIVELINESS_KEYEXPR_ALL: &str = "test/liveliness/querying-subscriber/brokered/*"; + + zenoh_util::try_init_log_from_env(); + + let _router = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Router)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Router ZID: {}", s.zid()); + s + }; + + let client1 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (1) ZID: {}", s.zid()); + s + }; + + let client2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (2) ZID: {}", s.zid()); + s + }; + + let client3 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (3) ZID: {}", s.zid()); + s + }; + + let token1 = ztimeout!(client2.liveliness().declare_token(LIVELINESS_KEYEXPR_1)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sub = ztimeout!(client1 + .liveliness() + .declare_subscriber(LIVELINESS_KEYEXPR_ALL) + .fetching(|cb| client1 + .liveliness() + .get(LIVELINESS_KEYEXPR_ALL) + .callback(cb) + .wait())) + .unwrap(); + tokio::time::sleep(SLEEP).await; + + let _token2 = ztimeout!(client3.liveliness().declare_token(LIVELINESS_KEYEXPR_2)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Put); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_2); + + drop(token1); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert_eq!(sample.kind(), SampleKind::Delete); + assert_eq!(sample.key_expr().as_str(), LIVELINESS_KEYEXPR_1); +} diff --git a/zenoh/src/api/admin.rs b/zenoh/src/api/admin.rs index 4c4d2a869e..9f2e073f75 100644 --- a/zenoh/src/api/admin.rs +++ b/zenoh/src/api/admin.rs @@ -31,6 +31,7 @@ use super::{ queryable::Query, sample::{DataInfo, Locality, SampleKind}, session::Session, + subscriber::SubscriberKind, }; macro_rules! ke_for_sure { @@ -162,11 +163,12 @@ impl TransportMulticastEventHandler for Handler { encoding: Some(Encoding::APPLICATION_JSON), ..Default::default() }; - self.session.handle_data( + self.session.execute_subscriber_callbacks( true, &expr, Some(info), serde_json::to_vec(&peer).unwrap().into(), + SubscriberKind::Subscriber, None, ); Ok(Arc::new(PeerHandler { @@ -207,7 +209,7 @@ impl TransportPeerEventHandler for PeerHandler { encoding: Some(Encoding::APPLICATION_JSON), ..Default::default() }; - self.session.handle_data( + self.session.execute_subscriber_callbacks( true, &self .expr @@ -215,6 +217,7 @@ impl TransportPeerEventHandler for PeerHandler { .with_suffix(&format!("/link/{}", s.finish())), Some(info), serde_json::to_vec(&link).unwrap().into(), + SubscriberKind::Subscriber, None, ); } @@ -226,7 +229,7 @@ impl TransportPeerEventHandler for PeerHandler { kind: SampleKind::Delete, ..Default::default() }; - self.session.handle_data( + self.session.execute_subscriber_callbacks( true, &self .expr @@ -234,6 +237,7 @@ impl TransportPeerEventHandler for PeerHandler { .with_suffix(&format!("/link/{}", s.finish())), Some(info), vec![0u8; 0].into(), + SubscriberKind::Subscriber, None, ); } @@ -245,8 +249,14 @@ impl TransportPeerEventHandler for PeerHandler { kind: SampleKind::Delete, ..Default::default() }; - self.session - .handle_data(true, &self.expr, Some(info), vec![0u8; 0].into(), None); + self.session.execute_subscriber_callbacks( + true, + &self.expr, + Some(info), + vec![0u8; 0].into(), + SubscriberKind::Subscriber, + None, + ); } fn as_any(&self) -> &dyn std::any::Any { diff --git a/zenoh/src/api/liveliness.rs b/zenoh/src/api/liveliness.rs index 91f5d4b227..6e8cc30483 100644 --- a/zenoh/src/api/liveliness.rs +++ b/zenoh/src/api/liveliness.rs @@ -21,30 +21,17 @@ use std::{ use zenoh_config::unwrap_or_default; use zenoh_core::{Resolvable, Resolve, Result as ZResult, Wait}; -use zenoh_keyexpr::keyexpr; -use zenoh_protocol::{ - core::Parameters, - network::{declare::subscriber::ext::SubscriberInfo, request}, -}; use super::{ handlers::{locked, DefaultHandler, IntoHandler}, key_expr::KeyExpr, - query::{QueryConsolidation, QueryTarget, Reply}, - sample::{Locality, Sample, SourceInfo}, + query::Reply, + sample::{Locality, Sample}, session::{Session, SessionRef, Undeclarable}, subscriber::{Subscriber, SubscriberInner}, Id, }; -#[zenoh_macros::unstable] -pub(crate) static PREFIX_LIVELINESS: &str = crate::net::routing::PREFIX_LIVELINESS; - -#[zenoh_macros::unstable] -lazy_static::lazy_static!( - pub(crate) static ref KE_PREFIX_LIVELINESS: &'static keyexpr = unsafe { keyexpr::from_str_unchecked(PREFIX_LIVELINESS) }; -); - /// A structure with functions to declare a /// [`LivelinessToken`](LivelinessToken), query /// existing [`LivelinessTokens`](LivelinessToken) @@ -552,21 +539,18 @@ where { #[zenoh_macros::unstable] fn wait(self) -> ::To { + use super::subscriber::SubscriberKind; + let key_expr = self.key_expr?; let session = self.session; let (callback, handler) = self.handler.into_handler(); session - .declare_subscriber_inner( - &key_expr, - &Some(KeyExpr::from(*KE_PREFIX_LIVELINESS)), - Locality::default(), - callback, - &SubscriberInfo::DEFAULT, - ) + .declare_liveliness_subscriber_inner(&key_expr, None, Locality::default(), callback) .map(|sub_state| Subscriber { subscriber: SubscriberInner { session, state: sub_state, + kind: SubscriberKind::LivelinessSubscriber, undeclare_on_drop: true, }, handler, @@ -755,20 +739,7 @@ where fn wait(self) -> ::To { let (callback, receiver) = self.handler.into_handler(); self.session - .query( - &self.key_expr?, - &Parameters::empty(), - &Some(KeyExpr::from(*KE_PREFIX_LIVELINESS)), - QueryTarget::DEFAULT, - QueryConsolidation::DEFAULT, - request::ext::QoSType::REQUEST.into(), - Locality::default(), - self.timeout, - None, - None, - SourceInfo::empty(), - callback, - ) + .liveliness_query(&self.key_expr?, self.timeout, callback) .map(|_| receiver) } } diff --git a/zenoh/src/api/publisher.rs b/zenoh/src/api/publisher.rs index 8a4330676f..96cedc960f 100644 --- a/zenoh/src/api/publisher.rs +++ b/zenoh/src/api/publisher.rs @@ -53,7 +53,10 @@ use super::{ sample::{DataInfo, Locality, QoS, Sample, SampleFields, SampleKind}, session::{SessionRef, Undeclarable}, }; -use crate::{api::Id, net::primitives::Primitives}; +use crate::{ + api::{subscriber::SubscriberKind, Id}, + net::primitives::Primitives, +}; pub(crate) struct PublisherState { pub(crate) id: Id, @@ -621,11 +624,12 @@ impl Publisher<'_> { )), }; - self.session.handle_data( + self.session.execute_subscriber_callbacks( true, &self.key_expr.to_wire(&self.session), Some(data_info), payload.into(), + SubscriberKind::Subscriber, attachment, ); } diff --git a/zenoh/src/api/query.rs b/zenoh/src/api/query.rs index 48e3674c85..e46d0a75ba 100644 --- a/zenoh/src/api/query.rs +++ b/zenoh/src/api/query.rs @@ -149,6 +149,11 @@ impl From for Result { } } +#[cfg(feature = "unstable")] +pub(crate) struct LivelinessQueryState { + pub(crate) callback: Callback<'static, Reply>, +} + pub(crate) struct QueryState { pub(crate) nb_final: usize, pub(crate) key_expr: KeyExpr<'static>, diff --git a/zenoh/src/api/session.rs b/zenoh/src/api/session.rs index 1c4ae2086f..14e0899a55 100644 --- a/zenoh/src/api/session.rs +++ b/zenoh/src/api/session.rs @@ -31,7 +31,11 @@ use zenoh_collections::SingleOrVec; use zenoh_config::{unwrap_or_default, wrappers::ZenohId, Config, Notifier}; use zenoh_core::{zconfigurable, zread, Resolvable, Resolve, ResolveClosure, ResolveFuture, Wait}; #[cfg(feature = "unstable")] -use zenoh_protocol::network::{declare::SubscriberId, ext}; +use zenoh_protocol::network::{ + declare::{DeclareToken, SubscriberId, TokenId, UndeclareToken}, + ext, + interest::InterestId, +}; use zenoh_protocol::{ core::{ key_expr::{keyexpr, OwnedKeyExpr}, @@ -46,8 +50,9 @@ use zenoh_protocol::{ DeclareQueryable, DeclareSubscriber, UndeclareQueryable, UndeclareSubscriber, }, interest::{InterestMode, InterestOptions}, - request::{self, ext::TargetType, Request}, - AtomicRequestId, Interest, Mapping, Push, RequestId, Response, ResponseFinal, + request::{self, ext::TargetType}, + AtomicRequestId, DeclareFinal, Interest, Mapping, Push, Request, RequestId, Response, + ResponseFinal, }, zenoh::{ query::{self, ext::QueryBodyType, Consolidation}, @@ -78,7 +83,7 @@ use super::{ queryable::{Query, QueryInner, QueryableBuilder, QueryableState}, sample::{DataInfo, DataInfoIntoSample, Locality, QoS, Sample, SampleKind}, selector::Selector, - subscriber::{SubscriberBuilder, SubscriberState}, + subscriber::{SubscriberBuilder, SubscriberKind, SubscriberState}, value::Value, Id, }; @@ -87,6 +92,7 @@ use super::{ liveliness::{Liveliness, LivelinessTokenState}, publisher::Publisher, publisher::{MatchingListenerState, MatchingStatus}, + query::LivelinessQueryState, sample::SourceInfo, }; #[cfg(feature = "unstable")] @@ -108,18 +114,26 @@ pub(crate) struct SessionState { pub(crate) primitives: Option>, // @TODO replace with MaybeUninit ?? pub(crate) expr_id_counter: AtomicExprId, // @TODO: manage rollover and uniqueness pub(crate) qid_counter: AtomicRequestId, + #[cfg(feature = "unstable")] + pub(crate) liveliness_qid_counter: AtomicRequestId, pub(crate) local_resources: HashMap, pub(crate) remote_resources: HashMap, #[cfg(feature = "unstable")] pub(crate) remote_subscribers: HashMap>, pub(crate) publishers: HashMap, + #[cfg(feature = "unstable")] + pub(crate) remote_tokens: HashMap>, + //pub(crate) publications: Vec, pub(crate) subscribers: HashMap>, + pub(crate) liveliness_subscribers: HashMap>, pub(crate) queryables: HashMap>, #[cfg(feature = "unstable")] pub(crate) tokens: HashMap>, #[cfg(feature = "unstable")] pub(crate) matching_listeners: HashMap>, pub(crate) queries: HashMap, + #[cfg(feature = "unstable")] + pub(crate) liveliness_queries: HashMap, pub(crate) aggregated_subscribers: Vec, pub(crate) aggregated_publishers: Vec, } @@ -133,18 +147,26 @@ impl SessionState { primitives: None, expr_id_counter: AtomicExprId::new(1), // Note: start at 1 because 0 is reserved for NO_RESOURCE qid_counter: AtomicRequestId::new(0), + #[cfg(feature = "unstable")] + liveliness_qid_counter: AtomicRequestId::new(0), local_resources: HashMap::new(), remote_resources: HashMap::new(), #[cfg(feature = "unstable")] remote_subscribers: HashMap::new(), publishers: HashMap::new(), + #[cfg(feature = "unstable")] + remote_tokens: HashMap::new(), + //publications: Vec::new(), subscribers: HashMap::new(), + liveliness_subscribers: HashMap::new(), queryables: HashMap::new(), #[cfg(feature = "unstable")] tokens: HashMap::new(), #[cfg(feature = "unstable")] matching_listeners: HashMap::new(), queries: HashMap::new(), + #[cfg(feature = "unstable")] + liveliness_queries: HashMap::new(), aggregated_subscribers, aggregated_publishers, } @@ -244,14 +266,32 @@ impl SessionState { self.remote_key_to_expr(key_expr) } } + + pub(crate) fn subscribers(&self, kind: SubscriberKind) -> &HashMap> { + match kind { + SubscriberKind::Subscriber => &self.subscribers, + SubscriberKind::LivelinessSubscriber => &self.liveliness_subscribers, + } + } + + pub(crate) fn subscribers_mut( + &mut self, + kind: SubscriberKind, + ) -> &mut HashMap> { + match kind { + SubscriberKind::Subscriber => &mut self.subscribers, + SubscriberKind::LivelinessSubscriber => &mut self.liveliness_subscribers, + } + } } impl fmt::Debug for SessionState { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, - "SessionState{{ subscribers: {} }}", - self.subscribers.len() + "SessionState{{ subscribers: {}, liveliness_subscribers: {} }}", + self.subscribers.len(), + self.liveliness_subscribers.len() ) } } @@ -259,7 +299,36 @@ impl fmt::Debug for SessionState { pub(crate) struct ResourceNode { pub(crate) key_expr: OwnedKeyExpr, pub(crate) subscribers: Vec>, + pub(crate) liveliness_subscribers: Vec>, } + +impl ResourceNode { + pub(crate) fn new(key_expr: OwnedKeyExpr) -> Self { + Self { + key_expr, + subscribers: Vec::new(), + liveliness_subscribers: Vec::new(), + } + } + + pub(crate) fn subscribers(&self, kind: SubscriberKind) -> &Vec> { + match kind { + SubscriberKind::Subscriber => &self.subscribers, + SubscriberKind::LivelinessSubscriber => &self.liveliness_subscribers, + } + } + + pub(crate) fn subscribers_mut( + &mut self, + kind: SubscriberKind, + ) -> &mut Vec> { + match kind { + SubscriberKind::Subscriber => &mut self.subscribers, + SubscriberKind::LivelinessSubscriber => &mut self.liveliness_subscribers, + } + } +} + pub(crate) enum Resource { Prefix { prefix: Box }, Node(ResourceNode), @@ -274,10 +343,7 @@ impl Resource { } } pub(crate) fn for_keyexpr(key_expr: OwnedKeyExpr) -> Self { - Self::Node(ResourceNode { - key_expr, - subscribers: Vec::new(), - }) + Self::Node(ResourceNode::new(key_expr)) } pub(crate) fn name(&self) -> &str { match self { @@ -881,15 +947,15 @@ impl Session { None => { let expr_id = state.expr_id_counter.fetch_add(1, Ordering::SeqCst); let mut res = Resource::new(Box::from(prefix)); - if let Resource::Node(ResourceNode { - key_expr, - subscribers, - .. - }) = &mut res - { - for sub in state.subscribers.values() { - if key_expr.intersects(&sub.key_expr) { - subscribers.push(sub.clone()); + if let Resource::Node(res_node) = &mut res { + for kind in [ + SubscriberKind::Subscriber, + SubscriberKind::LivelinessSubscriber, + ] { + for sub in state.subscribers(kind).values() { + if res_node.key_expr.intersects(&sub.key_expr) { + res_node.subscribers_mut(kind).push(sub.clone()); + } } } } @@ -1014,7 +1080,7 @@ impl Session { pub(crate) fn declare_subscriber_inner( &self, key_expr: &KeyExpr, - scope: &Option, + scope: Option<&KeyExpr>, origin: Locality, callback: Callback<'static, Sample>, info: &SubscriberInfo, @@ -1031,61 +1097,63 @@ impl Session { id, remote_id: id, key_expr: key_expr.clone().into_owned(), - scope: scope.clone().map(|e| e.into_owned()), + scope: scope.map(|e| e.clone().into_owned()), origin, callback, }; - #[cfg(not(feature = "unstable"))] let declared_sub = origin != Locality::SessionLocal; - #[cfg(feature = "unstable")] - let declared_sub = origin != Locality::SessionLocal - && !key_expr - .as_str() - .starts_with(crate::api::liveliness::PREFIX_LIVELINESS); - - let declared_sub = - declared_sub - .then(|| { - match state - .aggregated_subscribers - .iter() - .find(|s| s.includes(&key_expr)) - { - Some(join_sub) => { - if let Some(joined_sub) = state.subscribers.values().find(|s| { + + let declared_sub = declared_sub + .then(|| { + match state + .aggregated_subscribers + .iter() + .find(|s| s.includes(&key_expr)) + { + Some(join_sub) => { + if let Some(joined_sub) = state + .subscribers(SubscriberKind::Subscriber) + .values() + .find(|s| { s.origin != Locality::SessionLocal && join_sub.includes(&s.key_expr) - }) { - sub_state.remote_id = joined_sub.remote_id; - None - } else { - Some(join_sub.clone().into()) - } + }) + { + sub_state.remote_id = joined_sub.remote_id; + None + } else { + Some(join_sub.clone().into()) } - None => { - if let Some(twin_sub) = state.subscribers.values().find(|s| { - s.origin != Locality::SessionLocal && s.key_expr == key_expr - }) { - sub_state.remote_id = twin_sub.remote_id; - None - } else { - Some(key_expr.clone()) - } + } + None => { + if let Some(twin_sub) = state + .subscribers(SubscriberKind::Subscriber) + .values() + .find(|s| s.origin != Locality::SessionLocal && s.key_expr == key_expr) + { + sub_state.remote_id = twin_sub.remote_id; + None + } else { + Some(key_expr.clone()) } } - }) - .flatten(); + } + }) + .flatten(); let sub_state = Arc::new(sub_state); - state.subscribers.insert(sub_state.id, sub_state.clone()); + state + .subscribers_mut(SubscriberKind::Subscriber) + .insert(sub_state.id, sub_state.clone()); for res in state .local_resources .values_mut() .filter_map(Resource::as_node_mut) { if key_expr.intersects(&res.key_expr) { - res.subscribers.push(sub_state.clone()); + res.subscribers_mut(SubscriberKind::Subscriber) + .push(sub_state.clone()); } } for res in state @@ -1094,7 +1162,8 @@ impl Session { .filter_map(Resource::as_node_mut) { if key_expr.intersects(&res.key_expr) { - res.subscribers.push(sub_state.clone()); + res.subscribers_mut(SubscriberKind::Subscriber) + .push(sub_state.clone()); } } @@ -1141,61 +1210,39 @@ impl Session { let state = zread!(self.state); self.update_status_up(&state, &key_expr) } - } else { - #[cfg(feature = "unstable")] - if key_expr - .as_str() - .starts_with(crate::api::liveliness::PREFIX_LIVELINESS) - { - let primitives = state.primitives.as_ref().unwrap().clone(); - drop(state); - - primitives.send_interest(Interest { - id, - mode: InterestMode::CurrentFuture, - options: InterestOptions::KEYEXPRS + InterestOptions::SUBSCRIBERS, - wire_expr: Some(key_expr.to_wire(self).to_owned()), - ext_qos: network::ext::QoSType::DEFAULT, - ext_tstamp: None, - ext_nodeid: network::ext::NodeIdType::DEFAULT, - }); - } } Ok(sub_state) } - pub(crate) fn undeclare_subscriber_inner(&self, sid: Id) -> ZResult<()> { + pub(crate) fn undeclare_subscriber_inner(&self, sid: Id, kind: SubscriberKind) -> ZResult<()> { let mut state = zwrite!(self.state); - if let Some(sub_state) = state.subscribers.remove(&sid) { + if let Some(sub_state) = state + .subscribers_mut(SubscriberKind::Subscriber) + .remove(&sid) + { trace!("undeclare_subscriber({:?})", sub_state); for res in state .local_resources .values_mut() .filter_map(Resource::as_node_mut) { - res.subscribers.retain(|sub| sub.id != sub_state.id); + res.subscribers_mut(kind) + .retain(|sub| sub.id != sub_state.id); } for res in state .remote_resources .values_mut() .filter_map(Resource::as_node_mut) { - res.subscribers.retain(|sub| sub.id != sub_state.id); + res.subscribers_mut(kind) + .retain(|sub| sub.id != sub_state.id); } - #[cfg(not(feature = "unstable"))] - let send_forget = sub_state.origin != Locality::SessionLocal; - #[cfg(feature = "unstable")] - let send_forget = sub_state.origin != Locality::SessionLocal - && !sub_state - .key_expr - .as_str() - .starts_with(crate::api::liveliness::PREFIX_LIVELINESS); - if send_forget { + if sub_state.origin != Locality::SessionLocal && kind == SubscriberKind::Subscriber { // Note: there might be several Subscribers on the same KeyExpr. // Before calling forget_subscriber(key_expr), check if this was the last one. - if !state.subscribers.values().any(|s| { + if !state.subscribers(kind).values().any(|s| { s.origin != Locality::SessionLocal && s.remote_id == sub_state.remote_id }) { let primitives = state.primitives.as_ref().unwrap().clone(); @@ -1220,11 +1267,7 @@ impl Session { } } else { #[cfg(feature = "unstable")] - if sub_state - .key_expr - .as_str() - .starts_with(crate::api::liveliness::PREFIX_LIVELINESS) - { + if kind == SubscriberKind::LivelinessSubscriber { let primitives = state.primitives.as_ref().unwrap().clone(); drop(state); @@ -1239,6 +1282,7 @@ impl Session { }); } } + Ok(()) } else { Err(zerror!("Unable to find subscriber").into()) @@ -1321,7 +1365,6 @@ impl Session { let mut state = zwrite!(self.state); tracing::trace!("declare_liveliness({:?})", key_expr); let id = self.runtime.next_id(); - let key_expr = KeyExpr::from(*crate::api::liveliness::KE_PREFIX_LIVELINESS / key_expr); let tok_state = Arc::new(LivelinessTokenState { id, key_expr: key_expr.clone().into_owned(), @@ -1335,15 +1378,83 @@ impl Session { ext_qos: declare::ext::QoSType::DECLARE, ext_tstamp: None, ext_nodeid: declare::ext::NodeIdType::DEFAULT, - body: DeclareBody::DeclareSubscriber(DeclareSubscriber { + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr: key_expr.to_wire(self).to_owned(), - ext_info: SubscriberInfo::DEFAULT, }), }); Ok(tok_state) } + #[cfg(feature = "unstable")] + pub(crate) fn declare_liveliness_subscriber_inner( + &self, + key_expr: &KeyExpr, + scope: Option<&KeyExpr>, + origin: Locality, + callback: Callback<'static, Sample>, + ) -> ZResult> { + let mut state = zwrite!(self.state); + trace!("declare_liveliness_subscriber({:?})", key_expr); + let id = self.runtime.next_id(); + let key_expr = match scope { + Some(scope) => scope / key_expr, + None => key_expr.clone(), + }; + + let sub_state = SubscriberState { + id, + remote_id: id, + key_expr: key_expr.clone().into_owned(), + scope: scope.map(|e| e.clone().into_owned()), + origin, + callback, + }; + + let sub_state = Arc::new(sub_state); + + state + .subscribers_mut(SubscriberKind::LivelinessSubscriber) + .insert(sub_state.id, sub_state.clone()); + + for res in state + .local_resources + .values_mut() + .filter_map(Resource::as_node_mut) + { + if key_expr.intersects(&res.key_expr) { + res.subscribers_mut(SubscriberKind::LivelinessSubscriber) + .push(sub_state.clone()); + } + } + + for res in state + .remote_resources + .values_mut() + .filter_map(Resource::as_node_mut) + { + if key_expr.intersects(&res.key_expr) { + res.subscribers_mut(SubscriberKind::LivelinessSubscriber) + .push(sub_state.clone()); + } + } + + let primitives = state.primitives.as_ref().unwrap().clone(); + drop(state); + + primitives.send_interest(Interest { + id, + mode: InterestMode::Future, + options: InterestOptions::KEYEXPRS + InterestOptions::TOKENS, + wire_expr: Some(key_expr.to_wire(self).to_owned()), + ext_qos: declare::ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: declare::ext::NodeIdType::DEFAULT, + }); + + Ok(sub_state) + } + #[zenoh_macros::unstable] pub(crate) fn undeclare_liveliness(&self, tid: Id) -> ZResult<()> { let mut state = zwrite!(self.state); @@ -1360,7 +1471,7 @@ impl Session { ext_qos: ext::QoSType::DECLARE, ext_tstamp: None, ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { + body: DeclareBody::UndeclareToken(UndeclareToken { id: tok_state.id, ext_wire_expr: WireExprType::null(), }), @@ -1530,12 +1641,13 @@ impl Session { } } - pub(crate) fn handle_data( + pub(crate) fn execute_subscriber_callbacks( &self, local: bool, key_expr: &WireExpr, info: Option, payload: ZBuf, + kind: SubscriberKind, attachment: Option, ) { let mut callbacks = SingleOrVec::default(); @@ -1543,7 +1655,7 @@ impl Session { if key_expr.suffix.is_empty() { match state.get_res(&key_expr.scope, key_expr.mapping, local) { Some(Resource::Node(res)) => { - for sub in &res.subscribers { + for sub in res.subscribers(kind) { if sub.origin == Locality::Any || (local == (sub.origin == Locality::SessionLocal)) { @@ -1593,7 +1705,7 @@ impl Session { } else { match state.wireexpr_to_keyexpr(key_expr, local) { Ok(key_expr) => { - for sub in state.subscribers.values() { + for sub in state.subscribers(kind).values() { if (sub.origin == Locality::Any || (local == (sub.origin == Locality::SessionLocal))) && key_expr.intersects(&sub.key_expr) @@ -1783,6 +1895,61 @@ impl Session { Ok(()) } + #[cfg(feature = "unstable")] + pub(crate) fn liveliness_query( + &self, + key_expr: &KeyExpr<'_>, + timeout: Duration, + callback: Callback<'static, Reply>, + ) -> ZResult<()> { + tracing::trace!("liveliness.get({}, {:?})", key_expr, timeout); + let mut state = zwrite!(self.state); + let id = state.liveliness_qid_counter.fetch_add(1, Ordering::SeqCst); + let token = self.task_controller.get_cancellation_token(); + self.task_controller + .spawn_with_rt(zenoh_runtime::ZRuntime::Net, { + let state = self.state.clone(); + let zid = self.runtime.zid(); + async move { + tokio::select! { + _ = tokio::time::sleep(timeout) => { + let mut state = zwrite!(state); + if let Some(query) = state.liveliness_queries.remove(&id) { + std::mem::drop(state); + tracing::debug!("Timeout on liveliness query {}! Send error and close.", id); + (query.callback)(Reply { + result: Err(Value::new("Timeout", Encoding::ZENOH_STRING).into()), + replier_id: Some(zid.into()), + }); + } + } + _ = token.cancelled() => {} + } + } + }); + + tracing::trace!("Register liveliness query {}", id); + let wexpr = key_expr.to_wire(self).to_owned(); + state + .liveliness_queries + .insert(id, LivelinessQueryState { callback }); + + let primitives = state.primitives.as_ref().unwrap().clone(); + drop(state); + + primitives.send_interest(Interest { + id, + mode: InterestMode::Current, + options: InterestOptions::KEYEXPRS + InterestOptions::TOKENS, + wire_expr: Some(wexpr.clone()), + ext_qos: request::ext::QoSType::DEFAULT, + ext_tstamp: None, + ext_nodeid: request::ext::NodeIdType::DEFAULT, + }); + + Ok(()) + } + #[allow(clippy::too_many_arguments)] pub(crate) fn handle_query( &self, @@ -2025,18 +2192,21 @@ impl Primitives for Session { let state = &mut zwrite!(self.state); match state.remote_key_to_expr(&m.wire_expr) { Ok(key_expr) => { - let mut subs = Vec::new(); - for sub in state.subscribers.values() { - if key_expr.intersects(&sub.key_expr) { - subs.push(sub.clone()); + let mut res_node = ResourceNode::new(key_expr.clone().into()); + for kind in [ + SubscriberKind::Subscriber, + SubscriberKind::LivelinessSubscriber, + ] { + for sub in state.subscribers(kind).values() { + if key_expr.intersects(&sub.key_expr) { + res_node.subscribers_mut(kind).push(sub.clone()); + } } } - let res = Resource::Node(ResourceNode { - key_expr: key_expr.into(), - subscribers: subs, - }); - state.remote_resources.insert(m.id, res); + state + .remote_resources + .insert(m.id, Resource::Node(res_node)); } Err(e) => error!( "Received Resource for invalid wire_expr `{}`: {}", @@ -2059,14 +2229,6 @@ impl Primitives for Session { Ok(expr) => { state.remote_subscribers.insert(m.id, expr.clone()); self.update_status_up(&state, &expr); - - if expr - .as_str() - .starts_with(crate::api::liveliness::PREFIX_LIVELINESS) - { - drop(state); - self.handle_data(false, &m.wire_expr, None, ZBuf::default(), None); - } } Err(err) => { tracing::error!( @@ -2084,24 +2246,6 @@ impl Primitives for Session { let mut state = zwrite!(self.state); if let Some(expr) = state.remote_subscribers.remove(&m.id) { self.update_status_down(&state, &expr); - - if expr - .as_str() - .starts_with(crate::api::liveliness::PREFIX_LIVELINESS) - { - drop(state); - let data_info = DataInfo { - kind: SampleKind::Delete, - ..Default::default() - }; - self.handle_data( - false, - &expr.to_wire(self), - Some(data_info), - ZBuf::default(), - None, - ); - } } else { tracing::error!("Received Undeclare Subscriber for unkown id: {}", m.id); } @@ -2113,14 +2257,121 @@ impl Primitives for Session { zenoh_protocol::network::DeclareBody::UndeclareQueryable(m) => { trace!("recv UndeclareQueryable {:?}", m.id); } - DeclareBody::DeclareToken(m) => { + zenoh_protocol::network::DeclareBody::DeclareToken(m) => { trace!("recv DeclareToken {:?}", m.id); + #[cfg(feature = "unstable")] + { + let mut state = zwrite!(self.state); + match state + .wireexpr_to_keyexpr(&m.wire_expr, false) + .map(|e| e.into_owned()) + { + Ok(key_expr) => { + if let Some(interest_id) = msg.interest_id { + if let Some(query) = state.liveliness_queries.get(&interest_id) { + let reply = Reply { + result: Ok(Sample { + key_expr, + payload: ZBytes::empty(), + kind: SampleKind::Put, + encoding: Encoding::default(), + timestamp: None, + qos: QoS::default(), + #[cfg(feature = "unstable")] + source_info: SourceInfo::empty(), + #[cfg(feature = "unstable")] + attachment: None, + }), + replier_id: None, + }; + + (query.callback)(reply); + } + } else { + state.remote_tokens.insert(m.id, key_expr.clone()); + + drop(state); + + self.execute_subscriber_callbacks( + false, + &m.wire_expr, + None, + ZBuf::default(), + SubscriberKind::LivelinessSubscriber, + #[cfg(feature = "unstable")] + None, + ); + } + } + Err(err) => { + tracing::error!("Received DeclareToken for unkown wire_expr: {}", err) + } + } + } } - DeclareBody::UndeclareToken(m) => { + zenoh_protocol::network::DeclareBody::UndeclareToken(m) => { trace!("recv UndeclareToken {:?}", m.id); + #[cfg(feature = "unstable")] + { + let mut state = zwrite!(self.state); + if let Some(key_expr) = state.remote_tokens.remove(&m.id) { + drop(state); + + let data_info = DataInfo { + kind: SampleKind::Delete, + ..Default::default() + }; + + self.execute_subscriber_callbacks( + false, + &key_expr.to_wire(self), + Some(data_info), + ZBuf::default(), + SubscriberKind::LivelinessSubscriber, + #[cfg(feature = "unstable")] + None, + ); + } else if m.ext_wire_expr.wire_expr != WireExpr::empty() { + match state + .wireexpr_to_keyexpr(&m.ext_wire_expr.wire_expr, false) + .map(|e| e.into_owned()) + { + Ok(key_expr) => { + drop(state); + + let data_info = DataInfo { + kind: SampleKind::Delete, + ..Default::default() + }; + + self.execute_subscriber_callbacks( + false, + &key_expr.to_wire(self), + Some(data_info), + ZBuf::default(), + SubscriberKind::LivelinessSubscriber, + #[cfg(feature = "unstable")] + None, + ); + } + Err(err) => { + tracing::error!( + "Received UndeclareToken for unkown wire_expr: {}", + err + ) + } + } + } + } } - DeclareBody::DeclareFinal(_) => { + DeclareBody::DeclareFinal(DeclareFinal) => { trace!("recv DeclareFinal {:?}", msg.interest_id); + + #[cfg(feature = "unstable")] + if let Some(interest_id) = msg.interest_id { + let mut state = zwrite!(self.state); + let _ = state.liveliness_queries.remove(&interest_id); + } } } } @@ -2137,11 +2388,12 @@ impl Primitives for Session { source_id: m.ext_sinfo.as_ref().map(|i| i.id.into()), source_sn: m.ext_sinfo.as_ref().map(|i| i.sn as u64), }; - self.handle_data( + self.execute_subscriber_callbacks( false, &msg.wire_expr, Some(info), m.payload, + SubscriberKind::Subscriber, m.ext_attachment.map(Into::into), ) } @@ -2154,11 +2406,12 @@ impl Primitives for Session { source_id: m.ext_sinfo.as_ref().map(|i| i.id.into()), source_sn: m.ext_sinfo.as_ref().map(|i| i.sn as u64), }; - self.handle_data( + self.execute_subscriber_callbacks( false, &msg.wire_expr, Some(info), ZBuf::empty(), + SubscriberKind::Subscriber, m.ext_attachment.map(Into::into), ) } diff --git a/zenoh/src/api/subscriber.rs b/zenoh/src/api/subscriber.rs index c77dbc8791..493df4a54c 100644 --- a/zenoh/src/api/subscriber.rs +++ b/zenoh/src/api/subscriber.rs @@ -78,6 +78,7 @@ impl fmt::Debug for SubscriberState { pub(crate) struct SubscriberInner<'a> { pub(crate) session: SessionRef<'a>, pub(crate) state: Arc, + pub(crate) kind: SubscriberKind, pub(crate) undeclare_on_drop: bool, } @@ -146,7 +147,7 @@ impl Wait for SubscriberUndeclaration<'_> { self.subscriber.undeclare_on_drop = false; self.subscriber .session - .undeclare_subscriber_inner(self.subscriber.state.id) + .undeclare_subscriber_inner(self.subscriber.state.id, self.subscriber.kind) } } @@ -162,7 +163,9 @@ impl IntoFuture for SubscriberUndeclaration<'_> { impl Drop for SubscriberInner<'_> { fn drop(&mut self) { if self.undeclare_on_drop { - let _ = self.session.undeclare_subscriber_inner(self.state.id); + let _ = self + .session + .undeclare_subscriber_inner(self.state.id, self.kind); } } } @@ -377,7 +380,7 @@ where session .declare_subscriber_inner( &key_expr, - &None, + None, self.origin, callback, &SubscriberInfo { @@ -388,6 +391,7 @@ where subscriber: SubscriberInner { session, state: sub_state, + kind: SubscriberKind::Subscriber, undeclare_on_drop: true, }, handler: receiver, @@ -540,3 +544,9 @@ impl DerefMut for Subscriber<'_, Handler> { /// A [`Subscriber`] that provides data through a `flume` channel. pub type FlumeSubscriber<'a> = Subscriber<'a, flume::Receiver>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SubscriberKind { + Subscriber, + LivelinessSubscriber, +} diff --git a/zenoh/src/net/routing/dispatcher/face.rs b/zenoh/src/net/routing/dispatcher/face.rs index 88f2abe9c1..4627a40654 100644 --- a/zenoh/src/net/routing/dispatcher/face.rs +++ b/zenoh/src/net/routing/dispatcher/face.rs @@ -287,10 +287,25 @@ impl Primitives for Face { ); } zenoh_protocol::network::DeclareBody::DeclareToken(m) => { - tracing::warn!("Received unsupported {m:?}") + declare_token( + ctrl_lock.as_ref(), + &self.tables, + &mut self.state.clone(), + m.id, + &m.wire_expr, + msg.ext_nodeid.node_id, + msg.interest_id, + ); } zenoh_protocol::network::DeclareBody::UndeclareToken(m) => { - tracing::warn!("Received unsupported {m:?}") + undeclare_token( + ctrl_lock.as_ref(), + &self.tables, + &mut self.state.clone(), + m.id, + &m.ext_wire_expr, + msg.ext_nodeid.node_id, + ); } zenoh_protocol::network::DeclareBody::DeclareFinal(_) => { if let Some(id) = msg.interest_id { diff --git a/zenoh/src/net/routing/dispatcher/interests.rs b/zenoh/src/net/routing/dispatcher/interests.rs index ab3764d14f..32724363f9 100644 --- a/zenoh/src/net/routing/dispatcher/interests.rs +++ b/zenoh/src/net/routing/dispatcher/interests.rs @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 ZettaScale Technology +// Copyright (c) 2024 ZettaScale Technology // // This program and the accompanying materials are made available under the // terms of the Eclipse Public License 2.0 which is available at diff --git a/zenoh/src/net/routing/dispatcher/mod.rs b/zenoh/src/net/routing/dispatcher/mod.rs index 0f42ae2aee..dc17b91b6b 100644 --- a/zenoh/src/net/routing/dispatcher/mod.rs +++ b/zenoh/src/net/routing/dispatcher/mod.rs @@ -23,3 +23,4 @@ pub mod pubsub; pub mod queries; pub mod resource; pub mod tables; +pub mod token; diff --git a/zenoh/src/net/routing/dispatcher/queries.rs b/zenoh/src/net/routing/dispatcher/queries.rs index 56e4ce9335..f1163c829d 100644 --- a/zenoh/src/net/routing/dispatcher/queries.rs +++ b/zenoh/src/net/routing/dispatcher/queries.rs @@ -21,6 +21,8 @@ use async_trait::async_trait; use tokio_util::sync::CancellationToken; use zenoh_buffers::ZBuf; use zenoh_config::WhatAmI; +#[cfg(feature = "stats")] +use zenoh_protocol::zenoh::reply::ReplyBody; use zenoh_protocol::{ core::{key_expr::keyexpr, Encoding, WireExpr}, network::{ @@ -31,7 +33,7 @@ use zenoh_protocol::{ }, response::{self, ext::ResponderIdType, Response, ResponseFinal}, }, - zenoh::{self, query::Consolidation, reply::ReplyBody, Put, Reply, RequestBody, ResponseBody}, + zenoh::{self, RequestBody, ResponseBody}, }; use zenoh_sync::get_mut_unchecked; use zenoh_util::Timed; @@ -572,58 +574,10 @@ pub fn route_query( let queries_lock = zwrite!(tables_ref.queries_lock); let route = compute_final_route(&rtables, &route, face, &mut expr, &ext_target, query); - let local_replies = - rtables - .hat_code - .compute_local_replies(&rtables, &prefix, expr.suffix, face); - let zid = rtables.zid; - let timeout = ext_timeout.unwrap_or(rtables.queries_default_timeout); - drop(queries_lock); drop(rtables); - for (wexpr, payload) in local_replies { - let payload = ResponseBody::Reply(Reply { - consolidation: Consolidation::DEFAULT, // @TODO: handle Del case - ext_unknown: vec![], // @TODO: handle unknown extensions - payload: ReplyBody::Put(Put { - // @TODO: handle Del case - timestamp: None, // @TODO: handle timestamp - encoding: Encoding::empty(), // @TODO: handle encoding - ext_sinfo: None, // @TODO: handle source info - ext_attachment: None, // @TODO: expose it in the API - #[cfg(feature = "shared-memory")] - ext_shm: None, - ext_unknown: vec![], // @TODO: handle unknown extensions - payload, - }), - }); - #[cfg(feature = "stats")] - if !admin { - inc_res_stats!(face, tx, user, payload) - } else { - inc_res_stats!(face, tx, admin, payload) - } - - face.primitives - .clone() - .send_response(RoutingContext::with_expr( - Response { - rid: qid, - wire_expr: wexpr, - payload, - ext_qos: response::ext::QoSType::DECLARE, - ext_tstamp: None, - ext_respid: Some(response::ext::ResponderIdType { - zid, - eid: 0, // 0 is reserved for routing core - }), - }, - expr.full_expr().to_string(), - )); - } - if route.is_empty() { tracing::debug!( "Send final reply {}:{} (no matching queryables or not master)", diff --git a/zenoh/src/net/routing/dispatcher/resource.rs b/zenoh/src/net/routing/dispatcher/resource.rs index e6b13dc2c8..a638c9a24f 100644 --- a/zenoh/src/net/routing/dispatcher/resource.rs +++ b/zenoh/src/net/routing/dispatcher/resource.rs @@ -57,6 +57,7 @@ pub(crate) struct SessionContext { pub(crate) remote_expr_id: Option, pub(crate) subs: Option, pub(crate) qabl: Option, + pub(crate) token: bool, pub(crate) in_interceptor_cache: Option>, pub(crate) e_interceptor_cache: Option>, } @@ -69,6 +70,7 @@ impl SessionContext { remote_expr_id: None, subs: None, qabl: None, + token: false, in_interceptor_cache: None, e_interceptor_cache: None, } diff --git a/zenoh/src/net/routing/dispatcher/token.rs b/zenoh/src/net/routing/dispatcher/token.rs new file mode 100644 index 0000000000..c563ce8802 --- /dev/null +++ b/zenoh/src/net/routing/dispatcher/token.rs @@ -0,0 +1,146 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +use std::sync::Arc; + +use zenoh_keyexpr::keyexpr; +use zenoh_protocol::{ + core::WireExpr, + network::{ + declare::{common::ext, TokenId}, + interest::InterestId, + }, +}; + +use super::{ + face::FaceState, + tables::{NodeId, TablesLock}, +}; +use crate::net::routing::{hat::HatTrait, router::Resource}; + +pub(crate) fn declare_token( + hat_code: &(dyn HatTrait + Send + Sync), + tables: &TablesLock, + face: &mut Arc, + id: TokenId, + expr: &WireExpr, + node_id: NodeId, + interest_id: Option, +) { + let rtables = zread!(tables.tables); + match rtables + .get_mapping(face, &expr.scope, expr.mapping) + .cloned() + { + Some(mut prefix) => { + tracing::debug!( + "{} Declare token {} ({}{})", + face, + id, + prefix.expr(), + expr.suffix + ); + let res = Resource::get_resource(&prefix, &expr.suffix); + let (mut res, mut wtables) = + if res.as_ref().map(|r| r.context.is_some()).unwrap_or(false) { + drop(rtables); + let wtables = zwrite!(tables.tables); + (res.unwrap(), wtables) + } else { + let mut fullexpr = prefix.expr(); + fullexpr.push_str(expr.suffix.as_ref()); + let mut matches = keyexpr::new(fullexpr.as_str()) + .map(|ke| Resource::get_matches(&rtables, ke)) + .unwrap_or_default(); + drop(rtables); + let mut wtables = zwrite!(tables.tables); + let mut res = + Resource::make_resource(&mut wtables, &mut prefix, expr.suffix.as_ref()); + matches.push(Arc::downgrade(&res)); + Resource::match_resource(&wtables, &mut res, matches); + (res, wtables) + }; + + hat_code.declare_token(&mut wtables, face, id, &mut res, node_id, interest_id); + drop(wtables); + } + None => tracing::error!( + "{} Declare token {} for unknown scope {}!", + face, + id, + expr.scope + ), + } +} + +pub(crate) fn undeclare_token( + hat_code: &(dyn HatTrait + Send + Sync), + tables: &TablesLock, + face: &mut Arc, + id: TokenId, + expr: &ext::WireExprType, + node_id: NodeId, +) { + let (res, mut wtables) = if expr.wire_expr.is_empty() { + (None, zwrite!(tables.tables)) + } else { + let rtables = zread!(tables.tables); + match rtables + .get_mapping(face, &expr.wire_expr.scope, expr.wire_expr.mapping) + .cloned() + { + Some(mut prefix) => { + match Resource::get_resource(&prefix, expr.wire_expr.suffix.as_ref()) { + Some(res) => { + drop(rtables); + (Some(res), zwrite!(tables.tables)) + } + None => { + // Here we create a Resource that will immediately be removed after treatment + // TODO this could be improved + let mut fullexpr = prefix.expr(); + fullexpr.push_str(expr.wire_expr.suffix.as_ref()); + let mut matches = keyexpr::new(fullexpr.as_str()) + .map(|ke| Resource::get_matches(&rtables, ke)) + .unwrap_or_default(); + drop(rtables); + let mut wtables = zwrite!(tables.tables); + let mut res = Resource::make_resource( + &mut wtables, + &mut prefix, + expr.wire_expr.suffix.as_ref(), + ); + matches.push(Arc::downgrade(&res)); + Resource::match_resource(&wtables, &mut res, matches); + (Some(res), wtables) + } + } + } + None => { + tracing::error!( + "{} Undeclare liveliness token with unknown scope {}", + face, + expr.wire_expr.scope + ); + return; + } + } + }; + + if let Some(res) = hat_code.undeclare_token(&mut wtables, face, id, res, node_id) { + tracing::debug!("{} Undeclare token {} ({})", face, id, res.expr()); + } else { + tracing::error!("{} Undeclare unknown token {}", face, id); + } +} diff --git a/zenoh/src/net/routing/hat/client/interests.rs b/zenoh/src/net/routing/hat/client/interests.rs index 3757677893..57e380ee12 100644 --- a/zenoh/src/net/routing/hat/client/interests.rs +++ b/zenoh/src/net/routing/hat/client/interests.rs @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 ZettaScale Technology +// Copyright (c) 2024 ZettaScale Technology // // This program and the accompanying materials are made available under the // terms of the Eclipse Public License 2.0 which is available at @@ -23,10 +23,11 @@ use zenoh_protocol::{ }; use zenoh_sync::get_mut_unchecked; -use super::{face_hat, face_hat_mut, HatCode, HatFace}; +use super::{face_hat, face_hat_mut, token::declare_token_interest, HatCode, HatFace}; use crate::net::routing::{ dispatcher::{ face::{FaceState, InterestState}, + interests::{CurrentInterest, CurrentInterestCleanup}, resource::Resource, tables::{Tables, TablesLock}, }, @@ -74,16 +75,32 @@ impl HatInterestTrait for HatCode { fn declare_interest( &self, tables: &mut Tables, - _tables_ref: &Arc, + tables_ref: &Arc, face: &mut Arc, id: InterestId, res: Option<&mut Arc>, mode: InterestMode, options: InterestOptions, ) { + if options.tokens() { + declare_token_interest( + tables, + face, + id, + res.as_ref().map(|r| (*r).clone()).as_mut(), + mode, + options.aggregate(), + ) + } face_hat_mut!(face) .remote_interests .insert(id, (res.as_ref().map(|res| (*res).clone()), options)); + + let interest = Arc::new(CurrentInterest { + src_face: face.clone(), + src_interest_id: id, + }); + for dst_face in tables .faces .values_mut() @@ -98,6 +115,14 @@ impl HatInterestTrait for HatCode { finalized: mode == InterestMode::Future, }, ); + if mode.current() && options.tokens() { + let dst_face_mut = get_mut_unchecked(dst_face); + let cancellation_token = dst_face_mut.task_controller.get_cancellation_token(); + dst_face_mut + .pending_current_interests + .insert(id, (interest.clone(), cancellation_token)); + CurrentInterestCleanup::spawn_interest_clean_up_task(dst_face, tables_ref, id); + } let wire_expr = res.as_ref().map(|res| Resource::decl_key(res, dst_face)); dst_face.primitives.send_interest(RoutingContext::with_expr( Interest { @@ -114,13 +139,34 @@ impl HatInterestTrait for HatCode { } if mode.current() { - face.primitives.send_declare(RoutingContext::new(Declare { - interest_id: Some(id), - ext_qos: ext::QoSType::DECLARE, - ext_tstamp: None, - ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::DeclareFinal(DeclareFinal), - })); + if options.tokens() { + if let Some(interest) = Arc::into_inner(interest) { + tracing::debug!( + "Propagate DeclareFinal {}:{}", + interest.src_face, + interest.src_interest_id + ); + interest + .src_face + .primitives + .clone() + .send_declare(RoutingContext::new(Declare { + interest_id: Some(interest.src_interest_id), + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareFinal(DeclareFinal), + })); + } + } else { + face.primitives.send_declare(RoutingContext::new(Declare { + interest_id: Some(id), + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareFinal(DeclareFinal), + })); + } } } diff --git a/zenoh/src/net/routing/hat/client/mod.rs b/zenoh/src/net/routing/hat/client/mod.rs index f41b36e584..0cbdd6d4bc 100644 --- a/zenoh/src/net/routing/hat/client/mod.rs +++ b/zenoh/src/net/routing/hat/client/mod.rs @@ -23,9 +23,10 @@ use std::{ sync::{atomic::AtomicU32, Arc}, }; +use token::{token_new_face, undeclare_client_token}; use zenoh_config::WhatAmI; use zenoh_protocol::network::{ - declare::{queryable::ext::QueryableInfoType, QueryableId, SubscriberId}, + declare::{queryable::ext::QueryableInfoType, QueryableId, SubscriberId, TokenId}, interest::{InterestId, InterestOptions}, Oam, }; @@ -56,6 +57,7 @@ use crate::net::{ mod interests; mod pubsub; mod queries; +mod token; macro_rules! face_hat { ($f:expr) => { @@ -105,6 +107,7 @@ impl HatBaseTrait for HatCode { interests_new_face(tables, &mut face.state); pubsub_new_face(tables, &mut face.state); queries_new_face(tables, &mut face.state); + token_new_face(tables, &mut face.state); Ok(()) } @@ -118,18 +121,27 @@ impl HatBaseTrait for HatCode { interests_new_face(tables, &mut face.state); pubsub_new_face(tables, &mut face.state); queries_new_face(tables, &mut face.state); + token_new_face(tables, &mut face.state); Ok(()) } fn close_face(&self, tables: &TablesLock, face: &mut Arc) { let mut wtables = zwrite!(tables.tables); let mut face_clone = face.clone(); + let face = get_mut_unchecked(face); + let hat_face = match face.hat.downcast_mut::() { + Some(hate_face) => hate_face, + None => { + tracing::error!("Error downcasting face hat in close_face!"); + return; + } + }; - face_hat_mut!(face).remote_interests.clear(); - face_hat_mut!(face).local_subs.clear(); - face_hat_mut!(face).local_qabls.clear(); + hat_face.remote_interests.clear(); + hat_face.local_subs.clear(); + hat_face.local_qabls.clear(); + hat_face.local_tokens.clear(); - let face = get_mut_unchecked(face); for res in face.remote_mappings.values_mut() { get_mut_unchecked(res).session_ctxs.remove(&face.id); Resource::clean(res); @@ -142,13 +154,7 @@ impl HatBaseTrait for HatCode { face.local_mappings.clear(); let mut subs_matches = vec![]; - for (_id, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_subs - .drain() - { + for (_id, mut res) in hat_face.remote_subs.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_subscription(&mut wtables, &mut face_clone, &mut res); @@ -170,13 +176,7 @@ impl HatBaseTrait for HatCode { } let mut qabls_matches = vec![]; - for (_id, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_qabls - .drain() - { + for (_id, mut res) in hat_face.remote_qabls.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_queryable(&mut wtables, &mut face_clone, &mut res); @@ -196,6 +196,11 @@ impl HatBaseTrait for HatCode { qabls_matches.push(res); } } + + for (_id, mut res) in hat_face.remote_tokens.drain() { + get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); + undeclare_client_token(&mut wtables, &mut face_clone, &mut res); + } drop(wtables); let mut matches_data_routes = vec![]; @@ -296,6 +301,8 @@ struct HatFace { remote_subs: HashMap>, local_qabls: HashMap, (QueryableId, QueryableInfoType)>, remote_qabls: HashMap>, + local_tokens: HashMap, TokenId>, + remote_tokens: HashMap>, } impl HatFace { @@ -307,6 +314,8 @@ impl HatFace { remote_subs: HashMap::new(), local_qabls: HashMap::new(), remote_qabls: HashMap::new(), + local_tokens: HashMap::new(), + remote_tokens: HashMap::new(), } } } diff --git a/zenoh/src/net/routing/hat/client/pubsub.rs b/zenoh/src/net/routing/hat/client/pubsub.rs index 7ba6005e5a..41dae88cdf 100644 --- a/zenoh/src/net/routing/hat/client/pubsub.rs +++ b/zenoh/src/net/routing/hat/client/pubsub.rs @@ -37,7 +37,7 @@ use crate::{ }, hat::{HatPubSubTrait, Sources}, router::{update_data_routes_from, RoutesIndexes}, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }, }; @@ -49,8 +49,7 @@ fn propagate_simple_subscription_to( sub_info: &SubscriberInfo, src_face: &mut Arc, ) { - if (src_face.id != dst_face.id - || (dst_face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS))) + if src_face.id != dst_face.id && !face_hat!(dst_face).local_subs.contains_key(res) && (src_face.whatami == WhatAmI::Client || dst_face.whatami == WhatAmI::Client) { @@ -201,22 +200,20 @@ pub(super) fn undeclare_client_subscription( } if client_subs.len() == 1 { let face = &mut client_subs[0]; - if !(face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS)) { - if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { - face.primitives.send_declare(RoutingContext::with_expr( - Declare { - interest_id: None, - ext_qos: ext::QoSType::DECLARE, - ext_tstamp: None, - ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { - id, - ext_wire_expr: WireExprType::null(), - }), - }, - res.expr(), - )); - } + if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); } } } diff --git a/zenoh/src/net/routing/hat/client/queries.rs b/zenoh/src/net/routing/hat/client/queries.rs index bc1fddbb3b..cebc04cd2f 100644 --- a/zenoh/src/net/routing/hat/client/queries.rs +++ b/zenoh/src/net/routing/hat/client/queries.rs @@ -18,14 +18,13 @@ use std::{ }; use ordered_float::OrderedFloat; -use zenoh_buffers::ZBuf; use zenoh_protocol::{ core::{ key_expr::{ include::{Includer, DEFAULT_INCLUDER}, OwnedKeyExpr, }, - WhatAmI, WireExpr, + WhatAmI, }, network::declare::{ common::ext::WireExprType, ext, queryable::ext::QueryableInfoType, Declare, DeclareBody, @@ -43,7 +42,7 @@ use crate::net::routing::{ }, hat::{HatQueriesTrait, Sources}, router::RoutesIndexes, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }; #[inline] @@ -86,11 +85,17 @@ fn propagate_simple_queryable( for mut dst_face in faces { let info = local_qabl_info(tables, res, &dst_face); let current = face_hat!(dst_face).local_qabls.get(res); - if (src_face.is_none() || src_face.as_ref().unwrap().id != dst_face.id) + if src_face + .as_ref() + .map(|src_face| dst_face.id != src_face.id) + .unwrap_or(true) && (current.is_none() || current.unwrap().1 != info) - && (src_face.is_none() - || src_face.as_ref().unwrap().whatami == WhatAmI::Client - || dst_face.whatami == WhatAmI::Client) + && src_face + .as_ref() + .map(|src_face| { + src_face.whatami == WhatAmI::Client || dst_face.whatami == WhatAmI::Client + }) + .unwrap_or(true) { let id = current .map(|c| c.0) @@ -360,44 +365,6 @@ impl HatQueriesTrait for HatCode { Arc::new(route) } - #[inline] - fn compute_local_replies( - &self, - tables: &Tables, - prefix: &Arc, - suffix: &str, - face: &Arc, - ) -> Vec<(WireExpr<'static>, ZBuf)> { - let mut result = vec![]; - // Only the first routing point in the query route - // should return the liveliness tokens - if face.whatami == WhatAmI::Client { - let key_expr = prefix.expr() + suffix; - let key_expr = match OwnedKeyExpr::try_from(key_expr) { - Ok(ke) => ke, - Err(e) => { - tracing::warn!("Invalid KE reached the system: {}", e); - return result; - } - }; - if key_expr.starts_with(PREFIX_LIVELINESS) { - let res = Resource::get_resource(prefix, suffix); - let matches = res - .as_ref() - .and_then(|res| res.context.as_ref()) - .map(|ctx| Cow::from(&ctx.matches)) - .unwrap_or_else(|| Cow::from(Resource::get_matches(tables, &key_expr))); - for mres in matches.iter() { - let mres = mres.upgrade().unwrap(); - if mres.session_ctxs.values().any(|ctx| ctx.subs.is_some()) { - result.push((Resource::get_best_key(&mres, "", face.id), ZBuf::default())); - } - } - } - } - result - } - fn get_query_routes_entries(&self, _tables: &Tables) -> RoutesIndexes { get_routes_entries() } diff --git a/zenoh/src/net/routing/hat/client/token.rs b/zenoh/src/net/routing/hat/client/token.rs new file mode 100644 index 0000000000..3b52bad36a --- /dev/null +++ b/zenoh/src/net/routing/hat/client/token.rs @@ -0,0 +1,383 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +use std::sync::{atomic::Ordering, Arc}; + +use zenoh_config::WhatAmI; +use zenoh_protocol::network::{ + declare::{common::ext::WireExprType, TokenId}, + ext, + interest::{InterestId, InterestMode}, + Declare, DeclareBody, DeclareToken, UndeclareToken, +}; +use zenoh_sync::get_mut_unchecked; + +use super::{face_hat, face_hat_mut, HatCode, HatFace}; +use crate::net::routing::{ + dispatcher::{face::FaceState, tables::Tables}, + hat::{CurrentFutureTrait, HatTokenTrait}, + router::{NodeId, Resource, SessionContext}, + RoutingContext, +}; + +#[inline] +fn propagate_simple_token_to( + _tables: &mut Tables, + dst_face: &mut Arc, + res: &Arc, + src_face: &mut Arc, +) { + if (src_face.id != dst_face.id || dst_face.whatami == WhatAmI::Client) + && !face_hat!(dst_face).local_tokens.contains_key(res) + && (src_face.whatami == WhatAmI::Client || dst_face.whatami == WhatAmI::Client) + { + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } +} + +fn propagate_simple_token(tables: &mut Tables, res: &Arc, src_face: &mut Arc) { + for mut dst_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + propagate_simple_token_to(tables, &mut dst_face, res, src_face); + } +} + +fn register_client_token( + _tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + // Register liveliness + { + let res = get_mut_unchecked(res); + match res.session_ctxs.get_mut(&face.id) { + Some(ctx) => { + if !ctx.token { + get_mut_unchecked(ctx).token = true; + } + } + None => { + let ctx = res + .session_ctxs + .entry(face.id) + .or_insert_with(|| Arc::new(SessionContext::new(face.clone()))); + get_mut_unchecked(ctx).token = true; + } + } + } + face_hat_mut!(face).remote_tokens.insert(id, res.clone()); +} + +fn declare_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, + interest_id: Option, +) { + register_client_token(tables, face, id, res); + + propagate_simple_token(tables, res, face); + + let wire_expr = Resource::decl_key(res, face); + if let Some(interest_id) = interest_id { + if let Some((interest, _)) = face.pending_current_interests.get(&interest_id) { + interest + .src_face + .primitives + .send_declare(RoutingContext::with_expr( + Declare { + interest_id: Some(interest.src_interest_id), + ext_qos: ext::QoSType::default(), + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::default(), + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + res.expr(), + )) + } + } +} + +#[inline] +fn client_tokens(res: &Arc) -> Vec> { + res.session_ctxs + .values() + .filter_map(|ctx| { + if ctx.token { + Some(ctx.face.clone()) + } else { + None + } + }) + .collect() +} + +fn propagate_forget_simple_token(tables: &mut Tables, res: &Arc) { + for face in tables.faces.values_mut() { + if let Some(id) = face_hat_mut!(face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } else if face_hat!(face) + .remote_interests + .values() + .any(|(r, o)| o.tokens() && r.as_ref().map(|r| r.matches(res)).unwrap_or(true)) + { + // Token has never been declared on this face. + // Send an Undeclare with a one shot generated id and a WireExpr ext. + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst), + ext_wire_expr: WireExprType { + wire_expr: Resource::get_best_key(res, "", face.id), + }, + }), + }, + res.expr(), + )); + } + } +} + +pub(super) fn undeclare_client_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, +) { + if !face_hat_mut!(face) + .remote_tokens + .values() + .any(|s| *s == *res) + { + if let Some(ctx) = get_mut_unchecked(res).session_ctxs.get_mut(&face.id) { + get_mut_unchecked(ctx).token = false; + } + + let mut client_tokens = client_tokens(res); + if client_tokens.is_empty() { + propagate_forget_simple_token(tables, res); + } + if client_tokens.len() == 1 { + let face = &mut client_tokens[0]; + if face.whatami != WhatAmI::Client { + if let Some(id) = face_hat_mut!(face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + } + } + } +} + +fn forget_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, +) -> Option> { + if let Some(mut res) = face_hat_mut!(face).remote_tokens.remove(&id) { + undeclare_client_token(tables, face, &mut res); + Some(res) + } else if let Some(mut res) = res { + undeclare_client_token(tables, face, &mut res); + Some(res) + } else { + None + } +} + +pub(super) fn token_new_face(tables: &mut Tables, face: &mut Arc) { + for src_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + for token in face_hat!(src_face).remote_tokens.values() { + propagate_simple_token_to(tables, face, token, &mut src_face.clone()); + } + } +} + +pub(crate) fn declare_token_interest( + tables: &mut Tables, + face: &mut Arc, + id: InterestId, + res: Option<&mut Arc>, + mode: InterestMode, + aggregate: bool, +) { + if mode.current() { + let interest_id = (!mode.future()).then_some(id); + if let Some(res) = res.as_ref() { + if aggregate { + if tables.faces.values().any(|src_face| { + face_hat!(src_face) + .remote_tokens + .values() + .any(|token| token.context.is_some() && token.matches(res)) + }) { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert((*res).clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(res, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + res.expr(), + )); + } + } else { + for src_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + for token in face_hat!(src_face).remote_tokens.values() { + if token.context.is_some() && token.matches(res) { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::default(), + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::default(), + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + res.expr(), + )) + } + } + } + } + } else { + for src_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + for token in face_hat!(src_face).remote_tokens.values() { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } +} + +impl HatTokenTrait for HatCode { + fn declare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, + _node_id: NodeId, + interest_id: Option, + ) { + declare_client_token(tables, face, id, res, interest_id); + } + + fn undeclare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, + _node_id: NodeId, + ) -> Option> { + forget_client_token(tables, face, id, res) + } +} diff --git a/zenoh/src/net/routing/hat/linkstate_peer/interests.rs b/zenoh/src/net/routing/hat/linkstate_peer/interests.rs index 413f06f67b..40bfb49780 100644 --- a/zenoh/src/net/routing/hat/linkstate_peer/interests.rs +++ b/zenoh/src/net/routing/hat/linkstate_peer/interests.rs @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 ZettaScale Technology +// Copyright (c) 2024 ZettaScale Technology // // This program and the accompanying materials are made available under the // terms of the Eclipse Public License 2.0 which is available at @@ -21,7 +21,8 @@ use zenoh_protocol::network::{ use zenoh_sync::get_mut_unchecked; use super::{ - face_hat_mut, pubsub::declare_sub_interest, queries::declare_qabl_interest, HatCode, HatFace, + face_hat_mut, pubsub::declare_sub_interest, queries::declare_qabl_interest, + token::declare_token_interest, HatCode, HatFace, }; use crate::net::routing::{ dispatcher::{ @@ -64,6 +65,16 @@ impl HatInterestTrait for HatCode { options.aggregate(), ) } + if options.tokens() { + declare_token_interest( + tables, + face, + id, + res.as_ref().map(|r| (*r).clone()).as_mut(), + mode, + options.aggregate(), + ) + } if mode.future() { face_hat_mut!(face) .remote_interests diff --git a/zenoh/src/net/routing/hat/linkstate_peer/mod.rs b/zenoh/src/net/routing/hat/linkstate_peer/mod.rs index bc10eaee8a..500ac29510 100644 --- a/zenoh/src/net/routing/hat/linkstate_peer/mod.rs +++ b/zenoh/src/net/routing/hat/linkstate_peer/mod.rs @@ -24,6 +24,7 @@ use std::{ time::Duration, }; +use token::{token_remove_node, undeclare_client_token}; use zenoh_config::{unwrap_or_default, ModeDependent, WhatAmI, WhatAmIMatcher}; use zenoh_protocol::{ common::ZExtBody, @@ -67,6 +68,7 @@ mod interests; mod network; mod pubsub; mod queries; +mod token; macro_rules! hat { ($t:expr) => { @@ -116,6 +118,7 @@ use face_hat_mut; struct HatTables { peer_subs: HashSet>, + peer_tokens: HashSet>, peer_qabls: HashSet>, peers_net: Option, peers_trees_task: Option, @@ -134,6 +137,7 @@ impl HatTables { fn new() -> Self { Self { peer_subs: HashSet::new(), + peer_tokens: HashSet::new(), peer_qabls: HashSet::new(), peers_net: None, peers_trees_task: None, @@ -157,6 +161,7 @@ impl HatTables { tracing::trace!("Compute routes"); pubsub::pubsub_tree_change(&mut tables, &new_childs); queries::queries_tree_change(&mut tables, &new_childs); + token::token_tree_change(&mut tables, &new_childs); tracing::trace!("Computations completed"); hat_mut!(tables).peers_trees_task = None; @@ -250,12 +255,20 @@ impl HatBaseTrait for HatCode { fn close_face(&self, tables: &TablesLock, face: &mut Arc) { let mut wtables = zwrite!(tables.tables); let mut face_clone = face.clone(); + let face = get_mut_unchecked(face); + let hat_face = match face.hat.downcast_mut::() { + Some(hate_face) => hate_face, + None => { + tracing::error!("Error downcasting face hat in close_face!"); + return; + } + }; - face_hat_mut!(face).remote_interests.clear(); - face_hat_mut!(face).local_subs.clear(); - face_hat_mut!(face).local_qabls.clear(); + hat_face.remote_interests.clear(); + hat_face.local_subs.clear(); + hat_face.local_qabls.clear(); + hat_face.local_tokens.clear(); - let face = get_mut_unchecked(face); for res in face.remote_mappings.values_mut() { get_mut_unchecked(res).session_ctxs.remove(&face.id); Resource::clean(res); @@ -268,13 +281,7 @@ impl HatBaseTrait for HatCode { face.local_mappings.clear(); let mut subs_matches = vec![]; - for (_id, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_subs - .drain() - { + for (_id, mut res) in hat_face.remote_subs.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_subscription(&mut wtables, &mut face_clone, &mut res); @@ -296,13 +303,7 @@ impl HatBaseTrait for HatCode { } let mut qabls_matches = vec![]; - for (_, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_qabls - .drain() - { + for (_, mut res) in hat_face.remote_qabls.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_queryable(&mut wtables, &mut face_clone, &mut res); @@ -322,6 +323,11 @@ impl HatBaseTrait for HatCode { qabls_matches.push(res); } } + + for (_id, mut res) in hat_face.remote_tokens.drain() { + get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); + undeclare_client_token(&mut wtables, &mut face_clone, &mut res); + } drop(wtables); let mut matches_data_routes = vec![]; @@ -377,6 +383,7 @@ impl HatBaseTrait for HatCode { for (_, removed_node) in changes.removed_nodes { pubsub_remove_node(tables, &removed_node.zid); queries_remove_node(tables, &removed_node.zid); + token_remove_node(tables, &removed_node.zid); } hat_mut!(tables).schedule_compute_trees(tables_ref.clone()); @@ -420,6 +427,7 @@ impl HatBaseTrait for HatCode { { pubsub_remove_node(tables, &removed_node.zid); queries_remove_node(tables, &removed_node.zid); + token_remove_node(tables, &removed_node.zid); } hat_mut!(tables).schedule_compute_trees(tables_ref.clone()); @@ -463,17 +471,17 @@ impl HatBaseTrait for HatCode { } struct HatContext { - router_subs: HashSet, peer_subs: HashSet, peer_qabls: HashMap, + peer_tokens: HashSet, } impl HatContext { fn new() -> Self { Self { - router_subs: HashSet::new(), peer_subs: HashSet::new(), peer_qabls: HashMap::new(), + peer_tokens: HashSet::new(), } } } @@ -484,6 +492,8 @@ struct HatFace { remote_interests: HashMap>, InterestOptions)>, local_subs: HashMap, SubscriberId>, remote_subs: HashMap>, + local_tokens: HashMap, SubscriberId>, + remote_tokens: HashMap>, local_qabls: HashMap, (QueryableId, QueryableInfoType)>, remote_qabls: HashMap>, } @@ -498,6 +508,8 @@ impl HatFace { remote_subs: HashMap::new(), local_qabls: HashMap::new(), remote_qabls: HashMap::new(), + local_tokens: HashMap::new(), + remote_tokens: HashMap::new(), } } } diff --git a/zenoh/src/net/routing/hat/linkstate_peer/pubsub.rs b/zenoh/src/net/routing/hat/linkstate_peer/pubsub.rs index a1ff061602..49bd026a31 100644 --- a/zenoh/src/net/routing/hat/linkstate_peer/pubsub.rs +++ b/zenoh/src/net/routing/hat/linkstate_peer/pubsub.rs @@ -45,7 +45,7 @@ use crate::net::routing::{ }, hat::{CurrentFutureTrait, HatPubSubTrait, Sources}, router::RoutesIndexes, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }; #[inline] @@ -62,7 +62,10 @@ fn send_sourced_subscription_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.unwrap().id { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let key_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -97,7 +100,7 @@ fn propagate_simple_subscription_to( sub_info: &SubscriberInfo, src_face: &mut Arc, ) { - if (src_face.id != dst_face.id || res.expr().starts_with(PREFIX_LIVELINESS)) + if (src_face.id != dst_face.id) && !face_hat!(dst_face).local_subs.contains_key(res) && dst_face.whatami == WhatAmI::Client { @@ -230,10 +233,8 @@ fn register_peer_subscription( propagate_sourced_subscription(tables, res, sub_info, Some(face), &peer); } - if tables.whatami == WhatAmI::Peer { - // Propagate subscription to clients - propagate_simple_subscription(tables, res, sub_info, face); - } + // Propagate subscription to clients + propagate_simple_subscription(tables, res, sub_info, face); } fn declare_peer_subscription( @@ -329,7 +330,10 @@ fn send_forget_sourced_subscription_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.unwrap().id { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let wire_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -447,9 +451,7 @@ fn unregister_peer_subscription(tables: &mut Tables, res: &mut Arc, pe .peer_subs .retain(|sub| !Arc::ptr_eq(sub, res)); - if tables.whatami == WhatAmI::Peer { - propagate_forget_simple_subscription(tables, res); - } + propagate_forget_simple_subscription(tables, res); } } @@ -492,7 +494,7 @@ pub(super) fn undeclare_client_subscription( if client_subs.len() == 1 && !peer_subs { let mut face = &mut client_subs[0]; - if !(face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS)) { + if face.whatami != WhatAmI::Client { if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { face.primitives.send_declare(RoutingContext::with_expr( Declare { @@ -571,10 +573,16 @@ pub(super) fn pubsub_remove_node(tables: &mut Tables, node: &ZenohIdProto) { } pub(super) fn pubsub_tree_change(tables: &mut Tables, new_childs: &[Vec]) { + let net = match hat!(tables).peers_net.as_ref() { + Some(net) => net, + None => { + tracing::error!("Error accessing peers_net in pubsub_tree_change!"); + return; + } + }; // propagate subs to new childs for (tree_sid, tree_childs) in new_childs.iter().enumerate() { if !tree_childs.is_empty() { - let net = hat!(tables).peers_net.as_ref().unwrap(); let tree_idx = NodeIndex::new(tree_sid); if net.graph.contains_node(tree_idx) { let tree_id = net.graph[tree_idx].zid; diff --git a/zenoh/src/net/routing/hat/linkstate_peer/queries.rs b/zenoh/src/net/routing/hat/linkstate_peer/queries.rs index 16ed7cc7ef..36fc03c03d 100644 --- a/zenoh/src/net/routing/hat/linkstate_peer/queries.rs +++ b/zenoh/src/net/routing/hat/linkstate_peer/queries.rs @@ -19,14 +19,13 @@ use std::{ use ordered_float::OrderedFloat; use petgraph::graph::NodeIndex; -use zenoh_buffers::ZBuf; use zenoh_protocol::{ core::{ key_expr::{ include::{Includer, DEFAULT_INCLUDER}, OwnedKeyExpr, }, - WhatAmI, WireExpr, ZenohIdProto, + WhatAmI, ZenohIdProto, }, network::{ declare::{ @@ -51,7 +50,7 @@ use crate::net::routing::{ }, hat::{CurrentFutureTrait, HatQueriesTrait, Sources}, router::RoutesIndexes, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }; #[inline] @@ -134,7 +133,11 @@ fn send_sourced_queryable_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.as_ref().unwrap().id { + if src_face + .as_ref() + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let key_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -170,7 +173,10 @@ fn propagate_simple_queryable( for mut dst_face in faces { let info = local_qabl_info(tables, res, &dst_face); let current = face_hat!(dst_face).local_qabls.get(res); - if (src_face.is_none() || src_face.as_ref().unwrap().id != dst_face.id) + if src_face + .as_ref() + .map(|src_face| dst_face.id != src_face.id) + .unwrap_or(true) && (current.is_none() || current.unwrap().1 != info) && dst_face.whatami == WhatAmI::Client && face_hat!(dst_face) @@ -259,10 +265,8 @@ fn register_peer_queryable( propagate_sourced_queryable(tables, res, qabl_info, face.as_deref_mut(), &peer); } - if tables.whatami == WhatAmI::Peer { - // Propagate queryable to clients - propagate_simple_queryable(tables, res, face); - } + // Propagate queryable to clients + propagate_simple_queryable(tables, res, face); } fn declare_peer_queryable( @@ -352,7 +356,10 @@ fn send_forget_sourced_queryable_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.unwrap().id { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let wire_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -470,9 +477,7 @@ fn unregister_peer_queryable(tables: &mut Tables, res: &mut Arc, peer: .peer_qabls .retain(|qabl| !Arc::ptr_eq(qabl, res)); - if tables.whatami == WhatAmI::Peer { - propagate_forget_simple_queryable(tables, res); - } + propagate_forget_simple_queryable(tables, res); } } @@ -602,10 +607,16 @@ pub(super) fn queries_remove_node(tables: &mut Tables, node: &ZenohIdProto) { } pub(super) fn queries_tree_change(tables: &mut Tables, new_childs: &[Vec]) { + let net = match hat!(tables).peers_net.as_ref() { + Some(net) => net, + None => { + tracing::error!("Error accessing peers_net in queries_tree_change!"); + return; + } + }; // propagate qabls to new childs for (tree_sid, tree_childs) in new_childs.iter().enumerate() { if !tree_childs.is_empty() { - let net = hat!(tables).peers_net.as_ref().unwrap(); let tree_idx = NodeIndex::new(tree_sid); if net.graph.contains_node(tree_idx) { let tree_id = net.graph[tree_idx].zid; @@ -939,48 +950,6 @@ impl HatQueriesTrait for HatCode { Arc::new(route) } - #[inline] - fn compute_local_replies( - &self, - tables: &Tables, - prefix: &Arc, - suffix: &str, - face: &Arc, - ) -> Vec<(WireExpr<'static>, ZBuf)> { - let mut result = vec![]; - // Only the first routing point in the query route - // should return the liveliness tokens - if face.whatami == WhatAmI::Client { - let key_expr = prefix.expr() + suffix; - let key_expr = match OwnedKeyExpr::try_from(key_expr) { - Ok(ke) => ke, - Err(e) => { - tracing::warn!("Invalid KE reached the system: {}", e); - return result; - } - }; - if key_expr.starts_with(PREFIX_LIVELINESS) { - let res = Resource::get_resource(prefix, suffix); - let matches = res - .as_ref() - .and_then(|res| res.context.as_ref()) - .map(|ctx| Cow::from(&ctx.matches)) - .unwrap_or_else(|| Cow::from(Resource::get_matches(tables, &key_expr))); - for mres in matches.iter() { - let mres = mres.upgrade().unwrap(); - if (mres.context.is_some() - && (!res_hat!(mres).router_subs.is_empty() - || !res_hat!(mres).peer_subs.is_empty())) - || mres.session_ctxs.values().any(|ctx| ctx.subs.is_some()) - { - result.push((Resource::get_best_key(&mres, "", face.id), ZBuf::default())); - } - } - } - } - result - } - fn get_query_routes_entries(&self, tables: &Tables) -> RoutesIndexes { get_routes_entries(tables) } diff --git a/zenoh/src/net/routing/hat/linkstate_peer/token.rs b/zenoh/src/net/routing/hat/linkstate_peer/token.rs new file mode 100644 index 0000000000..0085d8deb0 --- /dev/null +++ b/zenoh/src/net/routing/hat/linkstate_peer/token.rs @@ -0,0 +1,717 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +use std::sync::{atomic::Ordering, Arc}; + +use petgraph::graph::NodeIndex; +use zenoh_protocol::{ + core::{WhatAmI, ZenohIdProto}, + network::{ + declare::{common::ext::WireExprType, TokenId}, + ext, + interest::{InterestId, InterestMode, InterestOptions}, + Declare, DeclareBody, DeclareToken, UndeclareToken, + }, +}; +use zenoh_sync::get_mut_unchecked; + +use super::{ + face_hat, face_hat_mut, get_peer, hat, hat_mut, network::Network, res_hat, res_hat_mut, + HatCode, HatContext, HatFace, HatTables, +}; +use crate::net::routing::{ + dispatcher::{face::FaceState, tables::Tables}, + hat::{CurrentFutureTrait, HatTokenTrait}, + router::{NodeId, Resource, SessionContext}, + RoutingContext, +}; + +#[inline] +fn send_sourced_token_to_net_childs( + tables: &Tables, + net: &Network, + childs: &[NodeIndex], + res: &Arc, + src_face: Option<&Arc>, + routing_context: NodeId, +) { + for child in childs { + if net.graph.contains_node(*child) { + match tables.get_face(&net.graph[*child].zid).cloned() { + Some(mut someface) => { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { + let key_expr = Resource::decl_key(res, &mut someface); + + someface.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType { + node_id: routing_context, + }, + body: DeclareBody::DeclareToken(DeclareToken { + id: 0, // Sourced tokens do not use ids + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } + } + None => tracing::trace!("Unable to find face for zid {}", net.graph[*child].zid), + } + } + } +} + +#[inline] +fn propagate_simple_token_to( + _tables: &mut Tables, + dst_face: &mut Arc, + res: &Arc, + _src_face: &mut Arc, +) { + if !face_hat!(dst_face).local_tokens.contains_key(res) && dst_face.whatami == WhatAmI::Client { + if dst_face.whatami != WhatAmI::Client { + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } else { + let matching_interests = face_hat!(dst_face) + .remote_interests + .values() + .filter(|(r, o)| o.tokens() && r.as_ref().map(|r| r.matches(res)).unwrap_or(true)) + .cloned() + .collect::>, InterestOptions)>>(); + + for (int_res, options) in matching_interests { + let res = if options.aggregate() { + int_res.as_ref().unwrap_or(res) + } else { + res + }; + if !face_hat!(dst_face).local_tokens.contains_key(res) { + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } + } + } + } +} + +fn propagate_simple_token(tables: &mut Tables, res: &Arc, src_face: &mut Arc) { + for mut dst_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + propagate_simple_token_to(tables, &mut dst_face, res, src_face); + } +} + +fn propagate_sourced_token( + tables: &Tables, + res: &Arc, + src_face: Option<&Arc>, + source: &ZenohIdProto, +) { + let net = hat!(tables).peers_net.as_ref().unwrap(); + match net.get_idx(source) { + Some(tree_sid) => { + if net.trees.len() > tree_sid.index() { + send_sourced_token_to_net_childs( + tables, + net, + &net.trees[tree_sid.index()].childs, + res, + src_face, + tree_sid.index() as NodeId, + ); + } else { + tracing::trace!( + "Propagating token {}: tree for node {} sid:{} not yet ready", + res.expr(), + tree_sid.index(), + source + ); + } + } + None => tracing::error!( + "Error propagating token {}: cannot get index of {}!", + res.expr(), + source + ), + } +} + +fn register_peer_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + peer: ZenohIdProto, +) { + if !res_hat!(res).peer_tokens.contains(&peer) { + // Register peer liveliness + { + res_hat_mut!(res).peer_tokens.insert(peer); + hat_mut!(tables).peer_tokens.insert(res.clone()); + } + + // Propagate liveliness to peers + propagate_sourced_token(tables, res, Some(face), &peer); + } + + // Propagate liveliness to clients + propagate_simple_token(tables, res, face); +} + +fn declare_peer_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + peer: ZenohIdProto, +) { + register_peer_token(tables, face, res, peer); +} + +fn register_client_token( + _tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + // Register liveliness + { + let res = get_mut_unchecked(res); + match res.session_ctxs.get_mut(&face.id) { + Some(ctx) => { + if !ctx.token { + get_mut_unchecked(ctx).token = true; + } + } + None => { + let ctx = res + .session_ctxs + .entry(face.id) + .or_insert_with(|| Arc::new(SessionContext::new(face.clone()))); + get_mut_unchecked(ctx).token = true; + } + } + } + face_hat_mut!(face).remote_tokens.insert(id, res.clone()); +} + +fn declare_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + register_client_token(tables, face, id, res); + let zid = tables.zid; + register_peer_token(tables, face, res, zid); +} + +#[inline] +fn remote_peer_tokens(tables: &Tables, res: &Arc) -> bool { + res.context.is_some() + && res_hat!(res) + .peer_tokens + .iter() + .any(|peer| peer != &tables.zid) +} + +#[inline] +fn client_tokens(res: &Arc) -> Vec> { + res.session_ctxs + .values() + .filter_map(|ctx| { + if ctx.token { + Some(ctx.face.clone()) + } else { + None + } + }) + .collect() +} + +#[inline] +fn remote_client_tokens(res: &Arc, face: &Arc) -> bool { + res.session_ctxs + .values() + .any(|ctx| ctx.face.id != face.id && ctx.token) +} + +#[inline] +fn send_forget_sourced_token_to_net_childs( + tables: &Tables, + net: &Network, + childs: &[NodeIndex], + res: &Arc, + src_face: Option<&Arc>, + routing_context: Option, +) { + for child in childs { + if net.graph.contains_node(*child) { + match tables.get_face(&net.graph[*child].zid).cloned() { + Some(mut someface) => { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { + let wire_expr = Resource::decl_key(res, &mut someface); + + someface.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType { + node_id: routing_context.unwrap_or(0), + }, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: 0, // Sourced tokens do not use ids + ext_wire_expr: WireExprType { wire_expr }, + }), + }, + res.expr(), + )); + } + } + None => tracing::trace!("Unable to find face for zid {}", net.graph[*child].zid), + } + } + } +} + +fn propagate_forget_simple_token(tables: &mut Tables, res: &Arc) { + for mut face in tables.faces.values().cloned() { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_tokens + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade().is_some_and(|m| { + m.context.is_some() + && (remote_client_tokens(&m, &face) || remote_peer_tokens(tables, &m)) + }) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + } + } + } +} + +fn propagate_forget_sourced_token( + tables: &Tables, + res: &Arc, + src_face: Option<&Arc>, + source: &ZenohIdProto, +) { + let net = hat!(tables).peers_net.as_ref().unwrap(); + match net.get_idx(source) { + Some(tree_sid) => { + if net.trees.len() > tree_sid.index() { + send_forget_sourced_token_to_net_childs( + tables, + net, + &net.trees[tree_sid.index()].childs, + res, + src_face, + Some(tree_sid.index() as NodeId), + ); + } else { + tracing::trace!( + "Propagating forget token {}: tree for node {} sid:{} not yet ready", + res.expr(), + tree_sid.index(), + source + ); + } + } + None => tracing::error!( + "Error propagating forget token {}: cannot get index of {}!", + res.expr(), + source + ), + } +} + +fn unregister_peer_token(tables: &mut Tables, res: &mut Arc, peer: &ZenohIdProto) { + res_hat_mut!(res).peer_tokens.retain(|token| token != peer); + + if res_hat!(res).peer_tokens.is_empty() { + hat_mut!(tables) + .peer_tokens + .retain(|token| !Arc::ptr_eq(token, res)); + + propagate_forget_simple_token(tables, res); + } +} + +fn undeclare_peer_token( + tables: &mut Tables, + face: Option<&Arc>, + res: &mut Arc, + peer: &ZenohIdProto, +) { + if res_hat!(res).peer_tokens.contains(peer) { + unregister_peer_token(tables, res, peer); + propagate_forget_sourced_token(tables, res, face, peer); + } +} + +fn forget_peer_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + peer: &ZenohIdProto, +) { + undeclare_peer_token(tables, Some(face), res, peer); +} + +pub(super) fn undeclare_client_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, +) { + if !face_hat_mut!(face) + .remote_tokens + .values() + .any(|s| *s == *res) + { + if let Some(ctx) = get_mut_unchecked(res).session_ctxs.get_mut(&face.id) { + get_mut_unchecked(ctx).token = false; + } + + let mut client_tokens = client_tokens(res); + let peer_tokens = remote_peer_tokens(tables, res); + if client_tokens.is_empty() { + undeclare_peer_token(tables, None, res, &tables.zid.clone()); + } + + if client_tokens.len() == 1 && !peer_tokens { + let mut face = &mut client_tokens[0]; + if face.whatami != WhatAmI::Client { + if let Some(id) = face_hat_mut!(face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_tokens + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade().is_some_and(|m| { + m.context.is_some() + && (remote_client_tokens(&m, face) + || remote_peer_tokens(tables, &m)) + }) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + } + } + } + } + } +} + +fn forget_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, +) -> Option> { + if let Some(mut res) = face_hat_mut!(face).remote_tokens.remove(&id) { + undeclare_client_token(tables, face, &mut res); + Some(res) + } else { + None + } +} + +pub(super) fn token_remove_node(tables: &mut Tables, node: &ZenohIdProto) { + for mut res in hat!(tables) + .peer_tokens + .iter() + .filter(|res| res_hat!(res).peer_tokens.contains(node)) + .cloned() + .collect::>>() + { + unregister_peer_token(tables, &mut res, node); + Resource::clean(&mut res) + } +} + +pub(super) fn token_tree_change(tables: &mut Tables, new_childs: &[Vec]) { + let net = match hat!(tables).peers_net.as_ref() { + Some(net) => net, + None => { + tracing::error!("Error accessing peers_net in token_tree_change!"); + return; + } + }; + // propagate tokens to new childs + for (tree_sid, tree_childs) in new_childs.iter().enumerate() { + if !tree_childs.is_empty() { + let tree_idx = NodeIndex::new(tree_sid); + if net.graph.contains_node(tree_idx) { + let tree_id = net.graph[tree_idx].zid; + + let tokens_res = &hat!(tables).peer_tokens; + + for res in tokens_res { + let tokens = &res_hat!(res).peer_tokens; + for token in tokens { + if *token == tree_id { + send_sourced_token_to_net_childs( + tables, + net, + tree_childs, + res, + None, + tree_sid as NodeId, + ); + } + } + } + } + } + } +} + +pub(crate) fn declare_token_interest( + tables: &mut Tables, + face: &mut Arc, + id: InterestId, + res: Option<&mut Arc>, + mode: InterestMode, + aggregate: bool, +) { + if mode.current() && face.whatami == WhatAmI::Client { + let interest_id = (!mode.future()).then_some(id); + if let Some(res) = res.as_ref() { + if aggregate { + if hat!(tables).peer_tokens.iter().any(|token| { + token.context.is_some() + && token.matches(res) + && (remote_client_tokens(token, face) || remote_peer_tokens(tables, token)) + }) { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert((*res).clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(res, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + res.expr(), + )); + } + } else { + for token in &hat!(tables).peer_tokens { + if token.context.is_some() + && token.matches(res) + && (remote_client_tokens(token, face) || remote_peer_tokens(tables, token)) + { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } else { + for token in &hat!(tables).peer_tokens { + if token.context.is_some() + && (remote_client_tokens(token, face) || remote_peer_tokens(tables, token)) + { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } +} + +impl HatTokenTrait for HatCode { + fn declare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, + node_id: NodeId, + _interest_id: Option, + ) { + if face.whatami != WhatAmI::Client { + if let Some(peer) = get_peer(tables, face, node_id) { + declare_peer_token(tables, face, res, peer) + } + } else { + declare_client_token(tables, face, id, res) + } + } + + fn undeclare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, + node_id: NodeId, + ) -> Option> { + if face.whatami != WhatAmI::Client { + if let Some(mut res) = res { + if let Some(peer) = get_peer(tables, face, node_id) { + forget_peer_token(tables, face, &mut res, &peer); + Some(res) + } else { + None + } + } else { + None + } + } else { + forget_client_token(tables, face, id) + } + } +} diff --git a/zenoh/src/net/routing/hat/mod.rs b/zenoh/src/net/routing/hat/mod.rs index b30e6e9277..f2175474d4 100644 --- a/zenoh/src/net/routing/hat/mod.rs +++ b/zenoh/src/net/routing/hat/mod.rs @@ -19,14 +19,13 @@ //! [Click here for Zenoh's documentation](../zenoh/index.html) use std::{any::Any, sync::Arc}; -use zenoh_buffers::ZBuf; use zenoh_config::{unwrap_or_default, Config, WhatAmI}; use zenoh_protocol::{ - core::{WireExpr, ZenohIdProto}, + core::ZenohIdProto, network::{ declare::{ queryable::ext::QueryableInfoType, subscriber::ext::SubscriberInfo, QueryableId, - SubscriberId, + SubscriberId, TokenId, }, interest::{InterestId, InterestMode, InterestOptions}, Oam, @@ -73,7 +72,7 @@ impl Sources { } pub(crate) trait HatTrait: - HatBaseTrait + HatInterestTrait + HatPubSubTrait + HatQueriesTrait + HatBaseTrait + HatInterestTrait + HatPubSubTrait + HatQueriesTrait + HatTokenTrait { } @@ -222,14 +221,6 @@ pub(crate) trait HatQueriesTrait { ) -> Arc; fn get_query_routes_entries(&self, tables: &Tables) -> RoutesIndexes; - - fn compute_local_replies( - &self, - tables: &Tables, - prefix: &Arc, - suffix: &str, - face: &Arc, - ) -> Vec<(WireExpr<'static>, ZBuf)>; } pub(crate) fn new_hat(whatami: WhatAmI, config: &Config) -> Box { @@ -246,6 +237,27 @@ pub(crate) fn new_hat(whatami: WhatAmI, config: &Config) -> Box, + id: TokenId, + res: &mut Arc, + node_id: NodeId, + interest_id: Option, + ); + + fn undeclare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, + node_id: NodeId, + ) -> Option>; +} + trait CurrentFutureTrait { fn future(&self) -> bool; fn current(&self) -> bool; diff --git a/zenoh/src/net/routing/hat/p2p_peer/interests.rs b/zenoh/src/net/routing/hat/p2p_peer/interests.rs index 0b058fb4b1..4fe8936cc7 100644 --- a/zenoh/src/net/routing/hat/p2p_peer/interests.rs +++ b/zenoh/src/net/routing/hat/p2p_peer/interests.rs @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 ZettaScale Technology +// Copyright (c) 2024 ZettaScale Technology // // This program and the accompanying materials are made available under the // terms of the Eclipse Public License 2.0 which is available at @@ -24,8 +24,8 @@ use zenoh_protocol::{ use zenoh_sync::get_mut_unchecked; use super::{ - face_hat, face_hat_mut, pubsub::declare_sub_interest, queries::declare_qabl_interest, HatCode, - HatFace, + face_hat, face_hat_mut, pubsub::declare_sub_interest, queries::declare_qabl_interest, + token::declare_token_interest, HatCode, HatFace, }; use crate::net::routing::{ dispatcher::{ @@ -107,6 +107,16 @@ impl HatInterestTrait for HatCode { options.aggregate(), ) } + if options.tokens() { + declare_token_interest( + tables, + face, + id, + res.as_ref().map(|r| (*r).clone()).as_mut(), + mode, + options.aggregate(), + ) + } face_hat_mut!(face) .remote_interests .insert(id, (res.as_ref().map(|res| (*res).clone()), options)); diff --git a/zenoh/src/net/routing/hat/p2p_peer/mod.rs b/zenoh/src/net/routing/hat/p2p_peer/mod.rs index 38ee54e0f6..eab2f393de 100644 --- a/zenoh/src/net/routing/hat/p2p_peer/mod.rs +++ b/zenoh/src/net/routing/hat/p2p_peer/mod.rs @@ -23,6 +23,7 @@ use std::{ sync::{atomic::AtomicU32, Arc}, }; +use token::{token_new_face, undeclare_client_token}; use zenoh_config::{unwrap_or_default, ModeDependent, WhatAmI, WhatAmIMatcher}; use zenoh_protocol::{ common::ZExtBody, @@ -30,7 +31,7 @@ use zenoh_protocol::{ declare::{ ext::{NodeIdType, QoSType}, queryable::ext::QueryableInfoType, - QueryableId, SubscriberId, + QueryableId, SubscriberId, TokenId, }, interest::{InterestId, InterestOptions}, oam::id::OAM_LINKSTATE, @@ -69,6 +70,7 @@ mod gossip; mod interests; mod pubsub; mod queries; +mod token; macro_rules! hat_mut { ($t:expr) => { @@ -150,6 +152,7 @@ impl HatBaseTrait for HatCode { interests_new_face(tables, &mut face.state); pubsub_new_face(tables, &mut face.state); queries_new_face(tables, &mut face.state); + token_new_face(tables, &mut face.state); Ok(()) } @@ -179,6 +182,7 @@ impl HatBaseTrait for HatCode { interests_new_face(tables, &mut face.state); pubsub_new_face(tables, &mut face.state); queries_new_face(tables, &mut face.state); + token_new_face(tables, &mut face.state); if face.state.whatami == WhatAmI::Peer { face.state @@ -197,12 +201,20 @@ impl HatBaseTrait for HatCode { fn close_face(&self, tables: &TablesLock, face: &mut Arc) { let mut wtables = zwrite!(tables.tables); let mut face_clone = face.clone(); + let face = get_mut_unchecked(face); + let hat_face = match face.hat.downcast_mut::() { + Some(hate_face) => hate_face, + None => { + tracing::error!("Error downcasting face hat in close_face!"); + return; + } + }; - face_hat_mut!(face).remote_interests.clear(); - face_hat_mut!(face).local_subs.clear(); - face_hat_mut!(face).local_qabls.clear(); + hat_face.remote_interests.clear(); + hat_face.local_subs.clear(); + hat_face.local_qabls.clear(); + hat_face.local_tokens.clear(); - let face = get_mut_unchecked(face); for res in face.remote_mappings.values_mut() { get_mut_unchecked(res).session_ctxs.remove(&face.id); Resource::clean(res); @@ -215,13 +227,7 @@ impl HatBaseTrait for HatCode { face.local_mappings.clear(); let mut subs_matches = vec![]; - for (_id, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_subs - .drain() - { + for (_id, mut res) in hat_face.remote_subs.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_subscription(&mut wtables, &mut face_clone, &mut res); @@ -243,13 +249,7 @@ impl HatBaseTrait for HatCode { } let mut qabls_matches = vec![]; - for (_id, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_qabls - .drain() - { + for (_id, mut res) in hat_face.remote_qabls.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_queryable(&mut wtables, &mut face_clone, &mut res); @@ -269,6 +269,11 @@ impl HatBaseTrait for HatCode { qabls_matches.push(res); } } + + for (_id, mut res) in hat_face.remote_tokens.drain() { + get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); + undeclare_client_token(&mut wtables, &mut face_clone, &mut res); + } drop(wtables); let mut matches_data_routes = vec![]; @@ -394,6 +399,8 @@ struct HatFace { remote_interests: HashMap>, InterestOptions)>, local_subs: HashMap, SubscriberId>, remote_subs: HashMap>, + local_tokens: HashMap, TokenId>, + remote_tokens: HashMap>, local_qabls: HashMap, (QueryableId, QueryableInfoType)>, remote_qabls: HashMap>, } @@ -405,6 +412,8 @@ impl HatFace { remote_interests: HashMap::new(), local_subs: HashMap::new(), remote_subs: HashMap::new(), + local_tokens: HashMap::new(), + remote_tokens: HashMap::new(), local_qabls: HashMap::new(), remote_qabls: HashMap::new(), } diff --git a/zenoh/src/net/routing/hat/p2p_peer/pubsub.rs b/zenoh/src/net/routing/hat/p2p_peer/pubsub.rs index 9cc2f05bf6..bc0a6f7de2 100644 --- a/zenoh/src/net/routing/hat/p2p_peer/pubsub.rs +++ b/zenoh/src/net/routing/hat/p2p_peer/pubsub.rs @@ -40,7 +40,7 @@ use crate::{ }, hat::{CurrentFutureTrait, HatPubSubTrait, Sources}, router::{update_data_routes_from, RoutesIndexes}, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }, }; @@ -52,8 +52,7 @@ fn propagate_simple_subscription_to( sub_info: &SubscriberInfo, src_face: &mut Arc, ) { - if (src_face.id != dst_face.id - || (dst_face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS))) + if (src_face.id != dst_face.id) && !face_hat!(dst_face).local_subs.contains_key(res) && (src_face.whatami == WhatAmI::Client || dst_face.whatami == WhatAmI::Client) { @@ -277,47 +276,45 @@ pub(super) fn undeclare_client_subscription( if client_subs.len() == 1 { let mut face = &mut client_subs[0]; - if !(face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS)) { - if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { - face.primitives.send_declare(RoutingContext::with_expr( - Declare { - interest_id: None, - ext_qos: ext::QoSType::DECLARE, - ext_tstamp: None, - ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { - id, - ext_wire_expr: WireExprType::null(), - }), - }, - res.expr(), - )); - } - for res in face_hat!(face) - .local_subs - .keys() - .cloned() - .collect::>>() - { - if !res.context().matches.iter().any(|m| { - m.upgrade() - .is_some_and(|m| m.context.is_some() && remote_client_subs(&m, face)) - }) { - if let Some(id) = face_hat_mut!(&mut face).local_subs.remove(&res) { - face.primitives.send_declare(RoutingContext::with_expr( - Declare { - interest_id: None, - ext_qos: ext::QoSType::DECLARE, - ext_tstamp: None, - ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { - id, - ext_wire_expr: WireExprType::null(), - }), - }, - res.expr(), - )); - } + if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_subs + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade() + .is_some_and(|m| m.context.is_some() && remote_client_subs(&m, face)) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_subs.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); } } } diff --git a/zenoh/src/net/routing/hat/p2p_peer/queries.rs b/zenoh/src/net/routing/hat/p2p_peer/queries.rs index cafe65b8c7..6a8ebbc8e6 100644 --- a/zenoh/src/net/routing/hat/p2p_peer/queries.rs +++ b/zenoh/src/net/routing/hat/p2p_peer/queries.rs @@ -18,14 +18,13 @@ use std::{ }; use ordered_float::OrderedFloat; -use zenoh_buffers::ZBuf; use zenoh_protocol::{ core::{ key_expr::{ include::{Includer, DEFAULT_INCLUDER}, OwnedKeyExpr, }, - WhatAmI, WireExpr, + WhatAmI, }, network::{ declare::{ @@ -46,7 +45,7 @@ use crate::net::routing::{ }, hat::{CurrentFutureTrait, HatQueriesTrait, Sources}, router::{update_query_routes_from, RoutesIndexes}, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }; #[inline] @@ -89,7 +88,10 @@ fn propagate_simple_queryable_to( ) { let info = local_qabl_info(tables, res, dst_face); let current = face_hat!(dst_face).local_qabls.get(res); - if (src_face.is_none() || src_face.as_ref().unwrap().id != dst_face.id) + if src_face + .as_ref() + .map(|src_face| dst_face.id != src_face.id) + .unwrap_or(true) && (current.is_none() || current.unwrap().1 != info) && (dst_face.whatami != WhatAmI::Client || face_hat!(dst_face) @@ -599,44 +601,6 @@ impl HatQueriesTrait for HatCode { Arc::new(route) } - #[inline] - fn compute_local_replies( - &self, - tables: &Tables, - prefix: &Arc, - suffix: &str, - face: &Arc, - ) -> Vec<(WireExpr<'static>, ZBuf)> { - let mut result = vec![]; - // Only the first routing point in the query route - // should return the liveliness tokens - if face.whatami == WhatAmI::Client { - let key_expr = prefix.expr() + suffix; - let key_expr = match OwnedKeyExpr::try_from(key_expr) { - Ok(ke) => ke, - Err(e) => { - tracing::warn!("Invalid KE reached the system: {}", e); - return result; - } - }; - if key_expr.starts_with(PREFIX_LIVELINESS) { - let res = Resource::get_resource(prefix, suffix); - let matches = res - .as_ref() - .and_then(|res| res.context.as_ref()) - .map(|ctx| Cow::from(&ctx.matches)) - .unwrap_or_else(|| Cow::from(Resource::get_matches(tables, &key_expr))); - for mres in matches.iter() { - let mres = mres.upgrade().unwrap(); - if mres.session_ctxs.values().any(|ctx| ctx.subs.is_some()) { - result.push((Resource::get_best_key(&mres, "", face.id), ZBuf::default())); - } - } - } - } - result - } - fn get_query_routes_entries(&self, _tables: &Tables) -> RoutesIndexes { get_routes_entries() } diff --git a/zenoh/src/net/routing/hat/p2p_peer/token.rs b/zenoh/src/net/routing/hat/p2p_peer/token.rs new file mode 100644 index 0000000000..65c351c812 --- /dev/null +++ b/zenoh/src/net/routing/hat/p2p_peer/token.rs @@ -0,0 +1,482 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +use std::sync::{atomic::Ordering, Arc}; + +use zenoh_config::WhatAmI; +use zenoh_protocol::network::{ + declare::{common::ext::WireExprType, TokenId}, + ext, + interest::{InterestId, InterestMode, InterestOptions}, + Declare, DeclareBody, DeclareToken, UndeclareToken, +}; +use zenoh_sync::get_mut_unchecked; + +use super::{face_hat, face_hat_mut, HatCode, HatFace}; +use crate::net::routing::{ + dispatcher::{face::FaceState, tables::Tables}, + hat::{CurrentFutureTrait, HatTokenTrait}, + router::{NodeId, Resource, SessionContext}, + RoutingContext, +}; + +#[inline] +fn propagate_simple_token_to( + _tables: &mut Tables, + dst_face: &mut Arc, + res: &Arc, + src_face: &mut Arc, +) { + if (src_face.id != dst_face.id || dst_face.whatami == WhatAmI::Client) + && !face_hat!(dst_face).local_tokens.contains_key(res) + && (src_face.whatami == WhatAmI::Client || dst_face.whatami == WhatAmI::Client) + { + if dst_face.whatami != WhatAmI::Client { + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } else { + let matching_interests = face_hat!(dst_face) + .remote_interests + .values() + .filter(|(r, o)| o.tokens() && r.as_ref().map(|r| r.matches(res)).unwrap_or(true)) + .cloned() + .collect::>, InterestOptions)>>(); + + for (int_res, options) in matching_interests { + let res = if options.aggregate() { + int_res.as_ref().unwrap_or(res) + } else { + res + }; + if !face_hat!(dst_face).local_tokens.contains_key(res) { + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } + } + } + } +} + +fn propagate_simple_token(tables: &mut Tables, res: &Arc, src_face: &mut Arc) { + for mut dst_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + propagate_simple_token_to(tables, &mut dst_face, res, src_face); + } +} + +fn register_client_token( + _tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + // Register liveliness + { + let res = get_mut_unchecked(res); + match res.session_ctxs.get_mut(&face.id) { + Some(ctx) => { + if !ctx.token { + get_mut_unchecked(ctx).token = true; + } + } + None => { + let ctx = res + .session_ctxs + .entry(face.id) + .or_insert_with(|| Arc::new(SessionContext::new(face.clone()))); + get_mut_unchecked(ctx).token = true; + } + } + } + face_hat_mut!(face).remote_tokens.insert(id, res.clone()); +} + +fn declare_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + register_client_token(tables, face, id, res); + + propagate_simple_token(tables, res, face); +} + +#[inline] +fn client_tokens(res: &Arc) -> Vec> { + res.session_ctxs + .values() + .filter_map(|ctx| { + if ctx.token { + Some(ctx.face.clone()) + } else { + None + } + }) + .collect() +} + +#[inline] +fn remote_client_tokens(res: &Arc, face: &Arc) -> bool { + res.session_ctxs + .values() + .any(|ctx| ctx.face.id != face.id && ctx.token) +} + +fn propagate_forget_simple_token(tables: &mut Tables, res: &Arc) { + for mut face in tables.faces.values().cloned() { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } else if face_hat!(face).remote_interests.values().any(|(r, o)| { + o.tokens() && r.as_ref().map(|r| r.matches(res)).unwrap_or(true) && !o.aggregate() + }) { + // Token has never been declared on this face. + // Send an Undeclare with a one shot generated id and a WireExpr ext. + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst), + ext_wire_expr: WireExprType { + wire_expr: Resource::get_best_key(res, "", face.id), + }, + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_tokens + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade() + .is_some_and(|m| m.context.is_some() && remote_client_tokens(&m, &face)) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } else if face_hat!(face).remote_interests.values().any(|(r, o)| { + o.tokens() + && r.as_ref().map(|r| r.matches(&res)).unwrap_or(true) + && !o.aggregate() + }) { + // Token has never been declared on this face. + // Send an Undeclare with a one shot generated id and a WireExpr ext. + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst), + ext_wire_expr: WireExprType { + wire_expr: Resource::get_best_key(&res, "", face.id), + }, + }), + }, + res.expr(), + )); + } + } + } + } +} + +pub(super) fn undeclare_client_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, +) { + if !face_hat_mut!(face) + .remote_tokens + .values() + .any(|s| *s == *res) + { + if let Some(ctx) = get_mut_unchecked(res).session_ctxs.get_mut(&face.id) { + get_mut_unchecked(ctx).token = false; + } + + let mut client_tokens = client_tokens(res); + if client_tokens.is_empty() { + propagate_forget_simple_token(tables, res); + } + + if client_tokens.len() == 1 { + let mut face = &mut client_tokens[0]; + if face.whatami != WhatAmI::Client { + if let Some(id) = face_hat_mut!(face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_tokens + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade() + .is_some_and(|m| m.context.is_some() && remote_client_tokens(&m, face)) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + } + } + } + } + } +} + +fn forget_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, +) -> Option> { + if let Some(mut res) = face_hat_mut!(face).remote_tokens.remove(&id) { + undeclare_client_token(tables, face, &mut res); + Some(res) + } else if let Some(mut res) = res { + undeclare_client_token(tables, face, &mut res); + Some(res) + } else { + None + } +} + +pub(super) fn token_new_face(tables: &mut Tables, face: &mut Arc) { + if face.whatami != WhatAmI::Client { + for mut src_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + for token in face_hat!(src_face.clone()).remote_tokens.values() { + propagate_simple_token_to(tables, face, token, &mut src_face); + } + } + } +} + +pub(crate) fn declare_token_interest( + tables: &mut Tables, + face: &mut Arc, + id: InterestId, + res: Option<&mut Arc>, + mode: InterestMode, + aggregate: bool, +) { + if mode.current() && face.whatami == WhatAmI::Client { + let interest_id = (!mode.future()).then_some(id); + if let Some(res) = res.as_ref() { + if aggregate { + if tables.faces.values().any(|src_face| { + face_hat!(src_face) + .remote_tokens + .values() + .any(|token| token.context.is_some() && token.matches(res)) + }) { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert((*res).clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(res, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + res.expr(), + )); + } + } else { + for src_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + for token in face_hat!(src_face).remote_tokens.values() { + if token.context.is_some() && token.matches(res) { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } + } else { + for src_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + for token in face_hat!(src_face).remote_tokens.values() { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } +} + +impl HatTokenTrait for HatCode { + fn declare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, + _node_id: NodeId, + _interest_id: Option, + ) { + declare_client_token(tables, face, id, res) + } + + fn undeclare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, + _node_id: NodeId, + ) -> Option> { + forget_client_token(tables, face, id, res) + } +} diff --git a/zenoh/src/net/routing/hat/router/interests.rs b/zenoh/src/net/routing/hat/router/interests.rs index a12201d7ad..f9f289bfa7 100644 --- a/zenoh/src/net/routing/hat/router/interests.rs +++ b/zenoh/src/net/routing/hat/router/interests.rs @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 ZettaScale Technology +// Copyright (c) 2024 ZettaScale Technology // // This program and the accompanying materials are made available under the // terms of the Eclipse Public License 2.0 which is available at @@ -24,7 +24,8 @@ use zenoh_protocol::{ use zenoh_sync::get_mut_unchecked; use super::{ - face_hat_mut, pubsub::declare_sub_interest, queries::declare_qabl_interest, HatCode, HatFace, + face_hat_mut, pubsub::declare_sub_interest, queries::declare_qabl_interest, + token::declare_token_interest, HatCode, HatFace, }; use crate::net::routing::{ dispatcher::{ @@ -74,6 +75,16 @@ impl HatInterestTrait for HatCode { options.aggregate(), ) } + if options.tokens() { + declare_token_interest( + tables, + face, + id, + res.as_ref().map(|r| (*r).clone()).as_mut(), + mode, + options.aggregate(), + ) + } if mode.future() { face_hat_mut!(face) .remote_interests diff --git a/zenoh/src/net/routing/hat/router/mod.rs b/zenoh/src/net/routing/hat/router/mod.rs index c3f51eadba..94352ea77d 100644 --- a/zenoh/src/net/routing/hat/router/mod.rs +++ b/zenoh/src/net/routing/hat/router/mod.rs @@ -25,12 +25,13 @@ use std::{ time::Duration, }; +use token::{token_linkstate_change, token_remove_node, undeclare_client_token}; use zenoh_config::{unwrap_or_default, ModeDependent, WhatAmI, WhatAmIMatcher}; use zenoh_protocol::{ common::ZExtBody, core::ZenohIdProto, network::{ - declare::{queryable::ext::QueryableInfoType, QueryableId, SubscriberId}, + declare::{queryable::ext::QueryableInfoType, QueryableId, SubscriberId, TokenId}, interest::{InterestId, InterestOptions}, oam::id::OAM_LINKSTATE, Oam, @@ -68,6 +69,7 @@ mod interests; mod network; mod pubsub; mod queries; +mod token; macro_rules! hat { ($t:expr) => { @@ -118,6 +120,8 @@ use face_hat_mut; struct HatTables { router_subs: HashSet>, peer_subs: HashSet>, + router_tokens: HashSet>, + peer_tokens: HashSet>, router_qabls: HashSet>, peer_qabls: HashSet>, routers_net: Option, @@ -148,6 +152,8 @@ impl HatTables { peer_subs: HashSet::new(), router_qabls: HashSet::new(), peer_qabls: HashSet::new(), + router_tokens: HashSet::new(), + peer_tokens: HashSet::new(), routers_net: None, peers_net: None, shared_nodes: vec![], @@ -278,6 +284,7 @@ impl HatTables { tracing::trace!("Compute routes"); pubsub::pubsub_tree_change(&mut tables, &new_childs, net_type); queries::queries_tree_change(&mut tables, &new_childs, net_type); + token::token_tree_change(&mut tables, &new_childs, net_type); tracing::trace!("Computations completed"); match net_type { @@ -419,12 +426,20 @@ impl HatBaseTrait for HatCode { fn close_face(&self, tables: &TablesLock, face: &mut Arc) { let mut wtables = zwrite!(tables.tables); let mut face_clone = face.clone(); + let face = get_mut_unchecked(face); + let hat_face = match face.hat.downcast_mut::() { + Some(hate_face) => hate_face, + None => { + tracing::error!("Error downcasting face hat in close_face!"); + return; + } + }; - face_hat_mut!(face).remote_interests.clear(); - face_hat_mut!(face).local_subs.clear(); - face_hat_mut!(face).local_qabls.clear(); + hat_face.remote_interests.clear(); + hat_face.local_subs.clear(); + hat_face.local_qabls.clear(); + hat_face.local_tokens.clear(); - let face = get_mut_unchecked(face); for res in face.remote_mappings.values_mut() { get_mut_unchecked(res).session_ctxs.remove(&face.id); Resource::clean(res); @@ -437,13 +452,7 @@ impl HatBaseTrait for HatCode { face.local_mappings.clear(); let mut subs_matches = vec![]; - for (_id, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_subs - .drain() - { + for (_id, mut res) in hat_face.remote_subs.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_subscription(&mut wtables, &mut face_clone, &mut res); @@ -465,13 +474,7 @@ impl HatBaseTrait for HatCode { } let mut qabls_matches = vec![]; - for (_, mut res) in face - .hat - .downcast_mut::() - .unwrap() - .remote_qabls - .drain() - { + for (_, mut res) in hat_face.remote_qabls.drain() { get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); undeclare_client_queryable(&mut wtables, &mut face_clone, &mut res); @@ -491,6 +494,11 @@ impl HatBaseTrait for HatCode { qabls_matches.push(res); } } + + for (_id, mut res) in hat_face.remote_tokens.drain() { + get_mut_unchecked(&mut res).session_ctxs.remove(&face.id); + undeclare_client_token(&mut wtables, &mut face_clone, &mut res); + } drop(wtables); let mut matches_data_routes = vec![]; @@ -550,6 +558,7 @@ impl HatBaseTrait for HatCode { { pubsub_remove_node(tables, &removed_node.zid, WhatAmI::Router); queries_remove_node(tables, &removed_node.zid, WhatAmI::Router); + token_remove_node(tables, &removed_node.zid, WhatAmI::Router); } if hat!(tables).full_net(WhatAmI::Peer) { @@ -577,6 +586,7 @@ impl HatBaseTrait for HatCode { &removed_node.zid, WhatAmI::Peer, ); + token_remove_node(tables, &removed_node.zid, WhatAmI::Peer); } hat_mut!(tables).shared_nodes = shared_nodes( @@ -598,6 +608,11 @@ impl HatBaseTrait for HatCode { &updated_node.zid, &updated_node.links, ); + token_linkstate_change( + tables, + &updated_node.zid, + &updated_node.links, + ); } } } @@ -657,6 +672,7 @@ impl HatBaseTrait for HatCode { { pubsub_remove_node(tables, &removed_node.zid, WhatAmI::Router); queries_remove_node(tables, &removed_node.zid, WhatAmI::Router); + token_remove_node(tables, &removed_node.zid, WhatAmI::Router); } if hat!(tables).full_net(WhatAmI::Peer) { @@ -679,6 +695,7 @@ impl HatBaseTrait for HatCode { { pubsub_remove_node(tables, &removed_node.zid, WhatAmI::Peer); queries_remove_node(tables, &removed_node.zid, WhatAmI::Peer); + token_remove_node(tables, &removed_node.zid, WhatAmI::Peer); } hat_mut!(tables).shared_nodes = shared_nodes( @@ -766,6 +783,8 @@ struct HatContext { peer_subs: HashSet, router_qabls: HashMap, peer_qabls: HashMap, + router_tokens: HashSet, + peer_tokens: HashSet, } impl HatContext { @@ -775,6 +794,8 @@ impl HatContext { peer_subs: HashSet::new(), router_qabls: HashMap::new(), peer_qabls: HashMap::new(), + router_tokens: HashSet::new(), + peer_tokens: HashSet::new(), } } } @@ -787,6 +808,8 @@ struct HatFace { remote_subs: HashMap>, local_qabls: HashMap, (QueryableId, QueryableInfoType)>, remote_qabls: HashMap>, + local_tokens: HashMap, TokenId>, + remote_tokens: HashMap>, } impl HatFace { @@ -799,6 +822,8 @@ impl HatFace { remote_subs: HashMap::new(), local_qabls: HashMap::new(), remote_qabls: HashMap::new(), + local_tokens: HashMap::new(), + remote_tokens: HashMap::new(), } } } diff --git a/zenoh/src/net/routing/hat/router/pubsub.rs b/zenoh/src/net/routing/hat/router/pubsub.rs index e13aade332..5c1ced2405 100644 --- a/zenoh/src/net/routing/hat/router/pubsub.rs +++ b/zenoh/src/net/routing/hat/router/pubsub.rs @@ -45,7 +45,7 @@ use crate::net::routing::{ }, hat::{CurrentFutureTrait, HatPubSubTrait, Sources}, router::RoutesIndexes, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }; #[inline] @@ -62,7 +62,10 @@ fn send_sourced_subscription_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.unwrap().id { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let key_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -98,8 +101,7 @@ fn propagate_simple_subscription_to( src_face: &mut Arc, full_peer_net: bool, ) { - if (src_face.id != dst_face.id - || (dst_face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS))) + if src_face.id != dst_face.id && !face_hat!(dst_face).local_subs.contains_key(res) && if full_peer_net { dst_face.whatami == WhatAmI::Client @@ -368,7 +370,10 @@ fn send_forget_sourced_subscription_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.unwrap().id { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let wire_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -622,51 +627,49 @@ pub(super) fn undeclare_client_subscription( if client_subs.len() == 1 && !router_subs && !peer_subs { let mut face = &mut client_subs[0]; - if !(face.whatami == WhatAmI::Client && res.expr().starts_with(PREFIX_LIVELINESS)) { - if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { - face.primitives.send_declare(RoutingContext::with_expr( - Declare { - interest_id: None, - ext_qos: ext::QoSType::DECLARE, - ext_tstamp: None, - ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { - id, - ext_wire_expr: WireExprType::null(), - }), - }, - res.expr(), - )); - } - for res in face_hat!(face) - .local_subs - .keys() - .cloned() - .collect::>>() - { - if !res.context().matches.iter().any(|m| { - m.upgrade().is_some_and(|m| { - m.context.is_some() - && (remote_client_subs(&m, face) - || remote_peer_subs(tables, &m) - || remote_router_subs(tables, &m)) - }) - }) { - if let Some(id) = face_hat_mut!(&mut face).local_subs.remove(&res) { - face.primitives.send_declare(RoutingContext::with_expr( - Declare { - interest_id: None, - ext_qos: ext::QoSType::DECLARE, - ext_tstamp: None, - ext_nodeid: ext::NodeIdType::DEFAULT, - body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { - id, - ext_wire_expr: WireExprType::null(), - }), - }, - res.expr(), - )); - } + if let Some(id) = face_hat_mut!(face).local_subs.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_subs + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade().is_some_and(|m| { + m.context.is_some() + && (remote_client_subs(&m, face) + || remote_peer_subs(tables, &m) + || remote_router_subs(tables, &m)) + }) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_subs.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareSubscriber(UndeclareSubscriber { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); } } } @@ -731,10 +734,16 @@ pub(super) fn pubsub_tree_change( new_childs: &[Vec], net_type: WhatAmI, ) { + let net = match hat!(tables).get_net(net_type) { + Some(net) => net, + None => { + tracing::error!("Error accessing net in pubsub_tree_change!"); + return; + } + }; // propagate subs to new childs for (tree_sid, tree_childs) in new_childs.iter().enumerate() { if !tree_childs.is_empty() { - let net = hat!(tables).get_net(net_type).unwrap(); let tree_idx = NodeIndex::new(tree_sid); if net.graph.contains_node(tree_idx) { let tree_id = net.graph[tree_idx].zid; diff --git a/zenoh/src/net/routing/hat/router/queries.rs b/zenoh/src/net/routing/hat/router/queries.rs index 9df58a32a5..5a89757c46 100644 --- a/zenoh/src/net/routing/hat/router/queries.rs +++ b/zenoh/src/net/routing/hat/router/queries.rs @@ -19,14 +19,13 @@ use std::{ use ordered_float::OrderedFloat; use petgraph::graph::NodeIndex; -use zenoh_buffers::ZBuf; use zenoh_protocol::{ core::{ key_expr::{ include::{Includer, DEFAULT_INCLUDER}, OwnedKeyExpr, }, - WhatAmI, WireExpr, ZenohIdProto, + WhatAmI, ZenohIdProto, }, network::{ declare::{ @@ -51,7 +50,7 @@ use crate::net::routing::{ }, hat::{CurrentFutureTrait, HatQueriesTrait, Sources}, router::RoutesIndexes, - RoutingContext, PREFIX_LIVELINESS, + RoutingContext, }; #[inline] @@ -202,7 +201,11 @@ fn send_sourced_queryable_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.as_ref().unwrap().id { + if src_face + .as_ref() + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let key_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -239,7 +242,10 @@ fn propagate_simple_queryable( for mut dst_face in faces { let info = local_qabl_info(tables, res, &dst_face); let current = face_hat!(dst_face).local_qabls.get(res); - if (src_face.is_none() || src_face.as_ref().unwrap().id != dst_face.id) + if src_face + .as_ref() + .map(|src_face| dst_face.id != src_face.id) + .unwrap_or(true) && (current.is_none() || current.unwrap().1 != info) && face_hat!(dst_face) .remote_interests @@ -249,11 +255,14 @@ fn propagate_simple_queryable( dst_face.whatami == WhatAmI::Client } else { dst_face.whatami != WhatAmI::Router - && (src_face.is_none() - || src_face.as_ref().unwrap().whatami != WhatAmI::Peer - || dst_face.whatami != WhatAmI::Peer - || hat!(tables) - .failover_brokering(src_face.as_ref().unwrap().zid, dst_face.zid)) + && src_face + .as_ref() + .map(|src_face| { + src_face.whatami != WhatAmI::Peer + || dst_face.whatami != WhatAmI::Peer + || hat!(tables).failover_brokering(src_face.zid, dst_face.zid) + }) + .unwrap_or(true) } { let id = current @@ -486,7 +495,10 @@ fn send_forget_sourced_queryable_to_net_childs( if net.graph.contains_node(*child) { match tables.get_face(&net.graph[*child].zid).cloned() { Some(mut someface) => { - if src_face.is_none() || someface.id != src_face.unwrap().id { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { let wire_expr = Resource::decl_key(res, &mut someface); someface.primitives.send_declare(RoutingContext::with_expr( @@ -953,10 +965,16 @@ pub(super) fn queries_tree_change( new_childs: &[Vec], net_type: WhatAmI, ) { + let net = match hat!(tables).get_net(net_type) { + Some(net) => net, + None => { + tracing::error!("Error accessing net in queries_tree_change!"); + return; + } + }; // propagate qabls to new childs for (tree_sid, tree_childs) in new_childs.iter().enumerate() { if !tree_childs.is_empty() { - let net = hat!(tables).get_net(net_type).unwrap(); let tree_idx = NodeIndex::new(tree_sid); if net.graph.contains_node(tree_idx) { let tree_id = net.graph[tree_idx].zid; @@ -1376,48 +1394,6 @@ impl HatQueriesTrait for HatCode { Arc::new(route) } - #[inline] - fn compute_local_replies( - &self, - tables: &Tables, - prefix: &Arc, - suffix: &str, - face: &Arc, - ) -> Vec<(WireExpr<'static>, ZBuf)> { - let mut result = vec![]; - // Only the first routing point in the query route - // should return the liveliness tokens - if face.whatami == WhatAmI::Client { - let key_expr = prefix.expr() + suffix; - let key_expr = match OwnedKeyExpr::try_from(key_expr) { - Ok(ke) => ke, - Err(e) => { - tracing::warn!("Invalid KE reached the system: {}", e); - return result; - } - }; - if key_expr.starts_with(PREFIX_LIVELINESS) { - let res = Resource::get_resource(prefix, suffix); - let matches = res - .as_ref() - .and_then(|res| res.context.as_ref()) - .map(|ctx| Cow::from(&ctx.matches)) - .unwrap_or_else(|| Cow::from(Resource::get_matches(tables, &key_expr))); - for mres in matches.iter() { - let mres = mres.upgrade().unwrap(); - if (mres.context.is_some() - && (!res_hat!(mres).router_subs.is_empty() - || !res_hat!(mres).peer_subs.is_empty())) - || mres.session_ctxs.values().any(|ctx| ctx.subs.is_some()) - { - result.push((Resource::get_best_key(&mres, "", face.id), ZBuf::default())); - } - } - } - } - result - } - fn get_query_routes_entries(&self, tables: &Tables) -> RoutesIndexes { get_routes_entries(tables) } diff --git a/zenoh/src/net/routing/hat/router/token.rs b/zenoh/src/net/routing/hat/router/token.rs new file mode 100644 index 0000000000..583a4dc336 --- /dev/null +++ b/zenoh/src/net/routing/hat/router/token.rs @@ -0,0 +1,1051 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +use std::sync::{atomic::Ordering, Arc}; + +use petgraph::graph::NodeIndex; +use zenoh_protocol::{ + core::{WhatAmI, ZenohIdProto}, + network::{ + declare::{common::ext::WireExprType, TokenId}, + ext, + interest::{InterestId, InterestMode, InterestOptions}, + Declare, DeclareBody, DeclareToken, UndeclareToken, + }, +}; +use zenoh_sync::get_mut_unchecked; + +use super::{ + face_hat, face_hat_mut, get_peer, get_router, hat, hat_mut, network::Network, res_hat, + res_hat_mut, HatCode, HatContext, HatFace, HatTables, +}; +use crate::net::routing::{ + dispatcher::{face::FaceState, tables::Tables}, + hat::{CurrentFutureTrait, HatTokenTrait}, + router::{NodeId, Resource, SessionContext}, + RoutingContext, +}; + +#[inline] +fn send_sourced_token_to_net_childs( + tables: &Tables, + net: &Network, + childs: &[NodeIndex], + res: &Arc, + src_face: Option<&Arc>, + routing_context: NodeId, +) { + for child in childs { + if net.graph.contains_node(*child) { + match tables.get_face(&net.graph[*child].zid).cloned() { + Some(mut someface) => { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { + let key_expr = Resource::decl_key(res, &mut someface); + + someface.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType { + node_id: routing_context, + }, + body: DeclareBody::DeclareToken(DeclareToken { + id: 0, // Sourced tokens do not use ids + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } + } + None => tracing::trace!("Unable to find face for zid {}", net.graph[*child].zid), + } + } + } +} + +#[inline] +fn propagate_simple_token_to( + tables: &mut Tables, + dst_face: &mut Arc, + res: &Arc, + src_face: &mut Arc, + full_peer_net: bool, +) { + if (src_face.id != dst_face.id || dst_face.whatami == WhatAmI::Client) + && !face_hat!(dst_face).local_tokens.contains_key(res) + && if full_peer_net { + dst_face.whatami == WhatAmI::Client + } else { + dst_face.whatami != WhatAmI::Router + && (src_face.whatami != WhatAmI::Peer + || dst_face.whatami != WhatAmI::Peer + || hat!(tables).failover_brokering(src_face.zid, dst_face.zid)) + } + { + let matching_interests = face_hat!(dst_face) + .remote_interests + .values() + .filter(|(r, o)| o.tokens() && r.as_ref().map(|r| r.matches(res)).unwrap_or(true)) + .cloned() + .collect::>, InterestOptions)>>(); + + for (int_res, options) in matching_interests { + let res = if options.aggregate() { + int_res.as_ref().unwrap_or(res) + } else { + res + }; + if !face_hat!(dst_face).local_tokens.contains_key(res) { + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } + } + } +} + +fn propagate_simple_token(tables: &mut Tables, res: &Arc, src_face: &mut Arc) { + let full_peer_net = hat!(tables).full_net(WhatAmI::Peer); + for mut dst_face in tables + .faces + .values() + .cloned() + .collect::>>() + { + propagate_simple_token_to(tables, &mut dst_face, res, src_face, full_peer_net); + } +} + +fn propagate_sourced_token( + tables: &Tables, + res: &Arc, + src_face: Option<&Arc>, + source: &ZenohIdProto, + net_type: WhatAmI, +) { + let net = hat!(tables).get_net(net_type).unwrap(); + match net.get_idx(source) { + Some(tree_sid) => { + if net.trees.len() > tree_sid.index() { + send_sourced_token_to_net_childs( + tables, + net, + &net.trees[tree_sid.index()].childs, + res, + src_face, + tree_sid.index() as NodeId, + ); + } else { + tracing::trace!( + "Propagating liveliness {}: tree for node {} sid:{} not yet ready", + res.expr(), + tree_sid.index(), + source + ); + } + } + None => tracing::error!( + "Error propagating token {}: cannot get index of {}!", + res.expr(), + source + ), + } +} + +fn register_router_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + router: ZenohIdProto, +) { + if !res_hat!(res).router_tokens.contains(&router) { + // Register router liveliness + { + res_hat_mut!(res).router_tokens.insert(router); + hat_mut!(tables).router_tokens.insert(res.clone()); + } + + // Propagate liveliness to routers + propagate_sourced_token(tables, res, Some(face), &router, WhatAmI::Router); + } + // Propagate liveliness to peers + if hat!(tables).full_net(WhatAmI::Peer) && face.whatami != WhatAmI::Peer { + register_peer_token(tables, face, res, tables.zid) + } + + // Propagate liveliness to clients + propagate_simple_token(tables, res, face); +} + +fn declare_router_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + router: ZenohIdProto, +) { + register_router_token(tables, face, res, router); +} + +fn register_peer_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + peer: ZenohIdProto, +) { + if !res_hat!(res).peer_tokens.contains(&peer) { + // Register peer liveliness + { + res_hat_mut!(res).peer_tokens.insert(peer); + hat_mut!(tables).peer_tokens.insert(res.clone()); + } + + // Propagate liveliness to peers + propagate_sourced_token(tables, res, Some(face), &peer, WhatAmI::Peer); + } +} + +fn declare_peer_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + peer: ZenohIdProto, +) { + register_peer_token(tables, face, res, peer); + let zid = tables.zid; + register_router_token(tables, face, res, zid); +} + +fn register_client_token( + _tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + // Register liveliness + { + let res = get_mut_unchecked(res); + match res.session_ctxs.get_mut(&face.id) { + Some(ctx) => { + if !ctx.token { + get_mut_unchecked(ctx).token = true; + } + } + None => { + let ctx = res + .session_ctxs + .entry(face.id) + .or_insert_with(|| Arc::new(SessionContext::new(face.clone()))); + get_mut_unchecked(ctx).token = true; + } + } + } + face_hat_mut!(face).remote_tokens.insert(id, res.clone()); +} + +fn declare_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, +) { + register_client_token(tables, face, id, res); + let zid = tables.zid; + register_router_token(tables, face, res, zid); +} + +#[inline] +fn remote_router_tokens(tables: &Tables, res: &Arc) -> bool { + res.context.is_some() + && res_hat!(res) + .router_tokens + .iter() + .any(|peer| peer != &tables.zid) +} + +#[inline] +fn remote_peer_tokens(tables: &Tables, res: &Arc) -> bool { + res.context.is_some() + && res_hat!(res) + .peer_tokens + .iter() + .any(|peer| peer != &tables.zid) +} + +#[inline] +fn client_tokens(res: &Arc) -> Vec> { + res.session_ctxs + .values() + .filter_map(|ctx| { + if ctx.token { + Some(ctx.face.clone()) + } else { + None + } + }) + .collect() +} + +#[inline] +fn remote_client_tokens(res: &Arc, face: &Arc) -> bool { + res.session_ctxs + .values() + .any(|ctx| ctx.face.id != face.id && ctx.token) +} + +#[inline] +fn send_forget_sourced_token_to_net_childs( + tables: &Tables, + net: &Network, + childs: &[NodeIndex], + res: &Arc, + src_face: Option<&Arc>, + routing_context: Option, +) { + for child in childs { + if net.graph.contains_node(*child) { + match tables.get_face(&net.graph[*child].zid).cloned() { + Some(mut someface) => { + if src_face + .map(|src_face| someface.id != src_face.id) + .unwrap_or(true) + { + let wire_expr = Resource::decl_key(res, &mut someface); + + someface.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType { + node_id: routing_context.unwrap_or(0), + }, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: 0, // Sourced tokens do not use ids + ext_wire_expr: WireExprType { wire_expr }, + }), + }, + res.expr(), + )); + } + } + None => tracing::trace!("Unable to find face for zid {}", net.graph[*child].zid), + } + } + } +} + +fn propagate_forget_simple_token(tables: &mut Tables, res: &Arc) { + for mut face in tables.faces.values().cloned() { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } else if face_hat!(face).remote_interests.values().any(|(r, o)| { + o.tokens() && r.as_ref().map(|r| r.matches(res)).unwrap_or(true) && !o.aggregate() + }) { + // Token has never been declared on this face. + // Send an Undeclare with a one shot generated id and a WireExpr ext. + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst), + ext_wire_expr: WireExprType { + wire_expr: Resource::get_best_key(res, "", face.id), + }, + }), + }, + res.expr(), + )); + } + for res in face_hat!(&mut face) + .local_tokens + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade().is_some_and(|m| { + m.context.is_some() + && (remote_client_tokens(&m, &face) + || remote_peer_tokens(tables, &m) + || remote_router_tokens(tables, &m)) + }) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } else if face_hat!(face).remote_interests.values().any(|(r, o)| { + o.tokens() + && r.as_ref().map(|r| r.matches(&res)).unwrap_or(true) + && !o.aggregate() + }) { + // Token has never been declared on this face. + // Send an Undeclare with a one shot generated id and a WireExpr ext. + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id: face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst), + ext_wire_expr: WireExprType { + wire_expr: Resource::get_best_key(&res, "", face.id), + }, + }), + }, + res.expr(), + )); + } + } + } + } +} + +fn propagate_forget_simple_token_to_peers(tables: &mut Tables, res: &Arc) { + if !hat!(tables).full_net(WhatAmI::Peer) + && res_hat!(res).router_tokens.len() == 1 + && res_hat!(res).router_tokens.contains(&tables.zid) + { + for mut face in tables + .faces + .values() + .cloned() + .collect::>>() + { + if face.whatami == WhatAmI::Peer + && face_hat!(face).local_tokens.contains_key(res) + && !res.session_ctxs.values().any(|s| { + face.zid != s.face.zid + && s.token + && (s.face.whatami == WhatAmI::Client + || (s.face.whatami == WhatAmI::Peer + && hat!(tables).failover_brokering(s.face.zid, face.zid))) + }) + { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + } + } + } +} + +fn propagate_forget_sourced_token( + tables: &Tables, + res: &Arc, + src_face: Option<&Arc>, + source: &ZenohIdProto, + net_type: WhatAmI, +) { + let net = hat!(tables).get_net(net_type).unwrap(); + match net.get_idx(source) { + Some(tree_sid) => { + if net.trees.len() > tree_sid.index() { + send_forget_sourced_token_to_net_childs( + tables, + net, + &net.trees[tree_sid.index()].childs, + res, + src_face, + Some(tree_sid.index() as NodeId), + ); + } else { + tracing::trace!( + "Propagating forget token {}: tree for node {} sid:{} not yet ready", + res.expr(), + tree_sid.index(), + source + ); + } + } + None => tracing::error!( + "Error propagating forget token {}: cannot get index of {}!", + res.expr(), + source + ), + } +} + +fn unregister_router_token(tables: &mut Tables, res: &mut Arc, router: &ZenohIdProto) { + res_hat_mut!(res) + .router_tokens + .retain(|token| token != router); + + if res_hat!(res).router_tokens.is_empty() { + hat_mut!(tables) + .router_tokens + .retain(|token| !Arc::ptr_eq(token, res)); + + if hat_mut!(tables).full_net(WhatAmI::Peer) { + undeclare_peer_token(tables, None, res, &tables.zid.clone()); + } + propagate_forget_simple_token(tables, res); + } + + propagate_forget_simple_token_to_peers(tables, res); +} + +fn undeclare_router_token( + tables: &mut Tables, + face: Option<&Arc>, + res: &mut Arc, + router: &ZenohIdProto, +) { + if res_hat!(res).router_tokens.contains(router) { + unregister_router_token(tables, res, router); + propagate_forget_sourced_token(tables, res, face, router, WhatAmI::Router); + } +} + +fn forget_router_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + router: &ZenohIdProto, +) { + undeclare_router_token(tables, Some(face), res, router); +} + +fn unregister_peer_token(tables: &mut Tables, res: &mut Arc, peer: &ZenohIdProto) { + res_hat_mut!(res).peer_tokens.retain(|token| token != peer); + + if res_hat!(res).peer_tokens.is_empty() { + hat_mut!(tables) + .peer_tokens + .retain(|token| !Arc::ptr_eq(token, res)); + } +} + +fn undeclare_peer_token( + tables: &mut Tables, + face: Option<&Arc>, + res: &mut Arc, + peer: &ZenohIdProto, +) { + if res_hat!(res).peer_tokens.contains(peer) { + unregister_peer_token(tables, res, peer); + propagate_forget_sourced_token(tables, res, face, peer, WhatAmI::Peer); + } +} + +fn forget_peer_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, + peer: &ZenohIdProto, +) { + undeclare_peer_token(tables, Some(face), res, peer); + let client_tokens = res.session_ctxs.values().any(|ctx| ctx.token); + let peer_tokens = remote_peer_tokens(tables, res); + let zid = tables.zid; + if !client_tokens && !peer_tokens { + undeclare_router_token(tables, None, res, &zid); + } +} + +pub(super) fn undeclare_client_token( + tables: &mut Tables, + face: &mut Arc, + res: &mut Arc, +) { + if !face_hat_mut!(face) + .remote_tokens + .values() + .any(|s| *s == *res) + { + if let Some(ctx) = get_mut_unchecked(res).session_ctxs.get_mut(&face.id) { + get_mut_unchecked(ctx).token = false; + } + + let mut client_tokens = client_tokens(res); + let router_tokens = remote_router_tokens(tables, res); + let peer_tokens = remote_peer_tokens(tables, res); + if client_tokens.is_empty() && !peer_tokens { + undeclare_router_token(tables, None, res, &tables.zid.clone()); + } else { + propagate_forget_simple_token_to_peers(tables, res); + } + + if client_tokens.len() == 1 && !router_tokens && !peer_tokens { + let mut face = &mut client_tokens[0]; + if face.whatami != WhatAmI::Client { + if let Some(id) = face_hat_mut!(face).local_tokens.remove(res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + for res in face_hat!(face) + .local_tokens + .keys() + .cloned() + .collect::>>() + { + if !res.context().matches.iter().any(|m| { + m.upgrade().is_some_and(|m| { + m.context.is_some() + && (remote_client_tokens(&m, face) + || remote_peer_tokens(tables, &m) + || remote_router_tokens(tables, &m)) + }) + }) { + if let Some(id) = face_hat_mut!(&mut face).local_tokens.remove(&res) { + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + } + } + } + } + } + } +} + +fn forget_client_token( + tables: &mut Tables, + face: &mut Arc, + id: TokenId, +) -> Option> { + if let Some(mut res) = face_hat_mut!(face).remote_tokens.remove(&id) { + undeclare_client_token(tables, face, &mut res); + Some(res) + } else { + None + } +} + +pub(super) fn token_remove_node(tables: &mut Tables, node: &ZenohIdProto, net_type: WhatAmI) { + match net_type { + WhatAmI::Router => { + for mut res in hat!(tables) + .router_tokens + .iter() + .filter(|res| res_hat!(res).router_tokens.contains(node)) + .cloned() + .collect::>>() + { + unregister_router_token(tables, &mut res, node); + Resource::clean(&mut res) + } + } + WhatAmI::Peer => { + for mut res in hat!(tables) + .peer_tokens + .iter() + .filter(|res| res_hat!(res).peer_tokens.contains(node)) + .cloned() + .collect::>>() + { + unregister_peer_token(tables, &mut res, node); + let client_tokens = res.session_ctxs.values().any(|ctx| ctx.token); + let peer_tokens = remote_peer_tokens(tables, &res); + if !client_tokens && !peer_tokens { + undeclare_router_token(tables, None, &mut res, &tables.zid.clone()); + } + Resource::clean(&mut res) + } + } + _ => (), + } +} + +pub(super) fn token_tree_change( + tables: &mut Tables, + new_childs: &[Vec], + net_type: WhatAmI, +) { + let net = match hat!(tables).get_net(net_type) { + Some(net) => net, + None => { + tracing::error!("Error accessing net in token_tree_change!"); + return; + } + }; + // propagate tokens to new childs + for (tree_sid, tree_childs) in new_childs.iter().enumerate() { + if !tree_childs.is_empty() { + let tree_idx = NodeIndex::new(tree_sid); + if net.graph.contains_node(tree_idx) { + let tree_id = net.graph[tree_idx].zid; + + let tokens_res = match net_type { + WhatAmI::Router => &hat!(tables).router_tokens, + _ => &hat!(tables).peer_tokens, + }; + + for res in tokens_res { + let tokens = match net_type { + WhatAmI::Router => &res_hat!(res).router_tokens, + _ => &res_hat!(res).peer_tokens, + }; + for token in tokens { + if *token == tree_id { + send_sourced_token_to_net_childs( + tables, + net, + tree_childs, + res, + None, + tree_sid as NodeId, + ); + } + } + } + } + } + } +} + +pub(super) fn token_linkstate_change( + tables: &mut Tables, + zid: &ZenohIdProto, + links: &[ZenohIdProto], +) { + if let Some(src_face) = tables.get_face(zid).cloned() { + if hat!(tables).router_peers_failover_brokering && src_face.whatami == WhatAmI::Peer { + for res in face_hat!(src_face).remote_tokens.values() { + let client_tokens = res + .session_ctxs + .values() + .any(|ctx| ctx.face.whatami == WhatAmI::Client && ctx.token); + if !remote_router_tokens(tables, res) && !client_tokens { + for ctx in get_mut_unchecked(&mut res.clone()) + .session_ctxs + .values_mut() + { + let dst_face = &mut get_mut_unchecked(ctx).face; + if dst_face.whatami == WhatAmI::Peer && src_face.zid != dst_face.zid { + if let Some(id) = face_hat!(dst_face).local_tokens.get(res).cloned() { + let forget = !HatTables::failover_brokering_to(links, dst_face.zid) + && { + let ctx_links = hat!(tables) + .peers_net + .as_ref() + .map(|net| net.get_links(dst_face.zid)) + .unwrap_or_else(|| &[]); + res.session_ctxs.values().any(|ctx2| { + ctx2.face.whatami == WhatAmI::Peer + && ctx2.token + && HatTables::failover_brokering_to( + ctx_links, + ctx2.face.zid, + ) + }) + }; + if forget { + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::UndeclareToken(UndeclareToken { + id, + ext_wire_expr: WireExprType::null(), + }), + }, + res.expr(), + )); + + face_hat_mut!(dst_face).local_tokens.remove(res); + } + } else if HatTables::failover_brokering_to(links, ctx.face.zid) { + let dst_face = &mut get_mut_unchecked(ctx).face; + let id = face_hat!(dst_face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(dst_face).local_tokens.insert(res.clone(), id); + let key_expr = Resource::decl_key(res, dst_face); + dst_face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id: None, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { + id, + wire_expr: key_expr, + }), + }, + res.expr(), + )); + } + } + } + } + } + } + } +} + +pub(crate) fn declare_token_interest( + tables: &mut Tables, + face: &mut Arc, + id: InterestId, + res: Option<&mut Arc>, + mode: InterestMode, + aggregate: bool, +) { + if mode.current() && face.whatami == WhatAmI::Client { + let interest_id = (!mode.future()).then_some(id); + if let Some(res) = res.as_ref() { + if aggregate { + if hat!(tables).router_tokens.iter().any(|token| { + token.context.is_some() + && token.matches(res) + && (remote_client_tokens(token, face) + || remote_peer_tokens(tables, token) + || remote_router_tokens(tables, token)) + }) { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert((*res).clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(res, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + res.expr(), + )); + } + } else { + for token in &hat!(tables).router_tokens { + if token.context.is_some() + && token.matches(res) + && (res_hat!(token) + .router_tokens + .iter() + .any(|r| *r != tables.zid) + || res_hat!(token).peer_tokens.iter().any(|r| *r != tables.zid) + || token.session_ctxs.values().any(|s| { + s.face.id != face.id + && s.token + && (s.face.whatami == WhatAmI::Client + || face.whatami == WhatAmI::Client + || (s.face.whatami == WhatAmI::Peer + && hat!(tables) + .failover_brokering(s.face.zid, face.zid))) + })) + { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } else { + for token in &hat!(tables).router_tokens { + if token.context.is_some() + && (res_hat!(token) + .router_tokens + .iter() + .any(|r| *r != tables.zid) + || res_hat!(token).peer_tokens.iter().any(|r| *r != tables.zid) + || token.session_ctxs.values().any(|s| { + s.token + && (s.face.whatami != WhatAmI::Peer + || face.whatami != WhatAmI::Peer + || hat!(tables).failover_brokering(s.face.zid, face.zid)) + })) + { + let id = if mode.future() { + let id = face_hat!(face).next_id.fetch_add(1, Ordering::SeqCst); + face_hat_mut!(face).local_tokens.insert(token.clone(), id); + id + } else { + 0 + }; + let wire_expr = Resource::decl_key(token, face); + face.primitives.send_declare(RoutingContext::with_expr( + Declare { + interest_id, + ext_qos: ext::QoSType::DECLARE, + ext_tstamp: None, + ext_nodeid: ext::NodeIdType::DEFAULT, + body: DeclareBody::DeclareToken(DeclareToken { id, wire_expr }), + }, + token.expr(), + )); + } + } + } + } +} + +impl HatTokenTrait for HatCode { + fn declare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: &mut Arc, + node_id: NodeId, + _interest_id: Option, + ) { + match face.whatami { + WhatAmI::Router => { + if let Some(router) = get_router(tables, face, node_id) { + declare_router_token(tables, face, res, router) + } + } + WhatAmI::Peer => { + if hat!(tables).full_net(WhatAmI::Peer) { + if let Some(peer) = get_peer(tables, face, node_id) { + declare_peer_token(tables, face, res, peer) + } + } else { + declare_client_token(tables, face, id, res) + } + } + _ => declare_client_token(tables, face, id, res), + } + } + + fn undeclare_token( + &self, + tables: &mut Tables, + face: &mut Arc, + id: TokenId, + res: Option>, + node_id: NodeId, + ) -> Option> { + match face.whatami { + WhatAmI::Router => { + if let Some(mut res) = res { + if let Some(router) = get_router(tables, face, node_id) { + forget_router_token(tables, face, &mut res, &router); + Some(res) + } else { + None + } + } else { + None + } + } + WhatAmI::Peer => { + if hat!(tables).full_net(WhatAmI::Peer) { + if let Some(mut res) = res { + if let Some(peer) = get_peer(tables, face, node_id) { + forget_peer_token(tables, face, &mut res, &peer); + Some(res) + } else { + None + } + } else { + None + } + } else { + forget_client_token(tables, face, id) + } + } + _ => forget_client_token(tables, face, id), + } + } +} diff --git a/zenoh/src/net/routing/mod.rs b/zenoh/src/net/routing/mod.rs index 9601465326..f4a5e4c2da 100644 --- a/zenoh/src/net/routing/mod.rs +++ b/zenoh/src/net/routing/mod.rs @@ -32,8 +32,6 @@ use zenoh_protocol::{ use self::{dispatcher::face::Face, router::Resource}; use super::runtime; -pub(crate) static PREFIX_LIVELINESS: &str = "@/liveliness"; - pub(crate) struct RoutingContext { pub(crate) msg: Msg, pub(crate) inface: OnceCell, diff --git a/zenoh/src/net/routing/router.rs b/zenoh/src/net/routing/router.rs index d11ab24b5b..f13d1a7b95 100644 --- a/zenoh/src/net/routing/router.rs +++ b/zenoh/src/net/routing/router.rs @@ -23,6 +23,7 @@ use zenoh_protocol::core::{WhatAmI, ZenohIdProto}; use zenoh_result::ZResult; use zenoh_transport::{multicast::TransportMulticast, unicast::TransportUnicast, TransportPeer}; +pub(crate) use super::dispatcher::token::*; pub use super::dispatcher::{pubsub::*, queries::*, resource::*}; use super::{ dispatcher::{ @@ -60,7 +61,6 @@ impl Router { }) } - #[allow(clippy::too_many_arguments)] pub fn init_link_state(&mut self, runtime: Runtime) { let ctrl_lock = zlock!(self.tables.ctrl_lock); let mut tables = zwrite!(self.tables.tables); diff --git a/zenoh/tests/liveliness.rs b/zenoh/tests/liveliness.rs index 00d181311e..dbd850da24 100644 --- a/zenoh/tests/liveliness.rs +++ b/zenoh/tests/liveliness.rs @@ -11,63 +11,317 @@ // Contributors: // ZettaScale Zenoh Team, // -#![cfg(feature = "unstable")] -use std::time::Duration; - -use zenoh::{ - config, - prelude::*, - sample::{Sample, SampleKind}, -}; +#[cfg(feature = "unstable")] use zenoh_core::ztimeout; +#[cfg(feature = "unstable")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn zenoh_liveliness() { +async fn test_liveliness_subscriber_clique() { + use std::time::Duration; + + use zenoh::{config, prelude::*, sample::SampleKind}; + use zenoh_config::WhatAmI; + use zenoh_link::EndPoint; const TIMEOUT: Duration = Duration::from_secs(60); const SLEEP: Duration = Duration::from_secs(1); + const PEER1_ENDPOINT: &str = "tcp/localhost:47447"; + const LIVELINESS_KEYEXPR: &str = "test/liveliness/subscriber/clique"; + + zenoh_util::try_init_log_from_env(); + + let peer1 = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (1) ZID: {}", s.zid()); + s + }; - let mut c1 = config::peer(); - c1.listen - .set_endpoints(vec!["tcp/localhost:47447".parse().unwrap()]) - .unwrap(); - c1.scouting.multicast.set_enabled(Some(false)).unwrap(); - let session1 = ztimeout!(zenoh::open(c1)).unwrap(); - let mut c2 = config::peer(); - c2.connect - .set_endpoints(vec!["tcp/localhost:47447".parse().unwrap()]) - .unwrap(); - c2.scouting.multicast.set_enabled(Some(false)).unwrap(); - let session2 = ztimeout!(zenoh::open(c2)).unwrap(); - - let sub = ztimeout!(session2 - .liveliness() - .declare_subscriber("zenoh_liveliness_test")) - .unwrap(); - - let token = ztimeout!(session1.liveliness().declare_token("zenoh_liveliness_test")).unwrap(); + let peer2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (2) ZID: {}", s.zid()); + s + }; + let sub = ztimeout!(peer1.liveliness().declare_subscriber(LIVELINESS_KEYEXPR)).unwrap(); tokio::time::sleep(SLEEP).await; - let replies = ztimeout!(session2.liveliness().get("zenoh_liveliness_test")).unwrap(); - let sample: Sample = ztimeout!(replies.recv_async()) - .unwrap() - .into_result() - .unwrap(); + let token = ztimeout!(peer2.liveliness().declare_token(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); assert!(sample.kind() == SampleKind::Put); - assert!(sample.key_expr().as_str() == "zenoh_liveliness_test"); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); - assert!(ztimeout!(replies.recv_async()).is_err()); + drop(token); + tokio::time::sleep(SLEEP).await; let sample = ztimeout!(sub.recv_async()).unwrap(); + assert!(sample.kind() == SampleKind::Delete); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); +} + +#[cfg(feature = "unstable")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_query_clique() { + use std::time::Duration; + + use zenoh::{config, prelude::*, sample::SampleKind}; + use zenoh_config::WhatAmI; + use zenoh_link::EndPoint; + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const PEER1_ENDPOINT: &str = "tcp/localhost:47448"; + const LIVELINESS_KEYEXPR: &str = "test/liveliness/query/clique"; + + zenoh_util::try_init_log_from_env(); + + let peer1 = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (1) ZID: {}", s.zid()); + s + }; + + let peer2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![PEER1_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (2) ZID: {}", s.zid()); + s + }; + + let _token = ztimeout!(peer1.liveliness().declare_token(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let get = ztimeout!(peer2.liveliness().get(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(get.recv_async()).unwrap().into_result().unwrap(); assert!(sample.kind() == SampleKind::Put); - assert!(sample.key_expr().as_str() == "zenoh_liveliness_test"); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); +} + +#[cfg(feature = "unstable")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_subscriber_brokered() { + use std::time::Duration; + + use zenoh::{config, prelude::*, sample::SampleKind}; + use zenoh_config::WhatAmI; + use zenoh_link::EndPoint; + + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const ROUTER_ENDPOINT: &str = "tcp/localhost:47449"; + const LIVELINESS_KEYEXPR: &str = "test/liveliness/subscriber/brokered"; + + zenoh_util::try_init_log_from_env(); + + let _router = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Router)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Router ZID: {}", s.zid()); + s + }; + + let client1 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (1) ZID: {}", s.zid()); + s + }; + + let client2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (2) ZID: {}", s.zid()); + s + }; + + let sub = ztimeout!(client1.liveliness().declare_subscriber(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let token = ztimeout!(client2.liveliness().declare_token(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert!(sample.kind() == SampleKind::Put); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); drop(token); + tokio::time::sleep(SLEEP).await; + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert!(sample.kind() == SampleKind::Delete); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); +} + +#[cfg(feature = "unstable")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_query_brokered() { + use std::time::Duration; + + use zenoh::{config, prelude::*, sample::SampleKind}; + use zenoh_config::WhatAmI; + use zenoh_link::EndPoint; + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const ROUTER_ENDPOINT: &str = "tcp/localhost:47450"; + const LIVELINESS_KEYEXPR: &str = "test/liveliness/query/brokered"; + + zenoh_util::try_init_log_from_env(); + + let _router = { + let mut c = config::default(); + c.listen + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Router)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Router ZID: {}", s.zid()); + s + }; + + let client1 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (1) ZID: {}", s.zid()); + s + }; + + let client2 = { + let mut c = config::default(); + c.connect + .set_endpoints(vec![ROUTER_ENDPOINT.parse::().unwrap()]) + .unwrap(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Client)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Client (2) ZID: {}", s.zid()); + s + }; + + let _token = ztimeout!(client1.liveliness().declare_token(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let get = ztimeout!(client2.liveliness().get(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(get.recv_async()).unwrap().into_result().unwrap(); + assert!(sample.kind() == SampleKind::Put); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); +} + +#[cfg(feature = "unstable")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_subscriber_local() { + use std::time::Duration; + + use zenoh::{config, prelude::*, sample::SampleKind}; + use zenoh_config::WhatAmI; + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const LIVELINESS_KEYEXPR: &str = "test/liveliness/subscriber/local"; + + zenoh_util::try_init_log_from_env(); + + let peer = { + let mut c = config::default(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (1) ZID: {}", s.zid()); + s + }; + + let sub = ztimeout!(peer.liveliness().declare_subscriber(LIVELINESS_KEYEXPR)).unwrap(); tokio::time::sleep(SLEEP).await; - let replies = ztimeout!(session2.liveliness().get("zenoh_liveliness_test")).unwrap(); - assert!(ztimeout!(replies.recv_async()).is_err()); + let token = ztimeout!(peer.liveliness().declare_token(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert!(sample.kind() == SampleKind::Put); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); + + drop(token); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(sub.recv_async()).unwrap(); + assert!(sample.kind() == SampleKind::Delete); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); +} + +#[cfg(feature = "unstable")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_liveliness_query_local() { + use std::time::Duration; + + use zenoh::{config, prelude::*, sample::SampleKind}; + use zenoh_config::WhatAmI; + const TIMEOUT: Duration = Duration::from_secs(60); + const SLEEP: Duration = Duration::from_secs(1); + const LIVELINESS_KEYEXPR: &str = "test/liveliness/query/local"; + + zenoh_util::try_init_log_from_env(); + + let peer = { + let mut c = config::default(); + c.scouting.multicast.set_enabled(Some(false)).unwrap(); + let _ = c.set_mode(Some(WhatAmI::Peer)); + let s = ztimeout!(zenoh::open(c)).unwrap(); + tracing::info!("Peer (1) ZID: {}", s.zid()); + s + }; - assert!(replies.try_recv().is_err()); + let _token = ztimeout!(peer.liveliness().declare_token(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let get = ztimeout!(peer.liveliness().get(LIVELINESS_KEYEXPR)).unwrap(); + tokio::time::sleep(SLEEP).await; + + let sample = ztimeout!(get.recv_async()).unwrap().into_result().unwrap(); + assert!(sample.kind() == SampleKind::Put); + assert!(sample.key_expr().as_str() == LIVELINESS_KEYEXPR); }