From 53331ba2275442f0f55551560806950fc25f24f9 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 10 Jul 2023 20:55:59 +0200 Subject: [PATCH 01/52] Document dht-cache --- dht-cache/src/domocache.rs | 42 +++++++++++++++++++++++++++++++++++--- dht-cache/src/lib.rs | 4 ++-- dht-cache/src/utils.rs | 3 +++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 0b4f012..3bf5e9a 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -1,3 +1,4 @@ +//! Cached access to the DHT use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use crate::utils; use futures::prelude::*; @@ -33,7 +34,7 @@ fn generate_rsa_key() -> (Vec, Vec) { (pem, der) } -// possible events returned by cache_loop_event() +/// Events returned by [DomoCache::cache_event_loop] #[derive(Debug)] pub enum DomoEvent { None, @@ -44,14 +45,22 @@ pub enum DomoEvent { // period at which we send messages containing our cache hash const SEND_CACHE_HASH_PERIOD: u8 = 5; +/// Full Cache Element #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct DomoCacheElement { + /// Free-form topic name pub topic_name: String, + /// Unique identifier of the element pub topic_uuid: String, + /// JSON-serializable Value pub value: Value, + /// If true the element could be expunged from the local cache pub deleted: bool, + /// Time of the first pubblication pub publication_timestamp: u128, + /// First peer publishing it pub publisher_peer_id: String, + /// If non-zero the element is republished as part of a cache sync pub republication_timestamp: u128, } @@ -79,6 +88,9 @@ impl Display for DomoCacheElement { } } +/// Cached access to the DHT +/// +/// It keeps an in-memory (or persistent) cache of the whole DHT. pub struct DomoCache { storage: SqlxStorage, pub cache: BTreeMap>, @@ -116,7 +128,7 @@ impl Hash for DomoCache { } impl DomoCache { - #[allow(unused)] + /// Apply a jsonpath expression over the elements of a topic. pub fn filter_with_topic_name( &self, topic_name: &str, @@ -232,6 +244,7 @@ impl DomoCache { (true, true) } + /// Get a value identified by its uuid within a topic. pub fn get_topic_uuid(&self, topic_name: &str, topic_uuid: &str) -> Result { let ret = self.read_cache_element(topic_name, topic_uuid); match ret { @@ -244,6 +257,7 @@ impl DomoCache { } } + /// Get all the values within a topic. pub fn get_topic_name(&self, topic_name: &str) -> Result { let s = r#"[]"#; let mut ret: Value = serde_json::from_str(s).unwrap(); @@ -266,6 +280,7 @@ impl DomoCache { } } + /// Return the whole cache as a JSON Value pub fn get_all(&self) -> Value { let s = r#"[]"#; let mut ret: Value = serde_json::from_str(s).unwrap(); @@ -363,6 +378,7 @@ impl DomoCache { } } + /// Print the status of the cache across the known peers. pub fn print_peers_cache(&self) { for (peer_id, peer_data) in self.peers_caches_state.iter() { println!( @@ -460,6 +476,10 @@ impl DomoCache { ); Continue } + + /// Cache event loop + /// + /// To be called as often as needed to keep the cache in-sync and receive new data. pub async fn cache_event_loop(&mut self) -> std::result::Result> { use Event::*; loop { @@ -486,6 +506,9 @@ impl DomoCache { } } + /// Instantiate a new cache + /// + /// See [sifis_config::Cache] for the available parameters. pub async fn new(conf: sifis_config::Cache) -> Result> { if conf.url.is_empty() { panic!("db_url needed"); @@ -556,6 +579,10 @@ impl DomoCache { Ok(c) } + /// Publish a volatile value on the DHT + /// + /// All the peers reachable will receive it. + /// Peers joining later would not receive it. pub async fn pub_value(&mut self, value: Value) { let topic = Topic::new("domo-volatile-data"); @@ -601,6 +628,7 @@ impl DomoCache { } } + /// Print the current DHT state pub fn print(&self) { for (topic_name, topic_name_map) in self.cache.iter() { let mut first = true; @@ -617,16 +645,22 @@ impl DomoCache { } } + /// Print the DHT current hash pub fn print_cache_hash(&self) { println!("Hash {}", self.get_cache_hash()) } + /// Compute the hash of the current DHT state pub fn get_cache_hash(&self) -> u64 { let mut s = DefaultHasher::new(); self.hash(&mut s); s.finish() } + /// Mark a persistent value as deleted + /// + /// It will not be propagated and it is expunged from the initial DHT state fed to new peers + /// joining. pub async fn delete_value(&mut self, topic_name: &str, topic_uuid: &str) { let mut value_to_set = serde_json::Value::Null; @@ -649,7 +683,9 @@ impl DomoCache { .await; } - // metodo chiamato dall'applicazione, metto in cache e pubblico + /// Write/Update a persistent value + /// + /// The value will be part of the initial DHT state a peer joining will receive. pub async fn write_value(&mut self, topic_name: &str, topic_uuid: &str, value: Value) { let timest = utils::get_epoch_ms(); let elem = DomoCacheElement { diff --git a/dht-cache/src/lib.rs b/dht-cache/src/lib.rs index ab62c28..29203b1 100644 --- a/dht-cache/src/lib.rs +++ b/dht-cache/src/lib.rs @@ -1,5 +1,5 @@ -extern crate core; - +//! Simple DHT/messaging system based on libp2p +//! pub mod domocache; pub mod domolibp2p; pub mod domopersistentstorage; diff --git a/dht-cache/src/utils.rs b/dht-cache/src/utils.rs index 76d6f69..88cce77 100644 --- a/dht-cache/src/utils.rs +++ b/dht-cache/src/utils.rs @@ -1,4 +1,7 @@ +//! Miscellaneous utilities use std::time::{SystemTime, UNIX_EPOCH}; + +/// Compute the dht timestamps pub fn get_epoch_ms() -> u128 { SystemTime::now() .duration_since(UNIX_EPOCH) From f175b7554023e21efd409f43f27152c4483cfea0 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 10 Jul 2023 21:53:33 +0200 Subject: [PATCH 02/52] Hide unneeded API surface --- dht-cache/src/domocache.rs | 6 +++--- dht-cache/src/domopersistentstorage.rs | 4 +++- dht-cache/src/lib.rs | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 3bf5e9a..8a60392 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -65,7 +65,7 @@ pub struct DomoCacheElement { } #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -pub struct DomoCacheStateMessage { +struct DomoCacheStateMessage { pub peer_id: String, pub cache_hash: u64, pub publication_timestamp: u128, @@ -94,7 +94,7 @@ impl Display for DomoCacheElement { pub struct DomoCache { storage: SqlxStorage, pub cache: BTreeMap>, - pub peers_caches_state: BTreeMap, + peers_caches_state: BTreeMap, pub publish_cache_counter: u8, pub last_cache_repub_timestamp: u128, pub swarm: libp2p::Swarm, @@ -602,7 +602,7 @@ impl DomoCache { self.client_tx_channel.send(ev).await.unwrap(); } - pub async fn gossip_pub(&mut self, mut m: DomoCacheElement, republished: bool) { + async fn gossip_pub(&mut self, mut m: DomoCacheElement, republished: bool) { let topic = Topic::new("domo-persistent-data"); if republished { diff --git a/dht-cache/src/domopersistentstorage.rs b/dht-cache/src/domopersistentstorage.rs index a50a7f6..e1f4eca 100644 --- a/dht-cache/src/domopersistentstorage.rs +++ b/dht-cache/src/domopersistentstorage.rs @@ -8,7 +8,7 @@ use sqlx::{ any::{AnyConnectOptions, AnyKind, AnyRow}, postgres::PgConnectOptions, sqlite::SqliteConnectOptions, - AnyConnection, ConnectOptions, Connection, Executor, Row, SqliteConnection, + AnyConnection, ConnectOptions, Executor, Row, }; use std::str::FromStr; @@ -76,7 +76,9 @@ impl SqlxStorage { } } + #[cfg(test)] pub async fn new_in_memory(db_table: &str) -> Self { + use sqlx::{Connection, SqliteConnection}; let conn = SqliteConnection::connect("sqlite::memory:").await.unwrap(); Self::with_connection(conn.into(), db_table, true).await diff --git a/dht-cache/src/lib.rs b/dht-cache/src/lib.rs index 29203b1..074ddcd 100644 --- a/dht-cache/src/lib.rs +++ b/dht-cache/src/lib.rs @@ -1,8 +1,8 @@ //! Simple DHT/messaging system based on libp2p //! pub mod domocache; -pub mod domolibp2p; -pub mod domopersistentstorage; +mod domolibp2p; +mod domopersistentstorage; pub mod utils; pub use libp2p::identity::Keypair; From 66c1af63196cb2550b8701e3f2f694eb7641f613 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 10 Jul 2023 22:02:10 +0200 Subject: [PATCH 03/52] Re-export to the root of dht-cache --- dht-cache/src/domocache.rs | 2 +- dht-cache/src/lib.rs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 8a60392..a156a95 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -509,7 +509,7 @@ impl DomoCache { /// Instantiate a new cache /// /// See [sifis_config::Cache] for the available parameters. - pub async fn new(conf: sifis_config::Cache) -> Result> { + pub async fn new(conf: crate::Config) -> Result> { if conf.url.is_empty() { panic!("db_url needed"); } diff --git a/dht-cache/src/lib.rs b/dht-cache/src/lib.rs index 074ddcd..d428620 100644 --- a/dht-cache/src/lib.rs +++ b/dht-cache/src/lib.rs @@ -6,3 +6,9 @@ mod domopersistentstorage; pub mod utils; pub use libp2p::identity::Keypair; + +#[doc(inline)] +pub use domocache::DomoCache as Cache; + +/// Cache configuration +pub use sifis_config::Cache as Config; From f9c876b18bd3d6d24fb6f84c19560fd26d4fbef0 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 20 Jul 2023 11:07:56 +0200 Subject: [PATCH 04/52] Initial error type --- dht-cache/Cargo.toml | 4 +++- dht-cache/src/domocache.rs | 23 +++++++++-------------- dht-cache/src/domolibp2p.rs | 13 +++++++------ dht-cache/src/lib.rs | 20 ++++++++++++++++++++ 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/dht-cache/Cargo.toml b/dht-cache/Cargo.toml index 1098d70..f1eb96c 100644 --- a/dht-cache/Cargo.toml +++ b/dht-cache/Cargo.toml @@ -24,10 +24,12 @@ time = "0.3.17" tokio = { version = "1.19.0", features = ["full"] } url = "2.2.2" rsa = "0.9" -pem-rfc7468 = { version = "0.7", features = ["alloc"] } +pem-rfc7468 = { version = "0.7", features = ["std"] } sifis-config = { path = "../dht-config" } openssl-sys = "*" libsqlite3-sys = "*" +thiserror = "1.0.43" +anyhow = "1.0.72" [package.metadata.cargo-udeps.ignore] diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index a156a95..20d99ff 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -1,6 +1,7 @@ //! Cached access to the DHT use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use crate::utils; +use crate::Error; use futures::prelude::*; use libp2p::gossipsub::IdentTopic as Topic; use libp2p::identity::Keypair; @@ -12,7 +13,6 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; -use std::error::Error; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::io::ErrorKind; @@ -162,10 +162,7 @@ impl DomoCache { } } - fn handle_volatile_data( - &self, - message: &str, - ) -> std::result::Result> { + fn handle_volatile_data(&self, message: &str) -> std::result::Result { let m: serde_json::Value = serde_json::from_str(message)?; Ok(DomoEvent::VolatileData(m)) } @@ -173,7 +170,7 @@ impl DomoCache { async fn handle_persistent_message_data( &mut self, message: &str, - ) -> std::result::Result> { + ) -> std::result::Result { let mut m: DomoCacheElement = serde_json::from_str(message)?; // rimetto a 0 il republication timestamp altrimenti cambia hash @@ -480,7 +477,7 @@ impl DomoCache { /// Cache event loop /// /// To be called as often as needed to keep the cache in-sync and receive new data. - pub async fn cache_event_loop(&mut self) -> std::result::Result> { + pub async fn cache_event_loop(&mut self) -> std::result::Result { use Event::*; loop { match self.inner_select().await { @@ -509,7 +506,7 @@ impl DomoCache { /// Instantiate a new cache /// /// See [sifis_config::Cache] for the available parameters. - pub async fn new(conf: crate::Config) -> Result> { + pub async fn new(conf: crate::Config) -> Result { if conf.url.is_empty() { panic!("db_url needed"); } @@ -525,24 +522,22 @@ impl DomoCache { let mut pkcs8_der = if let Some(pk_path) = private_key_file { match std::fs::read(&pk_path) { Ok(pem) => { - let der = pem_rfc7468::decode_vec(&pem) - .map_err(|e| format!("Couldn't decode pem: {e:?}"))?; + let der = pem_rfc7468::decode_vec(&pem)?; der.1 } Err(e) if e.kind() == ErrorKind::NotFound => { // Generate a new key and put it into the file at the given path let (pem, der) = generate_rsa_key(); - std::fs::write(pk_path, pem).expect("Couldn't save "); + std::fs::write(pk_path, pem)?; der } - Err(e) => Err(format!("Couldn't load key file: {e:?}"))?, + Err(e) => Err(e)?, } } else { generate_rsa_key().1 }; - let local_key_pair = Keypair::rsa_from_pkcs8(&mut pkcs8_der) - .map_err(|e| format!("Couldn't load key: {e:?}"))?; + let local_key_pair = Keypair::rsa_from_pkcs8(&mut pkcs8_der)?; let swarm = crate::domolibp2p::start(shared_key, local_key_pair, loopback_only) .await diff --git a/dht-cache/src/domolibp2p.rs b/dht-cache/src/domolibp2p.rs index 6196ec7..9553945 100644 --- a/dht-cache/src/domolibp2p.rs +++ b/dht-cache/src/domolibp2p.rs @@ -23,12 +23,13 @@ use libp2p::Transport; use libp2p::{identity, mdns, swarm::NetworkBehaviour, PeerId, Swarm}; use libp2p::swarm::SwarmBuilder; -use std::error::Error; use std::time::Duration; +use crate::Error; + const KEY_SIZE: usize = 32; -fn parse_hex_key(s: &str) -> Result<[u8; KEY_SIZE], String> { +fn parse_hex_key(s: &str) -> Result<[u8; KEY_SIZE], Error> { if s.len() == KEY_SIZE * 2 { let mut r = [0u8; KEY_SIZE]; for i in 0..KEY_SIZE { @@ -37,16 +38,16 @@ fn parse_hex_key(s: &str) -> Result<[u8; KEY_SIZE], String> { Ok(res) => { r[i] = res; } - Err(_e) => return Err(String::from("Error while parsing")), + Err(_e) => return Err(Error::Hex("Error while parsing".into())), } } Ok(r) } else { - Err(format!( + Err(Error::Hex(format!( "Len Error: expected {} but got {}", KEY_SIZE * 2, s.len() - )) + ))) } } @@ -72,7 +73,7 @@ pub async fn start( shared_key: String, local_key_pair: identity::Keypair, loopback_only: bool, -) -> Result, Box> { +) -> Result, Error> { let local_peer_id = PeerId::from(local_key_pair.public()); // Create a Gossipsub topic diff --git a/dht-cache/src/lib.rs b/dht-cache/src/lib.rs index d428620..6d82e1d 100644 --- a/dht-cache/src/lib.rs +++ b/dht-cache/src/lib.rs @@ -12,3 +12,23 @@ pub use domocache::DomoCache as Cache; /// Cache configuration pub use sifis_config::Cache as Config; + +/// Error type +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum Error { + #[error("I/O error")] + Io(#[from] std::io::Error), + #[error("JSON serialization-deserialization error")] + Json(#[from] serde_json::Error), + #[error("Cannot decode the identity key")] + Identity(#[from] libp2p::identity::DecodingError), + #[error("Cannot decode the PEM information")] + Pem(#[from] pem_rfc7468::Error), + #[error("Cannot parse the hex string: {0}")] + Hex(String), + #[error("Connection setup error")] + Transport(#[from] libp2p::TransportError), + #[error("Cannot parse the multiaddr")] + MultiAddr(#[from] libp2p::multiaddr::Error), +} From c4f778eb982dd041d53da09a30b7e9d2f26e65a1 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 20 Jul 2023 11:16:00 +0200 Subject: [PATCH 05/52] Simplify the DomoEvent handling --- src/domobroker.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/domobroker.rs b/src/domobroker.rs index d54369a..de81bb9 100644 --- a/src/domobroker.rs +++ b/src/domobroker.rs @@ -18,7 +18,7 @@ pub struct DomoBroker { enum Event { WebSocket(SyncWebSocketDomoMessage), Rest(RestMessage), - Cache(Result>), + Cache(DomoEvent), Continue, } @@ -50,7 +50,7 @@ impl DomoBroker { }, m = self.domo_cache.cache_event_loop() => { - return Cache(m); + return Cache(m.unwrap_or(DomoEvent::None)); } } @@ -242,10 +242,10 @@ impl DomoBroker { } } - fn handle_cache_event_loop(&mut self, m: Result>) -> DomoEvent { + fn handle_cache_event_loop(&mut self, m: DomoEvent) -> DomoEvent { match m { - Ok(DomoEvent::None) => DomoEvent::None, - Ok(DomoEvent::PersistentData(m)) => { + DomoEvent::None => DomoEvent::None, + DomoEvent::PersistentData(m) => { println!( "Persistent message received {} {}", m.topic_name, m.topic_uuid @@ -263,7 +263,7 @@ impl DomoBroker { println!("SENT DATA ON WS {}", get_epoch_ms()); DomoEvent::PersistentData(m2) } - Ok(DomoEvent::VolatileData(m)) => { + DomoEvent::VolatileData(m) => { println!("Volatile message {m}"); let m2 = m.clone(); @@ -274,7 +274,6 @@ impl DomoBroker { DomoEvent::VolatileData(m2) } - _ => DomoEvent::None, } } } From 308d5b61041c62ed4c9968ce1e4a6defc57f3b48 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 20 Jul 2023 13:24:21 +0200 Subject: [PATCH 06/52] wip: Add get_peers_stats --- dht-cache/src/domocache.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 20d99ff..974278e 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -385,6 +385,15 @@ impl DomoCache { } } + /// Get the currently seen peers + /// + /// And their known hash and timestamp + pub fn get_peers_stats(&self) -> impl Iterator { + self.peers_caches_state + .values() + .map(|v| (v.peer_id.as_str(), v.cache_hash, v.publication_timestamp)) + } + async fn inner_select(&mut self) -> Event { use Event::*; tokio::select!( From 390743ad3c388d524205720afac9643df4f50a59 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 25 Jul 2023 09:43:23 +0200 Subject: [PATCH 07/52] Hide even more of the DomoCache --- dht-cache/src/domocache.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 974278e..7cd1094 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -93,11 +93,11 @@ impl Display for DomoCacheElement { /// It keeps an in-memory (or persistent) cache of the whole DHT. pub struct DomoCache { storage: SqlxStorage, - pub cache: BTreeMap>, + pub(crate) cache: BTreeMap>, peers_caches_state: BTreeMap, - pub publish_cache_counter: u8, - pub last_cache_repub_timestamp: u128, - pub swarm: libp2p::Swarm, + pub(crate) publish_cache_counter: u8, + pub(crate) last_cache_repub_timestamp: u128, + pub(crate) swarm: libp2p::Swarm, pub is_persistent_cache: bool, pub local_peer_id: String, client_tx_channel: Sender, From 505fd2e9826a2c306050d5f156b6ec1f97f94cc1 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 25 Jul 2023 20:27:53 +0200 Subject: [PATCH 08/52] Move the data structures in a separate module --- dht-cache/src/data.rs | 55 ++++++++++++++++++++++++++++++++++ dht-cache/src/domocache.rs | 60 +++----------------------------------- dht-cache/src/lib.rs | 1 + 3 files changed, 60 insertions(+), 56 deletions(-) create mode 100644 dht-cache/src/data.rs diff --git a/dht-cache/src/data.rs b/dht-cache/src/data.rs new file mode 100644 index 0000000..c83dde2 --- /dev/null +++ b/dht-cache/src/data.rs @@ -0,0 +1,55 @@ +//! Cached access to the DHT +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::fmt::{Display, Formatter}; + +/// Events returned by [DomoCache::cache_event_loop] +#[derive(Debug)] +pub enum DomoEvent { + None, + VolatileData(serde_json::Value), + PersistentData(DomoCacheElement), +} + +/// Full Cache Element +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub struct DomoCacheElement { + /// Free-form topic name + pub topic_name: String, + /// Unique identifier of the element + pub topic_uuid: String, + /// JSON-serializable Value + pub value: Value, + /// If true the element could be expunged from the local cache + pub deleted: bool, + /// Time of the first pubblication + pub publication_timestamp: u128, + /// First peer publishing it + pub publisher_peer_id: String, + /// If non-zero the element is republished as part of a cache sync + pub republication_timestamp: u128, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub(crate) struct DomoCacheStateMessage { + pub peer_id: String, + pub cache_hash: u64, + pub publication_timestamp: u128, +} + +impl Display for DomoCacheElement { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "(topic_name: {}, topic_uuid:{}, \ + value: {}, deleted: {}, publication_timestamp: {}, \ + peer_id: {})", + self.topic_name, + self.topic_uuid, + self.value, + self.deleted, + self.publication_timestamp, + self.publisher_peer_id + ) + } +} diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 7cd1094..a72e93a 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -1,4 +1,5 @@ //! Cached access to the DHT +pub use crate::data::*; use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use crate::utils; use crate::Error; @@ -9,11 +10,9 @@ use libp2p::mdns; use libp2p::swarm::SwarmEvent; use rsa::pkcs8::EncodePrivateKey; use rsa::RsaPrivateKey; -use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; -use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::io::ErrorKind; use std::time::Duration; @@ -21,6 +20,9 @@ use time::OffsetDateTime; use tokio::sync::mpsc; use tokio::sync::mpsc::{Receiver, Sender}; +// period at which we send messages containing our cache hash +const SEND_CACHE_HASH_PERIOD: u8 = 120; + fn generate_rsa_key() -> (Vec, Vec) { let mut rng = rand::thread_rng(); let bits = 2048; @@ -34,60 +36,6 @@ fn generate_rsa_key() -> (Vec, Vec) { (pem, der) } -/// Events returned by [DomoCache::cache_event_loop] -#[derive(Debug)] -pub enum DomoEvent { - None, - VolatileData(serde_json::Value), - PersistentData(DomoCacheElement), -} - -// period at which we send messages containing our cache hash -const SEND_CACHE_HASH_PERIOD: u8 = 5; - -/// Full Cache Element -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -pub struct DomoCacheElement { - /// Free-form topic name - pub topic_name: String, - /// Unique identifier of the element - pub topic_uuid: String, - /// JSON-serializable Value - pub value: Value, - /// If true the element could be expunged from the local cache - pub deleted: bool, - /// Time of the first pubblication - pub publication_timestamp: u128, - /// First peer publishing it - pub publisher_peer_id: String, - /// If non-zero the element is republished as part of a cache sync - pub republication_timestamp: u128, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -struct DomoCacheStateMessage { - pub peer_id: String, - pub cache_hash: u64, - pub publication_timestamp: u128, -} - -impl Display for DomoCacheElement { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "(topic_name: {}, topic_uuid:{}, \ - value: {}, deleted: {}, publication_timestamp: {}, \ - peer_id: {})", - self.topic_name, - self.topic_uuid, - self.value, - self.deleted, - self.publication_timestamp, - self.publisher_peer_id - ) - } -} - /// Cached access to the DHT /// /// It keeps an in-memory (or persistent) cache of the whole DHT. diff --git a/dht-cache/src/lib.rs b/dht-cache/src/lib.rs index 6d82e1d..f4efa81 100644 --- a/dht-cache/src/lib.rs +++ b/dht-cache/src/lib.rs @@ -1,5 +1,6 @@ //! Simple DHT/messaging system based on libp2p //! +mod data; pub mod domocache; mod domolibp2p; mod domopersistentstorage; From a9a8aa5ccbe34e392f455a570fcfb3c62cb287d4 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 27 Jul 2023 14:39:38 +0200 Subject: [PATCH 09/52] Derive Default for DomoCacheElement --- dht-cache/src/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dht-cache/src/data.rs b/dht-cache/src/data.rs index c83dde2..733519c 100644 --- a/dht-cache/src/data.rs +++ b/dht-cache/src/data.rs @@ -12,7 +12,7 @@ pub enum DomoEvent { } /// Full Cache Element -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Default, Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct DomoCacheElement { /// Free-form topic name pub topic_name: String, From 8bbfcd41387935022b43cb056fa1479dc4a9657b Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 27 Jul 2023 14:42:29 +0200 Subject: [PATCH 10/52] Move the rsa key generator in domolibp2p --- dht-cache/src/domocache.rs | 16 +--------------- dht-cache/src/domolibp2p.rs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index a72e93a..d9ee50a 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -1,5 +1,6 @@ //! Cached access to the DHT pub use crate::data::*; +use crate::domolibp2p::generate_rsa_key; use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use crate::utils; use crate::Error; @@ -8,8 +9,6 @@ use libp2p::gossipsub::IdentTopic as Topic; use libp2p::identity::Keypair; use libp2p::mdns; use libp2p::swarm::SwarmEvent; -use rsa::pkcs8::EncodePrivateKey; -use rsa::RsaPrivateKey; use serde_json::Value; use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; @@ -23,19 +22,6 @@ use tokio::sync::mpsc::{Receiver, Sender}; // period at which we send messages containing our cache hash const SEND_CACHE_HASH_PERIOD: u8 = 120; -fn generate_rsa_key() -> (Vec, Vec) { - let mut rng = rand::thread_rng(); - let bits = 2048; - let private_key = RsaPrivateKey::new(&mut rng, bits).expect("failed to generate a key"); - let pem = private_key - .to_pkcs8_pem(Default::default()) - .unwrap() - .as_bytes() - .to_vec(); - let der = private_key.to_pkcs8_der().unwrap().as_bytes().to_vec(); - (pem, der) -} - /// Cached access to the DHT /// /// It keeps an in-memory (or persistent) cache of the whole DHT. diff --git a/dht-cache/src/domolibp2p.rs b/dht-cache/src/domolibp2p.rs index 9553945..b2eb99c 100644 --- a/dht-cache/src/domolibp2p.rs +++ b/dht-cache/src/domolibp2p.rs @@ -8,6 +8,9 @@ use libp2p::gossipsub::{ }; use libp2p::{gossipsub, tcp}; +use rsa::pkcs8::EncodePrivateKey; +use rsa::RsaPrivateKey; + use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; // @@ -69,6 +72,19 @@ pub fn build_transport( .boxed() } +pub fn generate_rsa_key() -> (Vec, Vec) { + let mut rng = rand::thread_rng(); + let bits = 2048; + let private_key = RsaPrivateKey::new(&mut rng, bits).expect("failed to generate a key"); + let pem = private_key + .to_pkcs8_pem(Default::default()) + .unwrap() + .as_bytes() + .to_vec(); + let der = private_key.to_pkcs8_der().unwrap().as_bytes().to_vec(); + (pem, der) +} + pub async fn start( shared_key: String, local_key_pair: identity::Keypair, From ce34b61c07d0b839177bf5db003a17401360bfef Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 28 Jul 2023 10:13:54 +0200 Subject: [PATCH 11/52] Make the pg test use the DOMO_DHT_TEST_DB env var if available --- dht-cache/src/domopersistentstorage.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dht-cache/src/domopersistentstorage.rs b/dht-cache/src/domopersistentstorage.rs index e1f4eca..1f4ef77 100644 --- a/dht-cache/src/domopersistentstorage.rs +++ b/dht-cache/src/domopersistentstorage.rs @@ -225,10 +225,16 @@ mod tests { let _s = super::SqlxStorage::new(&db_config).await; } + fn get_pg_db() -> String { + std::env::var("DOMO_DHT_TEST_DB").unwrap_or_else(|_| { + "postgres://postgres:mysecretpassword@localhost/postgres".to_string() + }) + } + #[tokio::test] async fn test_pgsql_db_connection() { let db_config = sifis_config::Cache { - url: "postgres://postgres:mysecretpassword@localhost/postgres".to_string(), + url: get_pg_db(), table: "domo_test_pgsql_connection".to_string(), persistent: true, ..Default::default() @@ -246,7 +252,7 @@ mod tests { assert_eq!(v.len(), 0); let db_config = sifis_config::Cache { - url: "postgres://postgres:mysecretpassword@localhost/postgres".to_string(), + url: get_pg_db(), table: "test_initial_get_all_elements".to_string(), persistent: true, ..Default::default() @@ -280,7 +286,7 @@ mod tests { assert_eq!(v[0], m); let db_config = sifis_config::Cache { - url: "postgres://postgres:mysecretpassword@localhost/postgres".to_string(), + url: get_pg_db(), table: "test_store".to_string(), persistent: true, ..Default::default() From 47d6527d5f8c837dcdc540ccd092be4e86804893 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 28 Jul 2023 10:29:50 +0200 Subject: [PATCH 12/52] Split the behavior creation from the swarm setup Should make easier testing the network component using libp2p_swarm_test. --- dht-cache/src/domolibp2p.rs | 60 +++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/dht-cache/src/domolibp2p.rs b/dht-cache/src/domolibp2p.rs index b2eb99c..2bf67e1 100644 --- a/dht-cache/src/domolibp2p.rs +++ b/dht-cache/src/domolibp2p.rs @@ -92,11 +92,6 @@ pub async fn start( ) -> Result, Error> { let local_peer_id = PeerId::from(local_key_pair.public()); - // Create a Gossipsub topic - let topic_persistent_data = Topic::new("domo-persistent-data"); - let topic_volatile_data = Topic::new("domo-volatile-data"); - let topic_config = Topic::new("domo-config"); - let arr = parse_hex_key(&shared_key)?; let psk = PreSharedKey::new(arr); @@ -104,6 +99,37 @@ pub async fn start( // Create a swarm to manage peers and events. let mut swarm = { + let behaviour = DomoBehaviour::new(&local_key_pair)?; + + SwarmBuilder::with_tokio_executor(transport, behaviour, local_peer_id).build() + }; + + if !loopback_only { + // Listen on all interfaces and whatever port the OS assigns. + swarm.listen_on("/ip4/0.0.0.0/tcp/0".parse()?)?; + } else { + // Listen only on loopack interface + swarm.listen_on("/ip4/127.0.0.1/tcp/0".parse()?)?; + } + + Ok(swarm) +} + +// We create a custom network behaviour that combines mDNS and gossipsub. +#[derive(NetworkBehaviour)] +#[behaviour(to_swarm = "OutEvent")] +pub struct DomoBehaviour { + pub mdns: libp2p::mdns::tokio::Behaviour, + pub gossipsub: gossipsub::Behaviour, +} + +impl DomoBehaviour { + pub fn new(local_key_pair: &crate::Keypair) -> Result { + let local_peer_id = PeerId::from(local_key_pair.public()); + let topic_persistent_data = Topic::new("domo-persistent-data"); + let topic_volatile_data = Topic::new("domo-volatile-data"); + let topic_config = Topic::new("domo-config"); + let mdnsconf = mdns::Config { ttl: Duration::from_secs(600), query_interval: Duration::from_secs(30), @@ -120,6 +146,7 @@ pub async fn start( }; // Set a custom gossipsub + // SAFETY: hard-coded configuration let gossipsub_config = gossipsub::ConfigBuilder::default() .heartbeat_interval(Duration::from_secs(3)) // This is set to aid debugging by not cluttering the log space .check_explicit_peers_ticks(10) @@ -131,7 +158,7 @@ pub async fn start( // build a gossipsub network behaviour let mut gossipsub = gossipsub::Behaviour::new( - MessageAuthenticity::Signed(local_key_pair), + MessageAuthenticity::Signed(local_key_pair.to_owned()), gossipsub_config, ) .expect("Correct configuration"); @@ -146,28 +173,9 @@ pub async fn start( gossipsub.subscribe(&topic_config).unwrap(); let behaviour = DomoBehaviour { mdns, gossipsub }; - //Swarm::new(transport, behaviour, local_peer_id) - - SwarmBuilder::with_tokio_executor(transport, behaviour, local_peer_id).build() - }; - if !loopback_only { - // Listen on all interfaces and whatever port the OS assigns. - swarm.listen_on("/ip4/0.0.0.0/tcp/0".parse()?)?; - } else { - // Listen only on loopack interface - swarm.listen_on("/ip4/127.0.0.1/tcp/0".parse()?)?; + Ok(behaviour) } - - Ok(swarm) -} - -// We create a custom network behaviour that combines mDNS and gossipsub. -#[derive(NetworkBehaviour)] -#[behaviour(to_swarm = "OutEvent")] -pub struct DomoBehaviour { - pub mdns: libp2p::mdns::tokio::Behaviour, - pub gossipsub: gossipsub::Behaviour, } #[allow(clippy::large_enum_variant)] From 81d40d92eb4b96123903e66f5a7791e865dca6a8 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 28 Jul 2023 22:20:24 +0200 Subject: [PATCH 13/52] wip: Abstract over the network code --- dht-cache/src/dht.rs | 255 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 dht-cache/src/dht.rs diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs new file mode 100644 index 0000000..6c78df4 --- /dev/null +++ b/dht-cache/src/dht.rs @@ -0,0 +1,255 @@ +//! DHT Abstraction +//! + +use crate::domolibp2p::{DomoBehaviour, OutEvent}; +use futures::prelude::*; +use libp2p::{gossipsub::IdentTopic as Topic, swarm::SwarmEvent, Swarm}; +use libp2p::{mdns, PeerId}; +use serde_json::Value; +use time::OffsetDateTime; +use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use tokio::task::JoinHandle; + +/// Network commands +#[derive(Debug)] +pub enum Command { + Broadcast(Value), + Publish(Value), + Stop, +} + +/// Network Events +#[derive(Debug)] +pub enum Event { + PersistentData(String), + VolatileData(String), + Config(String), + Discovered(Vec), +} + +fn handle_command(swarm: &mut Swarm, cmd: Command) -> bool { + use Command::*; + match cmd { + Broadcast(val) => { + let topic = Topic::new("domo-volatile-data"); + let m = serde_json::to_string(&val).unwrap(); + + if let Err(e) = swarm.behaviour_mut().gossipsub.publish(topic, m.as_bytes()) { + log::info!("Publish error: {e:?}"); + } + true + } + Publish(val) => { + let topic = Topic::new("domo-persistent-data"); + let m2 = serde_json::to_string(&val).unwrap(); + + if let Err(e) = swarm + .behaviour_mut() + .gossipsub + .publish(topic, m2.as_bytes()) + { + log::info!("Publish error: {e:?}"); + } + true + } + Stop => false, + } +} + +fn handle_swarm_event( + swarm: &mut Swarm, + event: SwarmEvent, + ev_send: &UnboundedSender, +) -> Result<(), ()> { + use Event::*; + + match event { + SwarmEvent::ExpiredListenAddr { address, .. } => { + log::info!("Address {address:?} expired"); + } + SwarmEvent::ConnectionEstablished { .. } => { + log::info!("Connection established ..."); + } + SwarmEvent::ConnectionClosed { .. } => { + log::info!("Connection closed"); + } + SwarmEvent::ListenerError { .. } => { + log::info!("Listener Error"); + } + SwarmEvent::OutgoingConnectionError { .. } => { + log::info!("Outgoing connection error"); + } + SwarmEvent::ListenerClosed { .. } => { + log::info!("Listener Closed"); + } + SwarmEvent::NewListenAddr { address, .. } => { + println!("Listening in {address:?}"); + } + SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Gossipsub( + libp2p::gossipsub::Event::Message { + propagation_source: _peer_id, + message_id: _id, + message, + }, + )) => { + let data = String::from_utf8(message.data).unwrap(); + match message.topic.as_str() { + "domo-persistent-data" => { + ev_send.send(PersistentData(data)).map_err(|_| ())?; + } + "domo-config" => { + ev_send.send(Config(data)).map_err(|_| ())?; + } + "domo-volatile-data" => { + ev_send.send(VolatileData(data)).map_err(|_| ())?; + } + _ => { + log::info!("Not able to recognize message"); + } + } + } + SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Mdns(mdns::Event::Expired(list))) => { + let local = OffsetDateTime::now_utc(); + + for (peer, _) in list { + log::info!("MDNS for peer {peer} expired {local:?}"); + } + } + SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Mdns(mdns::Event::Discovered(list))) => { + let local = OffsetDateTime::now_utc(); + let peers = list + .iter() + .map(|(peer, _)| { + swarm.behaviour_mut().gossipsub.add_explicit_peer(peer); + log::info!("Discovered peer {peer} {local:?}"); + peer.to_owned() + }) + .collect(); + ev_send.send(Discovered(peers)).map_err(|_| ())?; + } + _ => {} + } + + Ok(()) +} + +/// Spawn a new task polling constantly for new swarm Events +pub fn dht_channel( + mut swarm: Swarm, +) -> ( + UnboundedSender, + UnboundedReceiver, + JoinHandle<()>, +) { + let (cmd_send, mut cmd_recv) = mpsc::unbounded_channel(); + let (mut ev_send, ev_recv) = mpsc::unbounded_channel(); + + let handle = tokio::task::spawn(async move { + loop { + tokio::select! { + cmd = cmd_recv.recv() => { + log::debug!("command {cmd:?}"); + if !cmd.is_some_and(|cmd| handle_command(&mut swarm, cmd)) { + log::debug!("Exiting cmd"); + return + } + } + ev = swarm.select_next_some() => { + if handle_swarm_event(&mut swarm, ev, &mut ev_send).is_err() { + log::debug!("Exiting ev"); + return + } + } + } + } + }); + + (cmd_send, ev_recv, handle) +} + +#[cfg(test)] +mod test { + use super::*; + use libp2p_swarm_test::SwarmExt; + use serde_json::json; + + #[tokio::test] + async fn multiple_peers() { + env_logger::init(); + let mut a = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + let mut b = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + let mut c = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + + for a in a.external_addresses() { + println!("{a:?}"); + } + + a.listen().await; + b.listen().await; + c.listen().await; + + a.connect(&mut b).await; + b.connect(&mut c).await; + c.connect(&mut a).await; + + let peers: Vec<_> = a.connected_peers().cloned().collect(); + + for peer in peers { + a.behaviour_mut().gossipsub.add_explicit_peer(&peer); + } + + let peers: Vec<_> = b.connected_peers().cloned().collect(); + + for peer in peers { + b.behaviour_mut().gossipsub.add_explicit_peer(&peer); + } + + let peers: Vec<_> = c.connected_peers().cloned().collect(); + + for peer in peers { + c.behaviour_mut().gossipsub.add_explicit_peer(&peer); + } + + let (a_s, mut ar, _) = dht_channel(a); + let (b_s, br, _) = dht_channel(b); + let (c_s, cr, _) = dht_channel(c); + + // Wait until peers are discovered + while let Some(ev) = ar.recv().await { + match ev { + Event::VolatileData(data) => log::info!("volatile {data}"), + Event::PersistentData(data) => log::info!("persistent {data}"), + Event::Config(cfg) => log::info!("config {cfg}"), + Event::Discovered(peers) => { + log::info!("found peers: {peers:?}"); + break; + } + } + } + + let msg = json!({"a": "value"}); + + a_s.send(Command::Broadcast(msg.clone())).unwrap(); + + for r in [br, cr].iter_mut() { + while let Some(ev) = r.recv().await { + match ev { + Event::VolatileData(data) => { + log::info!("volatile {data}"); + let val: Value = serde_json::from_str(&data).unwrap(); + assert_eq!(val, msg); + break; + } + Event::PersistentData(data) => log::info!("persistent {data}"), + Event::Config(cfg) => log::info!("config {cfg}"), + Event::Discovered(peers) => { + log::info!("found peers: {peers:?}"); + } + } + } + } + + drop(b_s); + drop(c_s); + } +} From 8b3ab42503ae3722430fba1322e01f31166b89e3 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sat, 29 Jul 2023 13:41:51 +0200 Subject: [PATCH 14/52] Fix documentation --- dht-config/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dht-config/src/lib.rs b/dht-config/src/lib.rs index 21c60d5..04d1c51 100644 --- a/dht-config/src/lib.rs +++ b/dht-config/src/lib.rs @@ -92,7 +92,7 @@ where } /// Set a custom config file path /// - /// By default uses the result of [Command::get_name] with the extension `.toml`. + /// By default uses the result of [clap::Command::get_name] with the extension `.toml`. pub fn with_config_path>(mut self, path: P) -> Self { self.default_path = Some(path.as_ref().to_owned()); self @@ -100,7 +100,7 @@ where /// Set a custom prefix for the env variable lookup /// - /// By default uses the result of [Command::get_name] with `_`. + /// By default uses the result of [clap::Command::get_name] with `_`. pub fn with_env_prefix(mut self, prefix: S) -> Self { self.prefix = Some(prefix.to_string()); self From 63310d9aab075da29dc89e730960bee117e20778 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 6 Aug 2023 16:11:43 +0200 Subject: [PATCH 15/52] wip: Rework the cache layer --- dht-cache/Cargo.toml | 5 + dht-cache/src/cache.rs | 358 +++++++++++++++++++++++++++++++++++ dht-cache/src/cache/local.rs | 269 ++++++++++++++++++++++++++ dht-cache/src/data.rs | 10 +- dht-cache/src/dht.rs | 48 ++++- dht-cache/src/lib.rs | 10 +- 6 files changed, 689 insertions(+), 11 deletions(-) create mode 100644 dht-cache/src/cache.rs create mode 100644 dht-cache/src/cache/local.rs diff --git a/dht-cache/Cargo.toml b/dht-cache/Cargo.toml index f1eb96c..b943dec 100644 --- a/dht-cache/Cargo.toml +++ b/dht-cache/Cargo.toml @@ -30,6 +30,11 @@ openssl-sys = "*" libsqlite3-sys = "*" thiserror = "1.0.43" anyhow = "1.0.72" +libp2p-swarm-test = "0.2.0" +tokio-stream = "0.1.14" + +[dev-dependencies] +env_logger = "0.10.0" [package.metadata.cargo-udeps.ignore] diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs new file mode 100644 index 0000000..01ca290 --- /dev/null +++ b/dht-cache/src/cache.rs @@ -0,0 +1,358 @@ +//! Cached access to the DHT + +mod local; + +use std::sync::Arc; +use std::{collections::BTreeMap, time::Duration}; + +use futures_util::{Stream, StreamExt}; +use libp2p::Swarm; +use serde_json::Value; +use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::RwLock; +use tokio::time; +use tokio_stream::wrappers::UnboundedReceiverStream; + +use crate::{ + cache::local::DomoCacheStateMessage, + data::DomoEvent, + dht::{dht_channel, Command, Event}, + domolibp2p::DomoBehaviour, + utils, Error, +}; + +use self::local::{DomoCacheElement, LocalCache, Query}; + +/// Cached DHT +/// +/// It keeps a local cache of the dht state and allow to query the persistent topics +pub struct Cache { + peer_id: String, + local: LocalCache, + cmd: UnboundedSender, +} + +impl Cache { + /// Send a volatile message + /// + /// Volatile messages are unstructured and do not persist in the DHT. + pub fn send(&self, value: &Value) -> Result<(), Error> { + self.cmd + .send(Command::Broadcast(value.to_owned())) + .map_err(|_| Error::Channel)?; + + Ok(()) + } + + /// Persist a value within the DHT + /// + /// It is identified by the topic and uuid value + pub async fn put(&self, topic: &str, uuid: &str, value: &Value) -> Result<(), Error> { + let elem = DomoCacheElement { + topic_name: topic.to_string(), + topic_uuid: uuid.to_string(), + value: value.to_owned(), + publication_timestamp: utils::get_epoch_ms(), + publisher_peer_id: self.peer_id.clone(), + ..Default::default() + }; + + self.local.put(&elem).await; + + self.cmd + .send(Command::Publish(serde_json::to_value(&elem)?)) + .map_err(|_| Error::Channel)?; + + Ok(()) + } + + /// Delete a value within the DHT + /// + /// It inserts the deletion entry and the entry value will be marked as deleted and removed + /// from the stored cache. + pub async fn del(&self, topic: &str, uuid: &str) -> Result<(), Error> { + let elem = DomoCacheElement { + topic_name: topic.to_string(), + topic_uuid: uuid.to_string(), + publication_timestamp: utils::get_epoch_ms(), + publisher_peer_id: self.peer_id.clone(), + deleted: true, + ..Default::default() + }; + + self.local.put(&elem).await; + + self.cmd + .send(Command::Publish(serde_json::to_value(&elem)?)) + .map_err(|_| Error::Channel)?; + + Ok(()) + } + + /// Query the local cache + pub fn query(&self, topic: &str) -> Query { + self.local.query(topic) + } +} + +#[derive(Default, Debug, Clone)] +pub(crate) struct PeersState { + list: BTreeMap, + last_repub_timestamp: u128, + repub_interval: u128, +} + +#[derive(Debug)] +enum CacheState { + Synced, + Desynced { is_leader: bool }, +} + +impl PeersState { + fn with_interval(repub_interval: u128) -> Self { + Self { + repub_interval, + ..Default::default() + } + } + + fn insert(&mut self, state: DomoCacheStateMessage) { + self.list.insert(state.peer_id.to_string(), state); + } + + async fn is_synchronized(&self, peer_id: &str, hash: u64) -> CacheState { + let cur_ts = utils::get_epoch_ms() - self.repub_interval; + let desync = self + .list + .values() + .find(|data| data.cache_hash != hash && data.publication_timestamp > cur_ts) + .is_some(); + + if desync { + CacheState::Desynced { + is_leader: self + .list + .values() + .find(|data| { + data.cache_hash == hash + && data.peer_id.as_str() < peer_id + && data.publication_timestamp > cur_ts + }) + .is_none(), + } + } else { + CacheState::Synced + } + } +} + +/// Join the dht and keep a local cache up to date +/// +/// the resend interval is expressed in milliseconds +pub fn cache_channel( + local: LocalCache, + swarm: Swarm, + resend_interval: u64, +) -> (Cache, impl Stream) { + let local_peer_id = swarm.local_peer_id().to_string(); + + let (cmd, r, _j) = dht_channel(swarm); + + let cache = Cache { + local: local.clone(), + cmd: cmd.clone(), + peer_id: local_peer_id.clone(), + }; + + let stream = UnboundedReceiverStream::new(r); + + let peers_state = Arc::new(RwLock::new(PeersState::with_interval( + resend_interval as u128, + ))); + + let local_read = local.clone(); + let cmd_update = cmd.clone(); + let peer_id = local_peer_id.clone(); + + tokio::task::spawn(async move { + let mut interval = time::interval(Duration::from_millis(resend_interval.max(100))); + while !cmd_update.is_closed() { + interval.tick().await; + let hash = local_read.get_hash().await; + let m = DomoCacheStateMessage { + peer_id: peer_id.clone(), + cache_hash: hash, + publication_timestamp: utils::get_epoch_ms(), + }; + + if cmd_update + .send(Command::Config(serde_json::to_value(&m).unwrap())) + .is_err() + { + break; + } + } + }); + + // TODO: refactor once async closures are stable + let events = stream.filter_map(move |ev| { + let local_write = local.clone(); + let peers_state = peers_state.clone(); + let peer_id = local_peer_id.clone(); + let cmd = cmd.clone(); + async move { + match ev { + Event::Config(cfg) => { + let m: DomoCacheStateMessage = serde_json::from_str(&cfg).unwrap(); + + let hash = local_write.get_hash().await; + + // SAFETY: only user + let mut peers_state = peers_state.write().await; + + // update the peers_caches_state + peers_state.insert(m); + + let sync_info = peers_state.is_synchronized(&peer_id, hash).await; + + log::debug!("local {peer_id:?} {sync_info:?} -> {peers_state:#?}"); + + if let CacheState::Desynced { is_leader } = sync_info { + if is_leader + && utils::get_epoch_ms() - peers_state.last_repub_timestamp + >= peers_state.repub_interval + { + local_write + .read_owned() + .await + .values() + .flat_map(|topic| topic.values()) + .for_each(|elem| { + let mut elem = elem.to_owned(); + log::debug!("resending {}", elem.topic_uuid); + elem.republication_timestamp = utils::get_epoch_ms(); + cmd.send(Command::Publish( + serde_json::to_value(&elem).unwrap(), + )) + .unwrap(); + }); + peers_state.last_repub_timestamp = utils::get_epoch_ms(); + } + } + + // check for desync + // republish the local cache if needed + None + } + Event::Discovered(who) => Some(DomoEvent::NewPeers( + who.into_iter().map(|w| w.to_string()).collect(), + )), + Event::VolatileData(data) => { + // TODO we swallow errors quietly here + serde_json::from_str(&data) + .ok() + .map(DomoEvent::VolatileData) + } + Event::PersistentData(data) => { + if let Ok(mut elem) = serde_json::from_str::(&data) { + if elem.republication_timestamp != 0 { + log::debug!("Retransmission"); + } + // TODO: do something with this value instead + elem.republication_timestamp = 0; + local_write + .try_put(&elem) + .await + .ok() + .map(|_| DomoEvent::PersistentData(elem)) + } else { + None + } + } + } + } + }); + + (cache, events) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::dht::test::*; + use std::{collections::HashSet, pin::pin}; + + #[tokio::test(flavor = "multi_thread")] + async fn syncronization() { + let [mut a, mut b, mut c] = make_peers().await; + let mut d = make_peer().await; + + connect_peer(&mut a, &mut d).await; + connect_peer(&mut b, &mut d).await; + connect_peer(&mut c, &mut d).await; + + let a_local_cache = LocalCache::new(); + let b_local_cache = LocalCache::new(); + let c_local_cache = LocalCache::new(); + let d_local_cache = LocalCache::new(); + + let mut expected: HashSet<_> = (0..10) + .into_iter() + .map(|uuid| format!("uuid-{uuid}")) + .collect(); + + tokio::task::spawn(async move { + let (a_c, a_ev) = cache_channel(a_local_cache, a, 1000); + let (_b_c, b_ev) = cache_channel(b_local_cache, b, 1000); + let (_c_c, c_ev) = cache_channel(c_local_cache, c, 1000); + + let mut a_ev = pin!(a_ev); + let mut b_ev = pin!(b_ev); + let mut c_ev = pin!(c_ev); + for uuid in 0..10 { + let _ = a_c + .put( + "Topic", + &format!("uuid-{uuid}"), + &serde_json::json!({"key": uuid}), + ) + .await; + } + + loop { + let (node, ev) = tokio::select! { + v = a_ev.next() => ("a", v.unwrap()), + v = b_ev.next() => ("b", v.unwrap()), + v = c_ev.next() => ("c", v.unwrap()), + }; + + match ev { + DomoEvent::PersistentData(data) => { + log::debug!("{node}: Got data {data:?}"); + } + _ => { + log::debug!("{node}: Other {ev:?}"); + } + } + } + }); + + log::info!("Adding D"); + + let (_d_c, d_ev) = cache_channel(d_local_cache, d, 1000); + + let mut d_ev = pin!(d_ev); + while !expected.is_empty() { + let ev = d_ev.next().await.unwrap(); + match ev { + DomoEvent::PersistentData(data) => { + assert!(expected.remove(&data.topic_uuid)); + log::warn!("d: Got data {data:?}"); + } + _ => { + log::warn!("d: Other {ev:?}"); + } + } + } + } +} diff --git a/dht-cache/src/cache/local.rs b/dht-cache/src/cache/local.rs new file mode 100644 index 0000000..3120fab --- /dev/null +++ b/dht-cache/src/cache/local.rs @@ -0,0 +1,269 @@ +//! Local in-memory cache + +pub use crate::data::*; +use serde_json::Value; +use std::collections::hash_map::DefaultHasher; +use std::collections::BTreeMap; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; +use tokio::sync::{OwnedRwLockReadGuard, RwLock}; + +/// Local cache +#[derive(Default, Clone, Debug)] +pub struct LocalCache(Arc>>>); + +impl LocalCache { + pub fn new() -> Self { + Default::default() + } + + /// Feeds a slice of this type into the given [`Hasher`]. + pub async fn hash(&self, state: &mut H) { + let cache = self.0.read().await; + for (topic_name, map_topic_name) in cache.iter() { + topic_name.hash(state); + + for (topic_uuid, value) in map_topic_name.iter() { + topic_uuid.hash(state); + value.to_string().hash(state); + } + } + } + + /// Put the element in the cache + /// + /// If it is already present overwrite it + pub async fn put(&self, elem: &DomoCacheElement) { + let mut cache = self.0.write().await; + let topic_name = elem.topic_name.clone(); + let topic_uuid = &elem.topic_uuid; + + cache + .entry(topic_name) + .and_modify(|topic| { + topic.insert(topic_uuid.to_owned(), elem.to_owned()); + }) + .or_insert_with(|| [(topic_uuid.to_owned(), elem.to_owned())].into()); + } + + /// Try to insert the element in the cache + /// + /// Return Err(()) if the element to insert is older than the one in the cache + pub async fn try_put(&self, elem: &DomoCacheElement) -> Result<(), ()> { + let mut cache = self.0.write().await; + let topic_name = elem.topic_name.clone(); + let topic_uuid = &elem.topic_uuid; + + let topic = cache.entry(topic_name).or_default(); + + if topic + .get(topic_uuid) + .is_some_and(|cur| elem.publication_timestamp <= cur.publication_timestamp) + { + Err(()) + } else { + topic.insert(topic_uuid.to_owned(), elem.to_owned()); + Ok(()) + } + } + + /// Retrieve an element by its uuid and topic + pub async fn get(&self, topic_name: &str, topic_uuid: &str) -> Option { + let cache = self.0.read().await; + + cache + .get(topic_name) + .and_then(|topic| topic.get(topic_uuid)) + .cloned() + } + + /// Instantiate a query over the local cache + pub fn query(&self, topic: &str) -> Query { + Query::new(topic, self.clone()) + } + + /// Compute the current hash value + pub async fn get_hash(&self) -> u64 { + let mut s = DefaultHasher::new(); + self.hash(&mut s).await; + s.finish() + } + + pub(crate) async fn read_owned( + &self, + ) -> OwnedRwLockReadGuard>> { + self.0.clone().read_owned().await + } +} + +/// Query the local DHT cache +#[derive(Clone)] +pub struct Query { + cache: LocalCache, + topic: String, + uuid: Option, +} + +impl Query { + /// Create a new query over a local cache + pub fn new(topic: &str, cache: LocalCache) -> Self { + Self { + topic: topic.to_owned(), + cache, + uuid: None, + } + } + /// Look up for a specific uuid + pub fn with_uuid(mut self, uuid: &str) -> Self { + self.uuid = Some(uuid.to_owned()); + self + } + + /// Execute the query and return a Value if found + pub async fn get(&self) -> Vec { + let cache = self.cache.0.read().await; + + if let Some(topics) = cache.get(&self.topic) { + if let Some(ref uuid) = self.uuid { + topics + .get(uuid) + .into_iter() + .map(|elem| elem.value.clone()) + .collect() + } else { + topics.values().map(|elem| elem.value.clone()).collect() + } + } else { + Vec::new() + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::data::DomoCacheElement; + use serde_json::*; + + fn make_test_element(topic_name: &str, topic_uuid: &str, value: &Value) -> DomoCacheElement { + DomoCacheElement { + topic_name: topic_name.to_owned(), + topic_uuid: topic_uuid.to_owned(), + value: value.to_owned(), + ..Default::default() + } + } + + #[tokio::test] + async fn hash() { + let cache = LocalCache::new(); + + let hash = cache.get_hash().await; + println!("{hash}"); + + let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); + cache.put(&elem).await; + + let hash2 = cache.get_hash().await; + println!("{hash2}"); + + assert_ne!(hash, hash2); + + let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": false})); + cache.put(&elem).await; + + let hash3 = cache.get_hash().await; + println!("{hash3}"); + + assert_ne!(hash2, hash3); + + let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); + cache.put(&elem).await; + + let hash4 = cache.get_hash().await; + println!("{hash4}"); + + assert_eq!(hash2, hash4); + } + + #[tokio::test] + async fn put() { + let cache = LocalCache::new(); + + let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); + + cache.put(&elem).await; + + let out = cache.get("Domo::Light", "luce-1").await.expect("element"); + + assert_eq!(out, elem); + + let elem2 = make_test_element("Domo::Light", "luce-1", &json!({ "connected": false})); + + cache.put(&elem2).await; + + let out = cache.get("Domo::Light", "luce-1").await.expect("element"); + + assert_eq!(out, elem2); + } + + #[tokio::test] + async fn try_put() { + let cache = LocalCache::new(); + + let mut elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); + + cache.try_put(&elem).await.unwrap(); + + let out = cache.get("Domo::Light", "luce-1").await.expect("element"); + + assert_eq!(out, elem); + + elem.publication_timestamp = 1; + + cache.try_put(&elem).await.expect("Update entry"); + + let out: DomoCacheElement = cache.get("Domo::Light", "luce-1").await.expect("element"); + + assert_eq!(out, elem); + + elem.publication_timestamp = 0; + + cache + .try_put(&elem) + .await + .expect_err("The update should fail"); + + let out: DomoCacheElement = cache.get("Domo::Light", "luce-1").await.expect("element"); + + assert_eq!(out.publication_timestamp, 1); + } + + #[tokio::test] + async fn query() { + let cache = LocalCache::new(); + + for item in 0..10 { + let elem = make_test_element( + "Domo::Light", + &format!("luce-{item}"), + &json!({ "connected": true, "count": item}), + ); + + cache.put(&elem).await; + } + + let q = cache.query("Domo::Light"); + + assert_eq!(q.get().await.len(), 10); + + assert_eq!(q.clone().with_uuid("not-existent").get().await.len(), 0); + + assert_eq!( + q.clone().with_uuid("luce-1").get().await[0] + .get("count") + .unwrap(), + 1 + ); + } +} diff --git a/dht-cache/src/data.rs b/dht-cache/src/data.rs index 733519c..b38141a 100644 --- a/dht-cache/src/data.rs +++ b/dht-cache/src/data.rs @@ -1,14 +1,19 @@ -//! Cached access to the DHT +//! Data types for interacting with the DHT +//! +//! The DHT may persist Elements indexed by a topic and an uuid or broadcast free-form messages. +//! +//! use serde::{Deserialize, Serialize}; use serde_json::Value; use std::fmt::{Display, Formatter}; -/// Events returned by [DomoCache::cache_event_loop] +/// Events of interest #[derive(Debug)] pub enum DomoEvent { None, VolatileData(serde_json::Value), PersistentData(DomoCacheElement), + NewPeers(Vec), } /// Full Cache Element @@ -30,6 +35,7 @@ pub struct DomoCacheElement { pub republication_timestamp: u128, } +/// Summary of the current state of the DHT according to a peer #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub(crate) struct DomoCacheStateMessage { pub peer_id: String, diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 6c78df4..f947ad7 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -13,6 +13,7 @@ use tokio::task::JoinHandle; /// Network commands #[derive(Debug)] pub enum Command { + Config(Value), Broadcast(Value), Publish(Value), Stop, @@ -52,6 +53,14 @@ fn handle_command(swarm: &mut Swarm, cmd: Command) -> bool { } true } + Config(val) => { + let topic = Topic::new("domo-config"); + let m = serde_json::to_string(&val).unwrap(); + if let Err(e) = swarm.behaviour_mut().gossipsub.publish(topic, m.as_bytes()) { + log::info!("Publish error: {e:?}"); + } + true + } Stop => false, } } @@ -139,7 +148,7 @@ pub fn dht_channel( ) -> ( UnboundedSender, UnboundedReceiver, - JoinHandle<()>, + JoinHandle>, ) { let (cmd_send, mut cmd_recv) = mpsc::unbounded_channel(); let (mut ev_send, ev_recv) = mpsc::unbounded_channel(); @@ -151,13 +160,13 @@ pub fn dht_channel( log::debug!("command {cmd:?}"); if !cmd.is_some_and(|cmd| handle_command(&mut swarm, cmd)) { log::debug!("Exiting cmd"); - return + return swarm } } ev = swarm.select_next_some() => { if handle_swarm_event(&mut swarm, ev, &mut ev_send).is_err() { log::debug!("Exiting ev"); - return + return swarm } } } @@ -168,20 +177,36 @@ pub fn dht_channel( } #[cfg(test)] -mod test { +pub(crate) mod test { use super::*; use libp2p_swarm_test::SwarmExt; use serde_json::json; - #[tokio::test] - async fn multiple_peers() { - env_logger::init(); + pub async fn make_peer() -> Swarm { + let mut a = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + a.listen().await; + + a + } + + pub async fn connect_peer(a: &mut Swarm, b: &mut Swarm) { + a.connect(b).await; + + let peers: Vec<_> = a.connected_peers().cloned().collect(); + + for peer in peers { + a.behaviour_mut().gossipsub.add_explicit_peer(&peer); + } + } + + pub async fn make_peers() -> [Swarm; 3] { + let _ = env_logger::builder().is_test(true).try_init(); let mut a = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); let mut b = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); let mut c = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); for a in a.external_addresses() { - println!("{a:?}"); + log::info!("{a:?}"); } a.listen().await; @@ -210,6 +235,13 @@ mod test { c.behaviour_mut().gossipsub.add_explicit_peer(&peer); } + [a, b, c] + } + + #[tokio::test] + async fn multiple_peers() { + let [a, b, c] = make_peers().await; + let (a_s, mut ar, _) = dht_channel(a); let (b_s, br, _) = dht_channel(b); let (c_s, cr, _) = dht_channel(c); diff --git a/dht-cache/src/lib.rs b/dht-cache/src/lib.rs index f4efa81..b5b3efa 100644 --- a/dht-cache/src/lib.rs +++ b/dht-cache/src/lib.rs @@ -1,6 +1,8 @@ //! Simple DHT/messaging system based on libp2p //! -mod data; +pub mod cache; +pub mod data; +pub mod dht; pub mod domocache; mod domolibp2p; mod domopersistentstorage; @@ -32,4 +34,10 @@ pub enum Error { Transport(#[from] libp2p::TransportError), #[error("Cannot parse the multiaddr")] MultiAddr(#[from] libp2p::multiaddr::Error), + #[error("Internal channel dropped")] + Channel, + #[error("Missing configuration")] + MissingConfig, + #[error("Invalid JSONPath expression")] + Jsonpath, } From 6afefdcc5c0b79f5375784262d86a9c096cbe3ba Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 9 Aug 2023 13:24:10 +0200 Subject: [PATCH 16/52] wip: Update the broker to handle the NewPeers event --- src/domobroker.rs | 1 + src/main.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/domobroker.rs b/src/domobroker.rs index de81bb9..f676e4a 100644 --- a/src/domobroker.rs +++ b/src/domobroker.rs @@ -274,6 +274,7 @@ impl DomoBroker { DomoEvent::VolatileData(m2) } + DomoEvent::NewPeers(_) => DomoEvent::None, } } } diff --git a/src/main.rs b/src/main.rs index 7fc3265..429c664 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,6 +65,9 @@ fn report_event(m: &DomoEvent) { DomoEvent::PersistentData(_v) => { println!("Persistent"); } + DomoEvent::NewPeers(peers) => { + println!("New peers {:#?}", peers); + } } } From c8c1a73a83f99684714879020e5ef71cb70f65e1 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 10 Aug 2023 19:16:04 +0200 Subject: [PATCH 17/52] fixup: use any() instead of find().is_some() --- dht-cache/src/cache.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 01ca290..19d7cbf 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -125,8 +125,7 @@ impl PeersState { let desync = self .list .values() - .find(|data| data.cache_hash != hash && data.publication_timestamp > cur_ts) - .is_some(); + .any(|data| data.cache_hash != hash && data.publication_timestamp > cur_ts); if desync { CacheState::Desynced { From 5351b75601c5cd677dff21ca816ba3aaf9be386f Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 10 Aug 2023 19:18:44 +0200 Subject: [PATCH 18/52] fixup: Move the comments to the right place --- dht-cache/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 19d7cbf..788097c 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -212,10 +212,12 @@ pub fn cache_channel( // update the peers_caches_state peers_state.insert(m); + // check for desync let sync_info = peers_state.is_synchronized(&peer_id, hash).await; log::debug!("local {peer_id:?} {sync_info:?} -> {peers_state:#?}"); + // republish the local cache if needed if let CacheState::Desynced { is_leader } = sync_info { if is_leader && utils::get_epoch_ms() - peers_state.last_repub_timestamp @@ -239,8 +241,6 @@ pub fn cache_channel( } } - // check for desync - // republish the local cache if needed None } Event::Discovered(who) => Some(DomoEvent::NewPeers( From 7cff78f9482fa799be6b94aacd8c6b5bdf57b563 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 10 Aug 2023 22:48:58 +0200 Subject: [PATCH 19/52] Wire in the persistent storage --- dht-cache/src/cache.rs | 1 + dht-cache/src/cache/local.rs | 77 ++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 788097c..02b3c4c 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -226,6 +226,7 @@ pub fn cache_channel( local_write .read_owned() .await + .mem .values() .flat_map(|topic| topic.values()) .for_each(|elem| { diff --git a/dht-cache/src/cache/local.rs b/dht-cache/src/cache/local.rs index 3120fab..a415647 100644 --- a/dht-cache/src/cache/local.rs +++ b/dht-cache/src/cache/local.rs @@ -1,6 +1,7 @@ //! Local in-memory cache pub use crate::data::*; +use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use serde_json::Value; use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; @@ -8,18 +9,56 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use tokio::sync::{OwnedRwLockReadGuard, RwLock}; +#[derive(Default)] +pub(crate) struct InnerCache { + pub mem: BTreeMap>, + pub store: Option, +} + +/// SAFETY: the SqlxStorage access is only over write() +unsafe impl std::marker::Sync for InnerCache {} + +impl InnerCache { + pub fn put(&mut self, elem: &DomoCacheElement) { + let topic_name = elem.topic_name.clone(); + let topic_uuid = &elem.topic_uuid; + + self.mem + .entry(topic_name) + .and_modify(|topic| { + topic.insert(topic_uuid.to_owned(), elem.to_owned()); + }) + .or_insert_with(|| [(topic_uuid.to_owned(), elem.to_owned())].into()); + } +} + /// Local cache -#[derive(Default, Clone, Debug)] -pub struct LocalCache(Arc>>>); +#[derive(Default, Clone)] +pub struct LocalCache(Arc>); impl LocalCache { + pub async fn with_config(db_config: &sifis_config::Cache) -> Self { + let mut inner = InnerCache::default(); + let mut store = SqlxStorage::new(db_config).await; + + for a in store.get_all_elements().await { + inner.put(&a); + } + + if db_config.persistent { + inner.store = Some(store); + } + + LocalCache(Arc::new(RwLock::new(inner))) + } + pub fn new() -> Self { Default::default() } /// Feeds a slice of this type into the given [`Hasher`]. pub async fn hash(&self, state: &mut H) { - let cache = self.0.read().await; + let cache = &self.0.read().await.mem; for (topic_name, map_topic_name) in cache.iter() { topic_name.hash(state); @@ -35,15 +74,12 @@ impl LocalCache { /// If it is already present overwrite it pub async fn put(&self, elem: &DomoCacheElement) { let mut cache = self.0.write().await; - let topic_name = elem.topic_name.clone(); - let topic_uuid = &elem.topic_uuid; - cache - .entry(topic_name) - .and_modify(|topic| { - topic.insert(topic_uuid.to_owned(), elem.to_owned()); - }) - .or_insert_with(|| [(topic_uuid.to_owned(), elem.to_owned())].into()); + if let Some(storage) = cache.store.as_mut() { + storage.store(&elem).await; + } + + cache.put(elem); } /// Try to insert the element in the cache @@ -54,9 +90,9 @@ impl LocalCache { let topic_name = elem.topic_name.clone(); let topic_uuid = &elem.topic_uuid; - let topic = cache.entry(topic_name).or_default(); + let topic = cache.mem.entry(topic_name).or_default(); - if topic + let e = if topic .get(topic_uuid) .is_some_and(|cur| elem.publication_timestamp <= cur.publication_timestamp) { @@ -64,7 +100,15 @@ impl LocalCache { } else { topic.insert(topic_uuid.to_owned(), elem.to_owned()); Ok(()) + }; + + if e.is_ok() { + if let Some(s) = cache.store.as_mut() { + s.store(&elem).await; + } } + + e } /// Retrieve an element by its uuid and topic @@ -72,6 +116,7 @@ impl LocalCache { let cache = self.0.read().await; cache + .mem .get(topic_name) .and_then(|topic| topic.get(topic_uuid)) .cloned() @@ -89,9 +134,7 @@ impl LocalCache { s.finish() } - pub(crate) async fn read_owned( - &self, - ) -> OwnedRwLockReadGuard>> { + pub(crate) async fn read_owned(&self) -> OwnedRwLockReadGuard { self.0.clone().read_owned().await } } @@ -123,7 +166,7 @@ impl Query { pub async fn get(&self) -> Vec { let cache = self.cache.0.read().await; - if let Some(topics) = cache.get(&self.topic) { + if let Some(topics) = cache.mem.get(&self.topic) { if let Some(ref uuid) = self.uuid { topics .get(uuid) From dfb94491442eeb56220b678de96f5fe53d5999d0 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 10 Aug 2023 22:53:51 +0200 Subject: [PATCH 20/52] Allow to not use a local db at all --- dht-cache/src/cache/local.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dht-cache/src/cache/local.rs b/dht-cache/src/cache/local.rs index a415647..3bb3624 100644 --- a/dht-cache/src/cache/local.rs +++ b/dht-cache/src/cache/local.rs @@ -37,16 +37,24 @@ impl InnerCache { pub struct LocalCache(Arc>); impl LocalCache { + /// Instantiate a local cache from the configuration provided + /// + /// If url is empty do not try to bootstrap the cache from the db + /// + /// TODO: propagate errors pub async fn with_config(db_config: &sifis_config::Cache) -> Self { let mut inner = InnerCache::default(); - let mut store = SqlxStorage::new(db_config).await; - for a in store.get_all_elements().await { - inner.put(&a); - } + if !db_config.url.is_empty() { + let mut store = SqlxStorage::new(db_config).await; - if db_config.persistent { - inner.store = Some(store); + for a in store.get_all_elements().await { + inner.put(&a); + } + + if db_config.persistent { + inner.store = Some(store); + } } LocalCache(Arc::new(RwLock::new(inner))) @@ -76,7 +84,7 @@ impl LocalCache { let mut cache = self.0.write().await; if let Some(storage) = cache.store.as_mut() { - storage.store(&elem).await; + storage.store(elem).await; } cache.put(elem); @@ -104,7 +112,7 @@ impl LocalCache { if e.is_ok() { if let Some(s) = cache.store.as_mut() { - s.store(&elem).await; + s.store(elem).await; } } From b673ce744e67dc1fabd0b29288cd612a2a2fcc1c Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 11 Aug 2023 09:20:53 +0200 Subject: [PATCH 21/52] fixup: Clippy lints --- dht-cache/src/cache.rs | 14 +++++--------- dht-cache/src/dht.rs | 4 ++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 02b3c4c..fddfe17 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -129,15 +129,11 @@ impl PeersState { if desync { CacheState::Desynced { - is_leader: self - .list - .values() - .find(|data| { - data.cache_hash == hash - && data.peer_id.as_str() < peer_id - && data.publication_timestamp > cur_ts - }) - .is_none(), + is_leader: !self.list.values().any(|data| { + data.cache_hash == hash + && data.peer_id.as_str() < peer_id + && data.publication_timestamp > cur_ts + }), } } else { CacheState::Synced diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index f947ad7..8111bc2 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -151,7 +151,7 @@ pub fn dht_channel( JoinHandle>, ) { let (cmd_send, mut cmd_recv) = mpsc::unbounded_channel(); - let (mut ev_send, ev_recv) = mpsc::unbounded_channel(); + let (ev_send, ev_recv) = mpsc::unbounded_channel(); let handle = tokio::task::spawn(async move { loop { @@ -164,7 +164,7 @@ pub fn dht_channel( } } ev = swarm.select_next_some() => { - if handle_swarm_event(&mut swarm, ev, &mut ev_send).is_err() { + if handle_swarm_event(&mut swarm, ev, &ev_send).is_err() { log::debug!("Exiting ev"); return swarm } From de440c027bf805a35fe8225f99c30d2135b2d1c5 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 11 Aug 2023 09:51:50 +0200 Subject: [PATCH 22/52] Test everything including pg integration --- .github/workflows/libp2p-rust-dht.yml | 30 +++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/libp2p-rust-dht.yml b/.github/workflows/libp2p-rust-dht.yml index 5214038..a86b58b 100644 --- a/.github/workflows/libp2p-rust-dht.yml +++ b/.github/workflows/libp2p-rust-dht.yml @@ -144,6 +144,19 @@ jobs: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: mysecretpassword + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/checkout@v3 @@ -179,7 +192,7 @@ jobs: RUSTFLAGS: "-Cinstrument-coverage" LLVM_PROFILE_FILE: "libp2p-rust-dht-%p-%m.profraw" run: | - cargo test --verbose + cargo test --verbose --all - name: Get coverage data for codecov run: | @@ -226,6 +239,19 @@ jobs: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: mysecretpassword + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/checkout@v3 @@ -271,7 +297,7 @@ jobs: RUSTFLAGS: "-Cinstrument-coverage" LLVM_PROFILE_FILE: "libp2p-rust-dht-%p-%m.profraw" run: | - cargo test --verbose + cargo test --verbose --all - name: Run grcov run: | From 75cb13b7ce940bc1c9671753ca971471e7e1f735 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 14 Aug 2023 19:30:19 +0200 Subject: [PATCH 23/52] wip: Add Builder --- dht-cache/src/cache.rs | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index fddfe17..e493052 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -13,6 +13,7 @@ use tokio::sync::RwLock; use tokio::time; use tokio_stream::wrappers::UnboundedReceiverStream; +use crate::domolibp2p::{self, generate_rsa_key}; use crate::{ cache::local::DomoCacheStateMessage, data::DomoEvent, @@ -23,6 +24,54 @@ use crate::{ use self::local::{DomoCacheElement, LocalCache, Query}; +/// Builder for a Cached DHT Node +// TODO: make it Clone +pub struct Builder { + cfg: crate::Config, +} + +impl Builder { + /// Create a new Builder from a [crate::Config] + pub fn from_config(cfg: crate::Config) -> Builder { + Builder { cfg } + } + + /// Instantiate a new DHT node a return + pub async fn make_channel( + self, + ) -> Result<(Cache, impl Stream), crate::Error> { + let loopback_only = self.cfg.loopback; + let shared_key = self.cfg.shared_key.clone(); + let private_key_file = self.cfg.private_key.as_ref(); + + // Create a random local key. + let mut pkcs8_der = if let Some(pk_path) = private_key_file { + match std::fs::read(pk_path) { + Ok(pem) => { + let der = pem_rfc7468::decode_vec(&pem)?; + der.1 + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // Generate a new key and put it into the file at the given path + let (pem, der) = generate_rsa_key(); + std::fs::write(pk_path, pem)?; + der + } + Err(e) => Err(e)?, + } + } else { + generate_rsa_key().1 + }; + + let local_key_pair = crate::Keypair::rsa_from_pkcs8(&mut pkcs8_der)?; + let swarm = domolibp2p::start(shared_key, local_key_pair, loopback_only).await?; + + let local = LocalCache::with_config(&self.cfg).await; + // TODO: add a configuration item for the resend interval + Ok(cache_channel(local, swarm, 1000)) + } +} + /// Cached DHT /// /// It keeps a local cache of the dht state and allow to query the persistent topics @@ -278,6 +327,17 @@ mod test { use crate::dht::test::*; use std::{collections::HashSet, pin::pin}; + #[tokio::test] + async fn builder() { + let cfg = crate::Config { + shared_key: "d061545647652562b4648f52e8373b3a417fc0df56c332154460da1801b341e9" + .to_owned(), + ..Default::default() + }; + + let (_cache, _events) = Builder::from_config(cfg).make_channel().await.unwrap(); + } + #[tokio::test(flavor = "multi_thread")] async fn syncronization() { let [mut a, mut b, mut c] = make_peers().await; From 5f02fdbba6c2324e892f7d6ca0d744f9bf4c36c7 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 14 Aug 2023 23:37:59 +0200 Subject: [PATCH 24/52] Refactor the psk logic --- dht-cache/src/cache.rs | 2 +- dht-cache/src/domocache.rs | 4 ++-- dht-cache/src/domolibp2p.rs | 13 ++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index e493052..9c772c3 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -41,7 +41,7 @@ impl Builder { self, ) -> Result<(Cache, impl Stream), crate::Error> { let loopback_only = self.cfg.loopback; - let shared_key = self.cfg.shared_key.clone(); + let shared_key = domolibp2p::parse_hex_key(&self.cfg.shared_key)?; let private_key_file = self.cfg.private_key.as_ref(); // Create a random local key. diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index d9ee50a..89999ea 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -1,6 +1,6 @@ //! Cached access to the DHT pub use crate::data::*; -use crate::domolibp2p::generate_rsa_key; +use crate::domolibp2p::{generate_rsa_key, parse_hex_key}; use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use crate::utils; use crate::Error; @@ -456,7 +456,7 @@ impl DomoCache { let is_persistent_cache = conf.persistent; let loopback_only = conf.loopback; - let shared_key = conf.shared_key.clone(); + let shared_key = parse_hex_key(&conf.shared_key)?; let private_key_file = conf.private_key.clone(); let storage = SqlxStorage::new(&conf).await; diff --git a/dht-cache/src/domolibp2p.rs b/dht-cache/src/domolibp2p.rs index 2bf67e1..08b3c3d 100644 --- a/dht-cache/src/domolibp2p.rs +++ b/dht-cache/src/domolibp2p.rs @@ -32,7 +32,7 @@ use crate::Error; const KEY_SIZE: usize = 32; -fn parse_hex_key(s: &str) -> Result<[u8; KEY_SIZE], Error> { +pub fn parse_hex_key(s: &str) -> Result { if s.len() == KEY_SIZE * 2 { let mut r = [0u8; KEY_SIZE]; for i in 0..KEY_SIZE { @@ -44,7 +44,9 @@ fn parse_hex_key(s: &str) -> Result<[u8; KEY_SIZE], Error> { Err(_e) => return Err(Error::Hex("Error while parsing".into())), } } - Ok(r) + let psk = PreSharedKey::new(r); + + Ok(psk) } else { Err(Error::Hex(format!( "Len Error: expected {} but got {}", @@ -86,16 +88,13 @@ pub fn generate_rsa_key() -> (Vec, Vec) { } pub async fn start( - shared_key: String, + shared_key: PreSharedKey, local_key_pair: identity::Keypair, loopback_only: bool, ) -> Result, Error> { let local_peer_id = PeerId::from(local_key_pair.public()); - let arr = parse_hex_key(&shared_key)?; - let psk = PreSharedKey::new(arr); - - let transport = build_transport(local_key_pair.clone(), psk); + let transport = build_transport(local_key_pair.clone(), shared_key); // Create a swarm to manage peers and events. let mut swarm = { From 030caee6d54928b6164e13468e71e59afc51fa9d Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 15 Aug 2023 00:11:37 +0200 Subject: [PATCH 25/52] fixme: move to dev-dependencies --- dht-cache/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dht-cache/Cargo.toml b/dht-cache/Cargo.toml index b943dec..b3536ea 100644 --- a/dht-cache/Cargo.toml +++ b/dht-cache/Cargo.toml @@ -30,11 +30,11 @@ openssl-sys = "*" libsqlite3-sys = "*" thiserror = "1.0.43" anyhow = "1.0.72" -libp2p-swarm-test = "0.2.0" tokio-stream = "0.1.14" [dev-dependencies] env_logger = "0.10.0" +libp2p-swarm-test = "0.2.0" [package.metadata.cargo-udeps.ignore] From 19817ac14fa883e477fb75acc454957d6984dc32 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 11:25:34 +0200 Subject: [PATCH 26/52] Use pnet in the ephemeral swarms to ensure tests do not interact due mdns --- dht-cache/Cargo.toml | 2 +- dht-cache/src/cache.rs | 6 +++--- dht-cache/src/dht.rs | 47 +++++++++++++++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/dht-cache/Cargo.toml b/dht-cache/Cargo.toml index b3536ea..1b09d69 100644 --- a/dht-cache/Cargo.toml +++ b/dht-cache/Cargo.toml @@ -35,7 +35,7 @@ tokio-stream = "0.1.14" [dev-dependencies] env_logger = "0.10.0" libp2p-swarm-test = "0.2.0" - +libp2p = { version = "0.52.0", features = ["plaintext"] } [package.metadata.cargo-udeps.ignore] normal = ["openssl-sys", "libsqlite3-sys", "libc"] diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 9c772c3..8aae5cb 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -338,10 +338,10 @@ mod test { let (_cache, _events) = Builder::from_config(cfg).make_channel().await.unwrap(); } - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn syncronization() { - let [mut a, mut b, mut c] = make_peers().await; - let mut d = make_peer().await; + let [mut a, mut b, mut c] = make_peers(2).await; + let mut d = make_peer(2).await; connect_peer(&mut a, &mut d).await; connect_peer(&mut b, &mut d).await; diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 8111bc2..2032dbb 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -178,12 +178,45 @@ pub fn dht_channel( #[cfg(test)] pub(crate) mod test { + use std::time::Duration; + use super::*; + use crate::Keypair; + use libp2p::core::transport::MemoryTransport; + use libp2p::core::upgrade::Version; + use libp2p::plaintext::PlainText2Config; + use libp2p::pnet::{PnetConfig, PreSharedKey}; + use libp2p::swarm::SwarmBuilder; + use libp2p::yamux; + use libp2p::Transport; use libp2p_swarm_test::SwarmExt; use serde_json::json; - pub async fn make_peer() -> Swarm { - let mut a = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + // like Swarm::new_ephemeral but with pnet variant + fn new_ephemeral( + behaviour_fn: impl FnOnce(Keypair) -> DomoBehaviour, + variant: u8, + ) -> Swarm { + let identity = Keypair::generate_ed25519(); + let peer_id = PeerId::from(identity.public()); + let psk = PreSharedKey::new([variant; 32]); + + let transport = MemoryTransport::default() + .or_transport(libp2p::tcp::async_io::Transport::default()) + .and_then(move |socket, _| PnetConfig::new(psk).handshake(socket)) + .upgrade(Version::V1) + .authenticate(PlainText2Config { + local_public_key: identity.public(), + }) + .multiplex(yamux::Config::default()) + .timeout(Duration::from_secs(20)) + .boxed(); + + SwarmBuilder::without_executor(transport, behaviour_fn(identity), peer_id).build() + } + + pub async fn make_peer(variant: u8) -> Swarm { + let mut a = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); a.listen().await; a @@ -199,11 +232,11 @@ pub(crate) mod test { } } - pub async fn make_peers() -> [Swarm; 3] { + pub async fn make_peers(variant: u8) -> [Swarm; 3] { let _ = env_logger::builder().is_test(true).try_init(); - let mut a = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); - let mut b = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); - let mut c = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + let mut a = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); + let mut b = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); + let mut c = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); for a in a.external_addresses() { log::info!("{a:?}"); @@ -240,7 +273,7 @@ pub(crate) mod test { #[tokio::test] async fn multiple_peers() { - let [a, b, c] = make_peers().await; + let [a, b, c] = make_peers(1).await; let (a_s, mut ar, _) = dht_channel(a); let (b_s, br, _) = dht_channel(b); From 1587597dda1c89d4cf8663bd05707be35d2b337e Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 11:43:28 +0200 Subject: [PATCH 27/52] Speed up the test execution --- dht-cache/src/cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 8aae5cb..f187332 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -358,9 +358,9 @@ mod test { .collect(); tokio::task::spawn(async move { - let (a_c, a_ev) = cache_channel(a_local_cache, a, 1000); - let (_b_c, b_ev) = cache_channel(b_local_cache, b, 1000); - let (_c_c, c_ev) = cache_channel(c_local_cache, c, 1000); + let (a_c, a_ev) = cache_channel(a_local_cache, a, 100); + let (_b_c, b_ev) = cache_channel(b_local_cache, b, 100); + let (_c_c, c_ev) = cache_channel(c_local_cache, c, 100); let mut a_ev = pin!(a_ev); let mut b_ev = pin!(b_ev); From 625a1bbb1f10f5c5dcb155876505933f961c3dfb Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 11:47:37 +0200 Subject: [PATCH 28/52] Speedup the test execution The resend task would not interfere with the rest this way. --- dht-cache/src/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index f187332..a45f412 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -327,7 +327,7 @@ mod test { use crate::dht::test::*; use std::{collections::HashSet, pin::pin}; - #[tokio::test] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn builder() { let cfg = crate::Config { shared_key: "d061545647652562b4648f52e8373b3a417fc0df56c332154460da1801b341e9" From c56cf1bc64ee24a1f4aac98005052836f3e13a06 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 11:51:37 +0200 Subject: [PATCH 29/52] fixup: Spurious comment --- dht-cache/src/cache.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index a45f412..cdf1fe5 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -251,7 +251,6 @@ pub fn cache_channel( let hash = local_write.get_hash().await; - // SAFETY: only user let mut peers_state = peers_state.write().await; // update the peers_caches_state From 4c9a15e713217d09665b38e07427593c1c4b1c51 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 11:53:07 +0200 Subject: [PATCH 30/52] fixup: is_syncronised is not async --- dht-cache/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index cdf1fe5..bddc144 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -169,7 +169,7 @@ impl PeersState { self.list.insert(state.peer_id.to_string(), state); } - async fn is_synchronized(&self, peer_id: &str, hash: u64) -> CacheState { + fn is_synchronized(&self, peer_id: &str, hash: u64) -> CacheState { let cur_ts = utils::get_epoch_ms() - self.repub_interval; let desync = self .list @@ -257,7 +257,7 @@ pub fn cache_channel( peers_state.insert(m); // check for desync - let sync_info = peers_state.is_synchronized(&peer_id, hash).await; + let sync_info = peers_state.is_synchronized(&peer_id, hash); log::debug!("local {peer_id:?} {sync_info:?} -> {peers_state:#?}"); From 691a18df892f1e5017da91c27e062ba8f71fda7c Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 12:01:04 +0200 Subject: [PATCH 31/52] Do not keep the write lock longer than needed --- dht-cache/src/cache.rs | 57 ++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index bddc144..a1f53d0 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -251,39 +251,42 @@ pub fn cache_channel( let hash = local_write.get_hash().await; - let mut peers_state = peers_state.write().await; + let republish = { + let mut peers_state = peers_state.write().await; - // update the peers_caches_state - peers_state.insert(m); + // update the peers_caches_state + peers_state.insert(m); - // check for desync - let sync_info = peers_state.is_synchronized(&peer_id, hash); + // check for desync + let sync_info = peers_state.is_synchronized(&peer_id, hash); - log::debug!("local {peer_id:?} {sync_info:?} -> {peers_state:#?}"); + log::debug!("local {peer_id:?} {sync_info:?} -> {peers_state:#?}"); + + if let CacheState::Desynced { is_leader } = sync_info { + is_leader + && utils::get_epoch_ms() - peers_state.last_repub_timestamp + >= peers_state.repub_interval + } else { + false + } + }; // republish the local cache if needed - if let CacheState::Desynced { is_leader } = sync_info { - if is_leader - && utils::get_epoch_ms() - peers_state.last_repub_timestamp - >= peers_state.repub_interval - { - local_write - .read_owned() - .await - .mem - .values() - .flat_map(|topic| topic.values()) - .for_each(|elem| { - let mut elem = elem.to_owned(); - log::debug!("resending {}", elem.topic_uuid); - elem.republication_timestamp = utils::get_epoch_ms(); - cmd.send(Command::Publish( - serde_json::to_value(&elem).unwrap(), - )) + if republish { + local_write + .read_owned() + .await + .mem + .values() + .flat_map(|topic| topic.values()) + .for_each(|elem| { + let mut elem = elem.to_owned(); + log::debug!("resending {}", elem.topic_uuid); + elem.republication_timestamp = utils::get_epoch_ms(); + cmd.send(Command::Publish(serde_json::to_value(&elem).unwrap())) .unwrap(); - }); - peers_state.last_repub_timestamp = utils::get_epoch_ms(); - } + }); + peers_state.write().await.last_repub_timestamp = utils::get_epoch_ms(); } None From c9fc8658977bf33d35f3c845694a81e896326b5c Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 12:27:48 +0200 Subject: [PATCH 32/52] Explicitly yield in the swarm task --- dht-cache/src/dht.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 2032dbb..6d29c6c 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -170,6 +170,7 @@ pub fn dht_channel( } } } + tokio::task::yield_now().await; } }); From 7e3c3ef7e6f642d57970bff0ec0066e9fd496cc2 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 13:32:09 +0200 Subject: [PATCH 33/52] Compact the CI --- .github/workflows/libp2p-rust-dht.yml | 98 ++------------------------- 1 file changed, 4 insertions(+), 94 deletions(-) diff --git a/.github/workflows/libp2p-rust-dht.yml b/.github/workflows/libp2p-rust-dht.yml index a86b58b..55c891c 100644 --- a/.github/workflows/libp2p-rust-dht.yml +++ b/.github/workflows/libp2p-rust-dht.yml @@ -233,47 +233,6 @@ jobs: exit 1 fi - weighted-code-coverage: - - needs: [build, docs] - - runs-on: ubuntu-latest - - services: - postgres: - image: postgres - env: - POSTGRES_PASSWORD: mysecretpassword - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - steps: - - uses: actions/checkout@v3 - - - name: Install protobuf compiler - run: | - sudo apt-get update - sudo apt-get install protobuf-compiler - - - name: Install Rust stable - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - - - name: Install grcov - env: - GRCOV_LINK: https://github.com/mozilla/grcov/releases/download - GRCOV_VERSION: v0.8.13 - GRCOV_BINARY: grcov-x86_64-unknown-linux-musl.tar.bz2 - run: | - curl -L "$GRCOV_LINK/$GRCOV_VERSION/$GRCOV_BINARY" | - tar xj -C $HOME/.cargo/bin - - name: Install weighted-code-coverage env: WCC_LINK: https://github.com/SoftengPoliTo/weighted-code-coverage/releases/download @@ -283,23 +242,7 @@ jobs: curl -L "$WCC_LINK/$WCC_VERSION/$WCC_BINARY" | tar xz -C $HOME/.cargo/bin - - name: Install llvm-tools-preview - run: | - rustup component add llvm-tools-preview - - # Not necessary on a newly created image, but strictly advised - - name: Run cargo clean - run: | - cargo clean - - - name: Run tests - env: - RUSTFLAGS: "-Cinstrument-coverage" - LLVM_PROFILE_FILE: "libp2p-rust-dht-%p-%m.profraw" - run: | - cargo test --verbose --all - - - name: Run grcov + - name: Run grcov to produce a coveralls json run: | grcov . --binary-path ./target/debug/ -t coveralls -s . --token YOUR_COVERALLS_TOKEN > coveralls.json @@ -318,7 +261,7 @@ jobs: audit: - needs: [code-coverage, weighted-code-coverage] + needs: [code-coverage] runs-on: ubuntu-latest @@ -348,7 +291,7 @@ jobs: deny: - needs: [code-coverage, weighted-code-coverage] + needs: [code-coverage] runs-on: ubuntu-latest @@ -394,7 +337,7 @@ jobs: udeps: - needs: [code-coverage, weighted-code-coverage] + needs: [code-coverage] runs-on: ubuntu-latest @@ -472,39 +415,6 @@ jobs: ################################## UNSAFE CHECKS LEVEL ######################### - valgrind: - - needs: cache-level - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Install protobuf compiler - run: | - sudo apt-get update - sudo apt-get install protobuf-compiler - - - name: Install valgrind - run: | - sudo apt-get install valgrind - - - name: Install cargo-valgrind - env: - VALGRIND_LINK: https://github.com/jfrimmel/cargo-valgrind/releases/download - VALGRIND_VERSION: 2.1.0 - run: | - curl -L "$VALGRIND_LINK/v$VALGRIND_VERSION/cargo-valgrind-$VALGRIND_VERSION-x86_64-unknown-linux-musl.tar.gz" | - tar xz -C $HOME/.cargo/bin - - # Usage of the `help` command as base command, please replace it - # with the effective command that valgrind has to analyze - - name: Run cargo-valgrind - run: | - cargo valgrind run -- --help - # cargo valgrind test - careful: needs: cache-level From 69056d76ad0d2715fab7a593c4d1972ab50f4851 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 14:39:37 +0200 Subject: [PATCH 34/52] Add a mean to access the list of peers --- dht-cache/src/cache.rs | 60 +++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index a1f53d0..01bfddc 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -78,9 +78,23 @@ impl Builder { pub struct Cache { peer_id: String, local: LocalCache, + peers: Arc>, cmd: UnboundedSender, } +/// Information regarding the known peers +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord)] +pub struct PeerInfo { + /// libp2p Identifier + pub peer_id: String, + /// Hash of its cache of the DHT + pub hash: u64, + /// Last time the peer updated its state + /// + /// TODO: use a better type + pub last_seen: u128, +} + impl Cache { /// Send a volatile message /// @@ -142,6 +156,20 @@ impl Cache { pub fn query(&self, topic: &str) -> Query { self.local.query(topic) } + + /// Get a list of the current peers + pub async fn peers(&self) -> Vec { + let peers = self.peers.read().await; + peers + .list + .values() + .map(|p| PeerInfo { + peer_id: p.peer_id.to_owned(), + hash: p.cache_hash, + last_seen: p.publication_timestamp, + }) + .collect() + } } #[derive(Default, Debug, Clone)] @@ -202,7 +230,12 @@ pub fn cache_channel( let (cmd, r, _j) = dht_channel(swarm); + let peers_state = Arc::new(RwLock::new(PeersState::with_interval( + resend_interval as u128, + ))); + let cache = Cache { + peers: peers_state.clone(), local: local.clone(), cmd: cmd.clone(), peer_id: local_peer_id.clone(), @@ -210,10 +243,6 @@ pub fn cache_channel( let stream = UnboundedReceiverStream::new(r); - let peers_state = Arc::new(RwLock::new(PeersState::with_interval( - resend_interval as u128, - ))); - let local_read = local.clone(); let cmd_update = cmd.clone(); let peer_id = local_peer_id.clone(); @@ -359,11 +388,18 @@ mod test { .map(|uuid| format!("uuid-{uuid}")) .collect(); - tokio::task::spawn(async move { - let (a_c, a_ev) = cache_channel(a_local_cache, a, 100); - let (_b_c, b_ev) = cache_channel(b_local_cache, b, 100); - let (_c_c, c_ev) = cache_channel(c_local_cache, c, 100); + let (a_c, a_ev) = cache_channel(a_local_cache, a, 100); + let (b_c, b_ev) = cache_channel(b_local_cache, b, 100); + let (c_c, c_ev) = cache_channel(c_local_cache, c, 100); + let mut expected_peers = [ + a_c.peer_id.clone(), + b_c.peer_id.clone(), + c_c.peer_id.clone(), + ]; + expected_peers.sort(); + + tokio::task::spawn(async move { let mut a_ev = pin!(a_ev); let mut b_ev = pin!(b_ev); let mut c_ev = pin!(c_ev); @@ -397,7 +433,7 @@ mod test { log::info!("Adding D"); - let (_d_c, d_ev) = cache_channel(d_local_cache, d, 1000); + let (d_c, d_ev) = cache_channel(d_local_cache, d, 1000); let mut d_ev = pin!(d_ev); while !expected.is_empty() { @@ -412,5 +448,11 @@ mod test { } } } + + let mut peers: Vec<_> = d_c.peers().await.into_iter().map(|p| p.peer_id).collect(); + + peers.sort(); + + assert_eq!(peers, expected_peers); } } From 5b4cadffec4c2ae34c6a1593e93f7472c7c68aee Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 15:10:55 +0200 Subject: [PATCH 35/52] Use more workers in the tests --- dht-cache/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 01bfddc..9536f59 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -358,7 +358,7 @@ mod test { use crate::dht::test::*; use std::{collections::HashSet, pin::pin}; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[tokio::test(flavor = "multi_thread", worker_threads = 3)] async fn builder() { let cfg = crate::Config { shared_key: "d061545647652562b4648f52e8373b3a417fc0df56c332154460da1801b341e9" @@ -369,7 +369,7 @@ mod test { let (_cache, _events) = Builder::from_config(cfg).make_channel().await.unwrap(); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn syncronization() { let [mut a, mut b, mut c] = make_peers(2).await; let mut d = make_peer(2).await; From cc9ca285a4f643bfbd2b02253058f3cfe88e29bd Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 18 Aug 2023 16:16:26 +0200 Subject: [PATCH 36/52] fixme: Check for a subset of peers --- dht-cache/src/cache.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 9536f59..1cb8a6b 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -392,12 +392,10 @@ mod test { let (b_c, b_ev) = cache_channel(b_local_cache, b, 100); let (c_c, c_ev) = cache_channel(c_local_cache, c, 100); - let mut expected_peers = [ - a_c.peer_id.clone(), - b_c.peer_id.clone(), - c_c.peer_id.clone(), - ]; - expected_peers.sort(); + let mut expected_peers = HashSet::new(); + expected_peers.insert(a_c.peer_id.clone()); + expected_peers.insert(b_c.peer_id.clone()); + expected_peers.insert(c_c.peer_id.clone()); tokio::task::spawn(async move { let mut a_ev = pin!(a_ev); @@ -449,10 +447,11 @@ mod test { } } - let mut peers: Vec<_> = d_c.peers().await.into_iter().map(|p| p.peer_id).collect(); + // d_c must had seen at least one of the expected peers + let peers: HashSet<_> = d_c.peers().await.into_iter().map(|p| p.peer_id).collect(); - peers.sort(); + log::info!("peers {peers:?}"); - assert_eq!(peers, expected_peers); + assert!(peers.is_subset(&expected_peers)); } } From aca4ae761e015c456073ec970f1feb1ae97a62a1 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 21 Aug 2023 08:46:54 +0200 Subject: [PATCH 37/52] Add Cache::get_hash --- dht-cache/src/cache.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 1cb8a6b..d874473 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -170,6 +170,11 @@ impl Cache { }) .collect() } + + /// Return the current cache hash + pub async fn get_hash(&self) -> u64 { + self.local.get_hash().await + } } #[derive(Default, Debug, Clone)] From 6cf94148985f0f60cd34ae10fcfed2a9b96386e3 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 21 Aug 2023 12:16:14 +0200 Subject: [PATCH 38/52] Apply the impl Into/owned Value pattern Co-authored-by: Edoardo Morandi --- dht-cache/src/cache.rs | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index d874473..22f7595 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -57,7 +57,7 @@ impl Builder { std::fs::write(pk_path, pem)?; der } - Err(e) => Err(e)?, + Err(e) => return Err(e.into()), } } else { generate_rsa_key().1 @@ -99,7 +99,7 @@ impl Cache { /// Send a volatile message /// /// Volatile messages are unstructured and do not persist in the DHT. - pub fn send(&self, value: &Value) -> Result<(), Error> { + pub fn send(&self, value: Value) -> Result<(), Error> { self.cmd .send(Command::Broadcast(value.to_owned())) .map_err(|_| Error::Channel)?; @@ -110,11 +110,16 @@ impl Cache { /// Persist a value within the DHT /// /// It is identified by the topic and uuid value - pub async fn put(&self, topic: &str, uuid: &str, value: &Value) -> Result<(), Error> { + pub async fn put( + &self, + topic: impl Into, + uuid: impl Into, + value: Value, + ) -> Result<(), Error> { let elem = DomoCacheElement { - topic_name: topic.to_string(), - topic_uuid: uuid.to_string(), - value: value.to_owned(), + topic_name: topic.into(), + topic_uuid: uuid.into(), + value, publication_timestamp: utils::get_epoch_ms(), publisher_peer_id: self.peer_id.clone(), ..Default::default() @@ -133,10 +138,14 @@ impl Cache { /// /// It inserts the deletion entry and the entry value will be marked as deleted and removed /// from the stored cache. - pub async fn del(&self, topic: &str, uuid: &str) -> Result<(), Error> { + pub async fn del( + &self, + topic: impl Into, + uuid: impl Into, + ) -> Result<(), Error> { let elem = DomoCacheElement { - topic_name: topic.to_string(), - topic_uuid: uuid.to_string(), + topic_name: topic.into(), + topic_uuid: uuid.into(), publication_timestamp: utils::get_epoch_ms(), publisher_peer_id: self.peer_id.clone(), deleted: true, @@ -317,6 +326,10 @@ pub fn cache_channel( let mut elem = elem.to_owned(); log::debug!("resending {}", elem.topic_uuid); elem.republication_timestamp = utils::get_epoch_ms(); + + // This cannot fail because `cmd` is the sender part of the + // `stream` we are currently reading. In practice, we are + // queueing the commands in order to read them later. cmd.send(Command::Publish(serde_json::to_value(&elem).unwrap())) .unwrap(); }); @@ -411,7 +424,7 @@ mod test { .put( "Topic", &format!("uuid-{uuid}"), - &serde_json::json!({"key": uuid}), + serde_json::json!({"key": uuid}), ) .await; } From 5f79a70313f6b702d0fddb1a44a4b0ab5bc781c7 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 21 Aug 2023 19:32:04 +0200 Subject: [PATCH 39/52] Use futures-concurrency --- dht-cache/Cargo.toml | 1 + dht-cache/src/cache.rs | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/dht-cache/Cargo.toml b/dht-cache/Cargo.toml index 1b09d69..0b61385 100644 --- a/dht-cache/Cargo.toml +++ b/dht-cache/Cargo.toml @@ -31,6 +31,7 @@ libsqlite3-sys = "*" thiserror = "1.0.43" anyhow = "1.0.72" tokio-stream = "0.1.14" +futures-concurrency = "7.4.1" [dev-dependencies] env_logger = "0.10.0" diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 22f7595..2557a53 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -374,6 +374,7 @@ pub fn cache_channel( mod test { use super::*; use crate::dht::test::*; + use futures_concurrency::prelude::*; use std::{collections::HashSet, pin::pin}; #[tokio::test(flavor = "multi_thread", worker_threads = 3)] @@ -416,9 +417,9 @@ mod test { expected_peers.insert(c_c.peer_id.clone()); tokio::task::spawn(async move { - let mut a_ev = pin!(a_ev); - let mut b_ev = pin!(b_ev); - let mut c_ev = pin!(c_ev); + let a_ev = pin!(a_ev); + let b_ev = pin!(b_ev); + let c_ev = pin!(c_ev); for uuid in 0..10 { let _ = a_c .put( @@ -429,13 +430,14 @@ mod test { .await; } - loop { - let (node, ev) = tokio::select! { - v = a_ev.next() => ("a", v.unwrap()), - v = b_ev.next() => ("b", v.unwrap()), - v = c_ev.next() => ("c", v.unwrap()), - }; + let mut s = ( + a_ev.map(|ev| ("a", ev)), + b_ev.map(|ev| ("b", ev)), + c_ev.map(|ev| ("c", ev)), + ) + .merge(); + while let Some((node, ev)) = s.next().await { match ev { DomoEvent::PersistentData(data) => { log::debug!("{node}: Got data {data:?}"); @@ -449,7 +451,7 @@ mod test { log::info!("Adding D"); - let (d_c, d_ev) = cache_channel(d_local_cache, d, 1000); + let (d_c, d_ev) = cache_channel(d_local_cache, d, 100); let mut d_ev = pin!(d_ev); while !expected.is_empty() { From c4000eb191e5f89e58ea0b412b6fedcce9afa086 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 21 Aug 2023 22:16:17 +0200 Subject: [PATCH 40/52] Signal when the nodes are available, not when they are discovered --- dht-cache/src/cache.rs | 129 +++++++++++++++++++++++------------------ dht-cache/src/dht.rs | 64 +++++++++++++++++--- 2 files changed, 128 insertions(+), 65 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 2557a53..ff16f62 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -16,14 +16,30 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use crate::domolibp2p::{self, generate_rsa_key}; use crate::{ cache::local::DomoCacheStateMessage, - data::DomoEvent, - dht::{dht_channel, Command, Event}, + dht::{dht_channel, Command, Event as DhtEvent}, domolibp2p::DomoBehaviour, utils, Error, }; use self::local::{DomoCacheElement, LocalCache, Query}; +/// DHT state change +#[derive(Debug)] +pub enum Event { + /// Persistent, structured data + /// + /// The information is persisted across nodes. + /// Newly joining nodes will receive it from other participants and + /// the local cache can be queried for it. + PersistentData(DomoCacheElement), + /// Volatile, unstructured data + /// + /// The information is transmitted across all the nodes participating + VolatileData(Value), + /// Notify the peer availability + ReadyPeers(Vec), +} + /// Builder for a Cached DHT Node // TODO: make it Clone pub struct Builder { @@ -37,9 +53,7 @@ impl Builder { } /// Instantiate a new DHT node a return - pub async fn make_channel( - self, - ) -> Result<(Cache, impl Stream), crate::Error> { + pub async fn make_channel(self) -> Result<(Cache, impl Stream), crate::Error> { let loopback_only = self.cfg.loopback; let shared_key = domolibp2p::parse_hex_key(&self.cfg.shared_key)?; let private_key_file = self.cfg.private_key.as_ref(); @@ -239,7 +253,7 @@ pub fn cache_channel( local: LocalCache, swarm: Swarm, resend_interval: u64, -) -> (Cache, impl Stream) { +) -> (Cache, impl Stream) { let local_peer_id = swarm.local_peer_id().to_string(); let (cmd, r, _j) = dht_channel(swarm); @@ -289,7 +303,7 @@ pub fn cache_channel( let cmd = cmd.clone(); async move { match ev { - Event::Config(cfg) => { + DhtEvent::Config(cfg) => { let m: DomoCacheStateMessage = serde_json::from_str(&cfg).unwrap(); let hash = local_write.get_hash().await; @@ -338,16 +352,16 @@ pub fn cache_channel( None } - Event::Discovered(who) => Some(DomoEvent::NewPeers( + DhtEvent::Discovered(_who) => None /* Some(DomoEvent::NewPeers( who.into_iter().map(|w| w.to_string()).collect(), - )), - Event::VolatileData(data) => { + ))*/, + DhtEvent::VolatileData(data) => { // TODO we swallow errors quietly here serde_json::from_str(&data) .ok() - .map(DomoEvent::VolatileData) + .map(Event::VolatileData) } - Event::PersistentData(data) => { + DhtEvent::PersistentData(data) => { if let Ok(mut elem) = serde_json::from_str::(&data) { if elem.republication_timestamp != 0 { log::debug!("Retransmission"); @@ -358,7 +372,15 @@ pub fn cache_channel( .try_put(&elem) .await .ok() - .map(|_| DomoEvent::PersistentData(elem)) + .map(|_| Event::PersistentData(elem)) + } else { + None + } + } + DhtEvent::Ready(peers) => { + if !peers.is_empty() { + Some(Event::ReadyPeers( + peers.into_iter().map(|p| p.to_string()).collect())) } else { None } @@ -400,75 +422,68 @@ mod test { let a_local_cache = LocalCache::new(); let b_local_cache = LocalCache::new(); let c_local_cache = LocalCache::new(); - let d_local_cache = LocalCache::new(); let mut expected: HashSet<_> = (0..10) .into_iter() .map(|uuid| format!("uuid-{uuid}")) .collect(); - let (a_c, a_ev) = cache_channel(a_local_cache, a, 100); - let (b_c, b_ev) = cache_channel(b_local_cache, b, 100); - let (c_c, c_ev) = cache_channel(c_local_cache, c, 100); + let (a_c, a_ev) = cache_channel(a_local_cache, a, 5000); + let (b_c, b_ev) = cache_channel(b_local_cache, b, 5000); + let (c_c, c_ev) = cache_channel(c_local_cache, c, 5000); let mut expected_peers = HashSet::new(); expected_peers.insert(a_c.peer_id.clone()); expected_peers.insert(b_c.peer_id.clone()); expected_peers.insert(c_c.peer_id.clone()); - tokio::task::spawn(async move { - let a_ev = pin!(a_ev); - let b_ev = pin!(b_ev); - let c_ev = pin!(c_ev); - for uuid in 0..10 { - let _ = a_c - .put( - "Topic", - &format!("uuid-{uuid}"), - serde_json::json!({"key": uuid}), - ) - .await; - } + let mut a_ev = pin!(a_ev); + let b_ev = pin!(b_ev); + let c_ev = pin!(c_ev); - let mut s = ( - a_ev.map(|ev| ("a", ev)), - b_ev.map(|ev| ("b", ev)), - c_ev.map(|ev| ("c", ev)), - ) - .merge(); - - while let Some((node, ev)) = s.next().await { - match ev { - DomoEvent::PersistentData(data) => { - log::debug!("{node}: Got data {data:?}"); - } - _ => { - log::debug!("{node}: Other {ev:?}"); - } + while let Some(ev) = a_ev.next().await { + match ev { + Event::ReadyPeers(peers) => { + log::info!("Ready peers {peers:?}"); + break; } + _ => log::debug!("waiting for ready {ev:?}"), } - }); - - log::info!("Adding D"); + } - let (d_c, d_ev) = cache_channel(d_local_cache, d, 100); + for uuid in 0..10 { + let _ = a_c + .put( + "Topic", + &format!("uuid-{uuid}"), + serde_json::json!({"key": uuid}), + ) + .await; + } + let mut s = ( + a_ev.map(|ev| ("a", ev)), + b_ev.map(|ev| ("b", ev)), + c_ev.map(|ev| ("c", ev)), + ) + .merge(); - let mut d_ev = pin!(d_ev); while !expected.is_empty() { - let ev = d_ev.next().await.unwrap(); + let (node, ev) = s.next().await.unwrap(); match ev { - DomoEvent::PersistentData(data) => { - assert!(expected.remove(&data.topic_uuid)); - log::warn!("d: Got data {data:?}"); + Event::PersistentData(data) => { + log::debug!("{node}: Got data {data:?}"); + if node == "c" { + assert!(expected.remove(&data.topic_uuid)); + } } _ => { - log::warn!("d: Other {ev:?}"); + log::debug!("{node}: Other {ev:?}"); } } } - // d_c must had seen at least one of the expected peers - let peers: HashSet<_> = d_c.peers().await.into_iter().map(|p| p.peer_id).collect(); + // c_c must had seen at least one of the expected peers + let peers: HashSet<_> = c_c.peers().await.into_iter().map(|p| p.peer_id).collect(); log::info!("peers {peers:?}"); diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 6d29c6c..5416722 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -1,6 +1,8 @@ //! DHT Abstraction //! +use std::time::Duration; + use crate::domolibp2p::{DomoBehaviour, OutEvent}; use futures::prelude::*; use libp2p::{gossipsub::IdentTopic as Topic, swarm::SwarmEvent, Swarm}; @@ -26,6 +28,7 @@ pub enum Event { VolatileData(String), Config(String), Discovered(Vec), + Ready(Vec), } fn handle_command(swarm: &mut Swarm, cmd: Command) -> bool { @@ -36,7 +39,7 @@ fn handle_command(swarm: &mut Swarm, cmd: Command) -> bool { let m = serde_json::to_string(&val).unwrap(); if let Err(e) = swarm.behaviour_mut().gossipsub.publish(topic, m.as_bytes()) { - log::info!("Publish error: {e:?}"); + log::info!("Boradcast error: {e:?}"); } true } @@ -57,7 +60,7 @@ fn handle_command(swarm: &mut Swarm, cmd: Command) -> bool { let topic = Topic::new("domo-config"); let m = serde_json::to_string(&val).unwrap(); if let Err(e) = swarm.behaviour_mut().gossipsub.publish(topic, m.as_bytes()) { - log::info!("Publish error: {e:?}"); + log::info!("Config error: {e:?}"); } true } @@ -117,6 +120,11 @@ fn handle_swarm_event( } } } + SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Gossipsub( + libp2p::gossipsub::Event::Subscribed { peer_id, topic }, + )) => { + log::debug!("Peer {peer_id} subscribed to {}", topic.as_str()); + } SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Mdns(mdns::Event::Expired(list))) => { let local = OffsetDateTime::now_utc(); @@ -154,16 +162,37 @@ pub fn dht_channel( let (ev_send, ev_recv) = mpsc::unbounded_channel(); let handle = tokio::task::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + let volatile = Topic::new("domo-volatile-data").hash(); + let persistent = Topic::new("domo-persistent-data").hash(); + let config = Topic::new("domo-config").hash(); loop { + log::trace!("Looping {}", swarm.local_peer_id()); tokio::select! { + // the mdns event is not enough to ensure we can send messages + _ = interval.tick() => { + log::debug!("{} Checking for peers", swarm.local_peer_id()); + let peers: Vec<_> = swarm.behaviour_mut().gossipsub.all_peers().filter_map(|(p, topics)| { + log::info!("{p}, {topics:?}"); + (topics.contains(&&volatile) && + topics.contains(&&persistent) && + topics.contains(&&config)).then( + ||p.to_owned()) + }).collect(); + if !peers.is_empty() && + ev_send.send(Event::Ready(peers)).is_err() { + return swarm; + } + } cmd = cmd_recv.recv() => { - log::debug!("command {cmd:?}"); + log::trace!("command {cmd:?}"); if !cmd.is_some_and(|cmd| handle_command(&mut swarm, cmd)) { log::debug!("Exiting cmd"); return swarm } } ev = swarm.select_next_some() => { + log::trace!("event {ev:?}"); if handle_swarm_event(&mut swarm, ev, &ev_send).is_err() { log::debug!("Exiting ev"); return swarm @@ -217,10 +246,9 @@ pub(crate) mod test { } pub async fn make_peer(variant: u8) -> Swarm { - let mut a = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); - a.listen().await; - - a + let mut swarm = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); + swarm.listen().await; + swarm } pub async fn connect_peer(a: &mut Swarm, b: &mut Swarm) { @@ -235,10 +263,15 @@ pub(crate) mod test { pub async fn make_peers(variant: u8) -> [Swarm; 3] { let _ = env_logger::builder().is_test(true).try_init(); + let mut a = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); let mut b = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); let mut c = new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap(), variant); - + /* + let mut a = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + let mut b = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + let mut c = Swarm::new_ephemeral(|identity| DomoBehaviour::new(&identity).unwrap()); + */ for a in a.external_addresses() { log::info!("{a:?}"); } @@ -251,8 +284,14 @@ pub(crate) mod test { b.connect(&mut c).await; c.connect(&mut a).await; + println!("a {}", a.local_peer_id()); + println!("b {}", b.local_peer_id()); + println!("c {}", c.local_peer_id()); + let peers: Vec<_> = a.connected_peers().cloned().collect(); + log::info!("Peers {peers:#?}"); + for peer in peers { a.behaviour_mut().gossipsub.add_explicit_peer(&peer); } @@ -280,6 +319,8 @@ pub(crate) mod test { let (b_s, br, _) = dht_channel(b); let (c_s, cr, _) = dht_channel(c); + log::info!("Waiting for peers"); + // Wait until peers are discovered while let Some(ev) = ar.recv().await { match ev { @@ -288,6 +329,9 @@ pub(crate) mod test { Event::Config(cfg) => log::info!("config {cfg}"), Event::Discovered(peers) => { log::info!("found peers: {peers:?}"); + } + Event::Ready(peers) => { + log::info!("ready peers: {peers:?}"); break; } } @@ -297,6 +341,7 @@ pub(crate) mod test { a_s.send(Command::Broadcast(msg.clone())).unwrap(); + log::info!("Sent volatile"); for r in [br, cr].iter_mut() { while let Some(ev) = r.recv().await { match ev { @@ -311,6 +356,9 @@ pub(crate) mod test { Event::Discovered(peers) => { log::info!("found peers: {peers:?}"); } + Event::Ready(peers) => { + log::info!("peers ready: {peers:?}"); + } } } } From b8a941041258272b25ae00ef61c117c9accd92d7 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 22 Aug 2023 10:09:07 +0200 Subject: [PATCH 41/52] Do not panic on non-utf8 data --- dht-cache/src/dht.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 5416722..239a29d 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -104,20 +104,23 @@ fn handle_swarm_event( message, }, )) => { - let data = String::from_utf8(message.data).unwrap(); - match message.topic.as_str() { - "domo-persistent-data" => { - ev_send.send(PersistentData(data)).map_err(|_| ())?; - } - "domo-config" => { - ev_send.send(Config(data)).map_err(|_| ())?; - } - "domo-volatile-data" => { - ev_send.send(VolatileData(data)).map_err(|_| ())?; - } - _ => { - log::info!("Not able to recognize message"); + if let Ok(data) = String::from_utf8(message.data) { + match message.topic.as_str() { + "domo-persistent-data" => { + ev_send.send(PersistentData(data)).map_err(|_| ())?; + } + "domo-config" => { + ev_send.send(Config(data)).map_err(|_| ())?; + } + "domo-volatile-data" => { + ev_send.send(VolatileData(data)).map_err(|_| ())?; + } + _ => { + log::info!("Not able to recognize message"); + } } + } else { + log::warn!("The message does not contain utf8 data"); } } SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Gossipsub( From d2e3540148c49e8edbf1d665ad8cb76fd625e38d Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 22 Aug 2023 10:17:04 +0200 Subject: [PATCH 42/52] use into_iter --- dht-cache/src/dht.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 239a29d..de26677 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -138,11 +138,11 @@ fn handle_swarm_event( SwarmEvent::Behaviour(crate::domolibp2p::OutEvent::Mdns(mdns::Event::Discovered(list))) => { let local = OffsetDateTime::now_utc(); let peers = list - .iter() + .into_iter() .map(|(peer, _)| { - swarm.behaviour_mut().gossipsub.add_explicit_peer(peer); + swarm.behaviour_mut().gossipsub.add_explicit_peer(&peer); log::info!("Discovered peer {peer} {local:?}"); - peer.to_owned() + peer }) .collect(); ev_send.send(Discovered(peers)).map_err(|_| ())?; From eef07a892c2faf141eff69fa364f6c5077b1b328 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 22 Aug 2023 10:22:46 +0200 Subject: [PATCH 43/52] Cleanup the peers check to be more readable --- dht-cache/src/dht.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index de26677..1b8f2c4 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -169,20 +169,31 @@ pub fn dht_channel( let volatile = Topic::new("domo-volatile-data").hash(); let persistent = Topic::new("domo-persistent-data").hash(); let config = Topic::new("domo-config").hash(); + + // Only peers that subscribed to all the topics are usable + let check_peers = |swarm: &mut Swarm| { + swarm + .behaviour_mut() + .gossipsub + .all_peers() + .filter_map(|(p, topics)| { + log::info!("{p}, {topics:?}"); + (topics.contains(&&volatile) + && topics.contains(&&persistent) + && topics.contains(&&config)) + .then(|| p.to_owned()) + }) + .collect() + }; + loop { log::trace!("Looping {}", swarm.local_peer_id()); tokio::select! { // the mdns event is not enough to ensure we can send messages _ = interval.tick() => { log::debug!("{} Checking for peers", swarm.local_peer_id()); - let peers: Vec<_> = swarm.behaviour_mut().gossipsub.all_peers().filter_map(|(p, topics)| { - log::info!("{p}, {topics:?}"); - (topics.contains(&&volatile) && - topics.contains(&&persistent) && - topics.contains(&&config)).then( - ||p.to_owned()) - }).collect(); - if !peers.is_empty() && + let peers: Vec<_> = check_peers(&mut swarm); + if !peers.is_empty() && ev_send.send(Event::Ready(peers)).is_err() { return swarm; } From 5f336c41e21a1e78e6a7bfef2f411837ac666988 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 22 Aug 2023 10:24:46 +0200 Subject: [PATCH 44/52] drop redundant drops --- dht-cache/src/dht.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index 1b8f2c4..ec30f12 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -330,8 +330,8 @@ pub(crate) mod test { let [a, b, c] = make_peers(1).await; let (a_s, mut ar, _) = dht_channel(a); - let (b_s, br, _) = dht_channel(b); - let (c_s, cr, _) = dht_channel(c); + let (_b_s, br, _) = dht_channel(b); + let (_c_s, cr, _) = dht_channel(c); log::info!("Waiting for peers"); @@ -376,8 +376,5 @@ pub(crate) mod test { } } } - - drop(b_s); - drop(c_s); } } From 33881b72ac7dad896621d25843c522834818082c Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Tue, 22 Aug 2023 10:25:45 +0200 Subject: [PATCH 45/52] Avoid redundant path --- dht-cache/src/domocache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dht-cache/src/domocache.rs b/dht-cache/src/domocache.rs index 89999ea..0e282b9 100644 --- a/dht-cache/src/domocache.rs +++ b/dht-cache/src/domocache.rs @@ -420,7 +420,7 @@ impl DomoCache { /// Cache event loop /// /// To be called as often as needed to keep the cache in-sync and receive new data. - pub async fn cache_event_loop(&mut self) -> std::result::Result { + pub async fn cache_event_loop(&mut self) -> Result { use Event::*; loop { match self.inner_select().await { From 339272f713c7a2cca240f8de5d99758da1263d35 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 21 Aug 2023 12:17:02 +0200 Subject: [PATCH 46/52] Use a worker task to persist the data --- dht-cache/src/cache/local.rs | 44 +++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/dht-cache/src/cache/local.rs b/dht-cache/src/cache/local.rs index 3bb3624..8df0114 100644 --- a/dht-cache/src/cache/local.rs +++ b/dht-cache/src/cache/local.rs @@ -7,17 +7,19 @@ use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; use std::hash::{Hash, Hasher}; use std::sync::Arc; +use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use tokio::sync::{OwnedRwLockReadGuard, RwLock}; +enum SqlxCommand { + Write(DomoCacheElement), +} + #[derive(Default)] pub(crate) struct InnerCache { pub mem: BTreeMap>, - pub store: Option, + store: Option>, } -/// SAFETY: the SqlxStorage access is only over write() -unsafe impl std::marker::Sync for InnerCache {} - impl InnerCache { pub fn put(&mut self, elem: &DomoCacheElement) { let topic_name = elem.topic_name.clone(); @@ -53,7 +55,14 @@ impl LocalCache { } if db_config.persistent { - inner.store = Some(store); + let (s, mut r) = unbounded_channel(); + + tokio::task::spawn(async move { + while let Some(SqlxCommand::Write(elem)) = r.recv().await { + store.store(&elem).await + } + }); + inner.store = Some(s); } } @@ -83,8 +92,8 @@ impl LocalCache { pub async fn put(&self, elem: &DomoCacheElement) { let mut cache = self.0.write().await; - if let Some(storage) = cache.store.as_mut() { - storage.store(elem).await; + if let Some(s) = cache.store.as_mut() { + let _ = s.send(SqlxCommand::Write(elem.to_owned())); } cache.put(elem); @@ -112,7 +121,7 @@ impl LocalCache { if e.is_ok() { if let Some(s) = cache.store.as_mut() { - s.store(elem).await; + let _ = s.send(SqlxCommand::Write(elem.to_owned())); } } @@ -317,4 +326,23 @@ mod test { 1 ); } + + #[tokio::test] + async fn persistence() { + let cfg = crate::Config { + ..Default::default() + }; + + let cache = LocalCache::with_config(&cfg).await; + + for item in 0..10 { + let elem = make_test_element( + "Domo::Light", + &format!("luce-{item}"), + &json!({ "connected": true, "count": item}), + ); + + cache.put(&elem).await; + } + } } From 7020d082b680b25914642a5ec4401e8cbca47458 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 24 Aug 2023 16:30:19 +0200 Subject: [PATCH 47/52] fixup: multiple_peer tests should wait for all of them Otherwise it would infiniloop it send the volatile message when only one of the two peers is ready. --- dht-cache/src/dht.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index ec30f12..d2bc9a1 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -222,6 +222,7 @@ pub fn dht_channel( #[cfg(test)] pub(crate) mod test { + use std::collections::HashSet; use std::time::Duration; use super::*; @@ -329,6 +330,9 @@ pub(crate) mod test { async fn multiple_peers() { let [a, b, c] = make_peers(1).await; + let mut expected_peers: HashSet<_> = + [b.local_peer_id().to_owned(), c.local_peer_id().to_owned()].into(); + let (a_s, mut ar, _) = dht_channel(a); let (_b_s, br, _) = dht_channel(b); let (_c_s, cr, _) = dht_channel(c); @@ -346,7 +350,13 @@ pub(crate) mod test { } Event::Ready(peers) => { log::info!("ready peers: {peers:?}"); - break; + for peer in peers { + expected_peers.remove(&peer); + } + + if expected_peers.is_empty() { + break; + } } } } From c7f0568bcb308dd76466ece65504def57f214b5f Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Thu, 24 Aug 2023 16:32:00 +0200 Subject: [PATCH 48/52] Take ownership of the elem in the local cache API as well --- dht-cache/src/cache.rs | 14 ++++---- dht-cache/src/cache/local.rs | 62 ++++++++++++++++++------------------ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index ff16f62..77be191 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -139,12 +139,12 @@ impl Cache { ..Default::default() }; - self.local.put(&elem).await; - self.cmd .send(Command::Publish(serde_json::to_value(&elem)?)) .map_err(|_| Error::Channel)?; + self.local.put(elem).await; + Ok(()) } @@ -166,12 +166,12 @@ impl Cache { ..Default::default() }; - self.local.put(&elem).await; - self.cmd .send(Command::Publish(serde_json::to_value(&elem)?)) .map_err(|_| Error::Channel)?; + self.local.put(elem).await; + Ok(()) } @@ -369,10 +369,10 @@ pub fn cache_channel( // TODO: do something with this value instead elem.republication_timestamp = 0; local_write - .try_put(&elem) + .try_put(elem.clone()) .await - .ok() - .map(|_| Event::PersistentData(elem)) + .then_some( + Event::PersistentData(elem)) } else { None } diff --git a/dht-cache/src/cache/local.rs b/dht-cache/src/cache/local.rs index 8df0114..05efe38 100644 --- a/dht-cache/src/cache/local.rs +++ b/dht-cache/src/cache/local.rs @@ -21,16 +21,18 @@ pub(crate) struct InnerCache { } impl InnerCache { - pub fn put(&mut self, elem: &DomoCacheElement) { - let topic_name = elem.topic_name.clone(); - let topic_uuid = &elem.topic_uuid; + pub fn put(&mut self, elem: DomoCacheElement) { + let topic_name = &elem.topic_name; + let topic_uuid = elem.topic_uuid.to_owned(); - self.mem - .entry(topic_name) - .and_modify(|topic| { - topic.insert(topic_uuid.to_owned(), elem.to_owned()); - }) - .or_insert_with(|| [(topic_uuid.to_owned(), elem.to_owned())].into()); + if let Some(topic) = self.mem.get_mut(topic_name) { + topic.insert(topic_uuid, elem); + } else { + self.mem.insert( + topic_name.to_owned(), + [(topic_uuid.to_owned(), elem)].into(), + ); + } } } @@ -51,7 +53,7 @@ impl LocalCache { let mut store = SqlxStorage::new(db_config).await; for a in store.get_all_elements().await { - inner.put(&a); + inner.put(a); } if db_config.persistent { @@ -61,6 +63,7 @@ impl LocalCache { while let Some(SqlxCommand::Write(elem)) = r.recv().await { store.store(&elem).await } + panic!("I'm out!"); }); inner.store = Some(s); } @@ -89,7 +92,7 @@ impl LocalCache { /// Put the element in the cache /// /// If it is already present overwrite it - pub async fn put(&self, elem: &DomoCacheElement) { + pub async fn put(&self, elem: DomoCacheElement) { let mut cache = self.0.write().await; if let Some(s) = cache.store.as_mut() { @@ -101,8 +104,8 @@ impl LocalCache { /// Try to insert the element in the cache /// - /// Return Err(()) if the element to insert is older than the one in the cache - pub async fn try_put(&self, elem: &DomoCacheElement) -> Result<(), ()> { + /// Return false if the element to insert is older than the one in the cache + pub async fn try_put(&self, elem: DomoCacheElement) -> bool { let mut cache = self.0.write().await; let topic_name = elem.topic_name.clone(); let topic_uuid = &elem.topic_uuid; @@ -113,15 +116,15 @@ impl LocalCache { .get(topic_uuid) .is_some_and(|cur| elem.publication_timestamp <= cur.publication_timestamp) { - Err(()) + false } else { - topic.insert(topic_uuid.to_owned(), elem.to_owned()); - Ok(()) + topic.insert(topic_uuid.to_owned(), elem.clone()); + true }; - if e.is_ok() { + if e { if let Some(s) = cache.store.as_mut() { - let _ = s.send(SqlxCommand::Write(elem.to_owned())); + let _ = s.send(SqlxCommand::Write(elem)); } } @@ -222,7 +225,7 @@ mod test { println!("{hash}"); let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); - cache.put(&elem).await; + cache.put(elem).await; let hash2 = cache.get_hash().await; println!("{hash2}"); @@ -230,7 +233,7 @@ mod test { assert_ne!(hash, hash2); let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": false})); - cache.put(&elem).await; + cache.put(elem).await; let hash3 = cache.get_hash().await; println!("{hash3}"); @@ -238,7 +241,7 @@ mod test { assert_ne!(hash2, hash3); let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); - cache.put(&elem).await; + cache.put(elem).await; let hash4 = cache.get_hash().await; println!("{hash4}"); @@ -252,7 +255,7 @@ mod test { let elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); - cache.put(&elem).await; + cache.put(elem.clone()).await; let out = cache.get("Domo::Light", "luce-1").await.expect("element"); @@ -260,7 +263,7 @@ mod test { let elem2 = make_test_element("Domo::Light", "luce-1", &json!({ "connected": false})); - cache.put(&elem2).await; + cache.put(elem2.clone()).await; let out = cache.get("Domo::Light", "luce-1").await.expect("element"); @@ -273,7 +276,7 @@ mod test { let mut elem = make_test_element("Domo::Light", "luce-1", &json!({ "connected": true})); - cache.try_put(&elem).await.unwrap(); + assert!(cache.try_put(elem.clone()).await); let out = cache.get("Domo::Light", "luce-1").await.expect("element"); @@ -281,7 +284,7 @@ mod test { elem.publication_timestamp = 1; - cache.try_put(&elem).await.expect("Update entry"); + assert!(cache.try_put(elem.clone()).await); let out: DomoCacheElement = cache.get("Domo::Light", "luce-1").await.expect("element"); @@ -289,10 +292,7 @@ mod test { elem.publication_timestamp = 0; - cache - .try_put(&elem) - .await - .expect_err("The update should fail"); + assert!(!cache.try_put(elem).await); let out: DomoCacheElement = cache.get("Domo::Light", "luce-1").await.expect("element"); @@ -310,7 +310,7 @@ mod test { &json!({ "connected": true, "count": item}), ); - cache.put(&elem).await; + cache.put(elem).await; } let q = cache.query("Domo::Light"); @@ -342,7 +342,7 @@ mod test { &json!({ "connected": true, "count": item}), ); - cache.put(&elem).await; + cache.put(elem).await; } } } From 37210e4eb5ea2255f029b36c079cf61bdca5d201 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 11 Sep 2023 12:30:32 +0200 Subject: [PATCH 49/52] Refine the Ready peers event --- dht-cache/src/dht.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dht-cache/src/dht.rs b/dht-cache/src/dht.rs index d2bc9a1..a221033 100644 --- a/dht-cache/src/dht.rs +++ b/dht-cache/src/dht.rs @@ -1,6 +1,7 @@ //! DHT Abstraction //! +use std::collections::HashSet; use std::time::Duration; use crate::domolibp2p::{DomoBehaviour, OutEvent}; @@ -28,7 +29,7 @@ pub enum Event { VolatileData(String), Config(String), Discovered(Vec), - Ready(Vec), + Ready(HashSet), } fn handle_command(swarm: &mut Swarm, cmd: Command) -> bool { @@ -169,7 +170,7 @@ pub fn dht_channel( let volatile = Topic::new("domo-volatile-data").hash(); let persistent = Topic::new("domo-persistent-data").hash(); let config = Topic::new("domo-config").hash(); - + let mut ready_peers = HashSet::new(); // Only peers that subscribed to all the topics are usable let check_peers = |swarm: &mut Swarm| { swarm @@ -177,7 +178,7 @@ pub fn dht_channel( .gossipsub .all_peers() .filter_map(|(p, topics)| { - log::info!("{p}, {topics:?}"); + log::debug!("{p}, {topics:?}"); (topics.contains(&&volatile) && topics.contains(&&persistent) && topics.contains(&&config)) @@ -192,10 +193,12 @@ pub fn dht_channel( // the mdns event is not enough to ensure we can send messages _ = interval.tick() => { log::debug!("{} Checking for peers", swarm.local_peer_id()); - let peers: Vec<_> = check_peers(&mut swarm); - if !peers.is_empty() && - ev_send.send(Event::Ready(peers)).is_err() { + let peers: HashSet<_> = check_peers(&mut swarm); + if !peers.is_empty() && ready_peers != peers { + ready_peers = peers.clone(); + if ev_send.send(Event::Ready(peers)).is_err() { return swarm; + } } } cmd = cmd_recv.recv() => { From 4fd2dbeab8a90a4f440547ee40f989afcd1bf843 Mon Sep 17 00:00:00 2001 From: Edoardo Morandi Date: Thu, 28 Sep 2023 09:11:03 +0200 Subject: [PATCH 50/52] fix: export structs from cache::local --- dht-cache/src/cache.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 77be191..45518ea 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -21,7 +21,8 @@ use crate::{ utils, Error, }; -use self::local::{DomoCacheElement, LocalCache, Query}; +use self::local::DomoCacheElement; +pub use self::local::{LocalCache, Query}; /// DHT state change #[derive(Debug)] From abff4709845637b725d473a1c8b37d461eb33fa5 Mon Sep 17 00:00:00 2001 From: Edoardo Morandi Date: Fri, 29 Sep 2023 09:53:44 +0200 Subject: [PATCH 51/52] feat!: overhaul dht_cache::cache::Query API --- dht-cache/src/cache.rs | 5 +- dht-cache/src/cache/local.rs | 155 ++++++++++++++++++++++++++--------- 2 files changed, 120 insertions(+), 40 deletions(-) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index 45518ea..b583f8c 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -22,7 +22,7 @@ use crate::{ }; use self::local::DomoCacheElement; -pub use self::local::{LocalCache, Query}; +pub use self::local::{LocalCache, Query, QueryGet, QueryGetIter}; /// DHT state change #[derive(Debug)] @@ -177,7 +177,8 @@ impl Cache { } /// Query the local cache - pub fn query(&self, topic: &str) -> Query { + #[must_use] + pub fn query<'a>(&'a self, topic: &'a str) -> Query<'a> { self.local.query(topic) } diff --git a/dht-cache/src/cache/local.rs b/dht-cache/src/cache/local.rs index 05efe38..73195eb 100644 --- a/dht-cache/src/cache/local.rs +++ b/dht-cache/src/cache/local.rs @@ -4,17 +4,17 @@ pub use crate::data::*; use crate::domopersistentstorage::{DomoPersistentStorage, SqlxStorage}; use serde_json::Value; use std::collections::hash_map::DefaultHasher; -use std::collections::BTreeMap; +use std::collections::{btree_map, BTreeMap}; use std::hash::{Hash, Hasher}; use std::sync::Arc; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; -use tokio::sync::{OwnedRwLockReadGuard, RwLock}; +use tokio::sync::{OwnedRwLockReadGuard, RwLock, RwLockReadGuard}; enum SqlxCommand { Write(DomoCacheElement), } -#[derive(Default)] +#[derive(Debug, Default)] pub(crate) struct InnerCache { pub mem: BTreeMap>, store: Option>, @@ -143,7 +143,7 @@ impl LocalCache { } /// Instantiate a query over the local cache - pub fn query(&self, topic: &str) -> Query { + pub fn query<'a>(&self, topic: &'a str) -> Query<'a> { Query::new(topic, self.clone()) } @@ -161,44 +161,120 @@ impl LocalCache { /// Query the local DHT cache #[derive(Clone)] -pub struct Query { +pub struct Query<'a> { cache: LocalCache, - topic: String, - uuid: Option, + topic: &'a str, } -impl Query { +impl<'a> Query<'a> { /// Create a new query over a local cache - pub fn new(topic: &str, cache: LocalCache) -> Self { - Self { - topic: topic.to_owned(), - cache, - uuid: None, - } + pub fn new(topic: &'a str, cache: LocalCache) -> Self { + Self { topic, cache } } - /// Look up for a specific uuid - pub fn with_uuid(mut self, uuid: &str) -> Self { - self.uuid = Some(uuid.to_owned()); - self + + /// Gets a value on a topic given a specific UUID. + /// + /// Keep in mind that the returned type holds a lock guard to the underlying data, be careful + /// to use it across yield points. + pub async fn get_by_uuid<'b>(&'b self, uuid: &'b str) -> Option> { + RwLockReadGuard::try_map(self.cache.0.read().await, |cache| { + cache + .mem + .get(self.topic) + .and_then(|tree| tree.get(uuid)) + .map(|cache_element| &cache_element.value) + }) + .ok() } - /// Execute the query and return a Value if found - pub async fn get(&self) -> Vec { - let cache = self.cache.0.read().await; - - if let Some(topics) = cache.mem.get(&self.topic) { - if let Some(ref uuid) = self.uuid { - topics - .get(uuid) - .into_iter() - .map(|elem| elem.value.clone()) - .collect() - } else { - topics.values().map(|elem| elem.value.clone()).collect() - } - } else { - Vec::new() - } + /// Gets the data stored for the topic. + /// + /// It returns an _iterable type_ that can be used to obtain pairs of UUID and values. + /// + /// Keep in mind that the returned type holds a lock guard to the underlying data, be careful + /// to use it across yield points. + /// + /// # Example + /// + /// ``` + /// # use sifis_dht::cache::Query; + /// # async fn handle_query(query: Query<'_>) { + /// let get = query.get().await; + /// for pair in &get { + /// let (uuid, value): (&str, &serde_json::Value) = pair; + /// println!("{uuid}, {value}"); + /// } + /// # } + /// ``` + #[inline] + pub async fn get(&self) -> QueryGet<'_> { + let lock = + RwLockReadGuard::try_map(self.cache.0.read().await, |cache| cache.mem.get(self.topic)) + .ok(); + + QueryGet(lock) + } +} + +#[derive(Debug)] +pub struct QueryGet<'a>(Option>>); + +impl<'a> QueryGet<'a> { + /// Iterate over queried pairs of UUIDs and values. + #[inline] + #[must_use] + pub fn iter(&'a self) -> QueryGetIter<'a> { + IntoIterator::into_iter(self) + } +} + +impl<'a> IntoIterator for &'a QueryGet<'a> { + type Item = (&'a str, &'a Value); + type IntoIter = QueryGetIter<'a>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + let values = self + .0 + .as_deref() + .map_or_else(Default::default, BTreeMap::iter); + + QueryGetIter(values) + } +} + +#[derive(Debug)] +pub struct QueryGetIter<'a>(btree_map::Iter<'a, String, DomoCacheElement>); + +impl<'a> Iterator for QueryGetIter<'a> { + type Item = (&'a str, &'a Value); + + #[inline] + fn next(&mut self) -> Option { + self.0 + .next() + .map(|(uuid, cache_element)| (&**uuid, &cache_element.value)) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } +} + +impl DoubleEndedIterator for QueryGetIter<'_> { + #[inline] + fn next_back(&mut self) -> Option { + self.0 + .next_back() + .map(|(uuid, cache_element)| (&**uuid, &cache_element.value)) + } +} + +impl ExactSizeIterator for QueryGetIter<'_> { + #[inline] + fn len(&self) -> usize { + self.0.len() } } @@ -315,12 +391,15 @@ mod test { let q = cache.query("Domo::Light"); - assert_eq!(q.get().await.len(), 10); + assert_eq!(q.get().await.iter().len(), 10); - assert_eq!(q.clone().with_uuid("not-existent").get().await.len(), 0); + assert!(q.get_by_uuid("not-existent").await.is_none()); assert_eq!( - q.clone().with_uuid("luce-1").get().await[0] + q.clone() + .get_by_uuid("luce-1") + .await + .unwrap() .get("count") .unwrap(), 1 From ff3cc9007ddc17cf9b63de1691ec336ffc380cfc Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 9 Oct 2023 11:35:28 +0200 Subject: [PATCH 52/52] Mark Cache as Clone --- dht-cache/src/cache.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dht-cache/src/cache.rs b/dht-cache/src/cache.rs index b583f8c..43706eb 100644 --- a/dht-cache/src/cache.rs +++ b/dht-cache/src/cache.rs @@ -90,6 +90,7 @@ impl Builder { /// Cached DHT /// /// It keeps a local cache of the dht state and allow to query the persistent topics +#[derive(Clone)] pub struct Cache { peer_id: String, local: LocalCache,