From db45d0ae6a2756d313af8d8aae9f065b4f08bbd3 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Mon, 27 Nov 2023 18:46:38 +0800 Subject: [PATCH 01/22] reduce the "already-connected peer" error logs from appearing --- clients/stellar-relay-lib/src/config.rs | 3 +- .../src/connection/connector/connector.rs | 3 - .../stellar-relay-lib/src/connection/mod.rs | 10 +- .../src/connection/overlay_connection.rs | 63 +++++---- .../src/connection/services.rs | 123 ++++++++---------- 5 files changed, 90 insertions(+), 112 deletions(-) diff --git a/clients/stellar-relay-lib/src/config.rs b/clients/stellar-relay-lib/src/config.rs index e34004e9a..ae369ca0d 100644 --- a/clients/stellar-relay-lib/src/config.rs +++ b/clients/stellar-relay-lib/src/config.rs @@ -48,7 +48,7 @@ impl StellarOverlayConfig { let address = std::str::from_utf8(&cfg.address) .map_err(|e| Error::ConfigError(format!("Address: {:?}", e)))?; - Ok(ConnectionInfo::new_with_timeout_and_retries( + Ok(ConnectionInfo::new_with_timeout( address, cfg.port, secret_key, @@ -57,7 +57,6 @@ impl StellarOverlayConfig { cfg.recv_scp_msgs, cfg.remote_called_us, cfg.timeout_in_secs, - cfg.retries, )) } } diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 44591e5d0..b890472aa 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -23,7 +23,6 @@ pub struct Connector { pub(crate) connection_auth: ConnectionAuth, pub(crate) timeout_in_secs: u64, - pub(crate) retries: u8, remote_called_us: bool, receive_tx_messages: bool, @@ -48,7 +47,6 @@ impl Debug for Connector { .field("hmac_keys_exist", &is_hmac_keys_filled) .field("connection_auth", &self.connection_auth) .field("timeout_in_secs", &self.timeout_in_secs) - .field("retries", &self.retries) .field("receive_tx_messages", &self.receive_tx_messages) .field("receive_scp_messages", &self.receive_scp_messages) .field("handshake_state", &self.handshake_state) @@ -137,7 +135,6 @@ impl Connector { hmac_keys: None, connection_auth, timeout_in_secs: conn_info.timeout_in_secs, - retries: conn_info.retries, remote_called_us: conn_info.remote_called_us, receive_tx_messages: conn_info.recv_tx_msgs, receive_scp_messages: conn_info.recv_scp_msgs, diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index 1da0f04d7..29c1c3c17 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -57,8 +57,6 @@ pub struct ConnectionInfo { pub remote_called_us: bool, /// how long to wait for the Stellar Node's messages. timeout_in_secs: u64, - /// number of retries to wait for the Stellar Node's messages and/or to connect back to it. - retries: u8, } impl Debug for ConnectionInfo { @@ -73,14 +71,13 @@ impl Debug for ConnectionInfo { .field("receive_scp_messages", &self.recv_scp_msgs) .field("remote_called_us", &self.remote_called_us) .field("timeout_in_seconds", &self.timeout_in_secs) - .field("retries", &self.retries) .finish() } } impl ConnectionInfo { #[allow(clippy::too_many_arguments)] - pub(crate) fn new_with_timeout_and_retries( + pub(crate) fn new_with_timeout( addr: &str, port: u32, secret_key: SecretKey, @@ -89,7 +86,6 @@ impl ConnectionInfo { recv_scp_msgs: bool, remote_called_us: bool, timeout_in_secs: u64, - retries: u8, ) -> Self { ConnectionInfo { address: addr.to_string(), @@ -100,7 +96,6 @@ impl ConnectionInfo { recv_scp_msgs, remote_called_us, timeout_in_secs, - retries, } } @@ -114,7 +109,7 @@ impl ConnectionInfo { recv_scp_msgs: bool, remote_called_us: bool, ) -> Self { - Self::new_with_timeout_and_retries( + Self::new_with_timeout( addr, port, secret_key, @@ -123,7 +118,6 @@ impl ConnectionInfo { recv_scp_msgs, remote_called_us, 10, - 3, ) } diff --git a/clients/stellar-relay-lib/src/connection/overlay_connection.rs b/clients/stellar-relay-lib/src/connection/overlay_connection.rs index 183c02de7..57cd4f7e4 100644 --- a/clients/stellar-relay-lib/src/connection/overlay_connection.rs +++ b/clients/stellar-relay-lib/src/connection/overlay_connection.rs @@ -48,35 +48,41 @@ impl StellarOverlayConnection { let res = self.relay_message_receiver.recv().await; match &res { - Some(StellarRelayMessage::Timeout) | Some(StellarRelayMessage::Error(_)) | None => { - log::info!("listen(): Reconnecting to {:?}...", &self.conn_info.address); - - match StellarOverlayConnection::connect( - self.local_node.clone(), - self.conn_info.clone(), - ) - .await - { - Ok(new_user) => { - self.actions_sender = new_user.actions_sender; - self.relay_message_receiver = new_user.relay_message_receiver; - log::info!( + Some(StellarRelayMessage::Timeout) | Some(StellarRelayMessage::Error(_)) | None => + // we want to keep reconnecting until it succeeds. + loop { + // Delaying reconnection gives time for the write and read streams to be dropped. + let timeout = self.conn_info.timeout_in_secs; + log::info!( + "listen(): Reconnecting to {:?} in {timeout} seconds", + &self.conn_info.address + ); + tokio::time::sleep(Duration::from_secs(timeout)).await; + + match StellarOverlayConnection::connect( + self.local_node.clone(), + self.conn_info.clone(), + ) + .await + { + Ok(new_user) => { + self.actions_sender = new_user.actions_sender; + self.relay_message_receiver = new_user.relay_message_receiver; + log::info!( "listen(): overlay connection reconnected to {:?}", &self.conn_info.address ); - return self.relay_message_receiver.recv().await - }, - Err(e) => { - log::error!( - "listen(): overlay connection failed to reconnect: {e:?}\n. Retrying in 3 seconds...", - ); - tokio::time::sleep(Duration::from_secs(3)).await; - }, - }; - }, + // break out of the loop since connection was successful. + return self.relay_message_receiver.recv().await + }, + Err(e) => { + log::error!("listen(): overlay connection failed to reconnect: {e:?}",); + continue + }, + }; + }, _ => {}, } - res } @@ -88,8 +94,6 @@ impl StellarOverlayConnection { ) -> Result { log::info!("connect(): Connecting to: {conn_info:?}"); - let retries = conn_info.retries; - let timeout_in_secs = conn_info.timeout_in_secs; // split the stream for easy handling of read and write let (rd, wr) = create_stream(&conn_info.address()).await?; // ------------------ prepare the channels @@ -110,10 +114,13 @@ impl StellarOverlayConnection { actions_sender.clone(), relay_message_sender, ); - // start the receiving_service - tokio::spawn(receiving_service(rd, actions_sender.clone(), timeout_in_secs, retries)); + // run the connector communication tokio::spawn(connection_handler(connector, actions_receiver, wr)); + + // start the receiving_service + tokio::spawn(receiving_service(rd, actions_sender.clone())); + // start the handshake actions_sender.send(ConnectorActions::SendHello).await?; Ok(overlay_connection) diff --git a/clients/stellar-relay-lib/src/connection/services.rs b/clients/stellar-relay-lib/src/connection/services.rs index dd503b9fa..db83af08c 100644 --- a/clients/stellar-relay-lib/src/connection/services.rs +++ b/clients/stellar-relay-lib/src/connection/services.rs @@ -151,11 +151,7 @@ async fn read_message( pub(crate) async fn receiving_service( mut r_stream: tcp::OwnedReadHalf, actions_sender: mpsc::Sender, - timeout_in_secs: u64, - retries: u8, ) { - let mut retry = 0; - let mut retry_read = 0; let mut proc_id = 0; // holds the number of bytes that were missing from the previous stellar message. @@ -167,19 +163,12 @@ pub(crate) async fn receiving_service( // check whether or not we should read the bytes as: // 1. the length of the next stellar message // 2. the remaining bytes of the previous stellar message - match timeout(Duration::from_secs(timeout_in_secs), r_stream.peek(&mut buff_for_peeking)) - .await - { - Ok(Ok(0)) => { - if retry_read >= retries { - log::error!("receiving_service(): proc_id: {proc_id}. Failed to read messages from the stream. Received 0 size more than {retries} times"); - return - } - retry_read += 1; + match r_stream.peek(&mut buff_for_peeking).await { + Ok(0) => { + log::error!("receiving_service(): proc_id: {proc_id}. Failed to read messages from the stream. Received 0 size"); + break }, - Ok(Ok(_)) if lack_bytes_from_prev == 0 => { - retry = 0; - retry_read = 0; + Ok(_) if lack_bytes_from_prev == 0 => { // if there are no more bytes lacking from the previous message, // then check the size of next stellar message. // If it's not enough, skip it. @@ -195,53 +184,49 @@ pub(crate) async fn receiving_service( // let's start reading the actual stellar message. readbuf = vec![0; expect_msg_len]; - log_error!( - read_message( - &mut r_stream, - &actions_sender, - &mut lack_bytes_from_prev, - &mut proc_id, - &mut readbuf, - expect_msg_len, - ) - .await, - format!("receiving_service(): proc_id: {proc_id}. Failed to read message") - ); + if let Err(e) = read_message( + &mut r_stream, + &actions_sender, + &mut lack_bytes_from_prev, + &mut proc_id, + &mut readbuf, + expect_msg_len, + ) + .await + { + log::error!( + "receiving_service(): proc_id: {proc_id}. Failed to read message: {e:?}" + ); + break + } }, - - Ok(Ok(_)) => { - retry = 0; - retry_read = 0; + Ok(_) => { // let's read the continuation number of bytes from the previous message. - log_error!( - read_unfinished_message( - &mut r_stream, - &actions_sender, - &mut lack_bytes_from_prev, - &mut proc_id, - &mut readbuf, - ).await, - format!("receiving_service(): proc_id:{proc_id}. Error occurred while reading unfinished stellar message") - ); - }, - Ok(Err(e)) => { - log::error!("receiving_service(): proc_id: {proc_id}. Error occurred while reading the stream: {e:?}"); - return - }, - Err(elapsed) => { - log::error!( - "receiving_service(): proc_id: {proc_id}. Timeout of {} seconds elapsed for reading messages from Stellar Node. Retry: #{retry}", - elapsed.to_string() - ); - - if retry >= retries { - log::error!("receiving_service(): proc_id: {proc_id}. Exhausted maximum retries for reading messages from Stellar Node."); - return + if let Err(e) = read_unfinished_message( + &mut r_stream, + &actions_sender, + &mut lack_bytes_from_prev, + &mut proc_id, + &mut readbuf, + ) + .await + { + log::error!( + "receiving_service(): proc_id: {proc_id}. Failed to read message: {e:?}" + ); + break } - retry += 1; + }, + Err(e) => { + log::error!("receiving_service(): proc_id: {proc_id}. Failed to read messages from the stream: {e:?}"); + break }, } } + + // stop the connection if stream does not return anything. + drop(r_stream); + return } async fn _write_to_stream( @@ -289,6 +274,7 @@ async fn _connection_handler( Ok(()) } + /// Handles actions for the connection. /// # Arguments /// * `connector` - the Connector that would send/handle messages to/from Stellar Node @@ -299,16 +285,12 @@ pub(crate) async fn connection_handler( mut actions_receiver: mpsc::Receiver, mut w_stream: tcp::OwnedWriteHalf, ) { - let mut retry = 0; - loop { match timeout(Duration::from_secs(connector.timeout_in_secs), actions_receiver.recv()).await { Ok(Some(ConnectorActions::Disconnect)) => { - log_error!( - w_stream.shutdown().await, - format!("connection_handler(): Failed to shutdown write half of stream:") - ); + log::info!("connection_handler(): Disconnecting...."); + drop(w_stream); drop(connector); drop(actions_receiver); return @@ -330,20 +312,19 @@ pub(crate) async fn connection_handler( log::warn!("connection_handler(): Unexpected empty response from receiver"); }, - Err(elapsed) => { + Err(_) => { log::error!( - "connection_handler(): Timeout of {} seconds elapsed for reading messages from Stellar Node. Retry: #{retry}", - elapsed.to_string() + "connection_handler(): Timeout elapsed for reading messages from Stellar Node.", ); + drop(w_stream); - if retry >= connector.retries { - log_error!( + log_error!( connector.send_to_user(StellarRelayMessage::Timeout).await, format!("connection_handler(): Exhausted maximum retries for receiving any actions from receiver.") ); - return - } - retry += 1; + drop(connector); + drop(actions_receiver); + return }, } } From 6be14854390f66bb9ee339e05b945e378b98b859 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Mon, 27 Nov 2023 20:40:43 +0800 Subject: [PATCH 02/22] drop stream and connector before returning --- .../src/connection/helper.rs | 13 ------------- .../src/connection/overlay_connection.rs | 13 +++++++------ .../src/connection/services.rs | 19 +++++++++---------- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/clients/stellar-relay-lib/src/connection/helper.rs b/clients/stellar-relay-lib/src/connection/helper.rs index 07e033362..57f9877b7 100644 --- a/clients/stellar-relay-lib/src/connection/helper.rs +++ b/clients/stellar-relay-lib/src/connection/helper.rs @@ -6,19 +6,6 @@ use substrate_stellar_sdk::{ SecretKey, XdrCodec, }; -/// a helpful macro to log an error (if it occurs) and return immediately. -macro_rules! log_error { - // expression, return value, extra log - ($res:expr, $log:expr) => { - if let Err(e) = $res { - log::error!("{:?}: {e:?}", $log); - return - } - }; -} - -pub(crate) use log_error; - /// Returns a new BigNumber with a pseudo-random value equal to or greater than 0 and less than 1. pub fn generate_random_nonce() -> Uint256 { let mut rng = rand::thread_rng(); diff --git a/clients/stellar-relay-lib/src/connection/overlay_connection.rs b/clients/stellar-relay-lib/src/connection/overlay_connection.rs index 57cd4f7e4..6b65b2b8b 100644 --- a/clients/stellar-relay-lib/src/connection/overlay_connection.rs +++ b/clients/stellar-relay-lib/src/connection/overlay_connection.rs @@ -49,9 +49,10 @@ impl StellarOverlayConnection { match &res { Some(StellarRelayMessage::Timeout) | Some(StellarRelayMessage::Error(_)) | None => - // we want to keep reconnecting until it succeeds. + // we want to keep reconnecting until it succeeds. loop { - // Delaying reconnection gives time for the write and read streams to be dropped. + // Delaying reconnection gives time for the write and read streams to be + // dropped. let timeout = self.conn_info.timeout_in_secs; log::info!( "listen(): Reconnecting to {:?} in {timeout} seconds", @@ -63,15 +64,15 @@ impl StellarOverlayConnection { self.local_node.clone(), self.conn_info.clone(), ) - .await + .await { Ok(new_user) => { self.actions_sender = new_user.actions_sender; self.relay_message_receiver = new_user.relay_message_receiver; log::info!( - "listen(): overlay connection reconnected to {:?}", - &self.conn_info.address - ); + "listen(): overlay connection reconnected to {:?}", + &self.conn_info.address + ); // break out of the loop since connection was successful. return self.relay_message_receiver.recv().await }, diff --git a/clients/stellar-relay-lib/src/connection/services.rs b/clients/stellar-relay-lib/src/connection/services.rs index db83af08c..df6f0dff5 100644 --- a/clients/stellar-relay-lib/src/connection/services.rs +++ b/clients/stellar-relay-lib/src/connection/services.rs @@ -2,7 +2,6 @@ use crate::{ connection::{ connector::{Connector, ConnectorActions}, helper::{time_now, to_base64_xdr_string}, - log_error, xdr_converter::get_xdr_message_length, }, Error, StellarRelayMessage, @@ -274,7 +273,6 @@ async fn _connection_handler( Ok(()) } - /// Handles actions for the connection. /// # Arguments /// * `connector` - the Connector that would send/handle messages to/from Stellar Node @@ -300,10 +298,12 @@ pub(crate) async fn connection_handler( if let Err(e) = _connection_handler(action, &mut connector, &mut w_stream).await { log::error!("connection_handler(): {e:?}"); - log_error!( - connector.send_to_user(StellarRelayMessage::Error(e.to_string())).await, - format!("connection_handler(): sending error message") - ); + drop(w_stream); + if let Err(e) = connector.send_to_user(StellarRelayMessage::Timeout).await { + log::error!("connection_handler(): failed to send timeout message"); + } + drop(connector); + drop(actions_receiver); return } }, @@ -318,10 +318,9 @@ pub(crate) async fn connection_handler( ); drop(w_stream); - log_error!( - connector.send_to_user(StellarRelayMessage::Timeout).await, - format!("connection_handler(): Exhausted maximum retries for receiving any actions from receiver.") - ); + if let Err(e) = connector.send_to_user(StellarRelayMessage::Timeout).await { + log::error!("connection_handler(): failed to send timeout message"); + } drop(connector); drop(actions_receiver); return From c40a5d01d7bc2830088bc2458c420536006f287c Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Tue, 28 Nov 2023 00:02:34 +0800 Subject: [PATCH 03/22] remove timeout; introduce "disconnect" message --- clients/stellar-relay-lib/examples/connect.rs | 2 +- .../src/connection/connector/connector.rs | 4 +-- .../stellar-relay-lib/src/connection/mod.rs | 3 +-- .../src/connection/overlay_connection.rs | 25 ++++++++++++++----- .../src/connection/services.rs | 17 ++++++++++--- clients/vault/src/oracle/agent.rs | 3 --- 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/clients/stellar-relay-lib/examples/connect.rs b/clients/stellar-relay-lib/examples/connect.rs index 0de4b5c2c..a33f3903d 100644 --- a/clients/stellar-relay-lib/examples/connect.rs +++ b/clients/stellar-relay-lib/examples/connect.rs @@ -61,7 +61,7 @@ async fn main() -> Result<(), Box> { StellarRelayMessage::Error(e) => { log::error!("Error: {:?}", e); }, - StellarRelayMessage::Timeout => { + StellarRelayMessage::Disconnect => { log::error!("timed out"); }, } diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index b890472aa..e2c2bd3c9 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -415,14 +415,14 @@ mod test { async fn connector_send_to_user_works() { let (_, _, connector, _, mut message_receiver) = create_connector(); - let message = StellarRelayMessage::Timeout; + let message = StellarRelayMessage::Error("test".to_string()); connector.send_to_user(message).await.unwrap(); let received_message = message_receiver.recv().await; assert!(received_message.is_some()); let message = received_message.unwrap(); match message { - StellarRelayMessage::Timeout => {}, + StellarRelayMessage::Error(_) => {}, _ => { panic!("Incorrect message received!!!") }, diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index 29c1c3c17..241b5ed49 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -40,8 +40,7 @@ pub enum StellarRelayMessage { msg: Box, }, Error(String), - /// The amount of time to wait for Stellar Node messages - Timeout, + Disconnect, } /// Config for connecting to Stellar Node diff --git a/clients/stellar-relay-lib/src/connection/overlay_connection.rs b/clients/stellar-relay-lib/src/connection/overlay_connection.rs index 6b65b2b8b..db65fdfd3 100644 --- a/clients/stellar-relay-lib/src/connection/overlay_connection.rs +++ b/clients/stellar-relay-lib/src/connection/overlay_connection.rs @@ -7,7 +7,7 @@ use crate::{ ConnectionInfo, Connector, Error, StellarRelayMessage, }; use substrate_stellar_sdk::types::StellarMessage; -use tokio::{sync::mpsc, time::Duration}; +use tokio::{sync::mpsc, task::JoinHandle, time::Duration}; pub struct StellarOverlayConnection { /// This is when we want to send stellar messages @@ -16,6 +16,7 @@ pub struct StellarOverlayConnection { relay_message_receiver: mpsc::Receiver, local_node: NodeInfo, conn_info: ConnectionInfo, + handles: Vec>, } impl StellarOverlayConnection { @@ -25,7 +26,13 @@ impl StellarOverlayConnection { local_node: NodeInfo, conn_info: ConnectionInfo, ) -> Self { - StellarOverlayConnection { actions_sender, relay_message_receiver, local_node, conn_info } + StellarOverlayConnection { + actions_sender, + relay_message_receiver, + local_node, + conn_info, + handles: vec![], + } } pub async fn send(&self, message: StellarMessage) -> Result<(), Error> { @@ -48,7 +55,7 @@ impl StellarOverlayConnection { let res = self.relay_message_receiver.recv().await; match &res { - Some(StellarRelayMessage::Timeout) | Some(StellarRelayMessage::Error(_)) | None => + Some(StellarRelayMessage::Error(_)) | None => // we want to keep reconnecting until it succeeds. loop { // Delaying reconnection gives time for the write and read streams to be @@ -82,6 +89,11 @@ impl StellarOverlayConnection { }, }; }, + Some(StellarRelayMessage::Disconnect) => + for handle in self.handles.iter() { + log::info!("listen(): thread {}", handle.is_finished()); + drop(handle); + }, _ => {}, } res @@ -103,7 +115,7 @@ impl StellarOverlayConnection { // this is a channel to communicate with the user/caller. let (relay_message_sender, relay_message_receiver) = mpsc::channel::(1024); - let overlay_connection = StellarOverlayConnection::new( + let mut overlay_connection = StellarOverlayConnection::new( actions_sender.clone(), relay_message_receiver, local_node, @@ -117,11 +129,12 @@ impl StellarOverlayConnection { ); // run the connector communication - tokio::spawn(connection_handler(connector, actions_receiver, wr)); + let x = tokio::spawn(connection_handler(connector, actions_receiver, wr)); // start the receiving_service - tokio::spawn(receiving_service(rd, actions_sender.clone())); + let y = tokio::spawn(receiving_service(rd, actions_sender.clone())); + overlay_connection.handles = vec![x, y]; // start the handshake actions_sender.send(ConnectorActions::SendHello).await?; Ok(overlay_connection) diff --git a/clients/stellar-relay-lib/src/connection/services.rs b/clients/stellar-relay-lib/src/connection/services.rs index df6f0dff5..70aa5a241 100644 --- a/clients/stellar-relay-lib/src/connection/services.rs +++ b/clients/stellar-relay-lib/src/connection/services.rs @@ -289,6 +289,11 @@ pub(crate) async fn connection_handler( Ok(Some(ConnectorActions::Disconnect)) => { log::info!("connection_handler(): Disconnecting...."); drop(w_stream); + + if let Err(e) = connector.send_to_user(StellarRelayMessage::Disconnect).await { + log::error!("connection_handler(): failed to send Disconnect message: {e:?}"); + } + drop(connector); drop(actions_receiver); return @@ -299,8 +304,10 @@ pub(crate) async fn connection_handler( log::error!("connection_handler(): {e:?}"); drop(w_stream); - if let Err(e) = connector.send_to_user(StellarRelayMessage::Timeout).await { - log::error!("connection_handler(): failed to send timeout message"); + if let Err(e) = + connector.send_to_user(StellarRelayMessage::Error(e.to_string())).await + { + log::error!("connection_handler(): failed to send error message: {e:?}"); } drop(connector); drop(actions_receiver); @@ -318,8 +325,10 @@ pub(crate) async fn connection_handler( ); drop(w_stream); - if let Err(e) = connector.send_to_user(StellarRelayMessage::Timeout).await { - log::error!("connection_handler(): failed to send timeout message"); + if let Err(e) = + connector.send_to_user(StellarRelayMessage::Error("Timeout".to_string())).await + { + log::error!("connection_handler(): failed to send timeout message: {e:?}"); } drop(connector); drop(actions_receiver); diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index b49da4948..090da1533 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -60,9 +60,6 @@ async fn handle_message( tracing::info!("handle_message(): Connected: via public key: {pub_key}"); tracing::info!("handle_message(): Connected: with {:#?}", node_info) }, - StellarRelayMessage::Timeout => { - tracing::error!("handle_message(): The Stellar Relay timed out. Failed to process message: {message:?}"); - }, _ => {}, } From c192eb086f1f49fb1fda5abfe30d7958fa35234c Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Wed, 29 Nov 2023 21:15:41 +0800 Subject: [PATCH 04/22] new connection change --- clients/stellar-relay-lib/examples/connect.rs | 51 ++- clients/stellar-relay-lib/src/config.rs | 8 +- .../connection/authentication/certificate.rs | 4 +- .../src/connection/authentication/tests.rs | 4 +- .../connection/connector/message_creation.rs | 5 +- .../connection/connector/message_handler.rs | 275 +++++++------- .../connection/connector/message_reader.rs | 199 ++++++++++ .../stellar-relay-lib/src/connection/error.rs | 5 + .../src/connection/handshake.rs | 2 +- .../stellar-relay-lib/src/connection/mod.rs | 22 +- .../src/connection/overlay_connection.rs | 281 --------------- .../src/connection/services.rs | 339 ------------------ .../src/connection/xdr_converter.rs | 2 +- clients/stellar-relay-lib/src/lib.rs | 83 ++++- clients/stellar-relay-lib/src/node/local.rs | 1 + clients/stellar-relay-lib/src/tests/mod.rs | 122 +++---- clients/vault/src/oracle/agent.rs | 100 ++++-- 17 files changed, 565 insertions(+), 938 deletions(-) create mode 100644 clients/stellar-relay-lib/src/connection/connector/message_reader.rs delete mode 100644 clients/stellar-relay-lib/src/connection/overlay_connection.rs delete mode 100644 clients/stellar-relay-lib/src/connection/services.rs diff --git a/clients/stellar-relay-lib/examples/connect.rs b/clients/stellar-relay-lib/examples/connect.rs index a33f3903d..7803f0ee9 100644 --- a/clients/stellar-relay-lib/examples/connect.rs +++ b/clients/stellar-relay-lib/examples/connect.rs @@ -1,7 +1,8 @@ use stellar_relay_lib::{ connect_to_stellar_overlay_network, sdk::types::{ScpStatementPledges, StellarMessage}, - StellarOverlayConfig, StellarRelayMessage, + StellarOverlayConfig, + helper::to_base64_xdr_string }; #[tokio::main] @@ -27,42 +28,32 @@ async fn main() -> Result<(), Box> { let mut overlay_connection = connect_to_stellar_overlay_network(cfg, &secret_key).await?; - while let Some(relay_message) = overlay_connection.listen().await { - match relay_message { - StellarRelayMessage::Connect { pub_key, node_info } => { - let pub_key = pub_key.to_encoding(); - let pub_key = std::str::from_utf8(&pub_key).expect("should work?"); - log::info!("Connected to Stellar Node: {pub_key}"); - log::info!("{:?}", node_info); - }, - StellarRelayMessage::Data { p_id: _, msg_type, msg } => match *msg { - StellarMessage::ScpMessage(msg) => { - let node_id = msg.statement.node_id.to_encoding(); - let node_id = base64::encode(&node_id); - let slot = msg.statement.slot_index; + let mut counter = 0; + while let Some(msg) = overlay_connection.listen().await { + counter+=1; + + match msg { + StellarMessage::ScpMessage(msg) => { + let node_id = msg.statement.node_id.to_encoding(); + let node_id = base64::encode(&node_id); + let slot = msg.statement.slot_index; - let stmt_type = match msg.statement.pledges { - ScpStatementPledges::ScpStPrepare(_) => "ScpStPrepare", - ScpStatementPledges::ScpStConfirm(_) => "ScpStConfirm", - ScpStatementPledges::ScpStExternalize(_) => "ScpStExternalize", - ScpStatementPledges::ScpStNominate(_) => "ScpStNominate ", - }; - log::info!( + let stmt_type = match msg.statement.pledges { + ScpStatementPledges::ScpStPrepare(_) => "ScpStPrepare", + ScpStatementPledges::ScpStConfirm(_) => "ScpStConfirm", + ScpStatementPledges::ScpStExternalize(_) => "ScpStExternalize", + ScpStatementPledges::ScpStNominate(_) => "ScpStNominate ", + }; + log::info!( "{} sent StellarMessage of type {} for ledger {}", node_id, stmt_type, slot ); - }, - _ => { - log::info!("rcv StellarMessage of type: {:?}", msg_type); - }, - }, - StellarRelayMessage::Error(e) => { - log::error!("Error: {:?}", e); }, - StellarRelayMessage::Disconnect => { - log::error!("timed out"); + other => { + let msg = StellarMessage::GetPeers; + let _ = overlay_connection.send_to_node(StellarMessage::GetPeers).await; }, } } diff --git a/clients/stellar-relay-lib/src/config.rs b/clients/stellar-relay-lib/src/config.rs index ae369ca0d..d85ebb14b 100644 --- a/clients/stellar-relay-lib/src/config.rs +++ b/clients/stellar-relay-lib/src/config.rs @@ -1,8 +1,10 @@ -use crate::{connection::Error, node::NodeInfo, ConnectionInfo, StellarOverlayConnection}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, BytesOrString}; use std::fmt::Debug; use substrate_stellar_sdk::SecretKey; +use crate::connection::{ConnectionInfo, Error}; +use crate::node::NodeInfo; +use crate::StellarOverlayConnection; /// The configuration structure of the StellarOverlay. /// It configures both the ConnectionInfo and the NodeInfo. @@ -42,7 +44,7 @@ impl StellarOverlayConfig { let public_key = secret_key.get_public().to_encoding(); let public_key = std::str::from_utf8(&public_key).unwrap(); log::info!( - "connection_info(): Connected to Stellar overlay network with public key: {public_key}" + "connection_info(): Connecting to Stellar overlay network using public key: {public_key}" ); let address = std::str::from_utf8(&cfg.address) @@ -129,7 +131,7 @@ impl ConnectionInfoCfg { } /// Triggers connection to the Stellar Node. -/// Returns the `StellarOverlayConnection` if connection is a success, otherwise an Error +/// Returns the `StellarStellarOverlayConnection` if connection is a success, otherwise an Error pub async fn connect_to_stellar_overlay_network( cfg: StellarOverlayConfig, secret_key: &str, diff --git a/clients/stellar-relay-lib/src/connection/authentication/certificate.rs b/clients/stellar-relay-lib/src/connection/authentication/certificate.rs index b454d458c..d3f69b60c 100644 --- a/clients/stellar-relay-lib/src/connection/authentication/certificate.rs +++ b/clients/stellar-relay-lib/src/connection/authentication/certificate.rs @@ -1,6 +1,6 @@ use crate::{ - connection::authentication::{BinarySha256Hash, ConnectionAuth, AUTH_CERT_EXPIRATION_LIMIT}, - Error, + connection::authentication::{BinarySha256Hash, ConnectionAuth, AUTH_CERT_EXPIRATION_LIMIT}, + Error, }; use sha2::{Digest, Sha256}; use substrate_stellar_sdk::{ diff --git a/clients/stellar-relay-lib/src/connection/authentication/tests.rs b/clients/stellar-relay-lib/src/connection/authentication/tests.rs index af1db5a29..5dbcbd760 100644 --- a/clients/stellar-relay-lib/src/connection/authentication/tests.rs +++ b/clients/stellar-relay-lib/src/connection/authentication/tests.rs @@ -1,5 +1,5 @@ use crate::{ - connection::{ + connection::{ authentication::{ certificate::{create_auth_cert, verify_remote_auth_cert}, shared_key::gen_shared_key, @@ -8,7 +8,7 @@ use crate::{ helper::{generate_random_nonce, time_now}, hmac::{create_receiving_mac_key, create_sending_mac_key, create_sha256_hmac, verify_hmac}, }, - Error, + Error, }; use substrate_stellar_sdk::{ network::Network, diff --git a/clients/stellar-relay-lib/src/connection/connector/message_creation.rs b/clients/stellar-relay-lib/src/connection/connector/message_creation.rs index 8e59b5325..73b6a773e 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_creation.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_creation.rs @@ -1,6 +1,5 @@ use crate::{ - connection::{authentication::create_auth_cert, handshake, hmac::create_sha256_hmac}, - xdr_converter, Connector, Error, + connection::{authentication::create_auth_cert, handshake, hmac::create_sha256_hmac, Connector, Error, xdr_converter}, }; use substrate_stellar_sdk::{ types::{AuthenticatedMessage, AuthenticatedMessageV0, HmacSha256Mac, StellarMessage}, @@ -74,4 +73,4 @@ impl Connector { local.node(), ) } -} +} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs index 3a71461c9..7230ce6d7 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs @@ -1,151 +1,138 @@ -use crate::{ - connection::{ - authentication::verify_remote_auth_cert, error_to_string, helper::time_now, hmac::HMacKeys, - xdr_converter::parse_authenticated_message, Connector, Xdr, - }, - node::RemoteInfo, - Error, StellarRelayMessage, -}; -use substrate_stellar_sdk::{ - types::{Hello, MessageType, StellarMessage}, - XdrCodec, -}; +use substrate_stellar_sdk::types::{Hello, MessageType, StellarMessage}; +use substrate_stellar_sdk::XdrCodec; + +use crate::connection::{Connector, Xdr, Error, helper::{error_to_string, time_now}, xdr_converter::parse_authenticated_message}; +use crate::connection::authentication::verify_remote_auth_cert; +use crate::connection::hmac::HMacKeys; + +use crate::node::RemoteInfo; impl Connector { - /// Processes the raw bytes from the stream - pub(crate) async fn process_raw_message(&mut self, xdr: Xdr) -> Result<(), Error> { - let (proc_id, data) = xdr; - let (auth_msg, msg_type) = parse_authenticated_message(&data)?; - - match msg_type { - MessageType::Transaction | MessageType::FloodAdvert if !self.receive_tx_messages() => { - self.increment_remote_sequence()?; - self.check_to_send_more(MessageType::Transaction).await?; - }, - - MessageType::ScpMessage if !self.receive_scp_messages() => { - self.increment_remote_sequence()?; - }, - - MessageType::ErrorMsg => match auth_msg.message { - StellarMessage::ErrorMsg(e) => { - log::error!( - "process_raw_message(): Received ErrorMsg: {}", + /// Processes the raw bytes from the stream + pub(super) async fn process_raw_message(&mut self, data: Xdr) -> Result, Error> { + let (auth_msg, msg_type) = parse_authenticated_message(&data)?; + + match msg_type { + MessageType::Transaction | MessageType::FloodAdvert if !self.receive_tx_messages() => { + self.increment_remote_sequence()?; + self.check_to_send_more(MessageType::Transaction).await?; + }, + + MessageType::ScpMessage if !self.receive_scp_messages() => { + self.increment_remote_sequence()?; + }, + + MessageType::ErrorMsg => match auth_msg.message { + StellarMessage::ErrorMsg(e) => { + log::error!( + "process_raw_message(): Received ErrorMsg during authentication: {}", error_to_string(e.clone()) ); - return Err(Error::OverlayError(e.code)) - }, - other => log::error!("process_raw_message(): Received ErroMsg other: {:?}", other), - }, - - _ => { - // we only verify the authenticated message when a handshake has been done. - if self.is_handshake_created() { - self.verify_auth(&auth_msg, &data[4..(data.len() - 32)])?; - self.increment_remote_sequence()?; - log::trace!( - "process_raw_message(): proc_id: {proc_id}. Processing {msg_type:?} message: auth verified" + return Err(Error::OverlayError(e.code)) + }, + other => log::error!("process_raw_message(): Received ErroMsg during authentication: {:?}", other), + }, + + _ => { + // we only verify the authenticated message when a handshake has been done. + if self.is_handshake_created() { + self.verify_auth(&auth_msg, &data[4..(data.len() - 32)])?; + self.increment_remote_sequence()?; + log::trace!( + "process_raw_message(): Processing {msg_type:?} message: auth verified" ); - } - - self.process_stellar_message(proc_id, auth_msg.message, msg_type).await?; - }, - } - Ok(()) - } - - /// Handles what to do next with the Stellar message. Mostly it will be sent back to the user - async fn process_stellar_message( - &mut self, - p_id: u32, - msg: StellarMessage, - msg_type: MessageType, - ) -> Result<(), Error> { - match msg { - StellarMessage::Hello(hello) => { - // update the node info based on the hello message - self.process_hello_message(hello)?; - - self.got_hello(); - - if self.remote_called_us() { - self.send_hello_message().await?; - } else { - self.send_auth_message().await?; - } - log::info!("process_stellar_message(): Hello message processed successfully"); - }, - - StellarMessage::Auth(_) => { - self.process_auth_message().await?; - }, - - StellarMessage::ErrorMsg(e) => { - self.send_to_user(StellarRelayMessage::Error(error_to_string(e))).await?; - }, - - other => { - log::trace!( - "process_stellar_message(): proc_id: {p_id}. Processing {msg_type:?} message: received from overlay" + } + + return self.process_stellar_message( auth_msg.message, msg_type).await; + }, + } + Ok(None) + } + + /// Returns a StellarMessage for the user/outsider. Else none if user/outsider do not need it. + /// This handles what to do with the Stellar message. + async fn process_stellar_message( + &mut self, + msg: StellarMessage, + msg_type: MessageType, + ) -> Result, Error> { + match msg { + StellarMessage::Hello(hello) => { + // update the node info based on the hello message + self.process_hello_message(hello)?; + + self.got_hello(); + + if self.remote_called_us() { + self.send_hello_message().await?; + } else { + self.send_auth_message().await?; + } + log::info!("process_stellar_message(): Hello message processed successfully"); + }, + + StellarMessage::Auth(_) => { + self.process_auth_message().await?; + }, + + StellarMessage::ErrorMsg(e) => { + log::error!("process_stellar_message(): received from overlay: {e:?}"); + return Ok(Some(StellarMessage::ErrorMsg(e))); + // self.send_to_user(StellarMessage::ErrorMsg(e)).await?; + }, + + other => { + log::trace!( + "process_stellar_message(): Processing {other:?} message: received from overlay" ); - self.send_to_user(StellarRelayMessage::Data { - p_id, - msg_type, - msg: Box::new(other), - }) - .await?; - self.check_to_send_more(msg_type).await?; - }, - } - Ok(()) - } - - async fn process_auth_message(&mut self) -> Result<(), Error> { - if self.remote_called_us() { - self.send_auth_message().await?; - } - - self.handshake_completed(); - - if let Some(remote) = self.remote() { - log::debug!("process_auth_message(): sending connect message: {remote:?}"); - self.send_to_user(StellarRelayMessage::Connect { - pub_key: remote.pub_key().clone(), - node_info: remote.node().clone(), - }) - .await?; - - self.enable_flow_controller( - self.local().node().overlay_version, - remote.node().overlay_version, - ); - } else { - log::warn!("process_auth_message(): No remote overlay version after handshake."); - } - - self.check_to_send_more(MessageType::Auth).await - } - - /// Updates the config based on the hello message that was received from the Stellar Node - fn process_hello_message(&mut self, hello: Hello) -> Result<(), Error> { - let mut network_id = self.connection_auth.network_id().to_xdr(); - - if !verify_remote_auth_cert(time_now(), &hello.peer_id, &hello.cert, &mut network_id) { - return Err(Error::AuthCertInvalid) - } - - let remote_info = RemoteInfo::new(&hello); - let shared_key = self.get_shared_key(remote_info.pub_key_ecdh()); - - self.set_hmac_keys(HMacKeys::new( - &shared_key, - self.local().nonce(), - remote_info.nonce(), - self.remote_called_us(), - )); - - self.set_remote(remote_info); - - Ok(()) - } + self.check_to_send_more(msg_type).await?; + return Ok(Some(other)); + }, + } + + Ok(None) + } + + async fn process_auth_message(&mut self) -> Result<(), Error> { + if self.remote_called_us() { + self.send_auth_message().await?; + } + + self.handshake_completed(); + + if let Some(remote) = self.remote() { + log::debug!("process_auth_message(): sending connect message: {remote:?}"); + self.enable_flow_controller( + self.local().node().overlay_version, + remote.node().overlay_version, + ); + } else { + log::warn!("process_auth_message(): No remote overlay version after handshake."); + } + + self.check_to_send_more(MessageType::Auth).await + } + + /// Updates the config based on the hello message that was received from the Stellar Node + fn process_hello_message(&mut self, hello: Hello) -> Result<(), Error> { + let mut network_id = self.connection_auth.network_id().to_xdr(); + + if !verify_remote_auth_cert(time_now(), &hello.peer_id, &hello.cert, &mut network_id) { + return Err(Error::AuthCertInvalid) + } + + let remote_info = RemoteInfo::new(&hello); + let shared_key = self.get_shared_key(remote_info.pub_key_ecdh()); + + self.set_hmac_keys(HMacKeys::new( + &shared_key, + self.local().nonce(), + remote_info.nonce(), + self.remote_called_us(), + )); + + self.set_remote(remote_info); + + Ok(()) + } } diff --git a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs new file mode 100644 index 000000000..f25420cb7 --- /dev/null +++ b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs @@ -0,0 +1,199 @@ +use std::time::Duration; +use crate::{ + connection::{ + authentication::verify_remote_auth_cert, error_to_string, helper::time_now, hmac::HMacKeys, + xdr_converter::{parse_authenticated_message, get_xdr_message_length}, Connector, Xdr,Error + }, + node::RemoteInfo +}; +use substrate_stellar_sdk::{ + types::{Hello, MessageType, StellarMessage}, + XdrCodec, +}; +use substrate_stellar_sdk::types::ErrorCode; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::tcp; +use tokio::sync::mpsc; +use tokio::time::timeout; +use crate::connection::to_base64_xdr_string; + + +/// Returns Xdr format of the `StellarMessage` sent from the Stellar Node +pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, timeout_in_secs:u64) -> Option { + let mut proc_id = 0; + + // holds the number of bytes that were missing from the previous stellar message. + let mut lack_bytes_from_prev = 0; + let mut readbuf: Vec = vec![]; + + let mut buff_for_peeking = vec![0; 4]; + loop { + // check whether or not we should read the bytes as: + // 1. the length of the next stellar message + // 2. the remaining bytes of the previous stellar message + match timeout(Duration::from_secs(timeout_in_secs), r_stream.peek(&mut buff_for_peeking)) + .await + { + Ok(Ok(0)) => { + log::error!("read_message_from_stellar(): proc_id: {proc_id}. Received 0 size"); + break + } + + Ok(Ok(_)) if lack_bytes_from_prev == 0 => { + // if there are no more bytes lacking from the previous message, + // then check the size of next stellar message. + // If it's not enough, skip it. + let expect_msg_len = next_message_length(r_stream).await; + log::trace!("read_message_from_stellar(): proc_id: {proc_id}. The next message length is {expect_msg_len}"); + + if expect_msg_len == 0 { + // there's nothing to read; wait for the next iteration + log::trace!("read_message_from_stellar(): proc_id: {proc_id}. Nothing left to read; waiting for next loop"); + continue + } + + // let's start reading the actual stellar message. + readbuf = vec![0; expect_msg_len]; + + match read_message( + r_stream, + &mut lack_bytes_from_prev, + &mut proc_id, + &mut readbuf, + expect_msg_len, + ).await { + Ok(None) => continue, + Ok(Some(xdr)) => return Some(xdr), + Err(e) => { + log::error!("read_message_from_stellar(): proc_id: {proc_id}. encountered error: {e:?}"); + break + } + } + } + + Ok(Ok(_)) => { + // let's read the continuation number of bytes from the previous message. + match read_unfinished_message( + r_stream, + &mut lack_bytes_from_prev, + &mut proc_id, + &mut readbuf, + ).await { + Ok(None) => continue, + Ok(Some(xdr)) => return Some(xdr), + Err(e) => { + log::error!("read_message_from_stellar(): proc_id: {proc_id}. encountered error: {e:?}"); + break + } + } + } + Ok(Err(e)) => { + log::error!("read_message_from_stellar(): proc_id: {proc_id}. encountered error: {e:?}"); + break + } + + Err(_) => { + log::error!("read_message_from_stellar(): proc_id: {proc_id}. timeout elapsed."); + break + } + } + } + + None +} + + +/// Returns Xdr when all bytes from the stream have successfully been converted; else None. +/// This reads a number of bytes based on the expected message length. +/// +/// # Arguments +/// * `r_stream` - the read stream for reading the xdr stellar message +/// * `lack_bytes_from_prev` - the number of bytes remaining, to complete the previous message +/// * `proc_id` - the process id, used for tracing. +/// * `readbuf` - the buffer that holds the bytes of the previous and incomplete message +/// * `xpect_msg_len` - the expected # of bytes of the Stellar message +async fn read_message( + r_stream: &mut tcp::OwnedReadHalf, + lack_bytes_from_prev: &mut usize, + proc_id: &mut u32, + readbuf: &mut Vec, + xpect_msg_len: usize, +) -> Result, Error> { + let actual_msg_len = read_stream(r_stream, readbuf).await?; + + // only when the message has the exact expected size bytes, should we send to user. + if actual_msg_len == xpect_msg_len { + return Ok(Some(readbuf.clone())) + } + + // The next bytes are remnants from the previous stellar message. + // save it and read it on the next loop. + *lack_bytes_from_prev = xpect_msg_len - actual_msg_len; + *readbuf = readbuf[0..actual_msg_len].to_owned(); + log::trace!( + "read_message(): proc_id: {} received only partial message. Need {} bytes to complete.", + proc_id, + lack_bytes_from_prev + ); + + Ok(None) +} + +/// Returns Xdr when all bytes from the stream have successfully been converted; else None. +/// Reads a continuation of bytes that belong to the previous message +/// +/// # Arguments +/// * `r_stream` - the read stream for reading the xdr stellar message +/// * `lack_bytes_from_prev` - the number of bytes remaining, to complete the previous message +/// * `proc_id` - the process id, used for tracing. +/// * `readbuf` - the buffer that holds the bytes of the previous and incomplete message +async fn read_unfinished_message( + r_stream: &mut tcp::OwnedReadHalf, + lack_bytes_from_prev: &mut usize, + proc_id: &mut u32, + readbuf: &mut Vec, +) -> Result, Error> { + // let's read the continuation number of bytes from the previous message. + let mut cont_buf = vec![0; *lack_bytes_from_prev]; + + let actual_msg_len = read_stream(r_stream, &mut cont_buf).await?; + + // this partial message completes the previous message. + if actual_msg_len == *lack_bytes_from_prev { + log::trace!("read_unfinished_message(): proc_id: {} received continuation from the previous message.", proc_id); + readbuf.append(&mut cont_buf); + + return Ok(Some(readbuf.clone())); + } + + // this partial message is not enough to complete the previous message. + if actual_msg_len > 0 { + *lack_bytes_from_prev -= actual_msg_len; + cont_buf = cont_buf[0..actual_msg_len].to_owned(); + readbuf.append(&mut cont_buf); + log::trace!( + "read_unfinished_message(): proc_id: {} not enough bytes to complete the previous message. Need {} bytes to complete.", + proc_id, + lack_bytes_from_prev + ); + } + + Ok(None) +} + +/// checks the length of the next stellar message. +async fn next_message_length(r_stream: &mut tcp::OwnedReadHalf) -> usize { + // let's check for messages. + let mut sizebuf = [0; 4]; + + if r_stream.read(&mut sizebuf).await.unwrap_or(0) == 0 { + return 0 + } + + get_xdr_message_length(&sizebuf) +} + +/// reads data from the stream and store to buffer +async fn read_stream(r_stream: &mut tcp::OwnedReadHalf, buffer: &mut [u8]) -> Result { + r_stream.read(buffer).await.map_err(|e| Error::ReadFailed(e.to_string())) +} diff --git a/clients/stellar-relay-lib/src/connection/error.rs b/clients/stellar-relay-lib/src/connection/error.rs index 9edbf7992..42d67965f 100644 --- a/clients/stellar-relay-lib/src/connection/error.rs +++ b/clients/stellar-relay-lib/src/connection/error.rs @@ -56,6 +56,9 @@ pub enum Error { #[error(display = "Received Error from Overlay: {:?}", _0)] OverlayError(ErrorCode), + + #[error(display = "Timeout elapsed")] + Timeout } impl From for Error { @@ -75,3 +78,5 @@ impl From for Error { Error::StellarSdkError(e) } } + + diff --git a/clients/stellar-relay-lib/src/connection/handshake.rs b/clients/stellar-relay-lib/src/connection/handshake.rs index 017454bee..d82a17b2d 100644 --- a/clients/stellar-relay-lib/src/connection/handshake.rs +++ b/clients/stellar-relay-lib/src/connection/handshake.rs @@ -1,6 +1,6 @@ use substrate_stellar_sdk::compound_types::LimitedString; -use crate::{node::NodeInfo, Error}; +use crate::{node::NodeInfo, connection::Error}; use substrate_stellar_sdk::{ types::{Auth, AuthCert, Hello, StellarMessage, Uint256}, PublicKey, diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index 241b5ed49..f255f7592 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -6,18 +6,15 @@ mod hmac; mod authentication; mod connector; pub mod helper; -mod overlay_connection; -mod services; pub mod xdr_converter; pub(crate) use connector::*; pub use error::Error; pub use helper::*; -pub use overlay_connection::*; use serde::Serialize; use std::fmt::{Debug, Formatter}; -type Xdr = (u32, Vec); +type Xdr = Vec; use crate::node::NodeInfo; use substrate_stellar_sdk::{ @@ -25,23 +22,6 @@ use substrate_stellar_sdk::{ PublicKey, SecretKey, }; -#[derive(Debug)] -/// Represents the messages that the connection creates bases on the Stellar Node -pub enum StellarRelayMessage { - /// Successfully connected to the node - Connect { - pub_key: PublicKey, - node_info: NodeInfo, - }, - /// Stellar messages from the node - Data { - p_id: u32, - msg_type: MessageType, - msg: Box, - }, - Error(String), - Disconnect, -} /// Config for connecting to Stellar Node #[derive(Clone, Serialize, PartialEq, Eq)] diff --git a/clients/stellar-relay-lib/src/connection/overlay_connection.rs b/clients/stellar-relay-lib/src/connection/overlay_connection.rs deleted file mode 100644 index db65fdfd3..000000000 --- a/clients/stellar-relay-lib/src/connection/overlay_connection.rs +++ /dev/null @@ -1,281 +0,0 @@ -use crate::{ - connection::{ - connector::ConnectorActions, - services::{connection_handler, create_stream, receiving_service}, - }, - node::NodeInfo, - ConnectionInfo, Connector, Error, StellarRelayMessage, -}; -use substrate_stellar_sdk::types::StellarMessage; -use tokio::{sync::mpsc, task::JoinHandle, time::Duration}; - -pub struct StellarOverlayConnection { - /// This is when we want to send stellar messages - actions_sender: mpsc::Sender, - /// For receiving stellar messages - relay_message_receiver: mpsc::Receiver, - local_node: NodeInfo, - conn_info: ConnectionInfo, - handles: Vec>, -} - -impl StellarOverlayConnection { - fn new( - actions_sender: mpsc::Sender, - relay_message_receiver: mpsc::Receiver, - local_node: NodeInfo, - conn_info: ConnectionInfo, - ) -> Self { - StellarOverlayConnection { - actions_sender, - relay_message_receiver, - local_node, - conn_info, - handles: vec![], - } - } - - pub async fn send(&self, message: StellarMessage) -> Result<(), Error> { - self.actions_sender - .send(ConnectorActions::SendMessage(Box::new(message))) - .await - .map_err(Error::from) - } - - pub async fn disconnect(&mut self) -> Result<(), Error> { - self.actions_sender - .send(ConnectorActions::Disconnect) - .await - .map_err(Error::from) - } - - /// Receives Stellar messages from the connection. - /// Restarts the connection when lost. - pub async fn listen(&mut self) -> Option { - let res = self.relay_message_receiver.recv().await; - - match &res { - Some(StellarRelayMessage::Error(_)) | None => - // we want to keep reconnecting until it succeeds. - loop { - // Delaying reconnection gives time for the write and read streams to be - // dropped. - let timeout = self.conn_info.timeout_in_secs; - log::info!( - "listen(): Reconnecting to {:?} in {timeout} seconds", - &self.conn_info.address - ); - tokio::time::sleep(Duration::from_secs(timeout)).await; - - match StellarOverlayConnection::connect( - self.local_node.clone(), - self.conn_info.clone(), - ) - .await - { - Ok(new_user) => { - self.actions_sender = new_user.actions_sender; - self.relay_message_receiver = new_user.relay_message_receiver; - log::info!( - "listen(): overlay connection reconnected to {:?}", - &self.conn_info.address - ); - // break out of the loop since connection was successful. - return self.relay_message_receiver.recv().await - }, - Err(e) => { - log::error!("listen(): overlay connection failed to reconnect: {e:?}",); - continue - }, - }; - }, - Some(StellarRelayMessage::Disconnect) => - for handle in self.handles.iter() { - log::info!("listen(): thread {}", handle.is_finished()); - drop(handle); - }, - _ => {}, - } - res - } - - /// Triggers connection to the Stellar Node. - /// Returns the UserControls for the user to send and receive Stellar messages. - pub(crate) async fn connect( - local_node: NodeInfo, - conn_info: ConnectionInfo, - ) -> Result { - log::info!("connect(): Connecting to: {conn_info:?}"); - - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(&conn_info.address()).await?; - // ------------------ prepare the channels - // this is a channel to communicate with the connection/config (this needs renaming) - let (actions_sender, actions_receiver) = mpsc::channel::(1024); - // this is a channel to communicate with the user/caller. - let (relay_message_sender, relay_message_receiver) = - mpsc::channel::(1024); - let mut overlay_connection = StellarOverlayConnection::new( - actions_sender.clone(), - relay_message_receiver, - local_node, - conn_info, - ); - let connector = Connector::new( - overlay_connection.local_node.clone(), - overlay_connection.conn_info.clone(), - actions_sender.clone(), - relay_message_sender, - ); - - // run the connector communication - let x = tokio::spawn(connection_handler(connector, actions_receiver, wr)); - - // start the receiving_service - let y = tokio::spawn(receiving_service(rd, actions_sender.clone())); - - overlay_connection.handles = vec![x, y]; - // start the handshake - actions_sender.send(ConnectorActions::SendHello).await?; - Ok(overlay_connection) - } - - pub fn get_actions_sender(&self) -> mpsc::Sender { - self.actions_sender.clone() - } - pub fn get_disconnect_action(&self) -> ConnectorActions { - ConnectorActions::Disconnect - } -} - -#[cfg(test)] -mod test { - use crate::{ - node::NodeInfo, ConnectionInfo, ConnectorActions, Error, StellarOverlayConnection, - StellarRelayMessage, - }; - use substrate_stellar_sdk::{ - network::TEST_NETWORK, - types::{MessageType, StellarMessage}, - SecretKey, - }; - use tokio::sync::mpsc; - - fn create_node_and_conn() -> (NodeInfo, ConnectionInfo) { - let secret = - SecretKey::from_encoding("SBLI7RKEJAEFGLZUBSCOFJHQBPFYIIPLBCKN7WVCWT4NEG2UJEW33N73") - .unwrap(); - let node_info = NodeInfo::new(19, 21, 19, "v19.1.0".to_string(), &TEST_NETWORK); - let conn_info = ConnectionInfo::new("34.235.168.98", 11625, secret, 0, false, true, false); - (node_info, conn_info) - } - - #[test] - fn create_stellar_overlay_connection_works() { - let (node_info, conn_info) = create_node_and_conn(); - - let (actions_sender, _) = mpsc::channel::(1024); - let (_, relay_message_receiver) = mpsc::channel::(1024); - - StellarOverlayConnection::new(actions_sender, relay_message_receiver, node_info, conn_info); - } - - #[tokio::test] - async fn stellar_overlay_connection_send_works() { - //arrange - let (node_info, conn_info) = create_node_and_conn(); - - let (actions_sender, mut actions_receiver) = mpsc::channel::(1024); - let (_, relay_message_receiver) = mpsc::channel::(1024); - - let overlay_connection = StellarOverlayConnection::new( - actions_sender.clone(), - relay_message_receiver, - node_info, - conn_info, - ); - let message_s = StellarMessage::GetPeers; - - //act - overlay_connection.send(message_s.clone()).await.expect("Should sent message"); - - //assert - let message = actions_receiver.recv().await.unwrap(); - if let ConnectorActions::SendMessage(message) = message { - assert_eq!(*message, message_s); - } else { - panic!("Incorrect stellar message") - } - } - - #[tokio::test] - async fn stellar_overlay_connection_listen_works() { - //arrange - let (node_info, conn_info) = create_node_and_conn(); - - let (actions_sender, _actions_receiver) = mpsc::channel::(1024); - let (relay_message_sender, relay_message_receiver) = - mpsc::channel::(1024); - - let mut overlay_connection = StellarOverlayConnection::new( - actions_sender.clone(), - relay_message_receiver, - node_info, - conn_info, - ); - - let expected_p_id = 2; - let expected_msg_type = MessageType::ErrorMsg; - relay_message_sender - .send(StellarRelayMessage::Data { - p_id: expected_p_id, - msg_type: expected_msg_type, - msg: Box::new(StellarMessage::GetPeers), - }) - .await - .expect("Stellar Relay message should be sent"); - - //act - let message = overlay_connection.listen().await.expect("Should receive some message"); - - //assert - match message { - StellarRelayMessage::Data { p_id, msg_type, msg } => { - assert_eq!(p_id, expected_p_id); - assert_eq!(msg_type, expected_msg_type); - assert_eq!(msg, Box::new(StellarMessage::GetPeers)) - }, - _ => panic!("wrong relay message received"), - } - } - - #[tokio::test] - async fn connect_should_fail_incorrect_address() { - let secret = - SecretKey::from_encoding("SBLI7RKEJAEFGLZUBSCOFJHQBPFYIIPLBCKN7WVCWT4NEG2UJEW33N73") - .unwrap(); - let node_info = NodeInfo::new(19, 21, 19, "v19.1.0".to_string(), &TEST_NETWORK); - let conn_info = - ConnectionInfo::new("incorrect address", 11625, secret, 0, false, true, false); - - let stellar_overlay_connection = - StellarOverlayConnection::connect(node_info, conn_info).await; - - assert!(stellar_overlay_connection.is_err()); - match stellar_overlay_connection.err().unwrap() { - Error::ConnectionFailed(_) => {}, - _ => { - panic!("Incorrect error") - }, - } - } - - #[tokio::test] - async fn stellar_overlay_connect_works() { - let (node_info, conn_info) = create_node_and_conn(); - let stellar_overlay_connection = - StellarOverlayConnection::connect(node_info, conn_info).await; - - assert!(stellar_overlay_connection.is_ok()); - } -} diff --git a/clients/stellar-relay-lib/src/connection/services.rs b/clients/stellar-relay-lib/src/connection/services.rs deleted file mode 100644 index 70aa5a241..000000000 --- a/clients/stellar-relay-lib/src/connection/services.rs +++ /dev/null @@ -1,339 +0,0 @@ -use crate::{ - connection::{ - connector::{Connector, ConnectorActions}, - helper::{time_now, to_base64_xdr_string}, - xdr_converter::get_xdr_message_length, - }, - Error, StellarRelayMessage, -}; -use substrate_stellar_sdk::types::StellarMessage; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::{tcp, TcpStream}, - sync::mpsc, - time::{timeout, Duration}, -}; - -/// For connecting to the StellarNode -pub(crate) async fn create_stream( - address: &str, -) -> Result<(tcp::OwnedReadHalf, tcp::OwnedWriteHalf), Error> { - let stream = TcpStream::connect(address) - .await - .map_err(|e| Error::ConnectionFailed(e.to_string()))?; - - Ok(stream.into_split()) -} - -/// checks the length of the next stellar message. -async fn next_message_length(r_stream: &mut tcp::OwnedReadHalf) -> usize { - // let's check for messages. - let mut sizebuf = [0; 4]; - - if r_stream.read(&mut sizebuf).await.unwrap_or(0) == 0 { - return 0 - } - - get_xdr_message_length(&sizebuf) -} - -/// reads data from the stream and store to buffer -async fn read_stream(r_stream: &mut tcp::OwnedReadHalf, buffer: &mut [u8]) -> Result { - r_stream.read(buffer).await.map_err(|e| Error::ReadFailed(e.to_string())) -} - -/// sends the HandleMessage action to the connector -async fn handle_message( - actions_sender: &mpsc::Sender, - proc_id: u32, - xdr_msg: Vec, -) -> Result<(), Error> { - actions_sender - .send(ConnectorActions::HandleMessage((proc_id, xdr_msg))) - .await - .map_err(Error::from) -} - -/// reads a continuation of bytes that belong to the previous message -/// -/// # Arguments -/// * `r_stream` - the read stream for reading the xdr stellar message -/// * `actions_sender` - the sender for actions a Connector must do -/// * `lack_bytes_from_prev` - the number of bytes remaining, to complete the previous message -/// * `proc_id` - the process id, used for tracing. -/// * `readbuf` - the buffer that holds the bytes of the previous and incomplete message -async fn read_unfinished_message( - r_stream: &mut tcp::OwnedReadHalf, - actions_sender: &mpsc::Sender, - lack_bytes_from_prev: &mut usize, - proc_id: &mut u32, - readbuf: &mut Vec, -) -> Result<(), Error> { - // let's read the continuation number of bytes from the previous message. - let mut cont_buf = vec![0; *lack_bytes_from_prev]; - - let actual_msg_len = read_stream(r_stream, &mut cont_buf).await?; - - // this partial message completes the previous message. - if actual_msg_len == *lack_bytes_from_prev { - log::trace!("read_unfinished_message(): proc_id: {} received continuation from the previous message.", proc_id); - readbuf.append(&mut cont_buf); - - handle_message(actions_sender, *proc_id, readbuf.clone()).await?; - - *lack_bytes_from_prev = 0; - readbuf.clear(); - *proc_id += 1; - - return Ok(()) - } - - // this partial message is not enough to complete the previous message. - if actual_msg_len > 0 { - *lack_bytes_from_prev -= actual_msg_len; - cont_buf = cont_buf[0..actual_msg_len].to_owned(); - readbuf.append(&mut cont_buf); - log::trace!( - "read_unfinished_message(): proc_id: {} not enough bytes to complete the previous message. Need {} bytes to complete.", - proc_id, - lack_bytes_from_prev - ); - } - - Ok(()) -} - -/// reads a number of bytes based on the expected message length. -/// -/// # Arguments -/// * `r_stream` - the read stream for reading the xdr stellar message -/// * `actions_sender` - the sender for actions a Connector must do -/// * `lack_bytes_from_prev` - the number of bytes remaining, to complete the previous message -/// * `proc_id` - the process id, used for tracing. -/// * `readbuf` - the buffer that holds the bytes of the previous and incomplete message -/// * `xpect_msg_len` - the expected # of bytes of the Stellar message -async fn read_message( - r_stream: &mut tcp::OwnedReadHalf, - actions_sender: &mpsc::Sender, - lack_bytes_from_prev: &mut usize, - proc_id: &mut u32, - readbuf: &mut Vec, - xpect_msg_len: usize, -) -> Result<(), Error> { - let actual_msg_len = read_stream(r_stream, readbuf).await?; - - // only when the message has the exact expected size bytes, should we send to user. - if actual_msg_len == xpect_msg_len { - handle_message(actions_sender, *proc_id, readbuf.clone()).await?; - readbuf.clear(); - *proc_id += 1; - return Ok(()) - } - - // The next bytes are remnants from the previous stellar message. - // save it and read it on the next loop. - *lack_bytes_from_prev = xpect_msg_len - actual_msg_len; - *readbuf = readbuf[0..actual_msg_len].to_owned(); - log::trace!( - "read_message(): proc_id: {} received only partial message. Need {} bytes to complete.", - proc_id, - lack_bytes_from_prev - ); - - Ok(()) -} - -/// This service is for RECEIVING a stellar message from the server. -/// # Arguments -/// * `r_stream` - the read stream for reading the xdr stellar message -/// * `tx_stream_reader` - the sender for handling the xdr stellar message -pub(crate) async fn receiving_service( - mut r_stream: tcp::OwnedReadHalf, - actions_sender: mpsc::Sender, -) { - let mut proc_id = 0; - - // holds the number of bytes that were missing from the previous stellar message. - let mut lack_bytes_from_prev = 0; - let mut readbuf: Vec = vec![]; - - let mut buff_for_peeking = vec![0; 4]; - loop { - // check whether or not we should read the bytes as: - // 1. the length of the next stellar message - // 2. the remaining bytes of the previous stellar message - match r_stream.peek(&mut buff_for_peeking).await { - Ok(0) => { - log::error!("receiving_service(): proc_id: {proc_id}. Failed to read messages from the stream. Received 0 size"); - break - }, - Ok(_) if lack_bytes_from_prev == 0 => { - // if there are no more bytes lacking from the previous message, - // then check the size of next stellar message. - // If it's not enough, skip it. - let expect_msg_len = next_message_length(&mut r_stream).await; - log::trace!("receiving_service(): proc_id: {proc_id}. The next message length is {expect_msg_len}"); - - if expect_msg_len == 0 { - // there's nothing to read; wait for the next iteration - log::trace!("receiving_service(): proc_id: {proc_id}. Nothing left to read; waiting for next loop"); - continue - } - - // let's start reading the actual stellar message. - readbuf = vec![0; expect_msg_len]; - - if let Err(e) = read_message( - &mut r_stream, - &actions_sender, - &mut lack_bytes_from_prev, - &mut proc_id, - &mut readbuf, - expect_msg_len, - ) - .await - { - log::error!( - "receiving_service(): proc_id: {proc_id}. Failed to read message: {e:?}" - ); - break - } - }, - Ok(_) => { - // let's read the continuation number of bytes from the previous message. - if let Err(e) = read_unfinished_message( - &mut r_stream, - &actions_sender, - &mut lack_bytes_from_prev, - &mut proc_id, - &mut readbuf, - ) - .await - { - log::error!( - "receiving_service(): proc_id: {proc_id}. Failed to read message: {e:?}" - ); - break - } - }, - Err(e) => { - log::error!("receiving_service(): proc_id: {proc_id}. Failed to read messages from the stream: {e:?}"); - break - }, - } - } - - // stop the connection if stream does not return anything. - drop(r_stream); - return -} - -async fn _write_to_stream( - msg: StellarMessage, - connector: &mut Connector, - w_stream: &mut tcp::OwnedWriteHalf, -) -> Result<(), Error> { - let xdr_msg = connector.create_xdr_message(msg)?; - w_stream - .write_all(&xdr_msg) - .await - .map_err(|e| Error::WriteFailed(e.to_string())) -} - -async fn _connection_handler( - actions: ConnectorActions, - connector: &mut Connector, - w_stream: &mut tcp::OwnedWriteHalf, -) -> Result<(), Error> { - match actions { - // start the connection to Stellar node with a 'hello' - ConnectorActions::SendHello => { - let msg = connector.create_hello_message(time_now())?; - - log::info!( - "_connection_handler(): Sending Hello Message: {}", - to_base64_xdr_string(&msg) - ); - _write_to_stream(msg, connector, w_stream).await?; - }, - - // write message to the stream - ConnectorActions::SendMessage(msg) => { - _write_to_stream(*msg, connector, w_stream).await?; - }, - - // handle incoming message from the stream - ConnectorActions::HandleMessage(xdr) => { - connector.process_raw_message(xdr).await?; - }, - - ConnectorActions::Disconnect => panic!("Should disconnect"), - } - - Ok(()) -} - -/// Handles actions for the connection. -/// # Arguments -/// * `connector` - the Connector that would send/handle messages to/from Stellar Node -/// * `receiver` - The receiver for actions that the Connector should do. -/// * `w_stream` -> the write half of the TcpStream to connect to the Stellar Node -pub(crate) async fn connection_handler( - mut connector: Connector, - mut actions_receiver: mpsc::Receiver, - mut w_stream: tcp::OwnedWriteHalf, -) { - loop { - match timeout(Duration::from_secs(connector.timeout_in_secs), actions_receiver.recv()).await - { - Ok(Some(ConnectorActions::Disconnect)) => { - log::info!("connection_handler(): Disconnecting...."); - drop(w_stream); - - if let Err(e) = connector.send_to_user(StellarRelayMessage::Disconnect).await { - log::error!("connection_handler(): failed to send Disconnect message: {e:?}"); - } - - drop(connector); - drop(actions_receiver); - return - }, - - Ok(Some(action)) => { - if let Err(e) = _connection_handler(action, &mut connector, &mut w_stream).await { - log::error!("connection_handler(): {e:?}"); - - drop(w_stream); - if let Err(e) = - connector.send_to_user(StellarRelayMessage::Error(e.to_string())).await - { - log::error!("connection_handler(): failed to send error message: {e:?}"); - } - drop(connector); - drop(actions_receiver); - return - } - }, - - Ok(None) => { - log::warn!("connection_handler(): Unexpected empty response from receiver"); - }, - - Err(_) => { - log::error!( - "connection_handler(): Timeout elapsed for reading messages from Stellar Node.", - ); - drop(w_stream); - - if let Err(e) = - connector.send_to_user(StellarRelayMessage::Error("Timeout".to_string())).await - { - log::error!("connection_handler(): failed to send timeout message: {e:?}"); - } - drop(connector); - drop(actions_receiver); - return - }, - } - } -} diff --git a/clients/stellar-relay-lib/src/connection/xdr_converter.rs b/clients/stellar-relay-lib/src/connection/xdr_converter.rs index cb25e724b..894bdc09f 100644 --- a/clients/stellar-relay-lib/src/connection/xdr_converter.rs +++ b/clients/stellar-relay-lib/src/connection/xdr_converter.rs @@ -56,7 +56,7 @@ macro_rules! parse_stellar_type { ($ref:ident, $struct_str:ident) => {{ use $crate::{ sdk::{types::$struct_str, XdrCodec}, - xdr_converter::Error, + connection2::xdr_converter::Error, }; let ret: Result<$struct_str, Error> = $struct_str::from_xdr($ref).map_err(|e| { diff --git a/clients/stellar-relay-lib/src/lib.rs b/clients/stellar-relay-lib/src/lib.rs index 952c2c4fc..b97f5926c 100644 --- a/clients/stellar-relay-lib/src/lib.rs +++ b/clients/stellar-relay-lib/src/lib.rs @@ -1,15 +1,86 @@ mod config; +// mod connection; mod connection; pub mod node; #[cfg(test)] mod tests; -pub(crate) use connection::{ - handshake::{self, HandshakeState}, - ConnectionInfo, Connector, ConnectorActions, -}; -pub use connection::{helper, xdr_converter, Error, StellarOverlayConnection, StellarRelayMessage}; - pub use substrate_stellar_sdk as sdk; +use substrate_stellar_sdk::types::StellarMessage; +use tokio::net::{tcp, TcpStream}; +use tokio::sync::mpsc; +use tokio::sync::mpsc::error::SendError; + pub use config::{connect_to_stellar_overlay_network, StellarOverlayConfig}; +pub use crate::connection::{Error, helper}; + + +use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; +use crate::node::NodeInfo; + +/// Used to send/receive messages to/from Stellar Node +pub struct StellarOverlayConnection { + sender: mpsc::Sender, + receiver: mpsc::Receiver +} + +impl StellarOverlayConnection { + pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { + self.sender.send(msg).await + } + + pub async fn listen(&mut self) -> Option { + self.receiver.recv().await + } + + pub fn is_alive(&self) -> bool { + !self.sender.is_closed() + } + + pub fn disconnect(&mut self) { + self.receiver.close(); + } +} + + +impl StellarOverlayConnection { + /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. + pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { + log::info!("connect(): connecting to {conn_info:?}"); + + // split the stream for easy handling of read and write + let (rd, wr) = create_stream(&conn_info.address()).await?; + + // this is a channel to communicate with the user/caller. + let (send_to_user_sender, send_to_user_receiver) = + mpsc::channel::(1024); + + let (send_to_node_sender, send_to_node_receiver) = + mpsc::channel::(1024); + + let mut connector = Connector::new( + local_node_info, + conn_info, + wr, + ); + connector.send_hello_message().await?; + + tokio::spawn(poll_messages_from_stellar(connector,rd,send_to_user_sender,send_to_node_receiver )); + + Ok(StellarOverlayConnection { + sender: send_to_node_sender, + receiver: send_to_user_receiver + }) + } +} + +async fn create_stream( + address: &str, +) -> Result<(tcp::OwnedReadHalf, tcp::OwnedWriteHalf), Error> { + let stream = TcpStream::connect(address) + .await + .map_err(|e| Error::ConnectionFailed(e.to_string()))?; + + Ok(stream.into_split()) +} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/node/local.rs b/clients/stellar-relay-lib/src/node/local.rs index c4d6c5409..a2d8cfff3 100644 --- a/clients/stellar-relay-lib/src/node/local.rs +++ b/clients/stellar-relay-lib/src/node/local.rs @@ -1,6 +1,7 @@ use crate::{connection::helper::generate_random_nonce, node::NodeInfo}; use substrate_stellar_sdk::types::Uint256; + #[derive(Debug, Clone)] pub struct LocalInfo { sequence: u64, diff --git a/clients/stellar-relay-lib/src/tests/mod.rs b/clients/stellar-relay-lib/src/tests/mod.rs index cf96ac17b..aac826a4b 100644 --- a/clients/stellar-relay-lib/src/tests/mod.rs +++ b/clients/stellar-relay-lib/src/tests/mod.rs @@ -4,12 +4,11 @@ use substrate_stellar_sdk::{ Hash, IntoHash, }; -use crate::{ - node::NodeInfo, ConnectionInfo, StellarOverlayConfig, StellarOverlayConnection, - StellarRelayMessage, -}; +use crate::{node::NodeInfo, ConnectionInfo, StellarOverlayConfig, StellarOverlayConnection}; use serial_test::serial; use tokio::{sync::Mutex, time::timeout}; +use tokio::time::sleep; +use crate::connection::to_base64_xdr_string; fn secret_key(is_mainnet: bool) -> String { let path = if is_mainnet { @@ -43,23 +42,23 @@ fn overlay_infos(is_mainnet: bool) -> (NodeInfo, ConnectionInfo) { ) } -#[tokio::test] -#[serial] -async fn stellar_overlay_connect_and_listen_connect_message() { - let (node_info, conn_info) = overlay_infos(false); - - let mut overlay_connection = - StellarOverlayConnection::connect(node_info.clone(), conn_info).await.unwrap(); - - let message = overlay_connection.listen().await.unwrap(); - if let StellarRelayMessage::Connect { pub_key: _x, node_info: y } = message { - assert_eq!(y.ledger_version, node_info.ledger_version); - } else { - panic!("Incorrect stellar relay message received"); - } - - overlay_connection.disconnect().await.expect("Should be able to disconnect"); -} +// #[tokio::test] +// #[serial] +// async fn stellar_overlay_connect_and_listen_connect_message() { +// let (node_info, conn_info) = overlay_infos(false); +// +// let mut overlay_connection = +// StellarOverlayConnection::connect(node_info.clone(), conn_info).await.unwrap(); +// +// let message = overlay_connection.listen().await.unwrap(); +// if let StellarRelayMessage::Connect { pub_key: _x, node_info: y } = message { +// assert_eq!(y.ledger_version, node_info.ledger_version); +// } else { +// panic!("Incorrect stellar relay message received"); +// } +// +// overlay_connection.disconnect(); +// } #[tokio::test] #[serial] @@ -76,18 +75,11 @@ async fn stellar_overlay_should_receive_scp_messages() { timeout(Duration::from_secs(300), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Some(relay_message) = ov_conn_locked.listen().await { - match relay_message { - StellarRelayMessage::Data { p_id: _, msg_type: _, msg } => match *msg { - StellarMessage::ScpMessage(msg) => { - scps_vec_clone.lock().await.push(msg); - ov_conn_locked.disconnect().await.expect("failed to disconnect"); - break - }, - _ => {}, - }, - _ => {}, - } + while let Some(msg) = ov_conn_locked.listen().await { + scps_vec_clone.lock().await.push(msg); + + ov_conn_locked.disconnect(); + break } }) .await @@ -96,6 +88,10 @@ async fn stellar_overlay_should_receive_scp_messages() { //assert //ensure that we receive some scp message from stellar node assert!(!scps_vec.lock().await.is_empty()); + + loop { + // nothing + } } #[tokio::test] @@ -121,37 +117,35 @@ async fn stellar_overlay_should_receive_tx_set() { timeout(Duration::from_secs(500), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Some(relay_message) = ov_conn_locked.listen().await { - match relay_message { - StellarRelayMessage::Data { p_id: _, msg_type: _, msg } => match *msg { - StellarMessage::ScpMessage(msg) => - if let ScpStatementPledges::ScpStExternalize(stmt) = &msg.statement.pledges - { - let tx_set_hash = get_tx_set_hash(stmt); - tx_set_hashes_clone.lock().await.push(tx_set_hash.clone()); - ov_conn_locked - .send(StellarMessage::GetTxSet(tx_set_hash)) - .await - .unwrap(); - }, - StellarMessage::TxSet(set) => { - let tx_set_hash = set.into_hash().expect("should return a hash"); - actual_tx_set_hashes_clone.lock().await.push(tx_set_hash); - - ov_conn_locked.disconnect().await.expect("failed to disconnect"); - break + while let Some(msg) = ov_conn_locked.listen().await { + match msg { + StellarMessage::ScpMessage(msg) => + if let ScpStatementPledges::ScpStExternalize(stmt) = &msg.statement.pledges + { + let tx_set_hash = get_tx_set_hash(stmt); + tx_set_hashes_clone.lock().await.push(tx_set_hash.clone()); + ov_conn_locked + .send_to_node(StellarMessage::GetTxSet(tx_set_hash)) + .await + .unwrap(); }, - StellarMessage::GeneralizedTxSet(set) => { - let tx_set_hash = set.into_hash().expect("should return a hash"); - actual_tx_set_hashes_clone.lock().await.push(tx_set_hash); + StellarMessage::TxSet(set) => { + let tx_set_hash = set.into_hash().expect("should return a hash"); + actual_tx_set_hashes_clone.lock().await.push(tx_set_hash); - ov_conn_locked.disconnect().await.expect("failed to disconnect"); - break - }, - _ => {}, + ov_conn_locked.disconnect(); + break + }, + StellarMessage::GeneralizedTxSet(set) => { + let tx_set_hash = set.into_hash().expect("should return a hash"); + actual_tx_set_hashes_clone.lock().await.push(tx_set_hash); + + ov_conn_locked.disconnect(); + break }, _ => {}, } + } }) .await @@ -175,11 +169,7 @@ async fn stellar_overlay_disconnect_works() { let mut overlay_connection = StellarOverlayConnection::connect(node_info.clone(), conn_info).await.unwrap(); - let message = overlay_connection.listen().await.unwrap(); - if let StellarRelayMessage::Connect { pub_key: _x, node_info: y } = message { - assert_eq!(y.ledger_version, node_info.ledger_version); - } else { - panic!("Incorrect stellar relay message received"); - } - overlay_connection.disconnect().await.unwrap(); + let _ = overlay_connection.listen().await.unwrap(); + + overlay_connection.disconnect(); } diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 090da1533..2cd8a62c8 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -5,12 +5,13 @@ use tokio::{ sync::{mpsc, RwLock}, time::{sleep, timeout}, }; +use tokio::sync::Mutex; +use tokio::time::error::Elapsed; +use tokio::time::Timeout; +use tracing::log; use runtime::ShutdownSender; -use stellar_relay_lib::{ - connect_to_stellar_overlay_network, sdk::types::StellarMessage, StellarOverlayConfig, - StellarRelayMessage, -}; +use stellar_relay_lib::{connect_to_stellar_overlay_network, sdk::types::StellarMessage, StellarOverlayConfig, StellarOverlayConnection}; use crate::oracle::{ collector::ScpMessageCollector, @@ -33,32 +34,22 @@ pub struct OracleAgent { /// * `collector` - used to collect envelopes and transaction sets /// * `message_sender` - used to send messages to Stellar Node async fn handle_message( - message: StellarRelayMessage, + message: StellarMessage, collector: Arc>, message_sender: &StellarMessageSender, ) -> Result<(), Error> { match message { - StellarRelayMessage::Data { p_id: _, msg_type: _, msg } => match *msg { - StellarMessage::ScpMessage(env) => { - collector.write().await.handle_envelope(env, message_sender).await?; - }, - StellarMessage::TxSet(set) => - if let Err(e) = collector.read().await.add_txset(set) { - tracing::error!(e); - }, - StellarMessage::GeneralizedTxSet(set) => { - if let Err(e) = collector.read().await.add_txset(set) { - tracing::error!(e); - } - }, - _ => {}, + StellarMessage::ScpMessage(env) => { + collector.write().await.handle_envelope(env, message_sender).await?; }, - StellarRelayMessage::Connect { pub_key, node_info } => { - let pub_key = pub_key.to_encoding(); - let pub_key = std::str::from_utf8(&pub_key).unwrap_or("****"); - - tracing::info!("handle_message(): Connected: via public key: {pub_key}"); - tracing::info!("handle_message(): Connected: with {:#?}", node_info) + StellarMessage::TxSet(set) => + if let Err(e) = collector.read().await.add_txset(set) { + tracing::error!(e); + }, + StellarMessage::GeneralizedTxSet(set) => { + if let Err(e) = collector.read().await.add_txset(set) { + tracing::error!(e); + } }, _ => {}, } @@ -73,13 +64,18 @@ pub async fn start_oracle_agent( config: StellarOverlayConfig, secret_key: &str, ) -> Result { + let timeout_in_secs = config.connection_info.timeout_in_secs; + let secret_key_copy = secret_key.to_string(); + tracing::info!("start_oracle_agent(): Starting connection to Stellar overlay network..."); - let mut overlay_conn = connect_to_stellar_overlay_network(config.clone(), secret_key).await?; + struct ConnectionWrapper (StellarOverlayConnection); - // Get action sender and disconnect action before moving `overlay_conn` into the closure - let actions_sender = overlay_conn.get_actions_sender(); - let disconnect_action = overlay_conn.get_disconnect_action(); + let overlay_connection = Arc::new(Mutex::new( + ConnectionWrapper(connect_to_stellar_overlay_network(config.clone(), secret_key).await?) + )); + + let ov_conn = overlay_connection.clone(); let (sender, mut receiver) = mpsc::channel(34); let collector = Arc::new(RwLock::new(ScpMessageCollector::new( @@ -95,16 +91,45 @@ pub async fn start_oracle_agent( let collector_clone = collector.clone(); service::spawn_cancelable(shutdown_clone.subscribe(), async move { let sender = sender_clone.clone(); + let mut ov_conn_locked = ov_conn.lock().await; + loop { + if !ov_conn_locked.0.is_alive() { + loop { + log::info!("start_oracle_agent(): restarting overlay connection in {timeout_in_secs} seconds"); + sleep(Duration::from_secs(timeout_in_secs)).await; + + match connect_to_stellar_overlay_network(config.clone(), &secret_key_copy).await { + Ok(new_ov_conn) => { + ov_conn_locked.0 = new_ov_conn; + break; + }, + Err(e) => { + tracing::error!("start_oracle_agent(): failed to create connection: {e:?}"); + } + } + } + } + tokio::select! { // runs the stellar-relay and listens to data to collect the scp messages and txsets. - Some(msg) = overlay_conn.listen() => { - handle_message(msg, collector_clone.clone(), &sender).await?; + result_msg = timeout(Duration::from_secs(timeout_in_secs), ov_conn_locked.0.listen()) => + match result_msg { + Ok(Some(stellar_msg)) => handle_message(stellar_msg, collector_clone.clone(), &sender).await?, + Ok(_) => {} + Err(_) => { + tracing::warn!("start_oracle_agent(): time elapsed from listening to Stellar Node."); + ov_conn_locked.0.disconnect(); + } }, - Some(msg) = receiver.recv() => { - // We received the instruction to send a message to the overlay network by the receiver - overlay_conn.send(msg).await?; + result_msg = timeout(Duration::from_secs(timeout_in_secs),receiver.recv())=> + match result_msg { + Ok(Some(msg)) => ov_conn_locked.0.send_to_node(msg).await?, + Ok(_) => {}, + Err(_) => { + tracing::warn!("start_oracle_agent(): time elapsed from listening to Stellar Node (indirectly)."); + } } } } @@ -113,11 +138,8 @@ pub async fn start_oracle_agent( }); tokio::spawn(on_shutdown(shutdown_sender.clone(), async move { - let result_sending_disconnect = - actions_sender.send(disconnect_action).await.map_err(Error::from); - if let Err(e) = result_sending_disconnect { - tracing::error!("start_oracle_agent(): Failed to send disconnect message: {:#?}", e); - }; + let mut ov_conn_locked = overlay_connection.lock().await; + ov_conn_locked.0.disconnect(); })); Ok(OracleAgent { From 142f551df242242494b03f640157eaf2cc1dcc71 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Wed, 29 Nov 2023 21:15:49 +0800 Subject: [PATCH 05/22] new connection change --- .../src/connection/connector/connector.rs | 265 +----------------- .../connection/connector/message_sender.rs | 46 +-- .../src/connection/connector/mod.rs | 75 ++++- 3 files changed, 106 insertions(+), 280 deletions(-) diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index e2c2bd3c9..a6ab39165 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -3,16 +3,21 @@ use substrate_stellar_sdk::{ types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType}, XdrCodec, }; +use substrate_stellar_sdk::types::StellarMessage; +use tokio::io::AsyncWriteExt; +use tokio::net::tcp::OwnedWriteHalf; use tokio::sync::mpsc; +use tokio::sync::mpsc::Sender; use crate::{ connection::{ authentication::{gen_shared_key, ConnectionAuth}, flow_controller::FlowController, hmac::{verify_hmac, HMacKeys}, + ConnectionInfo, handshake::HandshakeState, + Error }, node::{LocalInfo, NodeInfo, RemoteInfo}, - ConnectionInfo, ConnectorActions, Error, HandshakeState, StellarRelayMessage, }; pub struct Connector { @@ -31,11 +36,8 @@ pub struct Connector { handshake_state: HandshakeState, flow_controller: FlowController, - /// a channel for writing xdr messages to stream. - actions_sender: mpsc::Sender, - - /// a channel for communicating back to the caller - relay_message_sender: mpsc::Sender, + /// for writing xdr messages to stream. + pub(crate) wr: OwnedWriteHalf, } impl Debug for Connector { @@ -55,15 +57,9 @@ impl Debug for Connector { } } -impl Drop for Connector { - fn drop(&mut self) { - log::trace!("dropped Connector: {:?}", self); - } -} - impl Connector { /// Verifies the AuthenticatedMessage, received from the Stellar Node - pub(crate) fn verify_auth( + pub(super) fn verify_auth( &self, auth_msg: &AuthenticatedMessageV0, body: &[u8], @@ -120,8 +116,7 @@ impl Connector { pub fn new( local_node: NodeInfo, conn_info: ConnectionInfo, - actions_sender: mpsc::Sender, - relay_message_sender: mpsc::Sender, + write_half_of_stream: OwnedWriteHalf, ) -> Self { let connection_auth = ConnectionAuth::new( &local_node.network_id, @@ -140,8 +135,7 @@ impl Connector { receive_scp_messages: conn_info.recv_scp_msgs, handshake_state: HandshakeState::Connecting, flow_controller: FlowController::default(), - actions_sender, - relay_message_sender, + wr: write_half_of_stream, } } @@ -206,14 +200,6 @@ impl Connector { self.handshake_state = HandshakeState::Completed; } - pub async fn send_to_user(&self, msg: StellarRelayMessage) -> Result<(), Error> { - self.relay_message_sender.send(msg).await.map_err(Error::from) - } - - pub async fn send_to_node(&self, action: ConnectorActions) -> Result<(), Error> { - self.actions_sender.send(action).await.map_err(Error::from) - } - pub fn inner_check_to_send_more(&mut self, msg_type: MessageType) -> bool { self.flow_controller.send_more(msg_type) } @@ -225,232 +211,3 @@ impl Connector { self.flow_controller.enable(local_overlay_version, remote_overlay_version) } } - -#[cfg(test)] -mod test { - use crate::{connection::hmac::HMacKeys, node::RemoteInfo, Connector, StellarOverlayConfig}; - - use substrate_stellar_sdk::{ - compound_types::LimitedString, - types::{Hello, MessageType}, - PublicKey, - }; - use tokio::sync::mpsc::{self, Receiver}; - - use crate::{ - connection::authentication::{create_auth_cert, ConnectionAuth}, - helper::time_now, - node::NodeInfo, - ConnectionInfo, ConnectorActions, StellarRelayMessage, - }; - - #[cfg(test)] - fn create_auth_cert_from_connection_auth( - connector_auth: &ConnectionAuth, - ) -> substrate_stellar_sdk::types::AuthCert { - let time_now = time_now(); - let new_auth_cert = create_auth_cert( - connector_auth.network_id(), - connector_auth.keypair(), - time_now, - connector_auth.pub_key_ecdh().clone(), - ) - .expect("should successfully create an auth cert"); - new_auth_cert - } - - #[cfg(test)] - fn create_connector() -> ( - NodeInfo, - ConnectionInfo, - Connector, - Receiver, - Receiver, - ) { - let cfg_file_path = "./resources/config/testnet/stellar_relay_config_sdftest1.json"; - let secret_key_path = "./resources/secretkey/stellar_secretkey_testnet"; - let secret_key = - std::fs::read_to_string(secret_key_path).expect("should be able to read file"); - - let cfg = - StellarOverlayConfig::try_from_path(cfg_file_path).expect("should create a config"); - let node_info = cfg.node_info(); - let conn_info = cfg.connection_info(&secret_key).expect("should create a connection info"); - // this is a channel to communicate with the connection/config (this needs renaming) - let (actions_sender, actions_receiver) = mpsc::channel::(1024); - // this is a channel to communicate with the user/caller. - let (relay_message_sender, relay_message_receiver) = - mpsc::channel::(1024); - let connector = Connector::new( - node_info.clone(), - conn_info.clone(), - actions_sender, - relay_message_sender, - ); - (node_info, conn_info, connector, actions_receiver, relay_message_receiver) - } - - #[test] - fn create_new_connector_works() { - let (node_info, _, connector, _, _) = create_connector(); - - let connector_local_node = connector.local.node(); - - assert_eq!(connector_local_node.ledger_version, node_info.ledger_version); - assert_eq!(connector_local_node.overlay_version, node_info.overlay_version); - assert_eq!(connector_local_node.overlay_min_version, node_info.overlay_min_version); - assert_eq!(connector_local_node.version_str, node_info.version_str); - assert_eq!(connector_local_node.network_id, node_info.network_id); - } - - #[test] - fn connector_local_sequence_works() { - let (_node_info, _, mut connector, _, _) = create_connector(); - assert_eq!(connector.local_sequence(), 0); - connector.increment_local_sequence(); - assert_eq!(connector.local_sequence(), 1); - } - - #[test] - fn connector_set_remote_works() { - let (_node_info, _, mut connector, _, _) = create_connector(); - - let connector_auth = &connector.connection_auth; - let new_auth_cert = create_auth_cert_from_connection_auth(connector_auth); - - let hello = Hello { - ledger_version: 0, - overlay_version: 0, - overlay_min_version: 0, - network_id: [0; 32], - version_str: LimitedString::<100_i32>::new(vec![]).unwrap(), - listening_port: 11625, - peer_id: PublicKey::PublicKeyTypeEd25519([0; 32]), - cert: new_auth_cert, - nonce: [0; 32], - }; - connector.set_remote(RemoteInfo::new(&hello)); - - assert!(connector.remote().is_some()); - } - - #[test] - fn connector_increment_remote_sequence_works() { - let (_node_info, _, mut connector, _, _) = create_connector(); - - let connector_auth = &connector.connection_auth; - let new_auth_cert = create_auth_cert_from_connection_auth(connector_auth); - - let hello = Hello { - ledger_version: 0, - overlay_version: 0, - overlay_min_version: 0, - network_id: [0; 32], - version_str: LimitedString::<100_i32>::new(vec![]).unwrap(), - listening_port: 11625, - peer_id: PublicKey::PublicKeyTypeEd25519([0; 32]), - cert: new_auth_cert, - nonce: [0; 32], - }; - connector.set_remote(RemoteInfo::new(&hello)); - assert_eq!(connector.remote().unwrap().sequence(), 0); - - connector.increment_remote_sequence().unwrap(); - connector.increment_remote_sequence().unwrap(); - connector.increment_remote_sequence().unwrap(); - assert_eq!(connector.remote().unwrap().sequence(), 3); - } - - #[test] - fn connector_get_and_set_hmac_keys_works() { - //arrange - let (_, _, mut connector, _, _) = create_connector(); - let connector_auth = &connector.connection_auth; - let new_auth_cert = create_auth_cert_from_connection_auth(connector_auth); - - let hello = Hello { - ledger_version: 0, - overlay_version: 0, - overlay_min_version: 0, - network_id: [0; 32], - version_str: LimitedString::<100_i32>::new(vec![]).unwrap(), - listening_port: 11625, - peer_id: PublicKey::PublicKeyTypeEd25519([0; 32]), - cert: new_auth_cert, - nonce: [0; 32], - }; - let remote = RemoteInfo::new(&hello); - let remote_nonce = remote.nonce(); - connector.set_remote(remote.clone()); - - let shared_key = connector.get_shared_key(remote.pub_key_ecdh()); - assert!(connector.hmac_keys().is_none()); - //act - connector.set_hmac_keys(HMacKeys::new( - &shared_key, - connector.local().nonce(), - remote_nonce, - connector.remote_called_us(), - )); - //assert - assert!(connector.hmac_keys().is_some()); - } - - #[test] - fn connector_method_works() { - let (_, conn_config, mut connector, _, _) = create_connector(); - - assert_eq!(connector.remote_called_us(), conn_config.remote_called_us); - assert_eq!(connector.receive_tx_messages(), conn_config.recv_tx_msgs); - assert_eq!(connector.receive_scp_messages(), conn_config.recv_scp_msgs); - - connector.got_hello(); - assert!(connector.is_handshake_created()); - - connector.handshake_completed(); - assert!(connector.is_handshake_created()); - } - - #[tokio::test] - async fn connector_send_to_user_works() { - let (_, _, connector, _, mut message_receiver) = create_connector(); - - let message = StellarRelayMessage::Error("test".to_string()); - connector.send_to_user(message).await.unwrap(); - - let received_message = message_receiver.recv().await; - assert!(received_message.is_some()); - let message = received_message.unwrap(); - match message { - StellarRelayMessage::Error(_) => {}, - _ => { - panic!("Incorrect message received!!!") - }, - } - } - - #[test] - fn enable_flow_controller_works() { - let (node_info, _, mut connector, _, _) = create_connector(); - - assert!(!connector.inner_check_to_send_more(MessageType::ScpMessage)); - connector.enable_flow_controller(node_info.overlay_version, node_info.overlay_version); - } - - #[tokio::test] - async fn connector_send_to_node_works() { - let (_, _, connector, mut actions_receiver, _) = create_connector(); - - connector.send_to_node(ConnectorActions::SendHello).await.unwrap(); - - let received_message = actions_receiver.recv().await; - assert!(received_message.is_some()); - let message = received_message.unwrap(); - match message { - ConnectorActions::SendHello => {}, - _ => { - panic!("Incorrect message received!!!") - }, - } - } -} diff --git a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs index 01964e168..8228e560d 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs @@ -1,14 +1,36 @@ -use crate::connection::connector::{Connector, ConnectorActions}; +use std::time::Duration; use substrate_stellar_sdk::types::{MessageType, SendMore, StellarMessage}; +use tokio::io::AsyncWriteExt; +use tokio::net::tcp::OwnedWriteHalf; +use tokio::time::{timeout, Timeout}; -use crate::{ - connection::flow_controller::MAX_FLOOD_MSG_CAP, handshake::create_auth_message, Error, -}; +use crate::connection::{Connector, Error, flow_controller::MAX_FLOOD_MSG_CAP, handshake::create_auth_message, helper::{time_now, to_base64_xdr_string}}; impl Connector { - /// Sends an xdr version of a wrapped AuthenticatedMessage ( StellarMessage ). - async fn send_stellar_message(&mut self, msg: StellarMessage) -> Result<(), Error> { - self.send_to_node(ConnectorActions::SendMessage(Box::new(msg))).await + pub async fn send_to_node(&mut self, msg: StellarMessage) -> Result<(), Error> { + let xdr_msg = &self.create_xdr_message(msg)?; + + match timeout( + Duration::from_secs(self.timeout_in_secs), + self.wr.write_all(&xdr_msg) + ).await { + Ok(res) => res.map_err(|e| Error::WriteFailed(e.to_string())), + Err(_) => Err(Error::Timeout) + } + } + + pub async fn send_hello_message(&mut self) -> Result<(), Error> { + let msg = self.create_hello_message(time_now())?; + log::info!("send_hello_message(): Sending Hello Message: {}",to_base64_xdr_string(&msg)); + + self.send_to_node(msg).await + } + + pub(super) async fn send_auth_message(&mut self) -> Result<(), Error> { + let msg = create_auth_message(); + log::info!("send_auth_message(): Sending Auth Message: {}", to_base64_xdr_string(&msg)); + + self.send_to_node(create_auth_message()).await } pub(super) async fn check_to_send_more( @@ -20,14 +42,6 @@ impl Connector { } let msg = StellarMessage::SendMore(SendMore { num_messages: MAX_FLOOD_MSG_CAP }); - self.send_stellar_message(msg).await - } - - pub(super) async fn send_hello_message(&mut self) -> Result<(), Error> { - self.send_to_node(ConnectorActions::SendHello).await.map_err(Error::from) - } - - pub(super) async fn send_auth_message(&mut self) -> Result<(), Error> { - self.send_stellar_message(create_auth_message()).await + self.send_to_node(msg).await } } diff --git a/clients/stellar-relay-lib/src/connection/connector/mod.rs b/clients/stellar-relay-lib/src/connection/connector/mod.rs index 238962764..ffa34f346 100644 --- a/clients/stellar-relay-lib/src/connection/connector/mod.rs +++ b/clients/stellar-relay-lib/src/connection/connector/mod.rs @@ -1,17 +1,72 @@ -use crate::connection::Xdr; -use substrate_stellar_sdk::types::StellarMessage; - mod connector; mod message_creation; mod message_handler; +mod message_reader; mod message_sender; pub(crate) use connector::Connector; -#[derive(Debug)] -pub enum ConnectorActions { - SendHello, - SendMessage(Box), - HandleMessage(Xdr), - Disconnect, -} +use substrate_stellar_sdk::types::StellarMessage; +use tokio::io::AsyncWriteExt; +use tokio::net::tcp; +use tokio::sync::mpsc; +use crate::connection::connector::message_reader::read_message_from_stellar; +use crate::connection::Error; + +/// Polls for messages coming from the Stellar Node and communicates it back to the user +/// +/// # Arguments +/// * `connector` - contains the config and necessary info for connecting to Stellar Node +/// * `r_stream` - the read half of the stream that is connected to Stellar Node +/// * `send_to_user_sender` - sends message from Stellar to the user +/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the stream. +pub(crate) async fn poll_messages_from_stellar( + mut connector: Connector, + mut r_stream: tcp::OwnedReadHalf, + send_to_user_sender: mpsc::Sender, + mut send_to_node_receiver: mpsc::Receiver +) { + log::info!("poll_messages_from_stellar(): started."); + + loop { + if send_to_user_sender.is_closed() { + log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); + // close this channel as communication to user was closed. + break; + } + + tokio::select! { + opt_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match opt_msg { + None => break, + Some(xdr) => match connector.process_raw_message(xdr).await { + Ok(Some(stellar_msg)) => { + // push message to user + if let Err(e) = send_to_user_sender.send(stellar_msg).await { + log::error!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); + break; + } + } + Ok(_) => {} + Err(e) => { + log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); + break; + } + } + }, + + // push message to Stellar Node + Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { + log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); + break; + } + } + } + + send_to_node_receiver.close(); + drop(send_to_user_sender); + // make sure to drop/shutdown all streams and channels + connector.wr.forget(); + drop(r_stream); + + log::info!("poll_messages_from_stellar(): stopped."); +} \ No newline at end of file From 0bec0975a6c7295b21b18c3d30b668d1a5c50105 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Thu, 30 Nov 2023 20:02:14 +0800 Subject: [PATCH 06/22] new changes --- clients/stellar-relay-lib/examples/connect.rs | 2 +- clients/stellar-relay-lib/src/config.rs | 3 + .../connection/connector/message_handler.rs | 8 +- .../connection/connector/message_reader.rs | 52 +++++------ .../src/connection/connector/mod.rs | 52 +++++++---- .../src/connection/helper.rs | 11 +++ .../src/connection/xdr_converter.rs | 2 +- clients/stellar-relay-lib/src/lib.rs | 58 ++++++++---- clients/vault/src/oracle/agent.rs | 89 ++++++++----------- 9 files changed, 152 insertions(+), 125 deletions(-) diff --git a/clients/stellar-relay-lib/examples/connect.rs b/clients/stellar-relay-lib/examples/connect.rs index 7803f0ee9..4fbb8e994 100644 --- a/clients/stellar-relay-lib/examples/connect.rs +++ b/clients/stellar-relay-lib/examples/connect.rs @@ -29,7 +29,7 @@ async fn main() -> Result<(), Box> { let mut overlay_connection = connect_to_stellar_overlay_network(cfg, &secret_key).await?; let mut counter = 0; - while let Some(msg) = overlay_connection.listen().await { + while let Ok(Some(msg)) = overlay_connection.listen().await { counter+=1; match msg { diff --git a/clients/stellar-relay-lib/src/config.rs b/clients/stellar-relay-lib/src/config.rs index d85ebb14b..d71310263 100644 --- a/clients/stellar-relay-lib/src/config.rs +++ b/clients/stellar-relay-lib/src/config.rs @@ -2,6 +2,9 @@ use serde::{Deserialize, Serialize}; use serde_with::{serde_as, BytesOrString}; use std::fmt::Debug; use substrate_stellar_sdk::SecretKey; +use substrate_stellar_sdk::types::StellarMessage; +use tokio::net::TcpStream; +use tokio::sync::mpsc; use crate::connection::{ConnectionInfo, Error}; use crate::node::NodeInfo; use crate::StellarOverlayConnection; diff --git a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs index 7230ce6d7..d1c43ae2b 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs @@ -1,4 +1,4 @@ -use substrate_stellar_sdk::types::{Hello, MessageType, StellarMessage}; +use substrate_stellar_sdk::types::{ErrorCode, Hello, MessageType, StellarMessage}; use substrate_stellar_sdk::XdrCodec; use crate::connection::{Connector, Xdr, Error, helper::{error_to_string, time_now}, xdr_converter::parse_authenticated_message}; @@ -28,7 +28,7 @@ impl Connector { "process_raw_message(): Received ErrorMsg during authentication: {}", error_to_string(e.clone()) ); - return Err(Error::OverlayError(e.code)) + return Err(Error::ConnectionFailed(error_to_string(e))) }, other => log::error!("process_raw_message(): Received ErroMsg during authentication: {:?}", other), }, @@ -77,8 +77,10 @@ impl Connector { StellarMessage::ErrorMsg(e) => { log::error!("process_stellar_message(): received from overlay: {e:?}"); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::ConnectionFailed(error_to_string(e))) + } return Ok(Some(StellarMessage::ErrorMsg(e))); - // self.send_to_user(StellarMessage::ErrorMsg(e)).await?; }, other => { diff --git a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs index f25420cb7..c9d5aeaee 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs @@ -19,9 +19,7 @@ use crate::connection::to_base64_xdr_string; /// Returns Xdr format of the `StellarMessage` sent from the Stellar Node -pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, timeout_in_secs:u64) -> Option { - let mut proc_id = 0; - +pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, timeout_in_secs:u64) -> Result { // holds the number of bytes that were missing from the previous stellar message. let mut lack_bytes_from_prev = 0; let mut readbuf: Vec = vec![]; @@ -31,12 +29,12 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, // check whether or not we should read the bytes as: // 1. the length of the next stellar message // 2. the remaining bytes of the previous stellar message - match timeout(Duration::from_secs(timeout_in_secs), r_stream.peek(&mut buff_for_peeking)) + match timeout(Duration::from_secs(60), r_stream.peek(&mut buff_for_peeking)) .await { Ok(Ok(0)) => { - log::error!("read_message_from_stellar(): proc_id: {proc_id}. Received 0 size"); - break + log::trace!("read_message_from_stellar(): ERROR: Received 0 size"); + return Err(Error::Disconnected); } Ok(Ok(_)) if lack_bytes_from_prev == 0 => { @@ -44,11 +42,11 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, // then check the size of next stellar message. // If it's not enough, skip it. let expect_msg_len = next_message_length(r_stream).await; - log::trace!("read_message_from_stellar(): proc_id: {proc_id}. The next message length is {expect_msg_len}"); + log::trace!("read_message_from_stellar(): The next message length is {expect_msg_len}"); if expect_msg_len == 0 { // there's nothing to read; wait for the next iteration - log::trace!("read_message_from_stellar(): proc_id: {proc_id}. Nothing left to read; waiting for next loop"); + log::trace!("read_message_from_stellar(): Nothing left to read; waiting for next loop"); continue } @@ -58,15 +56,14 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, match read_message( r_stream, &mut lack_bytes_from_prev, - &mut proc_id, &mut readbuf, expect_msg_len, ).await { Ok(None) => continue, - Ok(Some(xdr)) => return Some(xdr), + Ok(Some(xdr)) => return Ok(xdr), Err(e) => { - log::error!("read_message_from_stellar(): proc_id: {proc_id}. encountered error: {e:?}"); - break + log::trace!("read_message_from_stellar(): ERROR: {e:?}"); + return Err(e); } } } @@ -76,30 +73,29 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, match read_unfinished_message( r_stream, &mut lack_bytes_from_prev, - &mut proc_id, &mut readbuf, ).await { Ok(None) => continue, - Ok(Some(xdr)) => return Some(xdr), + Ok(Some(xdr)) => return Ok(xdr), Err(e) => { - log::error!("read_message_from_stellar(): proc_id: {proc_id}. encountered error: {e:?}"); - break + log::trace!("read_message_from_stellar(): ERROR: {e:?}"); + return Err(e); } } } Ok(Err(e)) => { - log::error!("read_message_from_stellar(): proc_id: {proc_id}. encountered error: {e:?}"); - break + log::trace!("read_message_from_stellar(): ERROR: {e:?}"); + return Err(Error::ReadFailed(e.to_string())); } Err(_) => { - log::error!("read_message_from_stellar(): proc_id: {proc_id}. timeout elapsed."); - break + log::error!("read_message_from_stellar(): timeout elapsed."); + return Err(Error::Timeout); } } } - None + Err(Error::ReadFailed("loop for reading has ended".to_string())) } @@ -109,13 +105,11 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, /// # Arguments /// * `r_stream` - the read stream for reading the xdr stellar message /// * `lack_bytes_from_prev` - the number of bytes remaining, to complete the previous message -/// * `proc_id` - the process id, used for tracing. /// * `readbuf` - the buffer that holds the bytes of the previous and incomplete message /// * `xpect_msg_len` - the expected # of bytes of the Stellar message async fn read_message( r_stream: &mut tcp::OwnedReadHalf, lack_bytes_from_prev: &mut usize, - proc_id: &mut u32, readbuf: &mut Vec, xpect_msg_len: usize, ) -> Result, Error> { @@ -131,9 +125,7 @@ async fn read_message( *lack_bytes_from_prev = xpect_msg_len - actual_msg_len; *readbuf = readbuf[0..actual_msg_len].to_owned(); log::trace!( - "read_message(): proc_id: {} received only partial message. Need {} bytes to complete.", - proc_id, - lack_bytes_from_prev + "read_message(): received only partial message. Need {lack_bytes_from_prev} bytes to complete." ); Ok(None) @@ -145,12 +137,10 @@ async fn read_message( /// # Arguments /// * `r_stream` - the read stream for reading the xdr stellar message /// * `lack_bytes_from_prev` - the number of bytes remaining, to complete the previous message -/// * `proc_id` - the process id, used for tracing. /// * `readbuf` - the buffer that holds the bytes of the previous and incomplete message async fn read_unfinished_message( r_stream: &mut tcp::OwnedReadHalf, lack_bytes_from_prev: &mut usize, - proc_id: &mut u32, readbuf: &mut Vec, ) -> Result, Error> { // let's read the continuation number of bytes from the previous message. @@ -160,7 +150,7 @@ async fn read_unfinished_message( // this partial message completes the previous message. if actual_msg_len == *lack_bytes_from_prev { - log::trace!("read_unfinished_message(): proc_id: {} received continuation from the previous message.", proc_id); + log::trace!("read_unfinished_message(): received continuation from the previous message."); readbuf.append(&mut cont_buf); return Ok(Some(readbuf.clone())); @@ -172,9 +162,7 @@ async fn read_unfinished_message( cont_buf = cont_buf[0..actual_msg_len].to_owned(); readbuf.append(&mut cont_buf); log::trace!( - "read_unfinished_message(): proc_id: {} not enough bytes to complete the previous message. Need {} bytes to complete.", - proc_id, - lack_bytes_from_prev + "read_unfinished_message(): not enough bytes to complete the previous message. Need {lack_bytes_from_prev} bytes to complete." ); } diff --git a/clients/stellar-relay-lib/src/connection/connector/mod.rs b/clients/stellar-relay-lib/src/connection/connector/mod.rs index ffa34f346..baf4d816a 100644 --- a/clients/stellar-relay-lib/src/connection/connector/mod.rs +++ b/clients/stellar-relay-lib/src/connection/connector/mod.rs @@ -4,14 +4,19 @@ mod message_handler; mod message_reader; mod message_sender; + +use std::time::Duration; pub(crate) use connector::Connector; use substrate_stellar_sdk::types::StellarMessage; -use tokio::io::AsyncWriteExt; -use tokio::net::tcp; +use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio::net::{tcp, TcpStream}; +use tokio::net::tcp::{OwnedReadHalf, ReuniteError}; use tokio::sync::mpsc; +use tokio::time::sleep; use crate::connection::connector::message_reader::read_message_from_stellar; use crate::connection::Error; +use crate::helper::{create_stream, to_base64_xdr_string}; /// Polls for messages coming from the Stellar Node and communicates it back to the user /// @@ -22,7 +27,8 @@ use crate::connection::Error; /// * `send_to_node_receiver` - receives message from user and writes it to the write half of the stream. pub(crate) async fn poll_messages_from_stellar( mut connector: Connector, - mut r_stream: tcp::OwnedReadHalf, + mut r_stream: OwnedReadHalf, + address: String, send_to_user_sender: mpsc::Sender, mut send_to_node_receiver: mpsc::Receiver ) { @@ -36,17 +42,18 @@ pub(crate) async fn poll_messages_from_stellar( } tokio::select! { - opt_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match opt_msg { - None => break, - Some(xdr) => match connector.process_raw_message(xdr).await { - Ok(Some(stellar_msg)) => { - // push message to user - if let Err(e) = send_to_user_sender.send(stellar_msg).await { - log::error!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); - break; - } - } - Ok(_) => {} + result_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match result_msg { + Err(e) => { + log::error!("poll_messages_from_stellar(): {e:?}"); + break; + }, + Ok(xdr) => match connector.process_raw_message(xdr).await { + Ok(Some(stellar_msg)) => + // push message to user + if let Err(e) = send_to_user_sender.send(stellar_msg).await { + log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); + }, + Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), Err(e) => { log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); break; @@ -57,16 +64,25 @@ pub(crate) async fn poll_messages_from_stellar( // push message to Stellar Node Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); - break; } } } + // make sure to drop/shutdown the stream + connector.wr.forget(); + drop(r_stream); send_to_node_receiver.close(); drop(send_to_user_sender); - // make sure to drop/shutdown all streams and channels - connector.wr.forget(); - drop(r_stream); + log::info!("poll_messages_from_stellar(): stopped."); +} + + +async fn restart(connector: &mut Connector, address: &str) -> Result { + // split the stream for easy handling of read and write + let (rd, wr) = create_stream(address).await?; + + connector.wr = wr; + Ok(rd) } \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/connection/helper.rs b/clients/stellar-relay-lib/src/connection/helper.rs index 57f9877b7..d909879f2 100644 --- a/clients/stellar-relay-lib/src/connection/helper.rs +++ b/clients/stellar-relay-lib/src/connection/helper.rs @@ -5,6 +5,7 @@ use substrate_stellar_sdk::{ types::{Error, Uint256}, SecretKey, XdrCodec, }; +use tokio::net::{tcp, TcpStream}; /// Returns a new BigNumber with a pseudo-random value equal to or greater than 0 and less than 1. pub fn generate_random_nonce() -> Uint256 { @@ -40,3 +41,13 @@ pub fn to_base64_xdr_string(msg: &T) -> String { let xdr = msg.to_base64_xdr(); String::from_utf8(xdr.clone()).unwrap_or(format!("{:?}", xdr)) } + +pub async fn create_stream( + address: &str, +) -> Result<(tcp::OwnedReadHalf, tcp::OwnedWriteHalf), crate::Error> { + let stream = TcpStream::connect(address) + .await + .map_err(|e| crate::Error::ConnectionFailed(e.to_string()))?; + + Ok(stream.into_split()) +} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/connection/xdr_converter.rs b/clients/stellar-relay-lib/src/connection/xdr_converter.rs index 894bdc09f..a067061c3 100644 --- a/clients/stellar-relay-lib/src/connection/xdr_converter.rs +++ b/clients/stellar-relay-lib/src/connection/xdr_converter.rs @@ -56,7 +56,7 @@ macro_rules! parse_stellar_type { ($ref:ident, $struct_str:ident) => {{ use $crate::{ sdk::{types::$struct_str, XdrCodec}, - connection2::xdr_converter::Error, + connection::xdr_converter::Error, }; let ret: Result<$struct_str, Error> = $struct_str::from_xdr($ref).map_err(|e| { diff --git a/clients/stellar-relay-lib/src/lib.rs b/clients/stellar-relay-lib/src/lib.rs index b97f5926c..e80799332 100644 --- a/clients/stellar-relay-lib/src/lib.rs +++ b/clients/stellar-relay-lib/src/lib.rs @@ -6,17 +6,19 @@ pub mod node; mod tests; pub use substrate_stellar_sdk as sdk; -use substrate_stellar_sdk::types::StellarMessage; +use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; use tokio::net::{tcp, TcpStream}; use tokio::sync::mpsc; -use tokio::sync::mpsc::error::SendError; - +use tokio::sync::mpsc::error::{SendError, TryRecvError}; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::oneshot; pub use config::{connect_to_stellar_overlay_network, StellarOverlayConfig}; pub use crate::connection::{Error, helper}; use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; +use crate::helper::{create_stream, error_to_string}; use crate::node::NodeInfo; /// Used to send/receive messages to/from Stellar Node @@ -30,27 +32,51 @@ impl StellarOverlayConnection { self.sender.send(msg).await } - pub async fn listen(&mut self) -> Option { - self.receiver.recv().await + pub async fn listen(&mut self) -> Result, Error> { + loop { + match self.receiver.try_recv() { + Ok(StellarMessage::ErrorMsg(e)) => { + log::error!("listen(): received error message: {e:?}"); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::ConnectionFailed(error_to_string(e))) + } + + return Ok(None) + }, + Ok(msg) => return Ok(Some(msg)), + Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), + Err(TryRecvError::Empty) => continue, + } + } } pub fn is_alive(&self) -> bool { - !self.sender.is_closed() + let result = self.sender.is_closed(); + + if result { + drop(self); + } + + !result } pub fn disconnect(&mut self) { + log::info!("disconnect(): closing channel"); self.receiver.close(); } } +impl Drop for StellarOverlayConnection { + fn drop(&mut self) { + self.disconnect(); + } +} impl StellarOverlayConnection { /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { log::info!("connect(): connecting to {conn_info:?}"); - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(&conn_info.address()).await?; // this is a channel to communicate with the user/caller. let (send_to_user_sender, send_to_user_receiver) = @@ -59,6 +85,10 @@ impl StellarOverlayConnection { let (send_to_node_sender, send_to_node_receiver) = mpsc::channel::(1024); + // split the stream for easy handling of read and write + let (rd, wr) = create_stream(&conn_info.address()).await?; + + let address = conn_info.address(); let mut connector = Connector::new( local_node_info, conn_info, @@ -66,21 +96,11 @@ impl StellarOverlayConnection { ); connector.send_hello_message().await?; - tokio::spawn(poll_messages_from_stellar(connector,rd,send_to_user_sender,send_to_node_receiver )); + tokio::spawn(poll_messages_from_stellar(connector,rd,address, send_to_user_sender,send_to_node_receiver)); Ok(StellarOverlayConnection { sender: send_to_node_sender, receiver: send_to_user_receiver }) } -} - -async fn create_stream( - address: &str, -) -> Result<(tcp::OwnedReadHalf, tcp::OwnedWriteHalf), Error> { - let stream = TcpStream::connect(address) - .await - .map_err(|e| Error::ConnectionFailed(e.to_string()))?; - - Ok(stream.into_split()) } \ No newline at end of file diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 2cd8a62c8..d1f3eb48c 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -12,6 +12,7 @@ use tracing::log; use runtime::ShutdownSender; use stellar_relay_lib::{connect_to_stellar_overlay_network, sdk::types::StellarMessage, StellarOverlayConfig, StellarOverlayConnection}; +use stellar_relay_lib::helper::to_base64_xdr_string; use crate::oracle::{ collector::ScpMessageCollector, @@ -69,83 +70,69 @@ pub async fn start_oracle_agent( tracing::info!("start_oracle_agent(): Starting connection to Stellar overlay network..."); - struct ConnectionWrapper (StellarOverlayConnection); + let mut overlay_conn = connect_to_stellar_overlay_network(config.clone(), secret_key).await?; - let overlay_connection = Arc::new(Mutex::new( - ConnectionWrapper(connect_to_stellar_overlay_network(config.clone(), secret_key).await?) - )); - - let ov_conn = overlay_connection.clone(); - - let (sender, mut receiver) = mpsc::channel(34); let collector = Arc::new(RwLock::new(ScpMessageCollector::new( config.is_public_network(), config.stellar_history_archive_urls(), ))); + let collector_clone = collector.clone(); + let shutdown_sender = ShutdownSender::default(); + let shutdown_sender_clone = shutdown_sender.clone(); + let shutdown_sender_clone2 = shutdown_sender.clone(); - let shutdown_clone = shutdown_sender.clone(); - // handle a message from the overlay network - let sender_clone = sender.clone(); + // disconnect signal sender + let (disconnect_signal_sender, mut disconnect_signal_receiver) = mpsc::channel::<()>(2); - let collector_clone = collector.clone(); - service::spawn_cancelable(shutdown_clone.subscribe(), async move { - let sender = sender_clone.clone(); - let mut ov_conn_locked = ov_conn.lock().await; + // handle a message from the overlay network + let (message_sender, mut message_receiver) = mpsc::channel::(34); + let message_sender_clone = message_sender.clone(); + service::spawn_cancelable(shutdown_sender_clone.subscribe(), async move { loop { - if !ov_conn_locked.0.is_alive() { - loop { - log::info!("start_oracle_agent(): restarting overlay connection in {timeout_in_secs} seconds"); - sleep(Duration::from_secs(timeout_in_secs)).await; - - match connect_to_stellar_overlay_network(config.clone(), &secret_key_copy).await { - Ok(new_ov_conn) => { - ov_conn_locked.0 = new_ov_conn; - break; - }, - Err(e) => { - tracing::error!("start_oracle_agent(): failed to create connection: {e:?}"); - } - } - } - } - tokio::select! { - // runs the stellar-relay and listens to data to collect the scp messages and txsets. - result_msg = timeout(Duration::from_secs(timeout_in_secs), ov_conn_locked.0.listen()) => - match result_msg { - Ok(Some(stellar_msg)) => handle_message(stellar_msg, collector_clone.clone(), &sender).await?, - Ok(_) => {} - Err(_) => { - tracing::warn!("start_oracle_agent(): time elapsed from listening to Stellar Node."); - ov_conn_locked.0.disconnect(); + result_msg= overlay_conn.listen() => match result_msg { + Ok(Some(msg)) => { + tracing::info!("start_oracle_agent(): handle message: {}", to_base64_xdr_string(&msg)); + handle_message( + msg, + collector_clone.clone(), + &message_sender_clone + ).await?; } + Ok(None) => {} + Err(e) => { + overlay_conn.disconnect(); + let _ = shutdown_sender_clone2.send(()); + return Ok(()); + } + }, + Some(msg) = message_receiver.recv() => if let Err(e) = overlay_conn.send_to_node(msg).await { + tracing::error!("start_oracle_agent(): failed to send msg to stellar node: {e:?}"); }, - result_msg = timeout(Duration::from_secs(timeout_in_secs),receiver.recv())=> - match result_msg { - Ok(Some(msg)) => ov_conn_locked.0.send_to_node(msg).await?, - Ok(_) => {}, - Err(_) => { - tracing::warn!("start_oracle_agent(): time elapsed from listening to Stellar Node (indirectly)."); - } + Some(_) = disconnect_signal_receiver.recv() => { + tracing::info!("start_oracle_agent(): disconnect signal received."); + + overlay_conn.disconnect(); } } } - #[allow(unreachable_code)] + Ok::<(), Error>(()) }); + tokio::spawn(on_shutdown(shutdown_sender.clone(), async move { - let mut ov_conn_locked = overlay_connection.lock().await; - ov_conn_locked.0.disconnect(); + tracing::info!("start_oracle_agent(): sending signal to shutdown overlay connection..."); + let _ = disconnect_signal_sender.send(()).await; })); Ok(OracleAgent { collector, is_public_network: false, - message_sender: Some(sender), + message_sender: Some(message_sender), shutdown_sender, }) } From ee24e88f51cbaf1693536ad82a877fa771f8c915 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Fri, 1 Dec 2023 12:59:00 +0800 Subject: [PATCH 07/22] cleanup --- .../src/connection/connector/connector.rs | 72 +++++++++++++- .../src/connection/connector/mod.rs | 82 +--------------- clients/stellar-relay-lib/src/lib.rs | 98 +------------------ clients/stellar-relay-lib/src/overlay.rs | 96 ++++++++++++++++++ 4 files changed, 170 insertions(+), 178 deletions(-) create mode 100644 clients/stellar-relay-lib/src/overlay.rs diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index a6ab39165..4631b90a2 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -5,7 +5,7 @@ use substrate_stellar_sdk::{ }; use substrate_stellar_sdk::types::StellarMessage; use tokio::io::AsyncWriteExt; -use tokio::net::tcp::OwnedWriteHalf; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::sync::mpsc; use tokio::sync::mpsc::Sender; @@ -19,6 +19,8 @@ use crate::{ }, node::{LocalInfo, NodeInfo, RemoteInfo}, }; +use crate::connection::connector::message_reader::read_message_from_stellar; +use crate::helper::create_stream; pub struct Connector { local: LocalInfo, @@ -211,3 +213,71 @@ impl Connector { self.flow_controller.enable(local_overlay_version, remote_overlay_version) } } + + +/// Polls for messages coming from the Stellar Node and communicates it back to the user +/// +/// # Arguments +/// * `connector` - contains the config and necessary info for connecting to Stellar Node +/// * `r_stream` - the read half of the stream that is connected to Stellar Node +/// * `send_to_user_sender` - sends message from Stellar to the user +/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the stream. +pub(crate) async fn poll_messages_from_stellar( + mut connector: Connector, + mut r_stream: OwnedReadHalf, + address: String, + send_to_user_sender: mpsc::Sender, + mut send_to_node_receiver: mpsc::Receiver +) { + log::info!("poll_messages_from_stellar(): started."); + + loop { + if send_to_user_sender.is_closed() { + log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); + // close this channel as communication to user was closed. + break; + } + + tokio::select! { + result_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match result_msg { + Err(e) => { + log::error!("poll_messages_from_stellar(): {e:?}"); + break; + }, + Ok(xdr) => match connector.process_raw_message(xdr).await { + Ok(Some(stellar_msg)) => + // push message to user + if let Err(e) = send_to_user_sender.send(stellar_msg).await { + log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); + }, + Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), + Err(e) => { + log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); + break; + } + } + }, + + // push message to Stellar Node + Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { + log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); + } + } + } + // make sure to drop/shutdown the stream + connector.wr.forget(); + drop(r_stream); + + send_to_node_receiver.close(); + drop(send_to_user_sender); + + log::info!("poll_messages_from_stellar(): stopped."); +} + +async fn restart(connector: &mut Connector, address: &str) -> Result { + // split the stream for easy handling of read and write + let (rd, wr) = create_stream(address).await?; + + connector.wr = wr; + Ok(rd) +} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/connection/connector/mod.rs b/clients/stellar-relay-lib/src/connection/connector/mod.rs index baf4d816a..97c65a694 100644 --- a/clients/stellar-relay-lib/src/connection/connector/mod.rs +++ b/clients/stellar-relay-lib/src/connection/connector/mod.rs @@ -4,85 +4,5 @@ mod message_handler; mod message_reader; mod message_sender; +pub(crate) use connector::*; -use std::time::Duration; -pub(crate) use connector::Connector; - -use substrate_stellar_sdk::types::StellarMessage; -use tokio::io::{AsyncWrite, AsyncWriteExt}; -use tokio::net::{tcp, TcpStream}; -use tokio::net::tcp::{OwnedReadHalf, ReuniteError}; -use tokio::sync::mpsc; -use tokio::time::sleep; -use crate::connection::connector::message_reader::read_message_from_stellar; -use crate::connection::Error; -use crate::helper::{create_stream, to_base64_xdr_string}; - -/// Polls for messages coming from the Stellar Node and communicates it back to the user -/// -/// # Arguments -/// * `connector` - contains the config and necessary info for connecting to Stellar Node -/// * `r_stream` - the read half of the stream that is connected to Stellar Node -/// * `send_to_user_sender` - sends message from Stellar to the user -/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the stream. -pub(crate) async fn poll_messages_from_stellar( - mut connector: Connector, - mut r_stream: OwnedReadHalf, - address: String, - send_to_user_sender: mpsc::Sender, - mut send_to_node_receiver: mpsc::Receiver -) { - log::info!("poll_messages_from_stellar(): started."); - - loop { - if send_to_user_sender.is_closed() { - log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); - // close this channel as communication to user was closed. - break; - } - - tokio::select! { - result_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match result_msg { - Err(e) => { - log::error!("poll_messages_from_stellar(): {e:?}"); - break; - }, - Ok(xdr) => match connector.process_raw_message(xdr).await { - Ok(Some(stellar_msg)) => - // push message to user - if let Err(e) = send_to_user_sender.send(stellar_msg).await { - log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); - }, - Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), - Err(e) => { - log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); - break; - } - } - }, - - // push message to Stellar Node - Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { - log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); - } - } - } - // make sure to drop/shutdown the stream - connector.wr.forget(); - drop(r_stream); - - send_to_node_receiver.close(); - drop(send_to_user_sender); - - - log::info!("poll_messages_from_stellar(): stopped."); -} - - -async fn restart(connector: &mut Connector, address: &str) -> Result { - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(address).await?; - - connector.wr = wr; - Ok(rd) -} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/lib.rs b/clients/stellar-relay-lib/src/lib.rs index e80799332..71f99ca15 100644 --- a/clients/stellar-relay-lib/src/lib.rs +++ b/clients/stellar-relay-lib/src/lib.rs @@ -4,103 +4,9 @@ mod connection; pub mod node; #[cfg(test)] mod tests; +mod overlay; pub use substrate_stellar_sdk as sdk; -use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; -use tokio::net::{tcp, TcpStream}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::error::{SendError, TryRecvError}; -use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::oneshot; - pub use config::{connect_to_stellar_overlay_network, StellarOverlayConfig}; pub use crate::connection::{Error, helper}; - - -use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; -use crate::helper::{create_stream, error_to_string}; -use crate::node::NodeInfo; - -/// Used to send/receive messages to/from Stellar Node -pub struct StellarOverlayConnection { - sender: mpsc::Sender, - receiver: mpsc::Receiver -} - -impl StellarOverlayConnection { - pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { - self.sender.send(msg).await - } - - pub async fn listen(&mut self) -> Result, Error> { - loop { - match self.receiver.try_recv() { - Ok(StellarMessage::ErrorMsg(e)) => { - log::error!("listen(): received error message: {e:?}"); - if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { - return Err(Error::ConnectionFailed(error_to_string(e))) - } - - return Ok(None) - }, - Ok(msg) => return Ok(Some(msg)), - Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), - Err(TryRecvError::Empty) => continue, - } - } - } - - pub fn is_alive(&self) -> bool { - let result = self.sender.is_closed(); - - if result { - drop(self); - } - - !result - } - - pub fn disconnect(&mut self) { - log::info!("disconnect(): closing channel"); - self.receiver.close(); - } -} - -impl Drop for StellarOverlayConnection { - fn drop(&mut self) { - self.disconnect(); - } -} - -impl StellarOverlayConnection { - /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. - pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { - log::info!("connect(): connecting to {conn_info:?}"); - - - // this is a channel to communicate with the user/caller. - let (send_to_user_sender, send_to_user_receiver) = - mpsc::channel::(1024); - - let (send_to_node_sender, send_to_node_receiver) = - mpsc::channel::(1024); - - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(&conn_info.address()).await?; - - let address = conn_info.address(); - let mut connector = Connector::new( - local_node_info, - conn_info, - wr, - ); - connector.send_hello_message().await?; - - tokio::spawn(poll_messages_from_stellar(connector,rd,address, send_to_user_sender,send_to_node_receiver)); - - Ok(StellarOverlayConnection { - sender: send_to_node_sender, - receiver: send_to_user_receiver - }) - } -} \ No newline at end of file +pub use overlay::StellarOverlayConnection; \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs new file mode 100644 index 000000000..ffb4566af --- /dev/null +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -0,0 +1,96 @@ +use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; +use tokio::net::{tcp, TcpStream}; +use tokio::sync::mpsc; +use tokio::sync::mpsc::error::{SendError, TryRecvError}; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::oneshot; + +use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; +use crate::Error; +use crate::helper::{create_stream, error_to_string}; +use crate::node::NodeInfo; + + +/// Used to send/receive messages to/from Stellar Node +pub struct StellarOverlayConnection { + sender: mpsc::Sender, + receiver: mpsc::Receiver +} + +impl StellarOverlayConnection { + pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { + self.sender.send(msg).await + } + + pub async fn listen(&mut self) -> Result, Error> { + loop { + match self.receiver.try_recv() { + Ok(StellarMessage::ErrorMsg(e)) => { + log::error!("listen(): received error message: {e:?}"); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::ConnectionFailed(error_to_string(e))) + } + + return Ok(None) + }, + Ok(msg) => return Ok(Some(msg)), + Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), + Err(TryRecvError::Empty) => continue, + } + } + } + + pub fn is_alive(&self) -> bool { + let result = self.sender.is_closed(); + + if result { + drop(self); + } + + !result + } + + pub fn disconnect(&mut self) { + log::info!("disconnect(): closing channel"); + self.receiver.close(); + } +} + +impl Drop for StellarOverlayConnection { + fn drop(&mut self) { + self.disconnect(); + } +} + +impl StellarOverlayConnection { + /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. + pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { + log::info!("connect(): connecting to {conn_info:?}"); + + + // this is a channel to communicate with the user/caller. + let (send_to_user_sender, send_to_user_receiver) = + mpsc::channel::(1024); + + let (send_to_node_sender, send_to_node_receiver) = + mpsc::channel::(1024); + + // split the stream for easy handling of read and write + let (rd, wr) = create_stream(&conn_info.address()).await?; + + let address = conn_info.address(); + let mut connector = Connector::new( + local_node_info, + conn_info, + wr, + ); + connector.send_hello_message().await?; + + tokio::spawn(poll_messages_from_stellar(connector,rd,address, send_to_user_sender,send_to_node_receiver)); + + Ok(StellarOverlayConnection { + sender: send_to_node_sender, + receiver: send_to_user_receiver + }) + } +} \ No newline at end of file From 56a6cfe1727287fc9904aedfedecc78f4aac920e Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Sat, 2 Dec 2023 22:25:28 +0800 Subject: [PATCH 08/22] added more meaningful errors --- .../connection/authentication/certificate.rs | 6 +++- .../connection/connector/message_handler.rs | 6 ++-- .../stellar-relay-lib/src/connection/error.rs | 31 ++++++++++++++++--- .../src/connection/handshake.rs | 6 +++- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/clients/stellar-relay-lib/src/connection/authentication/certificate.rs b/clients/stellar-relay-lib/src/connection/authentication/certificate.rs index d3f69b60c..a8b3c16da 100644 --- a/clients/stellar-relay-lib/src/connection/authentication/certificate.rs +++ b/clients/stellar-relay-lib/src/connection/authentication/certificate.rs @@ -51,7 +51,11 @@ pub fn create_auth_cert( let raw_sig_data = hash.finalize().to_vec(); - let signature: Signature = Signature::new(keypair.create_signature(raw_sig_data).to_vec())?; + let signature: Signature = Signature::new(keypair.create_signature(raw_sig_data).to_vec()) + .map_err(|e| { + log::error!("create_auth_cert(): {e:?}"); + Error::AuthSignatureFailed + })?; Ok(AuthCert { pubkey: pub_key_ecdh, expiration, sig: signature }) } diff --git a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs index d1c43ae2b..a1630b1dd 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs @@ -28,7 +28,7 @@ impl Connector { "process_raw_message(): Received ErrorMsg during authentication: {}", error_to_string(e.clone()) ); - return Err(Error::ConnectionFailed(error_to_string(e))) + return Err(Error::from(e)) }, other => log::error!("process_raw_message(): Received ErroMsg during authentication: {:?}", other), }, @@ -76,9 +76,9 @@ impl Connector { }, StellarMessage::ErrorMsg(e) => { - log::error!("process_stellar_message(): received from overlay: {e:?}"); + log::error!("process_stellar_message(): Received ErrorMsg during authentication: {e:?}"); if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { - return Err(Error::ConnectionFailed(error_to_string(e))) + return Err(Error::from(e)); } return Ok(Some(StellarMessage::ErrorMsg(e))); }, diff --git a/clients/stellar-relay-lib/src/connection/error.rs b/clients/stellar-relay-lib/src/connection/error.rs index 42d67965f..1e4a2a437 100644 --- a/clients/stellar-relay-lib/src/connection/error.rs +++ b/clients/stellar-relay-lib/src/connection/error.rs @@ -3,18 +3,25 @@ use crate::connection::xdr_converter::Error as XDRError; use substrate_stellar_sdk::{types::ErrorCode, StellarSdkError}; use tokio::sync; +use crate::helper::error_to_string; #[derive(Debug, err_derive::Error)] pub enum Error { - #[error(display = "Authentication Certification: Expired")] + #[error(display = "Auth Certificate: Expired")] AuthCertExpired, - #[error(display = "Authentication Certification: Not Found")] + #[error(display = "Auth Certificate: Not Found")] AuthCertNotFound, - #[error(display = "Authentication Certification: Invalid")] + #[error(display = "Auth Certificate: Invalid")] AuthCertInvalid, + #[error(display = "Auth Certificate Creation: Signature Failed")] + AuthSignatureFailed, + + #[error(display = "Authentication Failed: {}", _0)] + AuthFailed(String), + #[error(display = "Connection: {}", _0)] ConnectionFailed(String), @@ -58,7 +65,10 @@ pub enum Error { OverlayError(ErrorCode), #[error(display = "Timeout elapsed")] - Timeout + Timeout, + + #[error(display = "Config Error: Version String too long")] + VersionStrTooLong } impl From for Error { @@ -79,4 +89,15 @@ impl From for Error { } } - +impl From for Error { + fn from(value: substrate_stellar_sdk::types::Error) -> Self { + match value.code { + ErrorCode::ErrConf => Self::ConfigError(error_to_string(value)), + ErrorCode::ErrAuth => Self::AuthFailed(error_to_string(value)), + other => { + log::error!("Stellar Node returned error: {}", error_to_string(value)); + Self::OverlayError(other) + } + } + } +} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/connection/handshake.rs b/clients/stellar-relay-lib/src/connection/handshake.rs index d82a17b2d..9b326b066 100644 --- a/clients/stellar-relay-lib/src/connection/handshake.rs +++ b/clients/stellar-relay-lib/src/connection/handshake.rs @@ -33,7 +33,11 @@ pub fn create_hello_message( overlay_version: node_info.overlay_version, overlay_min_version: node_info.overlay_min_version, network_id: node_info.network_id, - version_str: LimitedString::<100>::new(version_str.clone())?, + version_str: LimitedString::<100>::new(version_str.clone()) + .map_err(|e| { + log::error!("create_hello_message(): {e:?}"); + Error::VersionStrTooLong + })?, listening_port: i32::try_from(listening_port).unwrap_or(11625), peer_id, cert, From d2487e495fadbfe07a1a0b0ccb30ffe898ef6225 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:54:19 +0800 Subject: [PATCH 09/22] new update --- .../src/connection/connector/connector.rs | 12 ++-- .../connection/connector/message_handler.rs | 3 +- .../connection/connector/message_reader.rs | 2 +- .../stellar-relay-lib/src/connection/mod.rs | 2 +- clients/stellar-relay-lib/src/overlay.rs | 7 +- clients/vault/src/oracle/agent.rs | 65 ++++++++++++++++--- 6 files changed, 70 insertions(+), 21 deletions(-) diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 4631b90a2..209c81e12 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -202,6 +202,10 @@ impl Connector { self.handshake_state = HandshakeState::Completed; } + pub fn get_handshake_state(&self) -> &HandshakeState { + &self.handshake_state + } + pub fn inner_check_to_send_more(&mut self, msg_type: MessageType) -> bool { self.flow_controller.send_more(msg_type) } @@ -273,11 +277,3 @@ pub(crate) async fn poll_messages_from_stellar( log::info!("poll_messages_from_stellar(): stopped."); } - -async fn restart(connector: &mut Connector, address: &str) -> Result { - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(address).await?; - - connector.wr = wr; - Ok(rd) -} \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs index d1c43ae2b..8d2ad99b0 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs @@ -68,7 +68,7 @@ impl Connector { } else { self.send_auth_message().await?; } - log::info!("process_stellar_message(): Hello message processed successfully"); + log::info!("process_stellar_message(): Hello message processed successfully: {:?}", self.get_handshake_state()); }, StellarMessage::Auth(_) => { @@ -101,6 +101,7 @@ impl Connector { } self.handshake_completed(); + log::info!("process_auth_message(): handshake completed"); if let Some(remote) = self.remote() { log::debug!("process_auth_message(): sending connect message: {remote:?}"); diff --git a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs index c9d5aeaee..2f553abe9 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs @@ -29,7 +29,7 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, // check whether or not we should read the bytes as: // 1. the length of the next stellar message // 2. the remaining bytes of the previous stellar message - match timeout(Duration::from_secs(60), r_stream.peek(&mut buff_for_peeking)) + match timeout(Duration::from_secs(timeout_in_secs), r_stream.peek(&mut buff_for_peeking)) .await { Ok(Ok(0)) => { diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index f255f7592..10ae16e0d 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -96,7 +96,7 @@ impl ConnectionInfo { recv_tx_msgs, recv_scp_msgs, remote_called_us, - 10, + 20, ) } diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs index ffb4566af..682267d71 100644 --- a/clients/stellar-relay-lib/src/overlay.rs +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -24,6 +24,10 @@ impl StellarOverlayConnection { pub async fn listen(&mut self) -> Result, Error> { loop { + if !self.is_alive() { + return Err(Error::Disconnected); + } + match self.receiver.try_recv() { Ok(StellarMessage::ErrorMsg(e)) => { log::error!("listen(): received error message: {e:?}"); @@ -51,7 +55,7 @@ impl StellarOverlayConnection { } pub fn disconnect(&mut self) { - log::info!("disconnect(): closing channel"); + log::info!("disconnect(): closing the overlay connection"); self.receiver.close(); } } @@ -67,7 +71,6 @@ impl StellarOverlayConnection { pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { log::info!("connect(): connecting to {conn_info:?}"); - // this is a channel to communicate with the user/caller. let (send_to_user_sender, send_to_user_receiver) = mpsc::channel::(1024); diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index d1f3eb48c..cf8f89592 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -89,12 +89,24 @@ pub async fn start_oracle_agent( let (message_sender, mut message_receiver) = mpsc::channel::(34); let message_sender_clone = message_sender.clone(); + let mut reconnection_increment = 0; service::spawn_cancelable(shutdown_sender_clone.subscribe(), async move { + let mut stop_overlay = false; loop { + if !overlay_conn.is_alive() { + tracing::info!("start_oracle_agent(): oracle is dead."); + let _ = shutdown_sender_clone2.send(()); + + message_receiver.close(); + disconnect_signal_receiver.close(); + + return Ok(()); + } + tokio::select! { - result_msg= overlay_conn.listen() => match result_msg { + result_msg = overlay_conn.listen() => { match result_msg { Ok(Some(msg)) => { - tracing::info!("start_oracle_agent(): handle message: {}", to_base64_xdr_string(&msg)); + //tracing::info!("start_oracle_agent(): handle message: {}", to_base64_xdr_string(&msg)); handle_message( msg, collector_clone.clone(), @@ -103,23 +115,60 @@ pub async fn start_oracle_agent( } Ok(None) => {} Err(e) => { + tracing::error!("start_oracle_agent(): received error: {e:?}"); + overlay_conn.disconnect(); let _ = shutdown_sender_clone2.send(()); return Ok(()); + // loop { + // let sleep_time = timeout_in_secs + reconnection_increment; + // tracing::info!("start_oracle_agent(): reconnect to Stellar in {sleep_time} seconds"); + // reconnection_increment += 5; // increment by 5 + // + // sleep(Duration::from_secs(timeout_in_secs + reconnection_increment)).await; + // + // match connect_to_stellar_overlay_network(config.clone(), &secret_key_copy).await { + // Ok(new_overlay_conn) => { + // overlay_conn = new_overlay_conn; + // + // // wait to be sure that a connection was established. + // sleep(Duration::from_secs(timeout_in_secs)).await; + // + // if overlay_conn.is_alive() { + // tracing::info!("start_oracle_agent(): new connection created..."); + // // a new connection was created; break out of this inner loop + // // and renew listening for messages + // break; + // } + // } + // Err(e) => tracing::warn!("start_oracle_agent(): reconnection failed: {e:?}") + // } + // } } - }, - Some(msg) = message_receiver.recv() => if let Err(e) = overlay_conn.send_to_node(msg).await { - tracing::error!("start_oracle_agent(): failed to send msg to stellar node: {e:?}"); + }}, + + result_msg = timeout(Duration::from_secs(10), message_receiver.recv()) => match result_msg { + Ok(Some(msg)) => if let Err(e) = overlay_conn.send_to_node(msg).await { + tracing::error!("start_oracle_agent(): failed to send msg to stellar node: {e:?}"); + }, + _ => {} }, - Some(_) = disconnect_signal_receiver.recv() => { - tracing::info!("start_oracle_agent(): disconnect signal received."); + result_msg = timeout(Duration::from_secs(10), disconnect_signal_receiver.recv()) => match result_msg { + Ok(Some(_)) => { + tracing::info!("start_oracle_agent(): disconnect signal received."); - overlay_conn.disconnect(); + overlay_conn.disconnect(); + let _ = shutdown_sender_clone2.send(()); + return Ok(()); + } + _ => {} } } } + tracing::info!("start_oracle_agent(): LOOP STOPPED!"); + Ok::<(), Error>(()) }); From 33e1a00b9a8b631f95432596f77dce4b5af0c6c4 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Mon, 4 Dec 2023 19:02:46 +0100 Subject: [PATCH 10/22] Fix compilation --- clients/stellar-relay-lib/src/tests/mod.rs | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/clients/stellar-relay-lib/src/tests/mod.rs b/clients/stellar-relay-lib/src/tests/mod.rs index aac826a4b..78507c805 100644 --- a/clients/stellar-relay-lib/src/tests/mod.rs +++ b/clients/stellar-relay-lib/src/tests/mod.rs @@ -4,11 +4,16 @@ use substrate_stellar_sdk::{ Hash, IntoHash, }; -use crate::{node::NodeInfo, ConnectionInfo, StellarOverlayConfig, StellarOverlayConnection}; +use crate::{ + connection::{to_base64_xdr_string, ConnectionInfo}, + node::NodeInfo, + StellarOverlayConfig, StellarOverlayConnection, +}; use serial_test::serial; -use tokio::{sync::Mutex, time::timeout}; -use tokio::time::sleep; -use crate::connection::to_base64_xdr_string; +use tokio::{ + sync::Mutex, + time::{sleep, timeout}, +}; fn secret_key(is_mainnet: bool) -> String { let path = if is_mainnet { @@ -75,7 +80,7 @@ async fn stellar_overlay_should_receive_scp_messages() { timeout(Duration::from_secs(300), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Some(msg) = ov_conn_locked.listen().await { + while let Ok(Some(msg)) = ov_conn_locked.listen().await { scps_vec_clone.lock().await.push(msg); ov_conn_locked.disconnect(); @@ -117,11 +122,10 @@ async fn stellar_overlay_should_receive_tx_set() { timeout(Duration::from_secs(500), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Some(msg) = ov_conn_locked.listen().await { - match msg { + while let Ok(Some(msg)) = ov_conn_locked.listen().await { + match msg { StellarMessage::ScpMessage(msg) => - if let ScpStatementPledges::ScpStExternalize(stmt) = &msg.statement.pledges - { + if let ScpStatementPledges::ScpStExternalize(stmt) = &msg.statement.pledges { let tx_set_hash = get_tx_set_hash(stmt); tx_set_hashes_clone.lock().await.push(tx_set_hash.clone()); ov_conn_locked @@ -145,7 +149,6 @@ async fn stellar_overlay_should_receive_tx_set() { }, _ => {}, } - } }) .await From a88c1622e1eec8887486818e436afb5bc455bc3a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Mon, 4 Dec 2023 19:17:10 +0100 Subject: [PATCH 11/22] Small refactoring --- .../src/connection/connector/connector.rs | 85 +++++---- .../connection/connector/message_sender.rs | 29 ++- clients/stellar-relay-lib/src/overlay.rs | 174 +++++++++--------- 3 files changed, 152 insertions(+), 136 deletions(-) diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 4631b90a2..0895a57e8 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -1,26 +1,26 @@ use std::fmt::{Debug, Formatter}; use substrate_stellar_sdk::{ - types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType}, + types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType, StellarMessage}, XdrCodec, }; -use substrate_stellar_sdk::types::StellarMessage; -use tokio::io::AsyncWriteExt; -use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::Sender; +use tokio::{ + io::AsyncWriteExt, + net::tcp::{OwnedReadHalf, OwnedWriteHalf}, + sync::{mpsc, mpsc::Sender}, +}; use crate::{ connection::{ authentication::{gen_shared_key, ConnectionAuth}, + connector::message_reader::read_message_from_stellar, flow_controller::FlowController, + handshake::HandshakeState, hmac::{verify_hmac, HMacKeys}, - ConnectionInfo, handshake::HandshakeState, - Error + ConnectionInfo, Error, }, + helper::create_stream, node::{LocalInfo, NodeInfo, RemoteInfo}, }; -use crate::connection::connector::message_reader::read_message_from_stellar; -use crate::helper::create_stream; pub struct Connector { local: LocalInfo, @@ -39,7 +39,7 @@ pub struct Connector { flow_controller: FlowController, /// for writing xdr messages to stream. - pub(crate) wr: OwnedWriteHalf, + pub(crate) write_stream_overlay: OwnedWriteHalf, } impl Debug for Connector { @@ -118,7 +118,7 @@ impl Connector { pub fn new( local_node: NodeInfo, conn_info: ConnectionInfo, - write_half_of_stream: OwnedWriteHalf, + write_stream_overlay: OwnedWriteHalf, ) -> Self { let connection_auth = ConnectionAuth::new( &local_node.network_id, @@ -137,7 +137,7 @@ impl Connector { receive_scp_messages: conn_info.recv_scp_msgs, handshake_state: HandshakeState::Connecting, flow_controller: FlowController::default(), - wr: write_half_of_stream, + write_stream_overlay, } } @@ -214,20 +214,19 @@ impl Connector { } } - /// Polls for messages coming from the Stellar Node and communicates it back to the user /// /// # Arguments /// * `connector` - contains the config and necessary info for connecting to Stellar Node -/// * `r_stream` - the read half of the stream that is connected to Stellar Node +/// * `read_stream_overlay` - the read half of the stream that is connected to Stellar Node /// * `send_to_user_sender` - sends message from Stellar to the user -/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the stream. +/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the +/// stream. pub(crate) async fn poll_messages_from_stellar( mut connector: Connector, - mut r_stream: OwnedReadHalf, - address: String, + mut read_stream_overlay: OwnedReadHalf, send_to_user_sender: mpsc::Sender, - mut send_to_node_receiver: mpsc::Receiver + mut send_to_node_receiver: mpsc::Receiver, ) { log::info!("poll_messages_from_stellar(): started."); @@ -235,38 +234,38 @@ pub(crate) async fn poll_messages_from_stellar( if send_to_user_sender.is_closed() { log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); // close this channel as communication to user was closed. - break; + break } tokio::select! { - result_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match result_msg { - Err(e) => { - log::error!("poll_messages_from_stellar(): {e:?}"); - break; - }, + result_msg = read_message_from_stellar(&mut read_stream_overlay, connector.timeout_in_secs) => match result_msg { + Err(e) => { + log::error!("poll_messages_from_stellar(): {e:?}"); + break; + }, Ok(xdr) => match connector.process_raw_message(xdr).await { - Ok(Some(stellar_msg)) => - // push message to user - if let Err(e) = send_to_user_sender.send(stellar_msg).await { - log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); - }, - Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), - Err(e) => { - log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); - break; - } - } + Ok(Some(stellar_msg)) => + // push message to user + if let Err(e) = send_to_user_sender.send(stellar_msg).await { + log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); + }, + Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), + Err(e) => { + log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); + break; + } + } }, // push message to Stellar Node Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { - log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); - } + log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); + } } } // make sure to drop/shutdown the stream - connector.wr.forget(); - drop(r_stream); + connector.write_stream_overlay.forget(); + drop(read_stream_overlay); send_to_node_receiver.close(); drop(send_to_user_sender); @@ -274,10 +273,10 @@ pub(crate) async fn poll_messages_from_stellar( log::info!("poll_messages_from_stellar(): stopped."); } -async fn restart(connector: &mut Connector, address: &str) -> Result { +async fn restart(connector: &mut Connector, address: &str) -> Result { // split the stream for easy handling of read and write let (rd, wr) = create_stream(address).await?; - connector.wr = wr; + connector.write_stream_overlay = wr; Ok(rd) -} \ No newline at end of file +} diff --git a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs index 8228e560d..7761aa096 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs @@ -1,10 +1,17 @@ use std::time::Duration; use substrate_stellar_sdk::types::{MessageType, SendMore, StellarMessage}; -use tokio::io::AsyncWriteExt; -use tokio::net::tcp::OwnedWriteHalf; -use tokio::time::{timeout, Timeout}; - -use crate::connection::{Connector, Error, flow_controller::MAX_FLOOD_MSG_CAP, handshake::create_auth_message, helper::{time_now, to_base64_xdr_string}}; +use tokio::{ + io::AsyncWriteExt, + net::tcp::OwnedWriteHalf, + time::{timeout, Timeout}, +}; + +use crate::connection::{ + flow_controller::MAX_FLOOD_MSG_CAP, + handshake::create_auth_message, + helper::{time_now, to_base64_xdr_string}, + Connector, Error, +}; impl Connector { pub async fn send_to_node(&mut self, msg: StellarMessage) -> Result<(), Error> { @@ -12,16 +19,18 @@ impl Connector { match timeout( Duration::from_secs(self.timeout_in_secs), - self.wr.write_all(&xdr_msg) - ).await { - Ok(res) => res.map_err(|e| Error::WriteFailed(e.to_string())), - Err(_) => Err(Error::Timeout) + self.write_stream_overlay.write_all(&xdr_msg), + ) + .await + { + Ok(res) => res.map_err(|e| Error::WriteFailed(e.to_string())), + Err(_) => Err(Error::Timeout), } } pub async fn send_hello_message(&mut self) -> Result<(), Error> { let msg = self.create_hello_message(time_now())?; - log::info!("send_hello_message(): Sending Hello Message: {}",to_base64_xdr_string(&msg)); + log::info!("send_hello_message(): Sending Hello Message: {}", to_base64_xdr_string(&msg)); self.send_to_node(msg).await } diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs index ffb4566af..d7bafc59c 100644 --- a/clients/stellar-relay-lib/src/overlay.rs +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -1,96 +1,104 @@ use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; -use tokio::net::{tcp, TcpStream}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::error::{SendError, TryRecvError}; -use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::oneshot; - -use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; -use crate::Error; -use crate::helper::{create_stream, error_to_string}; -use crate::node::NodeInfo; - +use tokio::{ + net::{tcp, TcpStream}, + sync::{ + mpsc, + mpsc::{ + error::{SendError, TryRecvError}, + Receiver, Sender, + }, + oneshot, + }, +}; + +use crate::{ + connection::{poll_messages_from_stellar, ConnectionInfo, Connector}, + helper::{create_stream, error_to_string}, + node::NodeInfo, + Error, +}; /// Used to send/receive messages to/from Stellar Node pub struct StellarOverlayConnection { - sender: mpsc::Sender, - receiver: mpsc::Receiver + sender: mpsc::Sender, + receiver: mpsc::Receiver, } impl StellarOverlayConnection { - pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { - self.sender.send(msg).await - } - - pub async fn listen(&mut self) -> Result, Error> { - loop { - match self.receiver.try_recv() { - Ok(StellarMessage::ErrorMsg(e)) => { - log::error!("listen(): received error message: {e:?}"); - if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { - return Err(Error::ConnectionFailed(error_to_string(e))) - } - - return Ok(None) - }, - Ok(msg) => return Ok(Some(msg)), - Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), - Err(TryRecvError::Empty) => continue, - } - } - } - - pub fn is_alive(&self) -> bool { - let result = self.sender.is_closed(); - - if result { - drop(self); - } - - !result - } - - pub fn disconnect(&mut self) { - log::info!("disconnect(): closing channel"); - self.receiver.close(); - } + pub async fn send_to_node(&self, msg: StellarMessage) -> Result<(), SendError> { + self.sender.send(msg).await + } + + pub async fn listen(&mut self) -> Result, Error> { + loop { + match self.receiver.try_recv() { + Ok(StellarMessage::ErrorMsg(e)) => { + log::error!("listen(): received error message: {e:?}"); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::ConnectionFailed(error_to_string(e))) + } + + return Ok(None) + }, + Ok(msg) => return Ok(Some(msg)), + Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), + Err(TryRecvError::Empty) => continue, + } + } + } + + pub fn is_alive(&self) -> bool { + let result = self.sender.is_closed(); + + if result { + drop(self); + } + + !result + } + + pub fn disconnect(&mut self) { + log::info!("disconnect(): closing channel"); + self.receiver.close(); + } } impl Drop for StellarOverlayConnection { - fn drop(&mut self) { - self.disconnect(); - } + fn drop(&mut self) { + self.disconnect(); + } } impl StellarOverlayConnection { - /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. - pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { - log::info!("connect(): connecting to {conn_info:?}"); - - - // this is a channel to communicate with the user/caller. - let (send_to_user_sender, send_to_user_receiver) = - mpsc::channel::(1024); - - let (send_to_node_sender, send_to_node_receiver) = - mpsc::channel::(1024); - - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(&conn_info.address()).await?; - - let address = conn_info.address(); - let mut connector = Connector::new( - local_node_info, - conn_info, - wr, - ); - connector.send_hello_message().await?; - - tokio::spawn(poll_messages_from_stellar(connector,rd,address, send_to_user_sender,send_to_node_receiver)); - - Ok(StellarOverlayConnection { - sender: send_to_node_sender, - receiver: send_to_user_receiver - }) - } -} \ No newline at end of file + /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. + pub async fn connect( + local_node_info: NodeInfo, + conn_info: ConnectionInfo, + ) -> Result { + log::info!("connect(): connecting to {conn_info:?}"); + + // this is a channel to communicate with the user/caller. + let (send_to_user_sender, send_to_user_receiver) = mpsc::channel::(1024); + + let (send_to_node_sender, send_to_node_receiver) = mpsc::channel::(1024); + + // split the stream for easy handling of read and write + let (read_stream_overlay, write_stream_overlay) = + create_stream(&conn_info.address()).await?; + + let mut connector = Connector::new(local_node_info, conn_info, write_stream_overlay); + connector.send_hello_message().await?; + + tokio::spawn(poll_messages_from_stellar( + connector, + read_stream_overlay, + send_to_user_sender, + send_to_node_receiver, + )); + + Ok(StellarOverlayConnection { + sender: send_to_node_sender, + receiver: send_to_user_receiver, + }) + } +} From 5a3d55e7a4678143433cd9506b3354127b3da4dd Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:37:21 +0800 Subject: [PATCH 12/22] fix test cases and run cargo fmt --- clients/stellar-relay-lib/src/config.rs | 11 +- .../connection/authentication/certificate.rs | 4 +- .../src/connection/authentication/tests.rs | 4 +- .../src/connection/connector/connector.rs | 61 ++-- .../connection/connector/message_creation.rs | 7 +- .../connection/connector/message_handler.rs | 273 +++++++++--------- .../connection/connector/message_reader.rs | 75 ++--- .../connection/connector/message_sender.rs | 23 +- .../src/connection/connector/mod.rs | 1 - .../stellar-relay-lib/src/connection/error.rs | 4 +- .../src/connection/handshake.rs | 2 +- .../src/connection/helper.rs | 2 +- .../stellar-relay-lib/src/connection/mod.rs | 8 +- .../src/connection/xdr_converter.rs | 2 +- clients/stellar-relay-lib/src/lib.rs | 10 +- clients/stellar-relay-lib/src/node/local.rs | 1 - clients/stellar-relay-lib/src/overlay.rs | 171 ++++++----- clients/stellar-relay-lib/src/tests/mod.rs | 29 +- 18 files changed, 337 insertions(+), 351 deletions(-) diff --git a/clients/stellar-relay-lib/src/config.rs b/clients/stellar-relay-lib/src/config.rs index d71310263..3312f047e 100644 --- a/clients/stellar-relay-lib/src/config.rs +++ b/clients/stellar-relay-lib/src/config.rs @@ -1,13 +1,12 @@ +use crate::{ + connection::{ConnectionInfo, Error}, + node::NodeInfo, + StellarOverlayConnection, +}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, BytesOrString}; use std::fmt::Debug; use substrate_stellar_sdk::SecretKey; -use substrate_stellar_sdk::types::StellarMessage; -use tokio::net::TcpStream; -use tokio::sync::mpsc; -use crate::connection::{ConnectionInfo, Error}; -use crate::node::NodeInfo; -use crate::StellarOverlayConnection; /// The configuration structure of the StellarOverlay. /// It configures both the ConnectionInfo and the NodeInfo. diff --git a/clients/stellar-relay-lib/src/connection/authentication/certificate.rs b/clients/stellar-relay-lib/src/connection/authentication/certificate.rs index d3f69b60c..b454d458c 100644 --- a/clients/stellar-relay-lib/src/connection/authentication/certificate.rs +++ b/clients/stellar-relay-lib/src/connection/authentication/certificate.rs @@ -1,6 +1,6 @@ use crate::{ - connection::authentication::{BinarySha256Hash, ConnectionAuth, AUTH_CERT_EXPIRATION_LIMIT}, - Error, + connection::authentication::{BinarySha256Hash, ConnectionAuth, AUTH_CERT_EXPIRATION_LIMIT}, + Error, }; use sha2::{Digest, Sha256}; use substrate_stellar_sdk::{ diff --git a/clients/stellar-relay-lib/src/connection/authentication/tests.rs b/clients/stellar-relay-lib/src/connection/authentication/tests.rs index 5dbcbd760..af1db5a29 100644 --- a/clients/stellar-relay-lib/src/connection/authentication/tests.rs +++ b/clients/stellar-relay-lib/src/connection/authentication/tests.rs @@ -1,5 +1,5 @@ use crate::{ - connection::{ + connection::{ authentication::{ certificate::{create_auth_cert, verify_remote_auth_cert}, shared_key::gen_shared_key, @@ -8,7 +8,7 @@ use crate::{ helper::{generate_random_nonce, time_now}, hmac::{create_receiving_mac_key, create_sending_mac_key, create_sha256_hmac, verify_hmac}, }, - Error, + Error, }; use substrate_stellar_sdk::{ network::Network, diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 209c81e12..643573b69 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -1,26 +1,24 @@ use std::fmt::{Debug, Formatter}; use substrate_stellar_sdk::{ - types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType}, + types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType, StellarMessage}, XdrCodec, }; -use substrate_stellar_sdk::types::StellarMessage; -use tokio::io::AsyncWriteExt; -use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::Sender; +use tokio::{ + net::tcp::{OwnedReadHalf, OwnedWriteHalf}, + sync::mpsc, +}; use crate::{ connection::{ authentication::{gen_shared_key, ConnectionAuth}, + connector::message_reader::read_message_from_stellar, flow_controller::FlowController, + handshake::HandshakeState, hmac::{verify_hmac, HMacKeys}, - ConnectionInfo, handshake::HandshakeState, - Error + ConnectionInfo, Error, }, node::{LocalInfo, NodeInfo, RemoteInfo}, }; -use crate::connection::connector::message_reader::read_message_from_stellar; -use crate::helper::create_stream; pub struct Connector { local: LocalInfo, @@ -218,20 +216,19 @@ impl Connector { } } - /// Polls for messages coming from the Stellar Node and communicates it back to the user /// /// # Arguments /// * `connector` - contains the config and necessary info for connecting to Stellar Node /// * `r_stream` - the read half of the stream that is connected to Stellar Node /// * `send_to_user_sender` - sends message from Stellar to the user -/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the stream. +/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the +/// stream. pub(crate) async fn poll_messages_from_stellar( mut connector: Connector, mut r_stream: OwnedReadHalf, - address: String, send_to_user_sender: mpsc::Sender, - mut send_to_node_receiver: mpsc::Receiver + mut send_to_node_receiver: mpsc::Receiver, ) { log::info!("poll_messages_from_stellar(): started."); @@ -239,33 +236,33 @@ pub(crate) async fn poll_messages_from_stellar( if send_to_user_sender.is_closed() { log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); // close this channel as communication to user was closed. - break; + break } tokio::select! { result_msg = read_message_from_stellar(&mut r_stream, connector.timeout_in_secs) => match result_msg { - Err(e) => { - log::error!("poll_messages_from_stellar(): {e:?}"); - break; - }, + Err(e) => { + log::error!("poll_messages_from_stellar(): {e:?}"); + break; + }, Ok(xdr) => match connector.process_raw_message(xdr).await { - Ok(Some(stellar_msg)) => - // push message to user - if let Err(e) = send_to_user_sender.send(stellar_msg).await { - log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); - }, - Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), - Err(e) => { - log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); - break; - } - } + Ok(Some(stellar_msg)) => + // push message to user + if let Err(e) = send_to_user_sender.send(stellar_msg).await { + log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); + }, + Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), + Err(e) => { + log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); + break; + } + } }, // push message to Stellar Node Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { - log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); - } + log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); + } } } // make sure to drop/shutdown the stream diff --git a/clients/stellar-relay-lib/src/connection/connector/message_creation.rs b/clients/stellar-relay-lib/src/connection/connector/message_creation.rs index 73b6a773e..16f1e3db1 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_creation.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_creation.rs @@ -1,5 +1,6 @@ -use crate::{ - connection::{authentication::create_auth_cert, handshake, hmac::create_sha256_hmac, Connector, Error, xdr_converter}, +use crate::connection::{ + authentication::create_auth_cert, handshake, hmac::create_sha256_hmac, xdr_converter, + Connector, Error, }; use substrate_stellar_sdk::{ types::{AuthenticatedMessage, AuthenticatedMessageV0, HmacSha256Mac, StellarMessage}, @@ -73,4 +74,4 @@ impl Connector { local.node(), ) } -} \ No newline at end of file +} diff --git a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs index 8d2ad99b0..0ccd39822 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs @@ -1,141 +1,156 @@ -use substrate_stellar_sdk::types::{ErrorCode, Hello, MessageType, StellarMessage}; -use substrate_stellar_sdk::XdrCodec; - -use crate::connection::{Connector, Xdr, Error, helper::{error_to_string, time_now}, xdr_converter::parse_authenticated_message}; -use crate::connection::authentication::verify_remote_auth_cert; -use crate::connection::hmac::HMacKeys; +use substrate_stellar_sdk::{ + types::{ErrorCode, Hello, MessageType, StellarMessage}, + XdrCodec, +}; + +use crate::connection::{ + authentication::verify_remote_auth_cert, + helper::{error_to_string, time_now}, + hmac::HMacKeys, + xdr_converter::parse_authenticated_message, + Connector, Error, Xdr, +}; use crate::node::RemoteInfo; impl Connector { - /// Processes the raw bytes from the stream - pub(super) async fn process_raw_message(&mut self, data: Xdr) -> Result, Error> { - let (auth_msg, msg_type) = parse_authenticated_message(&data)?; - - match msg_type { - MessageType::Transaction | MessageType::FloodAdvert if !self.receive_tx_messages() => { - self.increment_remote_sequence()?; - self.check_to_send_more(MessageType::Transaction).await?; - }, - - MessageType::ScpMessage if !self.receive_scp_messages() => { - self.increment_remote_sequence()?; - }, - - MessageType::ErrorMsg => match auth_msg.message { - StellarMessage::ErrorMsg(e) => { - log::error!( + /// Processes the raw bytes from the stream + pub(super) async fn process_raw_message( + &mut self, + data: Xdr, + ) -> Result, Error> { + let (auth_msg, msg_type) = parse_authenticated_message(&data)?; + + match msg_type { + MessageType::Transaction | MessageType::FloodAdvert if !self.receive_tx_messages() => { + self.increment_remote_sequence()?; + self.check_to_send_more(MessageType::Transaction).await?; + }, + + MessageType::ScpMessage if !self.receive_scp_messages() => { + self.increment_remote_sequence()?; + }, + + MessageType::ErrorMsg => match auth_msg.message { + StellarMessage::ErrorMsg(e) => { + log::error!( "process_raw_message(): Received ErrorMsg during authentication: {}", error_to_string(e.clone()) ); - return Err(Error::ConnectionFailed(error_to_string(e))) - }, - other => log::error!("process_raw_message(): Received ErroMsg during authentication: {:?}", other), - }, - - _ => { - // we only verify the authenticated message when a handshake has been done. - if self.is_handshake_created() { - self.verify_auth(&auth_msg, &data[4..(data.len() - 32)])?; - self.increment_remote_sequence()?; - log::trace!( + return Err(Error::ConnectionFailed(error_to_string(e))) + }, + other => log::error!( + "process_raw_message(): Received ErroMsg during authentication: {:?}", + other + ), + }, + + _ => { + // we only verify the authenticated message when a handshake has been done. + if self.is_handshake_created() { + self.verify_auth(&auth_msg, &data[4..(data.len() - 32)])?; + self.increment_remote_sequence()?; + log::trace!( "process_raw_message(): Processing {msg_type:?} message: auth verified" ); - } - - return self.process_stellar_message( auth_msg.message, msg_type).await; - }, - } - Ok(None) - } - - /// Returns a StellarMessage for the user/outsider. Else none if user/outsider do not need it. - /// This handles what to do with the Stellar message. - async fn process_stellar_message( - &mut self, - msg: StellarMessage, - msg_type: MessageType, - ) -> Result, Error> { - match msg { - StellarMessage::Hello(hello) => { - // update the node info based on the hello message - self.process_hello_message(hello)?; - - self.got_hello(); - - if self.remote_called_us() { - self.send_hello_message().await?; - } else { - self.send_auth_message().await?; - } - log::info!("process_stellar_message(): Hello message processed successfully: {:?}", self.get_handshake_state()); - }, - - StellarMessage::Auth(_) => { - self.process_auth_message().await?; - }, - - StellarMessage::ErrorMsg(e) => { - log::error!("process_stellar_message(): received from overlay: {e:?}"); - if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { - return Err(Error::ConnectionFailed(error_to_string(e))) - } - return Ok(Some(StellarMessage::ErrorMsg(e))); - }, - - other => { - log::trace!( + } + + return self.process_stellar_message(auth_msg.message, msg_type).await + }, + } + Ok(None) + } + + /// Returns a StellarMessage for the user/outsider. Else none if user/outsider do not need it. + /// This handles what to do with the Stellar message. + async fn process_stellar_message( + &mut self, + msg: StellarMessage, + msg_type: MessageType, + ) -> Result, Error> { + match msg { + StellarMessage::Hello(hello) => { + // update the node info based on the hello message + self.process_hello_message(hello)?; + + self.got_hello(); + + if self.remote_called_us() { + self.send_hello_message().await?; + } else { + self.send_auth_message().await?; + } + log::info!( + "process_stellar_message(): Hello message processed successfully: {:?}", + self.get_handshake_state() + ); + }, + + StellarMessage::Auth(_) => { + self.process_auth_message().await?; + }, + + StellarMessage::ErrorMsg(e) => { + log::error!("process_stellar_message(): received from overlay: {e:?}"); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::ConnectionFailed(error_to_string(e))) + } + return Ok(Some(StellarMessage::ErrorMsg(e))) + }, + + other => { + log::trace!( "process_stellar_message(): Processing {other:?} message: received from overlay" ); - self.check_to_send_more(msg_type).await?; - return Ok(Some(other)); - }, - } - - Ok(None) - } - - async fn process_auth_message(&mut self) -> Result<(), Error> { - if self.remote_called_us() { - self.send_auth_message().await?; - } - - self.handshake_completed(); - log::info!("process_auth_message(): handshake completed"); - - if let Some(remote) = self.remote() { - log::debug!("process_auth_message(): sending connect message: {remote:?}"); - self.enable_flow_controller( - self.local().node().overlay_version, - remote.node().overlay_version, - ); - } else { - log::warn!("process_auth_message(): No remote overlay version after handshake."); - } - - self.check_to_send_more(MessageType::Auth).await - } - - /// Updates the config based on the hello message that was received from the Stellar Node - fn process_hello_message(&mut self, hello: Hello) -> Result<(), Error> { - let mut network_id = self.connection_auth.network_id().to_xdr(); - - if !verify_remote_auth_cert(time_now(), &hello.peer_id, &hello.cert, &mut network_id) { - return Err(Error::AuthCertInvalid) - } - - let remote_info = RemoteInfo::new(&hello); - let shared_key = self.get_shared_key(remote_info.pub_key_ecdh()); - - self.set_hmac_keys(HMacKeys::new( - &shared_key, - self.local().nonce(), - remote_info.nonce(), - self.remote_called_us(), - )); - - self.set_remote(remote_info); - - Ok(()) - } + self.check_to_send_more(msg_type).await?; + return Ok(Some(other)) + }, + } + + Ok(None) + } + + async fn process_auth_message(&mut self) -> Result<(), Error> { + if self.remote_called_us() { + self.send_auth_message().await?; + } + + self.handshake_completed(); + log::info!("process_auth_message(): handshake completed"); + + if let Some(remote) = self.remote() { + log::debug!("process_auth_message(): sending connect message: {remote:?}"); + self.enable_flow_controller( + self.local().node().overlay_version, + remote.node().overlay_version, + ); + } else { + log::warn!("process_auth_message(): No remote overlay version after handshake."); + } + + self.check_to_send_more(MessageType::Auth).await + } + + /// Updates the config based on the hello message that was received from the Stellar Node + fn process_hello_message(&mut self, hello: Hello) -> Result<(), Error> { + let mut network_id = self.connection_auth.network_id().to_xdr(); + + if !verify_remote_auth_cert(time_now(), &hello.peer_id, &hello.cert, &mut network_id) { + return Err(Error::AuthCertInvalid) + } + + let remote_info = RemoteInfo::new(&hello); + let shared_key = self.get_shared_key(remote_info.pub_key_ecdh()); + + self.set_hmac_keys(HMacKeys::new( + &shared_key, + self.local().nonce(), + remote_info.nonce(), + self.remote_called_us(), + )); + + self.set_remote(remote_info); + + Ok(()) + } } diff --git a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs index 2f553abe9..4146b13d4 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs @@ -1,25 +1,13 @@ +use crate::connection::{xdr_converter::get_xdr_message_length, Error, Xdr}; use std::time::Duration; -use crate::{ - connection::{ - authentication::verify_remote_auth_cert, error_to_string, helper::time_now, hmac::HMacKeys, - xdr_converter::{parse_authenticated_message, get_xdr_message_length}, Connector, Xdr,Error - }, - node::RemoteInfo -}; -use substrate_stellar_sdk::{ - types::{Hello, MessageType, StellarMessage}, - XdrCodec, -}; -use substrate_stellar_sdk::types::ErrorCode; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::tcp; -use tokio::sync::mpsc; -use tokio::time::timeout; -use crate::connection::to_base64_xdr_string; +use tokio::{io::AsyncReadExt, net::tcp, time::timeout}; /// Returns Xdr format of the `StellarMessage` sent from the Stellar Node -pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, timeout_in_secs:u64) -> Result { +pub(super) async fn read_message_from_stellar( + r_stream: &mut tcp::OwnedReadHalf, + timeout_in_secs: u64, +) -> Result { // holds the number of bytes that were missing from the previous stellar message. let mut lack_bytes_from_prev = 0; let mut readbuf: Vec = vec![]; @@ -34,19 +22,23 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, { Ok(Ok(0)) => { log::trace!("read_message_from_stellar(): ERROR: Received 0 size"); - return Err(Error::Disconnected); - } + return Err(Error::Disconnected) + }, Ok(Ok(_)) if lack_bytes_from_prev == 0 => { // if there are no more bytes lacking from the previous message, // then check the size of next stellar message. // If it's not enough, skip it. let expect_msg_len = next_message_length(r_stream).await; - log::trace!("read_message_from_stellar(): The next message length is {expect_msg_len}"); + log::trace!( + "read_message_from_stellar(): The next message length is {expect_msg_len}" + ); if expect_msg_len == 0 { // there's nothing to read; wait for the next iteration - log::trace!("read_message_from_stellar(): Nothing left to read; waiting for next loop"); + log::trace!( + "read_message_from_stellar(): Nothing left to read; waiting for next loop" + ); continue } @@ -58,47 +50,44 @@ pub(super) async fn read_message_from_stellar(r_stream: &mut tcp::OwnedReadHalf, &mut lack_bytes_from_prev, &mut readbuf, expect_msg_len, - ).await { + ) + .await + { Ok(None) => continue, Ok(Some(xdr)) => return Ok(xdr), Err(e) => { - log::trace!("read_message_from_stellar(): ERROR: {e:?}"); - return Err(e); - } + log::trace!("read_message_from_stellar(): ERROR: {e:?}"); + return Err(e) + }, } - } + }, Ok(Ok(_)) => { // let's read the continuation number of bytes from the previous message. - match read_unfinished_message( - r_stream, - &mut lack_bytes_from_prev, - &mut readbuf, - ).await { + match read_unfinished_message(r_stream, &mut lack_bytes_from_prev, &mut readbuf) + .await + { Ok(None) => continue, Ok(Some(xdr)) => return Ok(xdr), Err(e) => { log::trace!("read_message_from_stellar(): ERROR: {e:?}"); - return Err(e); - } + return Err(e) + }, } - } + }, Ok(Err(e)) => { log::trace!("read_message_from_stellar(): ERROR: {e:?}"); - return Err(Error::ReadFailed(e.to_string())); - } + return Err(Error::ReadFailed(e.to_string())) + }, Err(_) => { log::error!("read_message_from_stellar(): timeout elapsed."); - return Err(Error::Timeout); - } + return Err(Error::Timeout) + }, } } - - Err(Error::ReadFailed("loop for reading has ended".to_string())) } - /// Returns Xdr when all bytes from the stream have successfully been converted; else None. /// This reads a number of bytes based on the expected message length. /// @@ -153,7 +142,7 @@ async fn read_unfinished_message( log::trace!("read_unfinished_message(): received continuation from the previous message."); readbuf.append(&mut cont_buf); - return Ok(Some(readbuf.clone())); + return Ok(Some(readbuf.clone())) } // this partial message is not enough to complete the previous message. diff --git a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs index 8228e560d..e7c475709 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs @@ -1,27 +1,28 @@ use std::time::Duration; use substrate_stellar_sdk::types::{MessageType, SendMore, StellarMessage}; -use tokio::io::AsyncWriteExt; -use tokio::net::tcp::OwnedWriteHalf; -use tokio::time::{timeout, Timeout}; +use tokio::{io::AsyncWriteExt, time::timeout}; -use crate::connection::{Connector, Error, flow_controller::MAX_FLOOD_MSG_CAP, handshake::create_auth_message, helper::{time_now, to_base64_xdr_string}}; +use crate::connection::{ + flow_controller::MAX_FLOOD_MSG_CAP, + handshake::create_auth_message, + helper::{time_now, to_base64_xdr_string}, + Connector, Error, +}; impl Connector { pub async fn send_to_node(&mut self, msg: StellarMessage) -> Result<(), Error> { let xdr_msg = &self.create_xdr_message(msg)?; - match timeout( - Duration::from_secs(self.timeout_in_secs), - self.wr.write_all(&xdr_msg) - ).await { - Ok(res) => res.map_err(|e| Error::WriteFailed(e.to_string())), - Err(_) => Err(Error::Timeout) + match timeout(Duration::from_secs(self.timeout_in_secs), self.wr.write_all(&xdr_msg)).await + { + Ok(res) => res.map_err(|e| Error::WriteFailed(e.to_string())), + Err(_) => Err(Error::Timeout), } } pub async fn send_hello_message(&mut self) -> Result<(), Error> { let msg = self.create_hello_message(time_now())?; - log::info!("send_hello_message(): Sending Hello Message: {}",to_base64_xdr_string(&msg)); + log::info!("send_hello_message(): Sending Hello Message: {}", to_base64_xdr_string(&msg)); self.send_to_node(msg).await } diff --git a/clients/stellar-relay-lib/src/connection/connector/mod.rs b/clients/stellar-relay-lib/src/connection/connector/mod.rs index 97c65a694..168456a1a 100644 --- a/clients/stellar-relay-lib/src/connection/connector/mod.rs +++ b/clients/stellar-relay-lib/src/connection/connector/mod.rs @@ -5,4 +5,3 @@ mod message_reader; mod message_sender; pub(crate) use connector::*; - diff --git a/clients/stellar-relay-lib/src/connection/error.rs b/clients/stellar-relay-lib/src/connection/error.rs index 42d67965f..32a0be384 100644 --- a/clients/stellar-relay-lib/src/connection/error.rs +++ b/clients/stellar-relay-lib/src/connection/error.rs @@ -58,7 +58,7 @@ pub enum Error { OverlayError(ErrorCode), #[error(display = "Timeout elapsed")] - Timeout + Timeout, } impl From for Error { @@ -78,5 +78,3 @@ impl From for Error { Error::StellarSdkError(e) } } - - diff --git a/clients/stellar-relay-lib/src/connection/handshake.rs b/clients/stellar-relay-lib/src/connection/handshake.rs index d82a17b2d..879c677c3 100644 --- a/clients/stellar-relay-lib/src/connection/handshake.rs +++ b/clients/stellar-relay-lib/src/connection/handshake.rs @@ -1,6 +1,6 @@ use substrate_stellar_sdk::compound_types::LimitedString; -use crate::{node::NodeInfo, connection::Error}; +use crate::{connection::Error, node::NodeInfo}; use substrate_stellar_sdk::{ types::{Auth, AuthCert, Hello, StellarMessage, Uint256}, PublicKey, diff --git a/clients/stellar-relay-lib/src/connection/helper.rs b/clients/stellar-relay-lib/src/connection/helper.rs index d909879f2..051357777 100644 --- a/clients/stellar-relay-lib/src/connection/helper.rs +++ b/clients/stellar-relay-lib/src/connection/helper.rs @@ -50,4 +50,4 @@ pub async fn create_stream( .map_err(|e| crate::Error::ConnectionFailed(e.to_string()))?; Ok(stream.into_split()) -} \ No newline at end of file +} diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index 10ae16e0d..fb4886592 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -13,16 +13,10 @@ pub use error::Error; pub use helper::*; use serde::Serialize; use std::fmt::{Debug, Formatter}; +use substrate_stellar_sdk::SecretKey; type Xdr = Vec; -use crate::node::NodeInfo; -use substrate_stellar_sdk::{ - types::{MessageType, StellarMessage}, - PublicKey, SecretKey, -}; - - /// Config for connecting to Stellar Node #[derive(Clone, Serialize, PartialEq, Eq)] pub struct ConnectionInfo { diff --git a/clients/stellar-relay-lib/src/connection/xdr_converter.rs b/clients/stellar-relay-lib/src/connection/xdr_converter.rs index a067061c3..cb25e724b 100644 --- a/clients/stellar-relay-lib/src/connection/xdr_converter.rs +++ b/clients/stellar-relay-lib/src/connection/xdr_converter.rs @@ -56,7 +56,7 @@ macro_rules! parse_stellar_type { ($ref:ident, $struct_str:ident) => {{ use $crate::{ sdk::{types::$struct_str, XdrCodec}, - connection::xdr_converter::Error, + xdr_converter::Error, }; let ret: Result<$struct_str, Error> = $struct_str::from_xdr($ref).map_err(|e| { diff --git a/clients/stellar-relay-lib/src/lib.rs b/clients/stellar-relay-lib/src/lib.rs index 71f99ca15..4579e9b25 100644 --- a/clients/stellar-relay-lib/src/lib.rs +++ b/clients/stellar-relay-lib/src/lib.rs @@ -2,11 +2,13 @@ mod config; // mod connection; mod connection; pub mod node; +mod overlay; #[cfg(test)] mod tests; -mod overlay; -pub use substrate_stellar_sdk as sdk; +pub use crate::connection::{ + handshake::HandshakeState, helper, xdr_converter, ConnectionInfo, Error, +}; pub use config::{connect_to_stellar_overlay_network, StellarOverlayConfig}; -pub use crate::connection::{Error, helper}; -pub use overlay::StellarOverlayConnection; \ No newline at end of file +pub use overlay::StellarOverlayConnection; +pub use substrate_stellar_sdk as sdk; diff --git a/clients/stellar-relay-lib/src/node/local.rs b/clients/stellar-relay-lib/src/node/local.rs index a2d8cfff3..c4d6c5409 100644 --- a/clients/stellar-relay-lib/src/node/local.rs +++ b/clients/stellar-relay-lib/src/node/local.rs @@ -1,7 +1,6 @@ use crate::{connection::helper::generate_random_nonce, node::NodeInfo}; use substrate_stellar_sdk::types::Uint256; - #[derive(Debug, Clone)] pub struct LocalInfo { sequence: u64, diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs index 682267d71..f1aaf8dbe 100644 --- a/clients/stellar-relay-lib/src/overlay.rs +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -1,99 +1,98 @@ use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; -use tokio::net::{tcp, TcpStream}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::error::{SendError, TryRecvError}; -use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::oneshot; - -use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; -use crate::Error; -use crate::helper::{create_stream, error_to_string}; -use crate::node::NodeInfo; - +use tokio::sync::{ + mpsc, + mpsc::error::{SendError, TryRecvError}, +}; + +use crate::{ + connection::{poll_messages_from_stellar, ConnectionInfo, Connector}, + helper::{create_stream, error_to_string}, + node::NodeInfo, + Error, +}; /// Used to send/receive messages to/from Stellar Node pub struct StellarOverlayConnection { - sender: mpsc::Sender, - receiver: mpsc::Receiver + sender: mpsc::Sender, + receiver: mpsc::Receiver, } impl StellarOverlayConnection { - pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { - self.sender.send(msg).await - } - - pub async fn listen(&mut self) -> Result, Error> { - loop { - if !self.is_alive() { - return Err(Error::Disconnected); - } - - match self.receiver.try_recv() { - Ok(StellarMessage::ErrorMsg(e)) => { - log::error!("listen(): received error message: {e:?}"); - if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { - return Err(Error::ConnectionFailed(error_to_string(e))) - } - - return Ok(None) - }, - Ok(msg) => return Ok(Some(msg)), - Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), - Err(TryRecvError::Empty) => continue, - } - } - } - - pub fn is_alive(&self) -> bool { - let result = self.sender.is_closed(); - - if result { - drop(self); - } - - !result - } - - pub fn disconnect(&mut self) { - log::info!("disconnect(): closing the overlay connection"); - self.receiver.close(); - } + pub async fn send_to_node(&self, msg: StellarMessage) -> Result<(), SendError> { + self.sender.send(msg).await + } + + pub async fn listen(&mut self) -> Result, Error> { + loop { + if !self.is_alive() { + return Err(Error::Disconnected) + } + + match self.receiver.try_recv() { + Ok(StellarMessage::ErrorMsg(e)) => { + log::error!("listen(): received error message: {e:?}"); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::ConnectionFailed(error_to_string(e))) + } + + return Ok(None) + }, + Ok(msg) => return Ok(Some(msg)), + Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), + Err(TryRecvError::Empty) => continue, + } + } + } + + pub fn is_alive(&self) -> bool { + let result = self.sender.is_closed(); + + if result { + drop(self); + } + + !result + } + + pub fn disconnect(&mut self) { + log::info!("disconnect(): closing the overlay connection"); + self.receiver.close(); + } } impl Drop for StellarOverlayConnection { - fn drop(&mut self) { - self.disconnect(); - } + fn drop(&mut self) { + self.disconnect(); + } } impl StellarOverlayConnection { - /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. - pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { - log::info!("connect(): connecting to {conn_info:?}"); - - // this is a channel to communicate with the user/caller. - let (send_to_user_sender, send_to_user_receiver) = - mpsc::channel::(1024); - - let (send_to_node_sender, send_to_node_receiver) = - mpsc::channel::(1024); - - // split the stream for easy handling of read and write - let (rd, wr) = create_stream(&conn_info.address()).await?; - - let address = conn_info.address(); - let mut connector = Connector::new( - local_node_info, - conn_info, - wr, - ); - connector.send_hello_message().await?; - - tokio::spawn(poll_messages_from_stellar(connector,rd,address, send_to_user_sender,send_to_node_receiver)); - - Ok(StellarOverlayConnection { - sender: send_to_node_sender, - receiver: send_to_user_receiver - }) - } -} \ No newline at end of file + /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. + pub async fn connect( + local_node_info: NodeInfo, + conn_info: ConnectionInfo, + ) -> Result { + log::info!("connect(): connecting to {conn_info:?}"); + + // this is a channel to communicate with the user/caller. + let (send_to_user_sender, send_to_user_receiver) = mpsc::channel::(1024); + + let (send_to_node_sender, send_to_node_receiver) = mpsc::channel::(1024); + + // split the stream for easy handling of read and write + let (rd, wr) = create_stream(&conn_info.address()).await?; + + let mut connector = Connector::new(local_node_info, conn_info, wr); + + tokio::spawn(async move { + let _ = connector.send_hello_message().await; + poll_messages_from_stellar(connector, rd, send_to_user_sender, send_to_node_receiver) + .await; + }); + + Ok(StellarOverlayConnection { + sender: send_to_node_sender, + receiver: send_to_user_receiver, + }) + } +} diff --git a/clients/stellar-relay-lib/src/tests/mod.rs b/clients/stellar-relay-lib/src/tests/mod.rs index aac826a4b..37f50e675 100644 --- a/clients/stellar-relay-lib/src/tests/mod.rs +++ b/clients/stellar-relay-lib/src/tests/mod.rs @@ -1,14 +1,13 @@ +use crate::{ + connection::ConnectionInfo, node::NodeInfo, StellarOverlayConfig, StellarOverlayConnection, +}; +use serial_test::serial; use std::{sync::Arc, time::Duration}; use substrate_stellar_sdk::{ types::{ScpStatementExternalize, ScpStatementPledges, StellarMessage}, Hash, IntoHash, }; - -use crate::{node::NodeInfo, ConnectionInfo, StellarOverlayConfig, StellarOverlayConnection}; -use serial_test::serial; use tokio::{sync::Mutex, time::timeout}; -use tokio::time::sleep; -use crate::connection::to_base64_xdr_string; fn secret_key(is_mainnet: bool) -> String { let path = if is_mainnet { @@ -60,7 +59,7 @@ fn overlay_infos(is_mainnet: bool) -> (NodeInfo, ConnectionInfo) { // overlay_connection.disconnect(); // } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] #[serial] async fn stellar_overlay_should_receive_scp_messages() { let (node_info, conn_info) = overlay_infos(false); @@ -75,7 +74,7 @@ async fn stellar_overlay_should_receive_scp_messages() { timeout(Duration::from_secs(300), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Some(msg) = ov_conn_locked.listen().await { + while let Ok(Some(msg)) = ov_conn_locked.listen().await { scps_vec_clone.lock().await.push(msg); ov_conn_locked.disconnect(); @@ -88,13 +87,9 @@ async fn stellar_overlay_should_receive_scp_messages() { //assert //ensure that we receive some scp message from stellar node assert!(!scps_vec.lock().await.is_empty()); - - loop { - // nothing - } } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] #[serial] async fn stellar_overlay_should_receive_tx_set() { //arrange @@ -117,11 +112,10 @@ async fn stellar_overlay_should_receive_tx_set() { timeout(Duration::from_secs(500), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Some(msg) = ov_conn_locked.listen().await { - match msg { + while let Ok(Some(msg)) = ov_conn_locked.listen().await { + match msg { StellarMessage::ScpMessage(msg) => - if let ScpStatementPledges::ScpStExternalize(stmt) = &msg.statement.pledges - { + if let ScpStatementPledges::ScpStExternalize(stmt) = &msg.statement.pledges { let tx_set_hash = get_tx_set_hash(stmt); tx_set_hashes_clone.lock().await.push(tx_set_hash.clone()); ov_conn_locked @@ -145,7 +139,6 @@ async fn stellar_overlay_should_receive_tx_set() { }, _ => {}, } - } }) .await @@ -161,7 +154,7 @@ async fn stellar_overlay_should_receive_tx_set() { assert!(expected_hashes.contains(&actual_hashes[0])) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] #[serial] async fn stellar_overlay_disconnect_works() { let (node_info, conn_info) = overlay_infos(false); From 3fdc156d7cbbfe9696356ec18b29b0ec6b6ad1a8 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 5 Dec 2023 19:52:16 +0100 Subject: [PATCH 13/22] Fix shutdown not propagated by passing the same shutdown sender to the oracle agent --- clients/vault/src/oracle/agent.rs | 18 ++++++++---------- clients/vault/src/system.rs | 14 ++++++++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index cf8f89592..398439698 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -2,17 +2,16 @@ use std::{sync::Arc, time::Duration}; use service::on_shutdown; use tokio::{ - sync::{mpsc, RwLock}, - time::{sleep, timeout}, + sync::{mpsc, Mutex, RwLock}, + time::{error::Elapsed, sleep, timeout, Timeout}, }; -use tokio::sync::Mutex; -use tokio::time::error::Elapsed; -use tokio::time::Timeout; use tracing::log; use runtime::ShutdownSender; -use stellar_relay_lib::{connect_to_stellar_overlay_network, sdk::types::StellarMessage, StellarOverlayConfig, StellarOverlayConnection}; -use stellar_relay_lib::helper::to_base64_xdr_string; +use stellar_relay_lib::{ + connect_to_stellar_overlay_network, helper::to_base64_xdr_string, sdk::types::StellarMessage, + StellarOverlayConfig, StellarOverlayConnection, +}; use crate::oracle::{ collector::ScpMessageCollector, @@ -64,6 +63,7 @@ async fn handle_message( pub async fn start_oracle_agent( config: StellarOverlayConfig, secret_key: &str, + shutdown_sender: ShutdownSender, ) -> Result { let timeout_in_secs = config.connection_info.timeout_in_secs; let secret_key_copy = secret_key.to_string(); @@ -78,7 +78,6 @@ pub async fn start_oracle_agent( ))); let collector_clone = collector.clone(); - let shutdown_sender = ShutdownSender::default(); let shutdown_sender_clone = shutdown_sender.clone(); let shutdown_sender_clone2 = shutdown_sender.clone(); @@ -100,7 +99,7 @@ pub async fn start_oracle_agent( message_receiver.close(); disconnect_signal_receiver.close(); - return Ok(()); + return Ok(()) } tokio::select! { @@ -172,7 +171,6 @@ pub async fn start_oracle_agent( Ok::<(), Error>(()) }); - tokio::spawn(on_shutdown(shutdown_sender.clone(), async move { tracing::info!("start_oracle_agent(): sending signal to shutdown overlay connection..."); let _ = disconnect_signal_sender.send(()).await; diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index 121fcdaa6..00e550ea4 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -394,6 +394,7 @@ impl VaultService { async fn create_oracle_agent( &self, is_public_network: bool, + shutdown_sender: ShutdownSender, ) -> Result, ServiceError> { let cfg_path = &self.config.stellar_overlay_config_filepath; let stellar_overlay_cfg = @@ -404,9 +405,13 @@ impl VaultService { return Err(ServiceError::IncompatibleNetwork) } - let oracle_agent = crate::oracle::start_oracle_agent(stellar_overlay_cfg, &self.secret_key) - .await - .expect("Failed to start oracle agent"); + let oracle_agent = crate::oracle::start_oracle_agent( + stellar_overlay_cfg, + &self.secret_key, + shutdown_sender, + ) + .await + .expect("Failed to start oracle agent"); Ok(Arc::new(oracle_agent)) } @@ -768,7 +773,8 @@ impl VaultService { .await; drop(wallet); - let oracle_agent = self.create_oracle_agent(is_public_network).await?; + let oracle_agent = + self.create_oracle_agent(is_public_network, self.shutdown.clone()).await?; self.execute_open_requests(oracle_agent.clone()); From eb664ecc9171ef17c4d8315c94cc2551cf00bd13 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:53:22 +0800 Subject: [PATCH 14/22] add retry counter and remove extra channel --- clients/service/src/lib.rs | 33 ++++- .../stellar-relay-lib/src/connection/mod.rs | 2 +- clients/stellar-relay-lib/src/overlay.rs | 5 + clients/vault/src/oracle/agent.rs | 131 ++++++------------ clients/vault/src/system.rs | 4 +- clients/vault/tests/helper/mod.rs | 13 +- 6 files changed, 87 insertions(+), 101 deletions(-) diff --git a/clients/service/src/lib.rs b/clients/service/src/lib.rs index 06258f540..b806b442b 100644 --- a/clients/service/src/lib.rs +++ b/clients/service/src/lib.rs @@ -1,10 +1,12 @@ use std::{fmt, sync::Arc, time::Duration}; +use std::time::SystemTime; use async_trait::async_trait; use futures::{future::Either, Future, FutureExt}; use governor::{Quota, RateLimiter}; use nonzero_ext::*; use tokio::sync::RwLock; +use tokio::time::sleep; pub use warp; pub use cli::{LoggingFormat, MonitoringConfig, RestartPolicy, ServiceConfig}; @@ -28,7 +30,7 @@ pub trait Service { spacewalk_parachain: SpacewalkParachain, config: Config, monitoring_config: MonitoringConfig, - shutdown: ShutdownSender, + shutdown: ShutdownSender ) -> Result where Self: Sized; @@ -65,9 +67,29 @@ impl ConnectionManager { } pub async fn start, InnerError: fmt::Display>( - &self, + &self ) -> Result<(), Error> { + let mut restart_delay_timer_in_secs = 20; // set default to 20 seconds for restart + let mut time_as_of_recording = SystemTime::now(); + loop { + let time_now = SystemTime::now(); + let _ = time_now.duration_since(time_as_of_recording).map(|duration| { + // Revert the counter if the restart happened more than 30 minutes (or 1800 seconds) ago + if duration.as_secs() > 1800 { + restart_delay_timer_in_secs = 20; + } + // Increase time by 10 seconds if a restart is triggered too frequently. + // This waits for delayed packets to be removed in the network, + // even though the connection on the client side is closed. + // Else, these straggler packets will interfere with the new connection. + // https://www.rfc-editor.org/rfc/rfc793#page-22 + else { + restart_delay_timer_in_secs += 10; + time_as_of_recording = time_now; + } + }); + tracing::info!("Version: {}", S::VERSION); tracing::info!( "Vault uses Substrate account with ID: {}", @@ -92,7 +114,7 @@ impl ConnectionManager { spacewalk_parachain, config, self.monitoring_config.clone(), - shutdown_tx.clone(), + shutdown_tx.clone() ) .await .map_err(|e| Error::StartServiceError(e))?; @@ -129,9 +151,14 @@ impl ConnectionManager { RestartPolicy::Never => return Err(Error::ClientShutdown), RestartPolicy::Always => { (self.increment_restart_counter)(); + + tracing::info!("Restarting in {restart_delay_timer_in_secs} seconds"); + sleep(Duration::from_secs(restart_delay_timer_in_secs)).await; + continue }, }; + } } } diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index fb4886592..2327721cd 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -90,7 +90,7 @@ impl ConnectionInfo { recv_tx_msgs, recv_scp_msgs, remote_called_us, - 20, + 10, ) } diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs index 0b64fe154..f34639f34 100644 --- a/clients/stellar-relay-lib/src/overlay.rs +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -1,6 +1,7 @@ use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; use tokio::sync::mpsc; use tokio::sync::mpsc::error::{SendError, TryRecvError}; +use tokio::sync::mpsc::Sender; use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; use crate::Error; @@ -15,6 +16,10 @@ pub struct StellarOverlayConnection { } impl StellarOverlayConnection { + pub fn sender(&self) -> Sender { + self.sender.clone() + } + pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { self.sender.send(msg).await } diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 398439698..1f0d0a043 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -2,15 +2,15 @@ use std::{sync::Arc, time::Duration}; use service::on_shutdown; use tokio::{ - sync::{mpsc, Mutex, RwLock}, - time::{error::Elapsed, sleep, timeout, Timeout}, + sync::{mpsc, RwLock}, + time::{ sleep, timeout}, }; -use tracing::log; +use tokio::sync::mpsc::error::TryRecvError; use runtime::ShutdownSender; use stellar_relay_lib::{ connect_to_stellar_overlay_network, helper::to_base64_xdr_string, sdk::types::StellarMessage, - StellarOverlayConfig, StellarOverlayConnection, + StellarOverlayConfig, }; use crate::oracle::{ @@ -65,12 +65,11 @@ pub async fn start_oracle_agent( secret_key: &str, shutdown_sender: ShutdownSender, ) -> Result { - let timeout_in_secs = config.connection_info.timeout_in_secs; - let secret_key_copy = secret_key.to_string(); - tracing::info!("start_oracle_agent(): Starting connection to Stellar overlay network..."); let mut overlay_conn = connect_to_stellar_overlay_network(config.clone(), secret_key).await?; + // use StellarOverlayConnection's sender to send message to Stellar + let sender = overlay_conn.sender(); let collector = Arc::new(RwLock::new(ScpMessageCollector::new( config.is_public_network(), @@ -79,107 +78,61 @@ pub async fn start_oracle_agent( let collector_clone = collector.clone(); let shutdown_sender_clone = shutdown_sender.clone(); + // a clone used to forcefully call a shutdown, when StellarOverlay disconnects. let shutdown_sender_clone2 = shutdown_sender.clone(); // disconnect signal sender let (disconnect_signal_sender, mut disconnect_signal_receiver) = mpsc::channel::<()>(2); - // handle a message from the overlay network - let (message_sender, mut message_receiver) = mpsc::channel::(34); - let message_sender_clone = message_sender.clone(); - - let mut reconnection_increment = 0; service::spawn_cancelable(shutdown_sender_clone.subscribe(), async move { - let mut stop_overlay = false; + let sender_clone = overlay_conn.sender(); loop { - if !overlay_conn.is_alive() { - tracing::info!("start_oracle_agent(): oracle is dead."); - let _ = shutdown_sender_clone2.send(()); - - message_receiver.close(); - disconnect_signal_receiver.close(); - - return Ok(()) + match disconnect_signal_receiver.try_recv() { + // if a disconnect signal was sent, disconnect from Stellar. + Ok(_) | Err(TryRecvError::Disconnected) => { + tracing::info!("start_oracle_agent(): disconnect overlay..."); + overlay_conn.disconnect(); + break; + } + Err(TryRecvError::Empty) => {} } - tokio::select! { - result_msg = overlay_conn.listen() => { match result_msg { - Ok(Some(msg)) => { - //tracing::info!("start_oracle_agent(): handle message: {}", to_base64_xdr_string(&msg)); - handle_message( - msg, - collector_clone.clone(), - &message_sender_clone - ).await?; - } - Ok(None) => {} - Err(e) => { - tracing::error!("start_oracle_agent(): received error: {e:?}"); - - overlay_conn.disconnect(); - let _ = shutdown_sender_clone2.send(()); - return Ok(()); - // loop { - // let sleep_time = timeout_in_secs + reconnection_increment; - // tracing::info!("start_oracle_agent(): reconnect to Stellar in {sleep_time} seconds"); - // reconnection_increment += 5; // increment by 5 - // - // sleep(Duration::from_secs(timeout_in_secs + reconnection_increment)).await; - // - // match connect_to_stellar_overlay_network(config.clone(), &secret_key_copy).await { - // Ok(new_overlay_conn) => { - // overlay_conn = new_overlay_conn; - // - // // wait to be sure that a connection was established. - // sleep(Duration::from_secs(timeout_in_secs)).await; - // - // if overlay_conn.is_alive() { - // tracing::info!("start_oracle_agent(): new connection created..."); - // // a new connection was created; break out of this inner loop - // // and renew listening for messages - // break; - // } - // } - // Err(e) => tracing::warn!("start_oracle_agent(): reconnection failed: {e:?}") - // } - // } + // listen for messages from Stellar + match overlay_conn.listen().await { + Ok(Some(msg)) => { + let msg_as_str = to_base64_xdr_string(&msg); + if let Err(e) = handle_message(msg, collector_clone.clone(),&sender_clone).await { + tracing::error!("start_oracle_agent(): failed to handle message: {msg_as_str}: {e:?}"); } - }}, - - result_msg = timeout(Duration::from_secs(10), message_receiver.recv()) => match result_msg { - Ok(Some(msg)) => if let Err(e) = overlay_conn.send_to_node(msg).await { - tracing::error!("start_oracle_agent(): failed to send msg to stellar node: {e:?}"); - }, - _ => {} - }, - - result_msg = timeout(Duration::from_secs(10), disconnect_signal_receiver.recv()) => match result_msg { - Ok(Some(_)) => { - tracing::info!("start_oracle_agent(): disconnect signal received."); - - overlay_conn.disconnect(); - let _ = shutdown_sender_clone2.send(()); - return Ok(()); + } + Ok(None) => {} + // connection got lost + Err(e) => { + overlay_conn.disconnect(); + tracing::error!("start_oracle_agent(): encounter error in overlay: {e:?}"); + + if let Err(e) = shutdown_sender_clone2.send(()) { + tracing::error!("start_oracle_agent(): Failed to send shutdown signal in thread: {e:?}"); } - _ => {} + break; } } } tracing::info!("start_oracle_agent(): LOOP STOPPED!"); - - Ok::<(), Error>(()) }); tokio::spawn(on_shutdown(shutdown_sender.clone(), async move { tracing::info!("start_oracle_agent(): sending signal to shutdown overlay connection..."); - let _ = disconnect_signal_sender.send(()).await; + if let Err(e) = disconnect_signal_sender.send(()).await { + tracing::warn!("start_oracle_agent(): failed to send disconnect signal: {e:?}"); + } })); Ok(OracleAgent { collector, is_public_network: false, - message_sender: Some(message_sender), + message_sender: Some(sender), shutdown_sender, }) } @@ -260,8 +213,9 @@ mod tests { #[ntest::timeout(1_800_000)] // timeout at 30 minutes #[serial] async fn test_get_proof_for_current_slot() { + let shutdown_sender = ShutdownSender::new(); let agent = - start_oracle_agent(get_test_stellar_relay_config(true), &get_test_secret_key(true)) + start_oracle_agent(get_test_stellar_relay_config(true), &get_test_secret_key(true), shutdown_sender) .await .expect("Failed to start agent"); sleep(Duration::from_secs(10)).await; @@ -286,8 +240,9 @@ mod tests { let scp_archive_storage = ScpArchiveStorage::default(); let tx_archive_storage = TransactionsArchiveStorage::default(); + let shutdown_sender = ShutdownSender::new(); let agent = - start_oracle_agent(get_test_stellar_relay_config(true), &get_test_secret_key(true)) + start_oracle_agent(get_test_stellar_relay_config(true), &get_test_secret_key(true), shutdown_sender) .await .expect("Failed to start agent"); @@ -320,7 +275,8 @@ mod tests { let modified_config = StellarOverlayConfig { stellar_history_archive_urls: archive_urls, ..base_config }; - let agent = start_oracle_agent(modified_config, &get_test_secret_key(true)) + let shutdown_sender = ShutdownSender::new(); + let agent = start_oracle_agent(modified_config, &get_test_secret_key(true), shutdown_sender) .await .expect("Failed to start agent"); @@ -347,7 +303,8 @@ mod tests { let modified_config: StellarOverlayConfig = StellarOverlayConfig { stellar_history_archive_urls: vec![], ..base_config }; - let agent = start_oracle_agent(modified_config, &get_test_secret_key(true)) + let shutdown = ShutdownSender::new(); + let agent = start_oracle_agent(modified_config, &get_test_secret_key(true), shutdown) .await .expect("Failed to start agent"); diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index 00e550ea4..88bd8db3f 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -394,7 +394,7 @@ impl VaultService { async fn create_oracle_agent( &self, is_public_network: bool, - shutdown_sender: ShutdownSender, + shutdown_sender: ShutdownSender ) -> Result, ServiceError> { let cfg_path = &self.config.stellar_overlay_config_filepath; let stellar_overlay_cfg = @@ -408,7 +408,7 @@ impl VaultService { let oracle_agent = crate::oracle::start_oracle_agent( stellar_overlay_cfg, &self.secret_key, - shutdown_sender, + shutdown_sender ) .await .expect("Failed to start oracle agent"); diff --git a/clients/vault/tests/helper/mod.rs b/clients/vault/tests/helper/mod.rs index 1cd84a33b..8b85e8cca 100644 --- a/clients/vault/tests/helper/mod.rs +++ b/clients/vault/tests/helper/mod.rs @@ -7,13 +7,9 @@ pub use helper::*; use async_trait::async_trait; use lazy_static::lazy_static; use primitives::CurrencyId; -use runtime::{ - integration::{ - default_provider_client, set_exchange_rate_and_wait, setup_provider, SubxtClient, - }, - types::FixedU128, - SpacewalkParachain, VaultId, -}; +use runtime::{integration::{ + default_provider_client, set_exchange_rate_and_wait, setup_provider, SubxtClient, +}, types::FixedU128, SpacewalkParachain, VaultId, ShutdownSender}; use sp_arithmetic::FixedPointNumber; use sp_keyring::AccountKeyring; use std::{future::Future, sync::Arc}; @@ -175,7 +171,8 @@ where .unwrap(), )); - let oracle_agent = start_oracle_agent(CFG.clone(), &SECRET_KEY) + let shutdown_tx = ShutdownSender::new(); + let oracle_agent = start_oracle_agent(CFG.clone(), &SECRET_KEY, shutdown_tx) .await .expect("failed to start agent"); let oracle_agent = Arc::new(oracle_agent); From bfde71a369e86db6ec89fb9abc931096924b794a Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Wed, 6 Dec 2023 20:27:07 +0800 Subject: [PATCH 15/22] cargo fmt and fix warnings --- clients/service/src/lib.rs | 22 +- clients/stellar-relay-lib/examples/connect.rs | 7 +- .../src/connection/connector/connector.rs | 17 +- .../connection/connector/message_handler.rs | 270 +++++++++--------- .../connection/connector/message_sender.rs | 12 +- .../stellar-relay-lib/src/connection/error.rs | 9 +- .../src/connection/handshake.rs | 9 +- .../stellar-relay-lib/src/connection/mod.rs | 22 -- clients/stellar-relay-lib/src/node/mod.rs | 19 -- clients/stellar-relay-lib/src/overlay.rs | 113 ++++---- clients/stellar-relay-lib/src/tests/mod.rs | 38 +-- clients/vault/src/oracle/agent.rs | 62 ++-- clients/vault/src/system.rs | 4 +- clients/vault/tests/helper/mod.rs | 10 +- 14 files changed, 305 insertions(+), 309 deletions(-) diff --git a/clients/service/src/lib.rs b/clients/service/src/lib.rs index b806b442b..9f611e810 100644 --- a/clients/service/src/lib.rs +++ b/clients/service/src/lib.rs @@ -1,12 +1,14 @@ -use std::{fmt, sync::Arc, time::Duration}; -use std::time::SystemTime; +use std::{ + fmt, + sync::Arc, + time::{Duration, SystemTime}, +}; use async_trait::async_trait; use futures::{future::Either, Future, FutureExt}; use governor::{Quota, RateLimiter}; use nonzero_ext::*; -use tokio::sync::RwLock; -use tokio::time::sleep; +use tokio::{sync::RwLock, time::sleep}; pub use warp; pub use cli::{LoggingFormat, MonitoringConfig, RestartPolicy, ServiceConfig}; @@ -30,7 +32,7 @@ pub trait Service { spacewalk_parachain: SpacewalkParachain, config: Config, monitoring_config: MonitoringConfig, - shutdown: ShutdownSender + shutdown: ShutdownSender, ) -> Result where Self: Sized; @@ -67,7 +69,7 @@ impl ConnectionManager { } pub async fn start, InnerError: fmt::Display>( - &self + &self, ) -> Result<(), Error> { let mut restart_delay_timer_in_secs = 20; // set default to 20 seconds for restart let mut time_as_of_recording = SystemTime::now(); @@ -75,7 +77,8 @@ impl ConnectionManager { loop { let time_now = SystemTime::now(); let _ = time_now.duration_since(time_as_of_recording).map(|duration| { - // Revert the counter if the restart happened more than 30 minutes (or 1800 seconds) ago + // Revert the counter if the restart happened more than 30 minutes (or 1800 seconds) + // ago if duration.as_secs() > 1800 { restart_delay_timer_in_secs = 20; } @@ -114,7 +117,7 @@ impl ConnectionManager { spacewalk_parachain, config, self.monitoring_config.clone(), - shutdown_tx.clone() + shutdown_tx.clone(), ) .await .map_err(|e| Error::StartServiceError(e))?; @@ -154,11 +157,10 @@ impl ConnectionManager { tracing::info!("Restarting in {restart_delay_timer_in_secs} seconds"); sleep(Duration::from_secs(restart_delay_timer_in_secs)).await; - + continue }, }; - } } } diff --git a/clients/stellar-relay-lib/examples/connect.rs b/clients/stellar-relay-lib/examples/connect.rs index 90d5b46ba..94a8ab1be 100644 --- a/clients/stellar-relay-lib/examples/connect.rs +++ b/clients/stellar-relay-lib/examples/connect.rs @@ -1,6 +1,5 @@ use stellar_relay_lib::{ connect_to_stellar_overlay_network, - helper::to_base64_xdr_string, sdk::types::{ScpStatementPledges, StellarMessage}, StellarOverlayConfig, }; @@ -28,10 +27,7 @@ async fn main() -> Result<(), Box> { let mut overlay_connection = connect_to_stellar_overlay_network(cfg, &secret_key).await?; - let mut counter = 0; while let Ok(Some(msg)) = overlay_connection.listen().await { - counter += 1; - match msg { StellarMessage::ScpMessage(msg) => { let node_id = msg.statement.node_id.to_encoding(); @@ -51,8 +47,7 @@ async fn main() -> Result<(), Box> { slot ); }, - other => { - let msg = StellarMessage::GetPeers; + _ => { let _ = overlay_connection.send_to_node(StellarMessage::GetPeers).await; }, } diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 55428018f..853a942f1 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -1,24 +1,24 @@ use std::fmt::{Debug, Formatter}; use substrate_stellar_sdk::{ - types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType}, + types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType, StellarMessage}, XdrCodec, }; -use substrate_stellar_sdk::types::StellarMessage; -use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::sync::mpsc; +use tokio::{ + net::tcp::{OwnedReadHalf, OwnedWriteHalf}, + sync::mpsc, +}; use crate::{ connection::{ authentication::{gen_shared_key, ConnectionAuth}, + connector::message_reader::read_message_from_stellar, flow_controller::FlowController, + handshake::HandshakeState, hmac::{verify_hmac, HMacKeys}, - ConnectionInfo, handshake::HandshakeState, - Error + ConnectionInfo, Error, }, node::{LocalInfo, NodeInfo, RemoteInfo}, }; -use crate::connection::connector::message_reader::read_message_from_stellar; - pub struct Connector { local: LocalInfo, @@ -212,7 +212,6 @@ impl Connector { } } - /// Polls for messages coming from the Stellar Node and communicates it back to the user /// /// # Arguments diff --git a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs index a1630b1dd..3333f09c5 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_handler.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_handler.rs @@ -1,140 +1,154 @@ -use substrate_stellar_sdk::types::{ErrorCode, Hello, MessageType, StellarMessage}; -use substrate_stellar_sdk::XdrCodec; - -use crate::connection::{Connector, Xdr, Error, helper::{error_to_string, time_now}, xdr_converter::parse_authenticated_message}; -use crate::connection::authentication::verify_remote_auth_cert; -use crate::connection::hmac::HMacKeys; +use substrate_stellar_sdk::{ + types::{ErrorCode, Hello, MessageType, StellarMessage}, + XdrCodec, +}; + +use crate::connection::{ + authentication::verify_remote_auth_cert, + helper::{error_to_string, time_now}, + hmac::HMacKeys, + xdr_converter::parse_authenticated_message, + Connector, Error, Xdr, +}; use crate::node::RemoteInfo; impl Connector { - /// Processes the raw bytes from the stream - pub(super) async fn process_raw_message(&mut self, data: Xdr) -> Result, Error> { - let (auth_msg, msg_type) = parse_authenticated_message(&data)?; - - match msg_type { - MessageType::Transaction | MessageType::FloodAdvert if !self.receive_tx_messages() => { - self.increment_remote_sequence()?; - self.check_to_send_more(MessageType::Transaction).await?; - }, - - MessageType::ScpMessage if !self.receive_scp_messages() => { - self.increment_remote_sequence()?; - }, - - MessageType::ErrorMsg => match auth_msg.message { - StellarMessage::ErrorMsg(e) => { - log::error!( + /// Processes the raw bytes from the stream + pub(super) async fn process_raw_message( + &mut self, + data: Xdr, + ) -> Result, Error> { + let (auth_msg, msg_type) = parse_authenticated_message(&data)?; + + match msg_type { + MessageType::Transaction | MessageType::FloodAdvert if !self.receive_tx_messages() => { + self.increment_remote_sequence()?; + self.check_to_send_more(MessageType::Transaction).await?; + }, + + MessageType::ScpMessage if !self.receive_scp_messages() => { + self.increment_remote_sequence()?; + }, + + MessageType::ErrorMsg => match auth_msg.message { + StellarMessage::ErrorMsg(e) => { + log::error!( "process_raw_message(): Received ErrorMsg during authentication: {}", error_to_string(e.clone()) ); - return Err(Error::from(e)) - }, - other => log::error!("process_raw_message(): Received ErroMsg during authentication: {:?}", other), - }, - - _ => { - // we only verify the authenticated message when a handshake has been done. - if self.is_handshake_created() { - self.verify_auth(&auth_msg, &data[4..(data.len() - 32)])?; - self.increment_remote_sequence()?; - log::trace!( + return Err(Error::from(e)) + }, + other => log::error!( + "process_raw_message(): Received ErroMsg during authentication: {:?}", + other + ), + }, + + _ => { + // we only verify the authenticated message when a handshake has been done. + if self.is_handshake_created() { + self.verify_auth(&auth_msg, &data[4..(data.len() - 32)])?; + self.increment_remote_sequence()?; + log::trace!( "process_raw_message(): Processing {msg_type:?} message: auth verified" ); - } - - return self.process_stellar_message( auth_msg.message, msg_type).await; - }, - } - Ok(None) - } - - /// Returns a StellarMessage for the user/outsider. Else none if user/outsider do not need it. - /// This handles what to do with the Stellar message. - async fn process_stellar_message( - &mut self, - msg: StellarMessage, - msg_type: MessageType, - ) -> Result, Error> { - match msg { - StellarMessage::Hello(hello) => { - // update the node info based on the hello message - self.process_hello_message(hello)?; - - self.got_hello(); - - if self.remote_called_us() { - self.send_hello_message().await?; - } else { - self.send_auth_message().await?; - } - log::info!("process_stellar_message(): Hello message processed successfully"); - }, - - StellarMessage::Auth(_) => { - self.process_auth_message().await?; - }, - - StellarMessage::ErrorMsg(e) => { - log::error!("process_stellar_message(): Received ErrorMsg during authentication: {e:?}"); - if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { - return Err(Error::from(e)); - } - return Ok(Some(StellarMessage::ErrorMsg(e))); - }, - - other => { - log::trace!( + } + + return self.process_stellar_message(auth_msg.message, msg_type).await + }, + } + Ok(None) + } + + /// Returns a StellarMessage for the user/outsider. Else none if user/outsider do not need it. + /// This handles what to do with the Stellar message. + async fn process_stellar_message( + &mut self, + msg: StellarMessage, + msg_type: MessageType, + ) -> Result, Error> { + match msg { + StellarMessage::Hello(hello) => { + // update the node info based on the hello message + self.process_hello_message(hello)?; + + self.got_hello(); + + if self.remote_called_us() { + self.send_hello_message().await?; + } else { + self.send_auth_message().await?; + } + log::info!("process_stellar_message(): Hello message processed successfully"); + }, + + StellarMessage::Auth(_) => { + self.process_auth_message().await?; + }, + + StellarMessage::ErrorMsg(e) => { + log::error!( + "process_stellar_message(): Received ErrorMsg during authentication: {e:?}" + ); + if e.code == ErrorCode::ErrConf || e.code == ErrorCode::ErrAuth { + return Err(Error::from(e)) + } + return Ok(Some(StellarMessage::ErrorMsg(e))) + }, + + other => { + log::trace!( "process_stellar_message(): Processing {other:?} message: received from overlay" ); - self.check_to_send_more(msg_type).await?; - return Ok(Some(other)); - }, - } - - Ok(None) - } - - async fn process_auth_message(&mut self) -> Result<(), Error> { - if self.remote_called_us() { - self.send_auth_message().await?; - } - - self.handshake_completed(); - - if let Some(remote) = self.remote() { - log::debug!("process_auth_message(): sending connect message: {remote:?}"); - self.enable_flow_controller( - self.local().node().overlay_version, - remote.node().overlay_version, - ); - } else { - log::warn!("process_auth_message(): No remote overlay version after handshake."); - } - - self.check_to_send_more(MessageType::Auth).await - } - - /// Updates the config based on the hello message that was received from the Stellar Node - fn process_hello_message(&mut self, hello: Hello) -> Result<(), Error> { - let mut network_id = self.connection_auth.network_id().to_xdr(); - - if !verify_remote_auth_cert(time_now(), &hello.peer_id, &hello.cert, &mut network_id) { - return Err(Error::AuthCertInvalid) - } - - let remote_info = RemoteInfo::new(&hello); - let shared_key = self.get_shared_key(remote_info.pub_key_ecdh()); - - self.set_hmac_keys(HMacKeys::new( - &shared_key, - self.local().nonce(), - remote_info.nonce(), - self.remote_called_us(), - )); - - self.set_remote(remote_info); - - Ok(()) - } + self.check_to_send_more(msg_type).await?; + return Ok(Some(other)) + }, + } + + Ok(None) + } + + async fn process_auth_message(&mut self) -> Result<(), Error> { + if self.remote_called_us() { + self.send_auth_message().await?; + } + + self.handshake_completed(); + + if let Some(remote) = self.remote() { + log::debug!("process_auth_message(): sending connect message: {remote:?}"); + self.enable_flow_controller( + self.local().node().overlay_version, + remote.node().overlay_version, + ); + } else { + log::warn!("process_auth_message(): No remote overlay version after handshake."); + } + + self.check_to_send_more(MessageType::Auth).await + } + + /// Updates the config based on the hello message that was received from the Stellar Node + fn process_hello_message(&mut self, hello: Hello) -> Result<(), Error> { + let mut network_id = self.connection_auth.network_id().to_xdr(); + + if !verify_remote_auth_cert(time_now(), &hello.peer_id, &hello.cert, &mut network_id) { + return Err(Error::AuthCertInvalid) + } + + let remote_info = RemoteInfo::new(&hello); + let shared_key = self.get_shared_key(remote_info.pub_key_ecdh()); + + self.set_hmac_keys(HMacKeys::new( + &shared_key, + self.local().nonce(), + remote_info.nonce(), + self.remote_called_us(), + )); + + self.set_remote(remote_info); + + Ok(()) + } } diff --git a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs index de7b5e2df..d5504e5c9 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs @@ -1,9 +1,13 @@ use std::time::Duration; use substrate_stellar_sdk::types::{MessageType, SendMore, StellarMessage}; -use tokio::io::AsyncWriteExt; -use tokio::time::timeout; - -use crate::connection::{Connector, Error, flow_controller::MAX_FLOOD_MSG_CAP, handshake::create_auth_message, helper::{time_now, to_base64_xdr_string}}; +use tokio::{io::AsyncWriteExt, time::timeout}; + +use crate::connection::{ + flow_controller::MAX_FLOOD_MSG_CAP, + handshake::create_auth_message, + helper::{time_now, to_base64_xdr_string}, + Connector, Error, +}; impl Connector { pub async fn send_to_node(&mut self, msg: StellarMessage) -> Result<(), Error> { diff --git a/clients/stellar-relay-lib/src/connection/error.rs b/clients/stellar-relay-lib/src/connection/error.rs index 1e4a2a437..d1afa62d6 100644 --- a/clients/stellar-relay-lib/src/connection/error.rs +++ b/clients/stellar-relay-lib/src/connection/error.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] //todo: remove after being tested and implemented -use crate::connection::xdr_converter::Error as XDRError; +use crate::{connection::xdr_converter::Error as XDRError, helper::error_to_string}; use substrate_stellar_sdk::{types::ErrorCode, StellarSdkError}; use tokio::sync; -use crate::helper::error_to_string; #[derive(Debug, err_derive::Error)] pub enum Error { @@ -68,7 +67,7 @@ pub enum Error { Timeout, #[error(display = "Config Error: Version String too long")] - VersionStrTooLong + VersionStrTooLong, } impl From for Error { @@ -97,7 +96,7 @@ impl From for Error { other => { log::error!("Stellar Node returned error: {}", error_to_string(value)); Self::OverlayError(other) - } + }, } } -} \ No newline at end of file +} diff --git a/clients/stellar-relay-lib/src/connection/handshake.rs b/clients/stellar-relay-lib/src/connection/handshake.rs index 58a2143bc..d9952fdac 100644 --- a/clients/stellar-relay-lib/src/connection/handshake.rs +++ b/clients/stellar-relay-lib/src/connection/handshake.rs @@ -33,11 +33,10 @@ pub fn create_hello_message( overlay_version: node_info.overlay_version, overlay_min_version: node_info.overlay_min_version, network_id: node_info.network_id, - version_str: LimitedString::<100>::new(version_str.clone()) - .map_err(|e| { - log::error!("create_hello_message(): {e:?}"); - Error::VersionStrTooLong - })?, + version_str: LimitedString::<100>::new(version_str.clone()).map_err(|e| { + log::error!("create_hello_message(): {e:?}"); + Error::VersionStrTooLong + })?, listening_port: i32::try_from(listening_port).unwrap_or(11625), peer_id, cert, diff --git a/clients/stellar-relay-lib/src/connection/mod.rs b/clients/stellar-relay-lib/src/connection/mod.rs index 2327721cd..ac342a8ed 100644 --- a/clients/stellar-relay-lib/src/connection/mod.rs +++ b/clients/stellar-relay-lib/src/connection/mod.rs @@ -72,28 +72,6 @@ impl ConnectionInfo { } } - #[cfg(test)] - pub(crate) fn new( - addr: &str, - port: u32, - secret_key: SecretKey, - auth_cert_expiration: u64, - recv_tx_msgs: bool, - recv_scp_msgs: bool, - remote_called_us: bool, - ) -> Self { - Self::new_with_timeout( - addr, - port, - secret_key, - auth_cert_expiration, - recv_tx_msgs, - recv_scp_msgs, - remote_called_us, - 10, - ) - } - pub fn address(&self) -> String { format!("{}:{}", self.address, self.port) } diff --git a/clients/stellar-relay-lib/src/node/mod.rs b/clients/stellar-relay-lib/src/node/mod.rs index 4886890f9..bd6fc642a 100644 --- a/clients/stellar-relay-lib/src/node/mod.rs +++ b/clients/stellar-relay-lib/src/node/mod.rs @@ -30,25 +30,6 @@ impl Debug for NodeInfo { } } -impl NodeInfo { - #[cfg(test)] - pub(crate) fn new( - ledger_version: u32, - overlay_version: u32, - overlay_min_version: u32, - version_str: String, - network: &Network, - ) -> NodeInfo { - NodeInfo { - ledger_version, - overlay_version, - overlay_min_version, - version_str: version_str.into_bytes(), - network_id: *network.get_id(), - } - } -} - impl From for NodeInfo { fn from(value: NodeInfoCfg) -> Self { let network: &Network = if value.is_pub_net { &PUBLIC_NETWORK } else { &TEST_NETWORK }; diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs index f34639f34..59b15e57f 100644 --- a/clients/stellar-relay-lib/src/overlay.rs +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -1,35 +1,40 @@ use substrate_stellar_sdk::types::{ErrorCode, StellarMessage}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::error::{SendError, TryRecvError}; -use tokio::sync::mpsc::Sender; - -use crate::connection::{ConnectionInfo, Connector, poll_messages_from_stellar}; -use crate::Error; -use crate::helper::{create_stream, error_to_string}; -use crate::node::NodeInfo; - +use tokio::sync::{ + mpsc, + mpsc::{ + error::{SendError, TryRecvError}, + Sender, + }, +}; + +use crate::{ + connection::{poll_messages_from_stellar, ConnectionInfo, Connector}, + helper::{create_stream, error_to_string}, + node::NodeInfo, + Error, +}; /// Used to send/receive messages to/from Stellar Node pub struct StellarOverlayConnection { - sender: mpsc::Sender, - receiver: mpsc::Receiver + sender: mpsc::Sender, + receiver: mpsc::Receiver, } impl StellarOverlayConnection { - pub fn sender(&self) -> Sender { - self.sender.clone() - } + pub fn sender(&self) -> Sender { + self.sender.clone() + } - pub async fn send_to_node(&self, msg:StellarMessage) -> Result<(), SendError> { - self.sender.send(msg).await - } + pub async fn send_to_node(&self, msg: StellarMessage) -> Result<(), SendError> { + self.sender.send(msg).await + } pub async fn listen(&mut self) -> Result, Error> { loop { - if !self.is_alive() { - self.disconnect(); - return Err(Error::Disconnected) - } + if !self.is_alive() { + self.disconnect(); + return Err(Error::Disconnected) + } match self.receiver.try_recv() { Ok(StellarMessage::ErrorMsg(e)) => { @@ -38,24 +43,24 @@ impl StellarOverlayConnection { return Err(Error::ConnectionFailed(error_to_string(e))) } - return Ok(None) - }, - Ok(msg) => return Ok(Some(msg)), - Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), - Err(TryRecvError::Empty) => continue, - } - } - } + return Ok(None) + }, + Ok(msg) => return Ok(Some(msg)), + Err(TryRecvError::Disconnected) => return Err(Error::Disconnected), + Err(TryRecvError::Empty) => continue, + } + } + } - pub fn is_alive(&self) -> bool { - let result = self.sender.is_closed(); + pub fn is_alive(&self) -> bool { + let result = self.sender.is_closed(); - if result { - drop(self); - } + if result { + drop(self); + } - !result - } + !result + } pub fn disconnect(&mut self) { log::info!("disconnect(): closing channel"); @@ -64,23 +69,23 @@ impl StellarOverlayConnection { } impl Drop for StellarOverlayConnection { - fn drop(&mut self) { - self.disconnect(); - } + fn drop(&mut self) { + self.disconnect(); + } } impl StellarOverlayConnection { - /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. - pub async fn connect(local_node_info: NodeInfo, conn_info: ConnectionInfo) -> Result { - log::info!("connect(): connecting to {conn_info:?}"); - + /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. + pub async fn connect( + local_node_info: NodeInfo, + conn_info: ConnectionInfo, + ) -> Result { + log::info!("connect(): connecting to {conn_info:?}"); - // this is a channel to communicate with the user/caller. - let (send_to_user_sender, send_to_user_receiver) = - mpsc::channel::(1024); + // this is a channel to communicate with the user/caller. + let (send_to_user_sender, send_to_user_receiver) = mpsc::channel::(1024); - let (send_to_node_sender, send_to_node_receiver) = - mpsc::channel::(1024); + let (send_to_node_sender, send_to_node_receiver) = mpsc::channel::(1024); // split the stream for easy handling of read and write let (read_stream_overlay, write_stream_overlay) = @@ -96,9 +101,9 @@ impl StellarOverlayConnection { send_to_node_receiver, )); - Ok(StellarOverlayConnection { - sender: send_to_node_sender, - receiver: send_to_user_receiver - }) - } -} \ No newline at end of file + Ok(StellarOverlayConnection { + sender: send_to_node_sender, + receiver: send_to_user_receiver, + }) + } +} diff --git a/clients/stellar-relay-lib/src/tests/mod.rs b/clients/stellar-relay-lib/src/tests/mod.rs index 37f50e675..aff162202 100644 --- a/clients/stellar-relay-lib/src/tests/mod.rs +++ b/clients/stellar-relay-lib/src/tests/mod.rs @@ -41,23 +41,27 @@ fn overlay_infos(is_mainnet: bool) -> (NodeInfo, ConnectionInfo) { ) } -// #[tokio::test] -// #[serial] -// async fn stellar_overlay_connect_and_listen_connect_message() { -// let (node_info, conn_info) = overlay_infos(false); -// -// let mut overlay_connection = -// StellarOverlayConnection::connect(node_info.clone(), conn_info).await.unwrap(); -// -// let message = overlay_connection.listen().await.unwrap(); -// if let StellarRelayMessage::Connect { pub_key: _x, node_info: y } = message { -// assert_eq!(y.ledger_version, node_info.ledger_version); -// } else { -// panic!("Incorrect stellar relay message received"); -// } -// -// overlay_connection.disconnect(); -// } +#[tokio::test] +#[serial] +async fn stellar_overlay_connect_and_listen_connect_message() { + let (node_info, conn_info) = overlay_infos(false); + + let mut overlay_connection = + StellarOverlayConnection::connect(node_info.clone(), conn_info).await.unwrap(); + + if let Some(message) = + overlay_connection.listen().await.expect("Should return a Stellar Message") + { + match message { + StellarMessage::ErrorMsg(_) => assert!(false), + _ => assert!(true), + } + } else { + assert!(false); + } + + overlay_connection.disconnect(); +} #[tokio::test(flavor = "multi_thread")] #[serial] diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 1f0d0a043..26693e894 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -2,10 +2,9 @@ use std::{sync::Arc, time::Duration}; use service::on_shutdown; use tokio::{ - sync::{mpsc, RwLock}, - time::{ sleep, timeout}, + sync::{mpsc, mpsc::error::TryRecvError, RwLock}, + time::{sleep, timeout}, }; -use tokio::sync::mpsc::error::TryRecvError; use runtime::ShutdownSender; use stellar_relay_lib::{ @@ -92,30 +91,36 @@ pub async fn start_oracle_agent( Ok(_) | Err(TryRecvError::Disconnected) => { tracing::info!("start_oracle_agent(): disconnect overlay..."); overlay_conn.disconnect(); - break; - } - Err(TryRecvError::Empty) => {} + break + }, + Err(TryRecvError::Empty) => {}, } // listen for messages from Stellar match overlay_conn.listen().await { Ok(Some(msg)) => { let msg_as_str = to_base64_xdr_string(&msg); - if let Err(e) = handle_message(msg, collector_clone.clone(),&sender_clone).await { - tracing::error!("start_oracle_agent(): failed to handle message: {msg_as_str}: {e:?}"); + if let Err(e) = + handle_message(msg, collector_clone.clone(), &sender_clone).await + { + tracing::error!( + "start_oracle_agent(): failed to handle message: {msg_as_str}: {e:?}" + ); } - } - Ok(None) => {} + }, + Ok(None) => {}, // connection got lost Err(e) => { overlay_conn.disconnect(); tracing::error!("start_oracle_agent(): encounter error in overlay: {e:?}"); if let Err(e) = shutdown_sender_clone2.send(()) { - tracing::error!("start_oracle_agent(): Failed to send shutdown signal in thread: {e:?}"); + tracing::error!( + "start_oracle_agent(): Failed to send shutdown signal in thread: {e:?}" + ); } - break; - } + break + }, } } @@ -124,7 +129,7 @@ pub async fn start_oracle_agent( tokio::spawn(on_shutdown(shutdown_sender.clone(), async move { tracing::info!("start_oracle_agent(): sending signal to shutdown overlay connection..."); - if let Err(e) = disconnect_signal_sender.send(()).await { + if let Err(e) = disconnect_signal_sender.send(()).await { tracing::warn!("start_oracle_agent(): failed to send disconnect signal: {e:?}"); } })); @@ -214,10 +219,13 @@ mod tests { #[serial] async fn test_get_proof_for_current_slot() { let shutdown_sender = ShutdownSender::new(); - let agent = - start_oracle_agent(get_test_stellar_relay_config(true), &get_test_secret_key(true), shutdown_sender) - .await - .expect("Failed to start agent"); + let agent = start_oracle_agent( + get_test_stellar_relay_config(true), + &get_test_secret_key(true), + shutdown_sender, + ) + .await + .expect("Failed to start agent"); sleep(Duration::from_secs(10)).await; // Wait until agent is caught up with the network. @@ -241,10 +249,13 @@ mod tests { let tx_archive_storage = TransactionsArchiveStorage::default(); let shutdown_sender = ShutdownSender::new(); - let agent = - start_oracle_agent(get_test_stellar_relay_config(true), &get_test_secret_key(true), shutdown_sender) - .await - .expect("Failed to start agent"); + let agent = start_oracle_agent( + get_test_stellar_relay_config(true), + &get_test_secret_key(true), + shutdown_sender, + ) + .await + .expect("Failed to start agent"); // This slot should be archived on the public network let target_slot = 44041116; @@ -276,9 +287,10 @@ mod tests { StellarOverlayConfig { stellar_history_archive_urls: archive_urls, ..base_config }; let shutdown_sender = ShutdownSender::new(); - let agent = start_oracle_agent(modified_config, &get_test_secret_key(true), shutdown_sender) - .await - .expect("Failed to start agent"); + let agent = + start_oracle_agent(modified_config, &get_test_secret_key(true), shutdown_sender) + .await + .expect("Failed to start agent"); // This slot should be archived on the public network let target_slot = 44041116; diff --git a/clients/vault/src/system.rs b/clients/vault/src/system.rs index 88bd8db3f..00e550ea4 100644 --- a/clients/vault/src/system.rs +++ b/clients/vault/src/system.rs @@ -394,7 +394,7 @@ impl VaultService { async fn create_oracle_agent( &self, is_public_network: bool, - shutdown_sender: ShutdownSender + shutdown_sender: ShutdownSender, ) -> Result, ServiceError> { let cfg_path = &self.config.stellar_overlay_config_filepath; let stellar_overlay_cfg = @@ -408,7 +408,7 @@ impl VaultService { let oracle_agent = crate::oracle::start_oracle_agent( stellar_overlay_cfg, &self.secret_key, - shutdown_sender + shutdown_sender, ) .await .expect("Failed to start oracle agent"); diff --git a/clients/vault/tests/helper/mod.rs b/clients/vault/tests/helper/mod.rs index 8b85e8cca..7747f0b10 100644 --- a/clients/vault/tests/helper/mod.rs +++ b/clients/vault/tests/helper/mod.rs @@ -7,9 +7,13 @@ pub use helper::*; use async_trait::async_trait; use lazy_static::lazy_static; use primitives::CurrencyId; -use runtime::{integration::{ - default_provider_client, set_exchange_rate_and_wait, setup_provider, SubxtClient, -}, types::FixedU128, SpacewalkParachain, VaultId, ShutdownSender}; +use runtime::{ + integration::{ + default_provider_client, set_exchange_rate_and_wait, setup_provider, SubxtClient, + }, + types::FixedU128, + ShutdownSender, SpacewalkParachain, VaultId, +}; use sp_arithmetic::FixedPointNumber; use sp_keyring::AccountKeyring; use std::{future::Future, sync::Arc}; From 461337d308188a35ab59ab8c8e37e7cf5a170cd6 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:52:29 +0800 Subject: [PATCH 16/22] fix failing test cases due to time elapsed with building proofs --- .../src/connection/connector/connector.rs | 45 ++++++++++--------- .../connection/connector/message_sender.rs | 1 + clients/vault/src/oracle/agent.rs | 1 - 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 853a942f1..3b412dd7e 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -5,7 +5,7 @@ use substrate_stellar_sdk::{ }; use tokio::{ net::tcp::{OwnedReadHalf, OwnedWriteHalf}, - sync::mpsc, + sync::{mpsc, mpsc::error::TryRecvError}, }; use crate::{ @@ -235,32 +235,37 @@ pub(crate) async fn poll_messages_from_stellar( break } - tokio::select! { - result_msg = read_message_from_stellar(&mut read_stream_overlay, connector.timeout_in_secs) => match result_msg { - Err(e) => { - log::error!("poll_messages_from_stellar(): {e:?}"); - break; + // check for messages from user. + match send_to_node_receiver.try_recv() { + Ok(msg) => + if let Err(e) = connector.send_to_node(msg).await { + log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); }, - Ok(xdr) => match connector.process_raw_message(xdr).await { - Ok(Some(stellar_msg)) => - // push message to user + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => {}, + } + + // check for messages from Stellar Node. + match read_message_from_stellar(&mut read_stream_overlay, connector.timeout_in_secs).await { + Err(e) => { + log::error!("poll_messages_from_stellar(): {e:?}"); + break + }, + Ok(xdr) => match connector.process_raw_message(xdr).await { + Ok(Some(stellar_msg)) => + // push message to user if let Err(e) = send_to_user_sender.send(stellar_msg).await { log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); }, - Ok(_) => log::info!("poll_messages_from_stellar(): no message for user"), - Err(e) => { - log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); - break; - } - } + Ok(_) => {}, + Err(e) => { + log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); + break + }, }, - - // push message to Stellar Node - Some(msg) = send_to_node_receiver.recv() => if let Err(e) = connector.send_to_node(msg).await { - log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); - } } } + // make sure to drop/shutdown the stream connector.write_stream_overlay.forget(); drop(read_stream_overlay); diff --git a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs index d5504e5c9..bf31d1b2e 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs @@ -11,6 +11,7 @@ use crate::connection::{ impl Connector { pub async fn send_to_node(&mut self, msg: StellarMessage) -> Result<(), Error> { + log::info!("send_to_node: sending message: {msg:?}"); let xdr_msg = &self.create_xdr_message(msg)?; match timeout( diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 26693e894..0812384cf 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -165,7 +165,6 @@ impl OracleAgent { let collector = collector.read().await; match collector.build_proof(slot, &stellar_sender).await { None => { - tracing::warn!("get_proof(): Failed to build proof for slot {slot}."); drop(collector); // give 10 seconds interval for every retry sleep(Duration::from_secs(10)).await; From e49b266167dbf8278456e0a0c595776758ecc8f5 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Thu, 7 Dec 2023 19:57:32 +0800 Subject: [PATCH 17/22] remove never-ending test and additional cleanup --- clients/service/src/lib.rs | 15 +- clients/stellar-relay-lib/README.md | 41 +-- clients/stellar-relay-lib/src/config.rs | 10 +- .../src/connection/connector/connector.rs | 267 ++++++++++++++---- .../connection/connector/message_reader.rs | 76 ++++- .../connection/connector/message_sender.rs | 1 - .../src/connection/connector/mod.rs | 1 + clients/stellar-relay-lib/src/tests/mod.rs | 22 -- clients/vault/src/oracle/agent.rs | 3 +- 9 files changed, 297 insertions(+), 139 deletions(-) diff --git a/clients/service/src/lib.rs b/clients/service/src/lib.rs index 9f611e810..97a7650c7 100644 --- a/clients/service/src/lib.rs +++ b/clients/service/src/lib.rs @@ -23,6 +23,9 @@ mod cli; mod error; mod trace; +const RESET_RESTART_TIME_IN_SECS: u64 = 1800; // reset the restart time in 30 minutes +const DEFAULT_RESTART_TIME_IN_SECS: u64 = 20; // default sleep time before restarting everything + #[async_trait] pub trait Service { const NAME: &'static str; @@ -71,7 +74,7 @@ impl ConnectionManager { pub async fn start, InnerError: fmt::Display>( &self, ) -> Result<(), Error> { - let mut restart_delay_timer_in_secs = 20; // set default to 20 seconds for restart + let mut restart_in_secs = DEFAULT_RESTART_TIME_IN_SECS; // set default to 20 seconds for restart let mut time_as_of_recording = SystemTime::now(); loop { @@ -79,8 +82,8 @@ impl ConnectionManager { let _ = time_now.duration_since(time_as_of_recording).map(|duration| { // Revert the counter if the restart happened more than 30 minutes (or 1800 seconds) // ago - if duration.as_secs() > 1800 { - restart_delay_timer_in_secs = 20; + if duration.as_secs() > RESET_RESTART_TIME_IN_SECS { + restart_in_secs = DEFAULT_RESTART_TIME_IN_SECS; } // Increase time by 10 seconds if a restart is triggered too frequently. // This waits for delayed packets to be removed in the network, @@ -88,7 +91,7 @@ impl ConnectionManager { // Else, these straggler packets will interfere with the new connection. // https://www.rfc-editor.org/rfc/rfc793#page-22 else { - restart_delay_timer_in_secs += 10; + restart_in_secs += 10; time_as_of_recording = time_now; } }); @@ -155,8 +158,8 @@ impl ConnectionManager { RestartPolicy::Always => { (self.increment_restart_counter)(); - tracing::info!("Restarting in {restart_delay_timer_in_secs} seconds"); - sleep(Duration::from_secs(restart_delay_timer_in_secs)).await; + tracing::info!("Restarting in {restart_in_secs} seconds"); + sleep(Duration::from_secs(restart_in_secs)).await; continue }, diff --git a/clients/stellar-relay-lib/README.md b/clients/stellar-relay-lib/README.md index e2bacb26e..c98557130 100644 --- a/clients/stellar-relay-lib/README.md +++ b/clients/stellar-relay-lib/README.md @@ -41,9 +41,7 @@ pub struct ConnectionInfoCfg { pub recv_scp_msgs: bool, pub remote_called_us: bool, /// how long to wait for the Stellar Node's messages. - timeout_in_secs: u64, - /// number of retries to wait for the Stellar Node's messages and/or to connect back to it. - retries:u8 + timeout_in_secs: u64 } ``` @@ -66,17 +64,9 @@ Create a connection using the `connect_to_stellar_overlay_network` function: ```rust let mut overlay_connection = stellar_relay_lib::connect_to_stellar_overlay_network(cfg, secret_key).await?; ``` -The `StellarOverlayConnection` has 2 async methods to interact with the Stellar Node: -* _`send(&self, message: StellarMessage)`_ -> for sending `StellarMessage`s to Stellar Node -* _`listen(&mut self)`_ -> for receiving `StellarRelayMessage`s from the Stellar Relay. - -### Interpreting the `StellarRelayMessage` -The `StellarRelayMessage` is an enum with the following variants: -* _`Connect`_ -> interprets a successful connection to Stellar Node. It contains the `PublicKey` and the `NodeInfo` -* _`Data`_ -> a wrapper of a `StellarMessage` and additional fields: the _message type_ and the unique `p_id`(process id) -* _`Timeout`_ -> Depends on the `timeout_in_secs` and `retries` defined in the `ConnectionInfo` (**10** and **3** by default). This message is returned after multiple retries have been done. -For example, Stellar Relay will wait for 10 seconds to read from the existing tcp stream before retrying again. After the 3rd retry, StellarRelay will create a new stream in 3 attempts, with an interval of 3 seconds. -* _`Error`_ -> a todo +The `StellarOverlayConnection` has 2 methods to interact with the Stellar Node: +* _`sender(&self)`_ -> used to send `StellarMessage`s to Stellar Node +* _`listen(&mut self)`_ -> async method for receiving `StellarMessage`s from the Stellar Node. ## Example In the `stellar-relay-lib` directory, run this command: @@ -102,25 +92,4 @@ and you should be able to see in the terminal: ... [2022-10-14T13:16:02Z INFO connect] R0E1U1RCTVY2UURYRkRHRDYyTUVITExIWlRQREk3N1UzUEZPRDJTRUxVNVJKREhRV0JSNU5OSzc= sent StellarMessage of type ScpStNominate for ledger 43109751 [2022-10-14T13:16:02Z INFO connect] R0NHQjJTMktHWUFSUFZJQTM3SFlaWFZSTTJZWlVFWEE2UzMzWlU1QlVEQzZUSFNCNjJMWlNUWUg= sent StellarMessage of type ScpStPrepare for ledger 43109751 -``` - -Here is an example in the terminal when disconnection/reconnection happens: -``` -[2022-10-17T05:56:47Z ERROR stellar_relay::connection::services] deadline has elapsed for reading messages from Stellar Node. Retry: 0 -[2022-10-17T05:56:47Z ERROR stellar_relay::connection::services] deadline has elapsed for receiving messages. Retry: 0 -[2022-10-17T05:56:57Z ERROR stellar_relay::connection::services] deadline has elapsed for reading messages from Stellar Node. Retry: 1 -[2022-10-17T05:56:57Z ERROR stellar_relay::connection::services] deadline has elapsed for receiving messages. Retry: 1 -[2022-10-17T05:57:07Z ERROR stellar_relay::connection::services] deadline has elapsed for reading messages from Stellar Node. Retry: 2 -[2022-10-17T05:57:07Z ERROR stellar_relay::connection::services] deadline has elapsed for receiving messages. Retry: 2 -[2022-10-17T05:57:17Z ERROR stellar_relay::connection::services] deadline has elapsed for reading messages from Stellar Node. Retry: 3 -[2022-10-17T05:57:17Z ERROR stellar_relay::connection::services] deadline has elapsed for receiving messages. Retry: 3 -[2022-10-17T05:57:17Z INFO stellar_relay::connection::user_controls] reconnecting to "135.181.16.110". -[2022-10-17T05:57:17Z ERROR stellar_relay::connection::user_controls] failed to reconnect! # of retries left: 2. Retrying in 3 seconds... -[2022-10-17T05:57:20Z INFO stellar_relay::connection::user_controls] reconnecting to "135.181.16.110". -[2022-10-17T05:57:20Z INFO stellar_relay::connection::services] Starting Handshake with Hello. -[2022-10-17T05:57:21Z INFO stellar_relay::connection::connector::message_handler] Hello message processed successfully -[2022-10-17T05:57:21Z INFO stellar_relay::connection::connector::message_handler] Handshake completed -``` - - -todo: add multiple tests \ No newline at end of file +``` \ No newline at end of file diff --git a/clients/stellar-relay-lib/src/config.rs b/clients/stellar-relay-lib/src/config.rs index 3312f047e..ce787e143 100644 --- a/clients/stellar-relay-lib/src/config.rs +++ b/clients/stellar-relay-lib/src/config.rs @@ -100,10 +100,6 @@ pub struct ConnectionInfoCfg { /// how long to wait for the Stellar Node's messages. #[serde(default = "ConnectionInfoCfg::default_timeout")] pub timeout_in_secs: u64, - - /// number of retries to wait for the Stellar Node's messages and/or to connect back to it. - #[serde(default = "ConnectionInfoCfg::default_retries")] - pub retries: u8, } impl ConnectionInfoCfg { @@ -126,14 +122,10 @@ impl ConnectionInfoCfg { fn default_timeout() -> u64 { 10 } - - fn default_retries() -> u8 { - 3 - } } /// Triggers connection to the Stellar Node. -/// Returns the `StellarStellarOverlayConnection` if connection is a success, otherwise an Error +/// Returns the `StellarOverlayConnection` if connection is a success, otherwise an Error pub async fn connect_to_stellar_overlay_network( cfg: StellarOverlayConfig, secret_key: &str, diff --git a/clients/stellar-relay-lib/src/connection/connector/connector.rs b/clients/stellar-relay-lib/src/connection/connector/connector.rs index 3b412dd7e..05e6a70c1 100644 --- a/clients/stellar-relay-lib/src/connection/connector/connector.rs +++ b/clients/stellar-relay-lib/src/connection/connector/connector.rs @@ -1,17 +1,13 @@ use std::fmt::{Debug, Formatter}; use substrate_stellar_sdk::{ - types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType, StellarMessage}, + types::{AuthenticatedMessageV0, Curve25519Public, HmacSha256Mac, MessageType}, XdrCodec, }; -use tokio::{ - net::tcp::{OwnedReadHalf, OwnedWriteHalf}, - sync::{mpsc, mpsc::error::TryRecvError}, -}; +use tokio::net::tcp::OwnedWriteHalf; use crate::{ connection::{ authentication::{gen_shared_key, ConnectionAuth}, - connector::message_reader::read_message_from_stellar, flow_controller::FlowController, handshake::HandshakeState, hmac::{verify_hmac, HMacKeys}, @@ -212,66 +208,215 @@ impl Connector { } } -/// Polls for messages coming from the Stellar Node and communicates it back to the user -/// -/// # Arguments -/// * `connector` - contains the config and necessary info for connecting to Stellar Node -/// * `read_stream_overlay` - the read half of the stream that is connected to Stellar Node -/// * `send_to_user_sender` - sends message from Stellar to the user -/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the -/// stream. -pub(crate) async fn poll_messages_from_stellar( - mut connector: Connector, - mut read_stream_overlay: OwnedReadHalf, - send_to_user_sender: mpsc::Sender, - mut send_to_node_receiver: mpsc::Receiver, -) { - log::info!("poll_messages_from_stellar(): started."); - - loop { - if send_to_user_sender.is_closed() { - log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); - // close this channel as communication to user was closed. - break - } +#[cfg(test)] +mod test { + use crate::{connection::hmac::HMacKeys, node::RemoteInfo, StellarOverlayConfig}; + use serial_test::serial; + + use substrate_stellar_sdk::{ + compound_types::LimitedString, + types::{Hello, MessageType}, + PublicKey, + }; + use tokio::{io::AsyncWriteExt, net::tcp::OwnedReadHalf}; + + use crate::{ + connection::{ + authentication::{create_auth_cert, ConnectionAuth}, + Connector, + }, + helper::{create_stream, time_now}, + node::NodeInfo, + ConnectionInfo, + }; + + fn create_auth_cert_from_connection_auth( + connector_auth: &ConnectionAuth, + ) -> substrate_stellar_sdk::types::AuthCert { + let time_now = time_now(); + let new_auth_cert = create_auth_cert( + connector_auth.network_id(), + connector_auth.keypair(), + time_now, + connector_auth.pub_key_ecdh().clone(), + ) + .expect("should successfully create an auth cert"); + new_auth_cert + } - // check for messages from user. - match send_to_node_receiver.try_recv() { - Ok(msg) => - if let Err(e) = connector.send_to_node(msg).await { - log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); - }, - Err(TryRecvError::Disconnected) => break, - Err(TryRecvError::Empty) => {}, - } + impl Connector { + fn shutdown(&mut self, read_half: OwnedReadHalf) { + let _ = self.write_stream_overlay.shutdown(); - // check for messages from Stellar Node. - match read_message_from_stellar(&mut read_stream_overlay, connector.timeout_in_secs).await { - Err(e) => { - log::error!("poll_messages_from_stellar(): {e:?}"); - break - }, - Ok(xdr) => match connector.process_raw_message(xdr).await { - Ok(Some(stellar_msg)) => - // push message to user - if let Err(e) = send_to_user_sender.send(stellar_msg).await { - log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); - }, - Ok(_) => {}, - Err(e) => { - log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); - break - }, - }, + drop(read_half); } } - // make sure to drop/shutdown the stream - connector.write_stream_overlay.forget(); - drop(read_stream_overlay); + async fn create_connector() -> (NodeInfo, ConnectionInfo, Connector, OwnedReadHalf) { + let cfg_file_path = "./resources/config/testnet/stellar_relay_config_sdftest1.json"; + let secret_key_path = "./resources/secretkey/stellar_secretkey_testnet"; + let secret_key = + std::fs::read_to_string(secret_key_path).expect("should be able to read file"); + + let cfg = + StellarOverlayConfig::try_from_path(cfg_file_path).expect("should create a config"); + let node_info = cfg.node_info(); + let conn_info = cfg.connection_info(&secret_key).expect("should create a connection info"); + // this is a channel to communicate with the connection/config (this needs renaming) + + let (read_half, write_half) = + create_stream(&conn_info.address()).await.expect("should return a stream"); + let connector = Connector::new(node_info.clone(), conn_info.clone(), write_half); + (node_info, conn_info, connector, read_half) + } + + #[tokio::test] + #[serial] + async fn create_new_connector_works() { + let (node_info, _, mut connector, read_half) = create_connector().await; - send_to_node_receiver.close(); - drop(send_to_user_sender); + let connector_local_node = connector.local.node(); - log::info!("poll_messages_from_stellar(): stopped."); + assert_eq!(connector_local_node.ledger_version, node_info.ledger_version); + assert_eq!(connector_local_node.overlay_version, node_info.overlay_version); + assert_eq!(connector_local_node.overlay_min_version, node_info.overlay_min_version); + assert_eq!(connector_local_node.version_str, node_info.version_str); + assert_eq!(connector_local_node.network_id, node_info.network_id); + + connector.shutdown(read_half); + } + + #[tokio::test] + #[serial] + async fn connector_local_sequence_works() { + let (_, _, mut connector, read_half) = create_connector().await; + assert_eq!(connector.local_sequence(), 0); + connector.increment_local_sequence(); + assert_eq!(connector.local_sequence(), 1); + + connector.shutdown(read_half); + } + + #[tokio::test] + #[serial] + async fn connector_set_remote_works() { + let (_, _, mut connector, read_half) = create_connector().await; + + let connector_auth = &connector.connection_auth; + let new_auth_cert = create_auth_cert_from_connection_auth(connector_auth); + + let hello = Hello { + ledger_version: 0, + overlay_version: 0, + overlay_min_version: 0, + network_id: [0; 32], + version_str: LimitedString::<100_i32>::new(vec![]).unwrap(), + listening_port: 11625, + peer_id: PublicKey::PublicKeyTypeEd25519([0; 32]), + cert: new_auth_cert, + nonce: [0; 32], + }; + connector.set_remote(RemoteInfo::new(&hello)); + + assert!(connector.remote().is_some()); + + connector.shutdown(read_half); + } + + #[tokio::test] + #[serial] + async fn connector_increment_remote_sequence_works() { + let (_, _, mut connector, read_half) = create_connector().await; + + let connector_auth = &connector.connection_auth; + let new_auth_cert = create_auth_cert_from_connection_auth(connector_auth); + + let hello = Hello { + ledger_version: 0, + overlay_version: 0, + overlay_min_version: 0, + network_id: [0; 32], + version_str: LimitedString::<100_i32>::new(vec![]).unwrap(), + listening_port: 11625, + peer_id: PublicKey::PublicKeyTypeEd25519([0; 32]), + cert: new_auth_cert, + nonce: [0; 32], + }; + connector.set_remote(RemoteInfo::new(&hello)); + assert_eq!(connector.remote().unwrap().sequence(), 0); + + connector.increment_remote_sequence().unwrap(); + connector.increment_remote_sequence().unwrap(); + connector.increment_remote_sequence().unwrap(); + assert_eq!(connector.remote().unwrap().sequence(), 3); + + connector.shutdown(read_half); + } + + #[tokio::test] + #[serial] + async fn connector_get_and_set_hmac_keys_works() { + //arrange + let (_, _, mut connector, read_half) = create_connector().await; + let connector_auth = &connector.connection_auth; + let new_auth_cert = create_auth_cert_from_connection_auth(connector_auth); + + let hello = Hello { + ledger_version: 0, + overlay_version: 0, + overlay_min_version: 0, + network_id: [0; 32], + version_str: LimitedString::<100_i32>::new(vec![]).unwrap(), + listening_port: 11625, + peer_id: PublicKey::PublicKeyTypeEd25519([0; 32]), + cert: new_auth_cert, + nonce: [0; 32], + }; + let remote = RemoteInfo::new(&hello); + let remote_nonce = remote.nonce(); + connector.set_remote(remote.clone()); + + let shared_key = connector.get_shared_key(remote.pub_key_ecdh()); + assert!(connector.hmac_keys().is_none()); + //act + connector.set_hmac_keys(HMacKeys::new( + &shared_key, + connector.local().nonce(), + remote_nonce, + connector.remote_called_us(), + )); + //assert + assert!(connector.hmac_keys().is_some()); + + connector.shutdown(read_half); + } + + #[tokio::test] + #[serial] + async fn connector_method_works() { + let (_, conn_config, mut connector, read_half) = create_connector().await; + + assert_eq!(connector.remote_called_us(), conn_config.remote_called_us); + assert_eq!(connector.receive_tx_messages(), conn_config.recv_tx_msgs); + assert_eq!(connector.receive_scp_messages(), conn_config.recv_scp_msgs); + + connector.got_hello(); + assert!(connector.is_handshake_created()); + + connector.handshake_completed(); + assert!(connector.is_handshake_created()); + + connector.shutdown(read_half); + } + + #[tokio::test] + #[serial] + async fn enable_flow_controller_works() { + let (node_info, _, mut connector, read_half) = create_connector().await; + + assert!(!connector.inner_check_to_send_more(MessageType::ScpMessage)); + connector.enable_flow_controller(node_info.overlay_version, node_info.overlay_version); + + connector.shutdown(read_half); + } } diff --git a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs index 4146b13d4..5c5b62085 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs @@ -1,10 +1,80 @@ -use crate::connection::{xdr_converter::get_xdr_message_length, Error, Xdr}; +use crate::connection::{xdr_converter::get_xdr_message_length, Connector, Error, Xdr}; use std::time::Duration; +use substrate_stellar_sdk::types::StellarMessage; -use tokio::{io::AsyncReadExt, net::tcp, time::timeout}; +use tokio::{ + io::AsyncReadExt, + net::{tcp, tcp::OwnedReadHalf}, + sync::{mpsc, mpsc::error::TryRecvError}, + time::timeout, +}; + +/// Polls for messages coming from the Stellar Node and communicates it back to the user +/// +/// # Arguments +/// * `connector` - contains the config and necessary info for connecting to Stellar Node +/// * `read_stream_overlay` - the read half of the stream that is connected to Stellar Node +/// * `send_to_user_sender` - sends message from Stellar to the user +/// * `send_to_node_receiver` - receives message from user and writes it to the write half of the +/// stream. +pub(crate) async fn poll_messages_from_stellar( + mut connector: Connector, + mut read_stream_overlay: OwnedReadHalf, + send_to_user_sender: mpsc::Sender, + mut send_to_node_receiver: mpsc::Receiver, +) { + log::info!("poll_messages_from_stellar(): started."); + + loop { + if send_to_user_sender.is_closed() { + log::info!("poll_messages_from_stellar(): closing receiver during disconnection"); + // close this channel as communication to user was closed. + break + } + + // check for messages from user. + match send_to_node_receiver.try_recv() { + Ok(msg) => + if let Err(e) = connector.send_to_node(msg).await { + log::error!("poll_messages_from_stellar(): Error occurred during sending message to node: {e:?}"); + }, + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => {}, + } + + // check for messages from Stellar Node. + match read_message_from_stellar(&mut read_stream_overlay, connector.timeout_in_secs).await { + Err(e) => { + log::error!("poll_messages_from_stellar(): {e:?}"); + break + }, + Ok(xdr) => match connector.process_raw_message(xdr).await { + Ok(Some(stellar_msg)) => + // push message to user + if let Err(e) = send_to_user_sender.send(stellar_msg).await { + log::warn!("poll_messages_from_stellar(): Error occurred during sending message to user: {e:?}"); + }, + Ok(_) => {}, + Err(e) => { + log::error!("poll_messages_from_stellar(): Error occurred during processing xdr message: {e:?}"); + break + }, + }, + } + } + + // make sure to drop/shutdown the stream + connector.write_stream_overlay.forget(); + drop(read_stream_overlay); + + send_to_node_receiver.close(); + drop(send_to_user_sender); + + log::info!("poll_messages_from_stellar(): stopped."); +} /// Returns Xdr format of the `StellarMessage` sent from the Stellar Node -pub(super) async fn read_message_from_stellar( +async fn read_message_from_stellar( r_stream: &mut tcp::OwnedReadHalf, timeout_in_secs: u64, ) -> Result { diff --git a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs index bf31d1b2e..d5504e5c9 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_sender.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_sender.rs @@ -11,7 +11,6 @@ use crate::connection::{ impl Connector { pub async fn send_to_node(&mut self, msg: StellarMessage) -> Result<(), Error> { - log::info!("send_to_node: sending message: {msg:?}"); let xdr_msg = &self.create_xdr_message(msg)?; match timeout( diff --git a/clients/stellar-relay-lib/src/connection/connector/mod.rs b/clients/stellar-relay-lib/src/connection/connector/mod.rs index 168456a1a..d1c5e3dfe 100644 --- a/clients/stellar-relay-lib/src/connection/connector/mod.rs +++ b/clients/stellar-relay-lib/src/connection/connector/mod.rs @@ -5,3 +5,4 @@ mod message_reader; mod message_sender; pub(crate) use connector::*; +pub(crate) use message_reader::poll_messages_from_stellar; diff --git a/clients/stellar-relay-lib/src/tests/mod.rs b/clients/stellar-relay-lib/src/tests/mod.rs index aff162202..c8fb68836 100644 --- a/clients/stellar-relay-lib/src/tests/mod.rs +++ b/clients/stellar-relay-lib/src/tests/mod.rs @@ -41,28 +41,6 @@ fn overlay_infos(is_mainnet: bool) -> (NodeInfo, ConnectionInfo) { ) } -#[tokio::test] -#[serial] -async fn stellar_overlay_connect_and_listen_connect_message() { - let (node_info, conn_info) = overlay_infos(false); - - let mut overlay_connection = - StellarOverlayConnection::connect(node_info.clone(), conn_info).await.unwrap(); - - if let Some(message) = - overlay_connection.listen().await.expect("Should return a Stellar Message") - { - match message { - StellarMessage::ErrorMsg(_) => assert!(false), - _ => assert!(true), - } - } else { - assert!(false); - } - - overlay_connection.disconnect(); -} - #[tokio::test(flavor = "multi_thread")] #[serial] async fn stellar_overlay_should_receive_scp_messages() { diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 0812384cf..ecd8ca7f1 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -80,7 +80,8 @@ pub async fn start_oracle_agent( // a clone used to forcefully call a shutdown, when StellarOverlay disconnects. let shutdown_sender_clone2 = shutdown_sender.clone(); - // disconnect signal sender + // disconnect signal sender tells the StellarOverlayConnection to close its TcpStream to Stellar + // Node let (disconnect_signal_sender, mut disconnect_signal_receiver) = mpsc::channel::<()>(2); service::spawn_cancelable(shutdown_sender_clone.subscribe(), async move { From fd5b72feb91443c14141a07a41c91999c3ec2b06 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Fri, 8 Dec 2023 00:07:22 +0800 Subject: [PATCH 18/22] support multi-thread in tests --- clients/vault/src/oracle/agent.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index ecd8ca7f1..18774e3e3 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -214,7 +214,7 @@ mod tests { use super::*; use serial_test::serial; - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] #[ntest::timeout(1_800_000)] // timeout at 30 minutes #[serial] async fn test_get_proof_for_current_slot() { @@ -242,7 +242,7 @@ mod tests { assert!(proof_result.is_ok(), "Failed to get proof for slot: {}", latest_slot); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] #[serial] async fn test_get_proof_for_archived_slot() { let scp_archive_storage = ScpArchiveStorage::default(); @@ -257,6 +257,7 @@ mod tests { .await .expect("Failed to start agent"); + sleep(Duration::from_secs(5)).await; // This slot should be archived on the public network let target_slot = 44041116; let proof = agent.get_proof(target_slot).await.expect("should return a proof"); @@ -270,7 +271,7 @@ mod tests { agent.stop().expect("Failed to stop the agent"); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] #[serial] async fn test_get_proof_for_archived_slot_with_fallback() { let scp_archive_storage = ScpArchiveStorage::default(); @@ -292,6 +293,7 @@ mod tests { .await .expect("Failed to start agent"); + sleep(Duration::from_secs(5)).await; // This slot should be archived on the public network let target_slot = 44041116; let proof = agent.get_proof(target_slot).await.expect("should return a proof"); @@ -305,7 +307,7 @@ mod tests { agent.stop().expect("Failed to stop the agent"); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] #[serial] async fn test_get_proof_for_archived_slot_fails_without_archives() { let scp_archive_storage = ScpArchiveStorage::default(); @@ -319,6 +321,7 @@ mod tests { let agent = start_oracle_agent(modified_config, &get_test_secret_key(true), shutdown) .await .expect("Failed to start agent"); + sleep(Duration::from_secs(5)).await; // This slot should be archived on the public network let target_slot = 44041116; From df86a24f2ecebd06c794fe6457221f979f02cfc2 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:17:11 +0800 Subject: [PATCH 19/22] https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419319850, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419311674, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419309778, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419305491, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419303814, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419301423, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419299133, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419287736, https://github.com/pendulum-chain/spacewalk/pull/454#discussion_r1419285478, https://github.com/pendulum-chain/spacewalk/pull/454#pullrequestreview-1770621811 --- clients/service/src/lib.rs | 9 ++- .../connection/connector/message_reader.rs | 2 +- .../stellar-relay-lib/src/connection/error.rs | 2 +- clients/stellar-relay-lib/src/overlay.rs | 78 +++++++++---------- clients/vault/src/oracle/agent.rs | 7 +- 5 files changed, 48 insertions(+), 50 deletions(-) diff --git a/clients/service/src/lib.rs b/clients/service/src/lib.rs index 97a7650c7..664e011f0 100644 --- a/clients/service/src/lib.rs +++ b/clients/service/src/lib.rs @@ -25,6 +25,7 @@ mod trace; const RESET_RESTART_TIME_IN_SECS: u64 = 1800; // reset the restart time in 30 minutes const DEFAULT_RESTART_TIME_IN_SECS: u64 = 20; // default sleep time before restarting everything +const RESTART_BACKOFF_DELAY: u64 = 10; #[async_trait] pub trait Service { @@ -75,11 +76,11 @@ impl ConnectionManager { &self, ) -> Result<(), Error> { let mut restart_in_secs = DEFAULT_RESTART_TIME_IN_SECS; // set default to 20 seconds for restart - let mut time_as_of_recording = SystemTime::now(); + let mut last_start_timestamp = SystemTime::now(); loop { let time_now = SystemTime::now(); - let _ = time_now.duration_since(time_as_of_recording).map(|duration| { + let _ = time_now.duration_since(last_start_timestamp).map(|duration| { // Revert the counter if the restart happened more than 30 minutes (or 1800 seconds) // ago if duration.as_secs() > RESET_RESTART_TIME_IN_SECS { @@ -91,8 +92,8 @@ impl ConnectionManager { // Else, these straggler packets will interfere with the new connection. // https://www.rfc-editor.org/rfc/rfc793#page-22 else { - restart_in_secs += 10; - time_as_of_recording = time_now; + restart_in_secs += RESTART_BACKOFF_DELAY; + last_start_timestamp = time_now; } }); diff --git a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs index 5c5b62085..8836fd998 100644 --- a/clients/stellar-relay-lib/src/connection/connector/message_reader.rs +++ b/clients/stellar-relay-lib/src/connection/connector/message_reader.rs @@ -70,7 +70,7 @@ pub(crate) async fn poll_messages_from_stellar( send_to_node_receiver.close(); drop(send_to_user_sender); - log::info!("poll_messages_from_stellar(): stopped."); + log::debug!("poll_messages_from_stellar(): stopped."); } /// Returns Xdr format of the `StellarMessage` sent from the Stellar Node diff --git a/clients/stellar-relay-lib/src/connection/error.rs b/clients/stellar-relay-lib/src/connection/error.rs index d1afa62d6..334600383 100644 --- a/clients/stellar-relay-lib/src/connection/error.rs +++ b/clients/stellar-relay-lib/src/connection/error.rs @@ -63,7 +63,7 @@ pub enum Error { #[error(display = "Received Error from Overlay: {:?}", _0)] OverlayError(ErrorCode), - #[error(display = "Timeout elapsed")] + #[error(display = "Encountered timeout")] Timeout, #[error(display = "Config Error: Version String too long")] diff --git a/clients/stellar-relay-lib/src/overlay.rs b/clients/stellar-relay-lib/src/overlay.rs index 59b15e57f..82b3f52ee 100644 --- a/clients/stellar-relay-lib/src/overlay.rs +++ b/clients/stellar-relay-lib/src/overlay.rs @@ -29,6 +29,38 @@ impl StellarOverlayConnection { self.sender.send(msg).await } + /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. + pub async fn connect( + local_node_info: NodeInfo, + conn_info: ConnectionInfo, + ) -> Result { + log::info!("connect(): connecting to {conn_info:?}"); + + // this is a channel to communicate with the user/caller. + let (send_to_user_sender, send_to_user_receiver) = mpsc::channel::(1024); + + let (send_to_node_sender, send_to_node_receiver) = mpsc::channel::(1024); + + // split the stream for easy handling of read and write + let (read_stream_overlay, write_stream_overlay) = + create_stream(&conn_info.address()).await?; + + let mut connector = Connector::new(local_node_info, conn_info, write_stream_overlay); + connector.send_hello_message().await?; + + tokio::spawn(poll_messages_from_stellar( + connector, + read_stream_overlay, + send_to_user_sender, + send_to_node_receiver, + )); + + Ok(StellarOverlayConnection { + sender: send_to_node_sender, + receiver: send_to_user_receiver, + }) + } + pub async fn listen(&mut self) -> Result, Error> { loop { if !self.is_alive() { @@ -52,18 +84,18 @@ impl StellarOverlayConnection { } } - pub fn is_alive(&self) -> bool { - let result = self.sender.is_closed(); + pub fn is_alive(&mut self) -> bool { + let is_closed = self.sender.is_closed(); - if result { - drop(self); + if is_closed { + self.disconnect(); } - !result + !is_closed } pub fn disconnect(&mut self) { - log::info!("disconnect(): closing channel"); + log::info!("disconnect(): closing connection to overlay network"); self.receiver.close(); } } @@ -73,37 +105,3 @@ impl Drop for StellarOverlayConnection { self.disconnect(); } } - -impl StellarOverlayConnection { - /// Returns an `StellarOverlayConnection` when a connection to Stellar Node is successful. - pub async fn connect( - local_node_info: NodeInfo, - conn_info: ConnectionInfo, - ) -> Result { - log::info!("connect(): connecting to {conn_info:?}"); - - // this is a channel to communicate with the user/caller. - let (send_to_user_sender, send_to_user_receiver) = mpsc::channel::(1024); - - let (send_to_node_sender, send_to_node_receiver) = mpsc::channel::(1024); - - // split the stream for easy handling of read and write - let (read_stream_overlay, write_stream_overlay) = - create_stream(&conn_info.address()).await?; - - let mut connector = Connector::new(local_node_info, conn_info, write_stream_overlay); - connector.send_hello_message().await?; - - tokio::spawn(poll_messages_from_stellar( - connector, - read_stream_overlay, - send_to_user_sender, - send_to_node_receiver, - )); - - Ok(StellarOverlayConnection { - sender: send_to_node_sender, - receiver: send_to_user_receiver, - }) - } -} diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 18774e3e3..555879fc2 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -124,12 +124,10 @@ pub async fn start_oracle_agent( }, } } - - tracing::info!("start_oracle_agent(): LOOP STOPPED!"); }); tokio::spawn(on_shutdown(shutdown_sender.clone(), async move { - tracing::info!("start_oracle_agent(): sending signal to shutdown overlay connection..."); + tracing::debug!("start_oracle_agent(): sending signal to shutdown overlay connection..."); if let Err(e) = disconnect_signal_sender.send(()).await { tracing::warn!("start_oracle_agent(): failed to send disconnect signal: {e:?}"); } @@ -218,6 +216,7 @@ mod tests { #[ntest::timeout(1_800_000)] // timeout at 30 minutes #[serial] async fn test_get_proof_for_current_slot() { + env_logger::init(); let shutdown_sender = ShutdownSender::new(); let agent = start_oracle_agent( get_test_stellar_relay_config(true), @@ -236,7 +235,7 @@ mod tests { } // use a future slot (2 slots ahead) to ensure enough messages can be collected // and to avoid "missed" messages. - latest_slot += 2; + latest_slot += 3; let proof_result = agent.get_proof(latest_slot).await; assert!(proof_result.is_ok(), "Failed to get proof for slot: {}", latest_slot); From 956da84d789add81b3cdfad0c82e99df04a0992e Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Mon, 11 Dec 2023 20:53:36 +0800 Subject: [PATCH 20/22] remove env logger --- clients/vault/src/oracle/agent.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 555879fc2..8b7248ee1 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -216,7 +216,6 @@ mod tests { #[ntest::timeout(1_800_000)] // timeout at 30 minutes #[serial] async fn test_get_proof_for_current_slot() { - env_logger::init(); let shutdown_sender = ShutdownSender::new(); let agent = start_oracle_agent( get_test_stellar_relay_config(true), From 3c967a03af08379cd1f6341675bc218ed71e1796 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Mon, 11 Dec 2023 23:46:07 +0800 Subject: [PATCH 21/22] reverting back to 2. Maybe 3 is too forward-looking --- clients/vault/src/oracle/agent.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/vault/src/oracle/agent.rs b/clients/vault/src/oracle/agent.rs index 8b7248ee1..58f58ca2c 100644 --- a/clients/vault/src/oracle/agent.rs +++ b/clients/vault/src/oracle/agent.rs @@ -234,7 +234,7 @@ mod tests { } // use a future slot (2 slots ahead) to ensure enough messages can be collected // and to avoid "missed" messages. - latest_slot += 3; + latest_slot += 2; let proof_result = agent.get_proof(latest_slot).await; assert!(proof_result.is_ok(), "Failed to get proof for slot: {}", latest_slot); From 2d1e6b30ae316ad4dda1f84ba3268c081a437aa8 Mon Sep 17 00:00:00 2001 From: b-yap <2826165+b-yap@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:07:12 +0800 Subject: [PATCH 22/22] https://github.com/pendulum-chain/spacewalk/actions/runs/7169885052/job/19521307640#step:13:66 --- clients/stellar-relay-lib/src/tests/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/stellar-relay-lib/src/tests/mod.rs b/clients/stellar-relay-lib/src/tests/mod.rs index c8fb68836..ad138ec2f 100644 --- a/clients/stellar-relay-lib/src/tests/mod.rs +++ b/clients/stellar-relay-lib/src/tests/mod.rs @@ -56,11 +56,10 @@ async fn stellar_overlay_should_receive_scp_messages() { timeout(Duration::from_secs(300), async move { let mut ov_conn_locked = ov_conn.lock().await; - while let Ok(Some(msg)) = ov_conn_locked.listen().await { + if let Ok(Some(msg)) = ov_conn_locked.listen().await { scps_vec_clone.lock().await.push(msg); ov_conn_locked.disconnect(); - break } }) .await