From 26c0e5dca8996b858b3560345aa00245693624d9 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 20:18:58 -0400 Subject: [PATCH 01/30] Adds an init_read_only() fn in zebra-state --- zebra-state/src/arbitrary.rs | 21 --------- zebra-state/src/lib.rs | 9 ++-- zebra-state/src/service.rs | 68 ++++++++++++++++++++++++++-- zebra-state/src/service/chain_tip.rs | 16 +++++-- 4 files changed, 81 insertions(+), 33 deletions(-) diff --git a/zebra-state/src/arbitrary.rs b/zebra-state/src/arbitrary.rs index 9f87c749c98..c2176296ca7 100644 --- a/zebra-state/src/arbitrary.rs +++ b/zebra-state/src/arbitrary.rs @@ -50,27 +50,6 @@ where } } -impl From for ChainTipBlock { - fn from(prepared: SemanticallyVerifiedBlock) -> Self { - let SemanticallyVerifiedBlock { - block, - hash, - height, - new_outputs: _, - transaction_hashes, - } = prepared; - - Self { - hash, - height, - time: block.header.time, - transactions: block.transactions.clone(), - transaction_hashes, - previous_block_hash: block.header.previous_block_hash, - } - } -} - impl SemanticallyVerifiedBlock { /// Returns a [`ContextuallyVerifiedBlock`] created from this block, /// with fake zero-valued spent UTXOs. diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 59088a931bd..58a83c9e15d 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -46,8 +46,10 @@ pub use request::{ }; pub use response::{KnownBlock, MinedTx, ReadResponse, Response}; pub use service::{ - chain_tip::{ChainTipChange, LatestChainTip, TipAction}, - check, init, spawn_init, + chain_tip::{ChainTipBlock, ChainTipChange, ChainTipSender, LatestChainTip, TipAction}, + check, init, init_read_only, + non_finalized_state::NonFinalizedState, + spawn_init, spawn_init_read_only, watch_receiver::WatchReceiver, OutputIndex, OutputLocation, TransactionIndex, TransactionLocation, }; @@ -76,7 +78,6 @@ pub use response::GetBlockTemplateChainInfo; #[cfg(any(test, feature = "proptest-impl"))] pub use service::{ arbitrary::{populated_state, CHAIN_TIP_UPDATE_WAIT_LIMIT}, - chain_tip::{ChainTipBlock, ChainTipSender}, finalized_state::{RawBytes, KV, MAX_ON_DISK_HEIGHT}, init_test, init_test_services, }; @@ -96,4 +97,4 @@ pub(crate) use config::hidden::{ write_database_format_version_to_disk, write_state_database_format_version_to_disk, }; -pub(crate) use request::ContextuallyVerifiedBlock; +pub use request::ContextuallyVerifiedBlock; diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 994e8ad6b32..8675cb45632 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -387,7 +387,7 @@ impl StateService { let read_service = ReadStateService::new( &finalized_state, - block_write_task, + Some(block_write_task), non_finalized_state_receiver, ); @@ -828,14 +828,14 @@ impl ReadStateService { /// and a watch channel for updating the shared recent non-finalized chain. pub(crate) fn new( finalized_state: &FinalizedState, - block_write_task: Arc>, + block_write_task: Option>>, non_finalized_state_receiver: watch::Receiver, ) -> Self { let read_service = Self { network: finalized_state.network(), db: finalized_state.db.clone(), non_finalized_state_receiver: WatchReceiver::new(non_finalized_state_receiver), - block_write_task: Some(block_write_task), + block_write_task, }; tracing::debug!("created new read-only state service"); @@ -1945,6 +1945,68 @@ pub fn init( ) } +/// Initialize a read state service from the provided [`Config`]. +/// Returns a read-only state service, +/// +/// Each `network` has its own separate on-disk database. +/// +/// To share access to the state, clone the returned [`ReadStateService`]. +pub fn init_read_only( + config: Config, + network: &Network, +) -> ( + ReadStateService, + ZebraDb, + tokio::sync::watch::Sender, +) { + let (non_finalized_state_sender, non_finalized_state_receiver) = + tokio::sync::watch::channel(NonFinalizedState::new(network)); + + #[cfg(feature = "elasticsearch")] + let finalized_state = { + let conn_pool = SingleNodeConnectionPool::new( + Url::parse(config.elasticsearch_url.as_str()) + .expect("configured elasticsearch url is invalid"), + ); + let transport = TransportBuilder::new(conn_pool) + .cert_validation(CertificateValidation::None) + .auth(Basic( + config.clone().elasticsearch_username, + config.clone().elasticsearch_password, + )) + .build() + .expect("elasticsearch transport builder should not fail"); + let elastic_db = Some(Elasticsearch::new(transport)); + + FinalizedState::new_with_debug(&config, network, true, elastic_db, true) + }; + + #[cfg(not(feature = "elasticsearch"))] + let finalized_state = { FinalizedState::new_with_debug(&config, network, true, true) }; + + let db = finalized_state.db.clone(); + + ( + ReadStateService::new(&finalized_state, None, non_finalized_state_receiver), + db, + non_finalized_state_sender, + ) +} + +/// Calls [`init_read_only`] with the provided [`Config`] and [`Network`] from a blocking task. +/// Returns a [`tokio::task::JoinHandle`] with a read state service and chain tip sender. +pub fn spawn_init_read_only( + config: Config, + network: &Network, +) -> tokio::task::JoinHandle<( + ReadStateService, + ZebraDb, + tokio::sync::watch::Sender, +)> { + let network = network.clone(); + tokio::task::spawn_blocking(move || init_read_only(config, &network)) +} + /// Calls [`init`] with the provided [`Config`] and [`Network`] from a blocking task. /// Returns a [`tokio::task::JoinHandle`] with a boxed state service, /// a read state service, and receivers for state chain tip updates. diff --git a/zebra-state/src/service/chain_tip.rs b/zebra-state/src/service/chain_tip.rs index 3f3468f4ca6..3fb016aa356 100644 --- a/zebra-state/src/service/chain_tip.rs +++ b/zebra-state/src/service/chain_tip.rs @@ -107,15 +107,15 @@ impl From for ChainTipBlock { } } -impl From for ChainTipBlock { - fn from(finalized: CheckpointVerifiedBlock) -> Self { - let CheckpointVerifiedBlock(SemanticallyVerifiedBlock { +impl From for ChainTipBlock { + fn from(prepared: SemanticallyVerifiedBlock) -> Self { + let SemanticallyVerifiedBlock { block, hash, height, + new_outputs: _, transaction_hashes, - .. - }) = finalized; + } = prepared; Self { hash, @@ -128,6 +128,12 @@ impl From for ChainTipBlock { } } +impl From for ChainTipBlock { + fn from(CheckpointVerifiedBlock(prepared): CheckpointVerifiedBlock) -> Self { + prepared.into() + } +} + /// A sender for changes to the non-finalized and finalized chain tips. #[derive(Debug)] pub struct ChainTipSender { From 281668bd828857ddb4a391c0d329456c172d1350 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 20:24:13 -0400 Subject: [PATCH 02/30] moves elasticsearch initialization to `FinalizedState::new_with_debug()` --- zebra-state/src/service.rs | 58 +--------------------- zebra-state/src/service/finalized_state.rs | 42 ++++++++++------ 2 files changed, 30 insertions(+), 70 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 8675cb45632..d6b3a416021 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -24,15 +24,6 @@ use std::{ time::{Duration, Instant}, }; -#[cfg(feature = "elasticsearch")] -use elasticsearch::{ - auth::Credentials::Basic, - cert::CertificateValidation, - http::transport::{SingleNodeConnectionPool, TransportBuilder}, - http::Url, - Elasticsearch, -}; - use futures::future::FutureExt; use tokio::sync::{oneshot, watch}; use tower::{util::BoxService, Service, ServiceExt}; @@ -319,29 +310,7 @@ impl StateService { checkpoint_verify_concurrency_limit: usize, ) -> (Self, ReadStateService, LatestChainTip, ChainTipChange) { let timer = CodeTimer::start(); - - #[cfg(feature = "elasticsearch")] - let finalized_state = { - let conn_pool = SingleNodeConnectionPool::new( - Url::parse(config.elasticsearch_url.as_str()) - .expect("configured elasticsearch url is invalid"), - ); - let transport = TransportBuilder::new(conn_pool) - .cert_validation(CertificateValidation::None) - .auth(Basic( - config.clone().elasticsearch_username, - config.clone().elasticsearch_password, - )) - .build() - .expect("elasticsearch transport builder should not fail"); - let elastic_db = Some(Elasticsearch::new(transport)); - - FinalizedState::new(&config, network, elastic_db) - }; - - #[cfg(not(feature = "elasticsearch"))] let finalized_state = { FinalizedState::new(&config, network) }; - timer.finish(module_path!(), line!(), "opening finalized state database"); let timer = CodeTimer::start(); @@ -1959,36 +1928,13 @@ pub fn init_read_only( ZebraDb, tokio::sync::watch::Sender, ) { + let finalized_state = { FinalizedState::new_with_debug(&config, network, true, true) }; let (non_finalized_state_sender, non_finalized_state_receiver) = tokio::sync::watch::channel(NonFinalizedState::new(network)); - #[cfg(feature = "elasticsearch")] - let finalized_state = { - let conn_pool = SingleNodeConnectionPool::new( - Url::parse(config.elasticsearch_url.as_str()) - .expect("configured elasticsearch url is invalid"), - ); - let transport = TransportBuilder::new(conn_pool) - .cert_validation(CertificateValidation::None) - .auth(Basic( - config.clone().elasticsearch_username, - config.clone().elasticsearch_password, - )) - .build() - .expect("elasticsearch transport builder should not fail"); - let elastic_db = Some(Elasticsearch::new(transport)); - - FinalizedState::new_with_debug(&config, network, true, elastic_db, true) - }; - - #[cfg(not(feature = "elasticsearch"))] - let finalized_state = { FinalizedState::new_with_debug(&config, network, true, true) }; - - let db = finalized_state.db.clone(); - ( ReadStateService::new(&finalized_state, None, non_finalized_state_receiver), - db, + finalized_state.db.clone(), non_finalized_state_sender, ) } diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 7256d89dac2..26e7b62a1f7 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -139,19 +139,8 @@ pub struct FinalizedState { impl FinalizedState { /// Returns an on-disk database instance for `config`, `network`, and `elastic_db`. /// If there is no existing database, creates a new database on disk. - pub fn new( - config: &Config, - network: &Network, - #[cfg(feature = "elasticsearch")] elastic_db: Option, - ) -> Self { - Self::new_with_debug( - config, - network, - false, - #[cfg(feature = "elasticsearch")] - elastic_db, - false, - ) + pub fn new(config: &Config, network: &Network) -> Self { + Self::new_with_debug(config, network, false, false) } /// Returns an on-disk database instance with the supplied production and debug settings. @@ -162,9 +151,34 @@ impl FinalizedState { config: &Config, network: &Network, debug_skip_format_upgrades: bool, - #[cfg(feature = "elasticsearch")] elastic_db: Option, read_only: bool, ) -> Self { + #[cfg(feature = "elasticsearch")] + let elastic_db = { + use elasticsearch::{ + auth::Credentials::Basic, + cert::CertificateValidation, + http::transport::{SingleNodeConnectionPool, TransportBuilder}, + http::Url, + Elasticsearch, + }; + + let conn_pool = SingleNodeConnectionPool::new( + Url::parse(config.elasticsearch_url.as_str()) + .expect("configured elasticsearch url is invalid"), + ); + let transport = TransportBuilder::new(conn_pool) + .cert_validation(CertificateValidation::None) + .auth(Basic( + config.clone().elasticsearch_username, + config.clone().elasticsearch_password, + )) + .build() + .expect("elasticsearch transport builder should not fail"); + + Some(Elasticsearch::new(transport)) + }; + let db = ZebraDb::new( config, STATE_DATABASE_KIND, From 33a3ccffe499ba1b894d04075b17efc8a23a9008 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 21:46:41 -0400 Subject: [PATCH 03/30] Updates callers of `FinalizedState::{new, new_with_debug}` to pass a bool to try enabling elasticsearch --- zebra-state/src/service.rs | 16 +++++++++++++-- zebra-state/src/service/finalized_state.rs | 20 ++++++++++++++++--- .../disk_format/tests/snapshot.rs | 2 +- .../src/service/finalized_state/tests/prop.rs | 4 ++-- .../zebra_db/block/tests/snapshot.rs | 2 +- .../service/non_finalized_state/tests/prop.rs | 2 +- .../non_finalized_state/tests/vectors.rs | 16 +++++++-------- zebra-state/src/tests/setup.rs | 2 +- 8 files changed, 45 insertions(+), 19 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index d6b3a416021..0fbe8d8eaad 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -310,7 +310,12 @@ impl StateService { checkpoint_verify_concurrency_limit: usize, ) -> (Self, ReadStateService, LatestChainTip, ChainTipChange) { let timer = CodeTimer::start(); - let finalized_state = { FinalizedState::new(&config, network) }; + let finalized_state = FinalizedState::new( + &config, + network, + #[cfg(feature = "elasticsearch")] + true, + ); timer.finish(module_path!(), line!(), "opening finalized state database"); let timer = CodeTimer::start(); @@ -1928,7 +1933,14 @@ pub fn init_read_only( ZebraDb, tokio::sync::watch::Sender, ) { - let finalized_state = { FinalizedState::new_with_debug(&config, network, true, true) }; + let finalized_state = FinalizedState::new_with_debug( + &config, + network, + true, + #[cfg(feature = "elasticsearch")] + false, + true, + ); let (non_finalized_state_sender, non_finalized_state_receiver) = tokio::sync::watch::channel(NonFinalizedState::new(network)); diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 26e7b62a1f7..f8c9bade5c1 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -139,8 +139,19 @@ pub struct FinalizedState { impl FinalizedState { /// Returns an on-disk database instance for `config`, `network`, and `elastic_db`. /// If there is no existing database, creates a new database on disk. - pub fn new(config: &Config, network: &Network) -> Self { - Self::new_with_debug(config, network, false, false) + pub fn new( + config: &Config, + network: &Network, + #[cfg(feature = "elasticsearch")] enable_elastic_db: bool, + ) -> Self { + Self::new_with_debug( + config, + network, + false, + #[cfg(feature = "elasticsearch")] + enable_elastic_db, + false, + ) } /// Returns an on-disk database instance with the supplied production and debug settings. @@ -151,10 +162,11 @@ impl FinalizedState { config: &Config, network: &Network, debug_skip_format_upgrades: bool, + #[cfg(feature = "elasticsearch")] enable_elastic_db: bool, read_only: bool, ) -> Self { #[cfg(feature = "elasticsearch")] - let elastic_db = { + let elastic_db = if enable_elastic_db { use elasticsearch::{ auth::Credentials::Basic, cert::CertificateValidation, @@ -177,6 +189,8 @@ impl FinalizedState { .expect("elasticsearch transport builder should not fail"); Some(Elasticsearch::new(transport)) + } else { + None }; let db = ZebraDb::new( diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs index 14a8dd6c2a7..eb12cf41f1b 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs @@ -62,7 +62,7 @@ fn test_raw_rocksdb_column_families_with_network(network: Network) { &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); // Snapshot the column family names diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs index d48761795f5..7ff2c3ac91b 100644 --- a/zebra-state/src/service/finalized_state/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/tests/prop.rs @@ -24,7 +24,7 @@ fn blocks_with_v5_transactions() -> Result<()> { .and_then(|v| v.parse().ok()) .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), |((chain, count, network, _history_tree) in PreparedChain::default())| { - let mut state = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] None); + let mut state = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); let mut height = Height(0); // use `count` to minimize test failures, so they are easier to diagnose for block in chain.iter().take(count) { @@ -65,7 +65,7 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<( .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), |((chain, _count, network, _history_tree) in PreparedChain::default().with_valid_commitments().no_shrink())| { - let mut state = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] None); + let mut state = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); let mut height = Height(0); let heartwood_height = NetworkUpgrade::Heartwood.activation_height(&network).unwrap(); let heartwood_height_plus1 = (heartwood_height + 1).unwrap(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index dbd0e7c1dae..c5f1ba371d5 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -169,7 +169,7 @@ fn test_block_and_transaction_data_with_network(network: Network) { &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); // Assert that empty databases are the same, regardless of the network. diff --git a/zebra-state/src/service/non_finalized_state/tests/prop.rs b/zebra-state/src/service/non_finalized_state/tests/prop.rs index d4ef55344c0..68f7d77588b 100644 --- a/zebra-state/src/service/non_finalized_state/tests/prop.rs +++ b/zebra-state/src/service/non_finalized_state/tests/prop.rs @@ -479,7 +479,7 @@ fn rejection_restores_internal_state_genesis() -> Result<()> { } ))| { let mut state = NonFinalizedState::new(&network); - let finalized_state = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] None); + let finalized_state = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); let fake_value_pool = ValueBalance::::fake_populated_pool(); finalized_state.set_finalized_value_pool(fake_value_pool); diff --git a/zebra-state/src/service/non_finalized_state/tests/vectors.rs b/zebra-state/src/service/non_finalized_state/tests/vectors.rs index 6f908f080c0..b489d6f94f0 100644 --- a/zebra-state/src/service/non_finalized_state/tests/vectors.rs +++ b/zebra-state/src/service/non_finalized_state/tests/vectors.rs @@ -157,7 +157,7 @@ fn best_chain_wins_for_network(network: Network) -> Result<()> { &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); state.commit_new_chain(block2.prepare(), &finalized_state)?; @@ -194,7 +194,7 @@ fn finalize_pops_from_best_chain_for_network(network: Network) -> Result<()> { &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); let fake_value_pool = ValueBalance::::fake_populated_pool(); @@ -245,7 +245,7 @@ fn commit_block_extending_best_chain_doesnt_drop_worst_chains_for_network( &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); let fake_value_pool = ValueBalance::::fake_populated_pool(); @@ -289,7 +289,7 @@ fn shorter_chain_can_be_best_chain_for_network(network: Network) -> Result<()> { &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); let fake_value_pool = ValueBalance::::fake_populated_pool(); @@ -334,7 +334,7 @@ fn longer_chain_with_more_work_wins_for_network(network: Network) -> Result<()> &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); let fake_value_pool = ValueBalance::::fake_populated_pool(); @@ -378,7 +378,7 @@ fn equal_length_goes_to_more_work_for_network(network: Network) -> Result<()> { &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); let fake_value_pool = ValueBalance::::fake_populated_pool(); @@ -426,7 +426,7 @@ fn history_tree_is_updated_for_network_upgrade( &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); state @@ -525,7 +525,7 @@ fn commitment_is_validated_for_network_upgrade(network: Network, network_upgrade &Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] - None, + false, ); state diff --git a/zebra-state/src/tests/setup.rs b/zebra-state/src/tests/setup.rs index d407726330e..cc53a0d7ee5 100644 --- a/zebra-state/src/tests/setup.rs +++ b/zebra-state/src/tests/setup.rs @@ -101,7 +101,7 @@ pub(crate) fn new_state_with_mainnet_genesis( // The tests that use this setup function also commit invalid blocks to the state. true, #[cfg(feature = "elasticsearch")] - None, + false, false, ); let non_finalized_state = NonFinalizedState::new(&network); From 995811e0d2a88b154b181c24c65d27236a18aaa6 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 20:32:27 -0400 Subject: [PATCH 04/30] Adds a non-finalized read state syncer to zebra-rpc --- zebra-rpc/Cargo.toml | 2 + zebra-rpc/src/lib.rs | 1 + zebra-rpc/src/sync.rs | 304 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 307 insertions(+) create mode 100644 zebra-rpc/src/sync.rs diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index 1ea74c342a9..6b151825da1 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -32,6 +32,8 @@ getblocktemplate-rpcs = [ # Experimental internal miner support internal-miner = [] +syncer = ["zebra-node-services/rpc-client"] + # Test-only features proptest-impl = [ "proptest", diff --git a/zebra-rpc/src/lib.rs b/zebra-rpc/src/lib.rs index 81a30d78e0c..d5b687913fd 100644 --- a/zebra-rpc/src/lib.rs +++ b/zebra-rpc/src/lib.rs @@ -9,6 +9,7 @@ pub mod constants; pub mod methods; pub mod queue; pub mod server; +pub mod sync; #[cfg(test)] mod tests; diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs new file mode 100644 index 00000000000..bc71316096f --- /dev/null +++ b/zebra-rpc/src/sync.rs @@ -0,0 +1,304 @@ +//! Syncer task for maintaining a non-finalized state in Zebra's ReadStateService via RPCs + +use std::{net::SocketAddr, sync::Arc, time::Duration}; + +use tower::BoxError; +use zebra_chain::{ + block::{self, Block, Height}, + parameters::Network, + serialization::ZcashDeserializeInto, +}; +use zebra_node_services::rpc_client::RpcRequestClient; +use zebra_state::{ + spawn_init_read_only, ChainTipBlock, ChainTipChange, ChainTipSender, CheckpointVerifiedBlock, + LatestChainTip, NonFinalizedState, ReadStateService, SemanticallyVerifiedBlock, ZebraDb, + MAX_BLOCK_REORG_HEIGHT, +}; + +use zebra_chain::diagnostic::task::WaitForPanics; + +use crate::methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHash}; + +/// Syncs non-finalized blocks in the best chain from a trusted Zebra node's RPC methods. +struct TrustedChainSync { + /// RPC client for calling Zebra's RPC methods. + rpc_client: RpcRequestClient, + /// Information about the next block height to request and how it should be processed. + cursor: SyncCursor, + /// The read state service + db: ZebraDb, + /// The non-finalized state - currently only contains the best chain. + non_finalized_state: NonFinalizedState, + /// The chain tip sender for updating [`LatestChainTip`] and [`ChainTipChange`] + chain_tip_sender: ChainTipSender, + /// The non-finalized state sender, for updating the [`ReadStateService`] when the non-finalized best chain changes. + non_finalized_state_sender: tokio::sync::watch::Sender, +} + +struct SyncCursor { + /// The best chain tip height in this process. + tip_height: Height, + /// The best chain tip hash in this process. + tip_hash: block::Hash, + /// The best chain tip hash in the Zebra node. + node_tip_hash: block::Hash, +} + +impl SyncCursor { + fn new(tip_height: Height, tip_hash: block::Hash, node_tip_hash: block::Hash) -> Self { + Self { + tip_height, + tip_hash, + node_tip_hash, + } + } +} + +impl TrustedChainSync { + /// Creates a new [`TrustedChainSync`] and starts syncing blocks from the node's non-finalized best chain. + pub async fn spawn( + rpc_address: SocketAddr, + db: ZebraDb, + non_finalized_state_sender: tokio::sync::watch::Sender, + ) -> (LatestChainTip, ChainTipChange, tokio::task::JoinHandle<()>) { + // TODO: Spawn a tokio task to do this in and return a `JoinHandle`. + let rpc_client = RpcRequestClient::new(rpc_address); + let non_finalized_state = NonFinalizedState::new(&db.network()); + let cursor = wait_for_new_blocks(&rpc_client, &non_finalized_state, &db).await; + let tip = { + let db = db.clone(); + tokio::task::spawn_blocking(move || db.tip_block()) + .wait_for_panics() + .await + .expect("checked for genesis block above") + }; + let chain_tip: ChainTipBlock = CheckpointVerifiedBlock::from(tip).into(); + let (chain_tip_sender, latest_chain_tip, chain_tip_change) = + ChainTipSender::new(chain_tip, &db.network()); + + let mut syncer = Self { + rpc_client, + cursor, + db, + non_finalized_state, + chain_tip_sender, + non_finalized_state_sender, + }; + + let sync_task = tokio::spawn(async move { + syncer.sync().await; + }); + + (latest_chain_tip, chain_tip_change, sync_task) + } + + /// Starts syncing blocks from the node's non-finalized best chain. + async fn sync(&mut self) { + loop { + let has_found_new_best_chain = self.sync_new_blocks().await; + + if has_found_new_best_chain { + self.non_finalized_state = NonFinalizedState::new(&self.db.network()); + let _ = self + .non_finalized_state_sender + .send(self.non_finalized_state.clone()); + } + + // TODO: Move the loop in `wait_for_new_blocks()` here, return the latest tip height every iteration, + // track the previous finalized tip height from the previous iteration, and if the latest tip height + // is higher and the chain has grown, despite there being no blocks to sync, read the latest block + // from the finalized state and send it to `chain_tip_sender`. + + // Wait until the best block hash in Zebra is different from the tip hash in this read state + self.cursor = self.wait_for_new_blocks().await; + } + } + + async fn sync_new_blocks(&mut self) -> bool { + loop { + let Some(block) = self + .rpc_client + // If we fail to get a block for any reason, we assume the block is missing and the chain hasn't grown, so there must have + // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block past the finalized tip. + // It should always deserialize successfully, but this resets the non-finalized state if it somehow fails. + // TODO: Check for the MISSING_BLOCK_ERROR_CODE? + .get_block(self.cursor.tip_height) + .await + .map(SemanticallyVerifiedBlock::from) + // If the next block's previous block hash doesn't match the expected hash, there must have + // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block + // past the finalized tip. + .filter(|block| block.block.header.previous_block_hash == self.cursor.tip_hash) + else { + break true; + }; + + let parent_hash = block.block.header.previous_block_hash; + if parent_hash != self.cursor.tip_hash { + break true; + } + + let block_hash = block.hash; + let block_height = block.height; + let commit_result = if self.non_finalized_state.chain_count() == 0 { + self.non_finalized_state + .commit_new_chain(block.clone(), &self.db) + } else { + self.non_finalized_state + .commit_block(block.clone(), &self.db) + }; + + if let Err(error) = commit_result { + tracing::warn!( + ?error, + ?block_hash, + "failed to commit block to non-finalized state" + ); + continue; + } + + update_channels( + block, + &self.non_finalized_state, + &mut self.non_finalized_state_sender, + &mut self.chain_tip_sender, + ); + + while self + .non_finalized_state + .best_chain_len() + .expect("just successfully inserted a non-finalized block above") + > MAX_BLOCK_REORG_HEIGHT + { + tracing::trace!("finalizing block past the reorg limit"); + self.non_finalized_state.finalize(); + } + + let _ = self + .non_finalized_state_sender + .send(self.non_finalized_state.clone()); + + self.cursor.tip_height = block_height; + self.cursor.tip_hash = block_hash; + + // If the block hash matches the output from the `getbestblockhash` RPC method, we can wait until + // the best block hash changes to get the next block. + if block_hash == self.cursor.node_tip_hash { + break false; + } + } + } + + /// Polls `getbestblockhash` RPC method until there are new blocks in the Zebra node's non-finalized state. + async fn wait_for_new_blocks(&self) -> SyncCursor { + wait_for_new_blocks(&self.rpc_client, &self.non_finalized_state, &self.db).await + } +} + +/// Accepts a [zebra-state configuration](zebra_state::Config), a [`Network`], and +/// the [`SocketAddr`] of a Zebra node's RPC server. +/// +/// Initializes a [`ReadStateService`] and a [`TrustedChainSync`] to update the +/// non-finalized best chain and the latest chain tip. +/// +/// Returns a [`ReadStateService`], [`LatestChainTip`], [`ChainTipChange`], and +/// a [`JoinHandle`](tokio::task::JoinHandle) for the sync task. +pub fn init_read_state_with_syncer( + config: zebra_state::Config, + network: &Network, + rpc_address: SocketAddr, +) -> tokio::task::JoinHandle< + Result< + ( + ReadStateService, + LatestChainTip, + ChainTipChange, + tokio::task::JoinHandle<()>, + ), + BoxError, + >, +> { + // TODO: Return an error or panic `if config.ephemeral == true`? (It'll panic anyway but it could be useful to say it's because the state is ephemeral). + let network = network.clone(); + tokio::spawn(async move { + let (read_state, db, non_finalized_state_sender) = + spawn_init_read_only(config, &network).await?; + let (latest_chain_tip, chain_tip_change, sync_task) = + TrustedChainSync::spawn(rpc_address, db, non_finalized_state_sender).await; + Ok((read_state, latest_chain_tip, chain_tip_change, sync_task)) + }) +} + +trait SyncerRpcMethods { + async fn get_best_block_hash(&self) -> Option; + async fn get_block(&self, height: block::Height) -> Option>; +} + +impl SyncerRpcMethods for RpcRequestClient { + async fn get_best_block_hash(&self) -> Option { + self.json_result_from_call("getbestblockhash", "[]") + .await + .map(|GetBlockHash(hash)| hash) + .ok() + } + + async fn get_block(&self, Height(height): Height) -> Option> { + self.json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) + .await + .ok() + .and_then(|HexData(raw_block)| raw_block.zcash_deserialize_into::().ok()) + .map(Arc::new) + } +} + +async fn wait_for_new_blocks( + rpc_client: &RpcRequestClient, + non_finalized_state: &NonFinalizedState, + db: &ZebraDb, +) -> SyncCursor { + // Wait until the best block hash in Zebra is different from the tip hash in this read state + loop { + // TODO: Use `getblockchaininfo` RPC instead to get the height and only break if the node tip height is above the + // tip height visible in this read state service? Or at least above the finalized tip height. + let Some(node_tip_hash) = rpc_client.get_best_block_hash().await else { + // Wait until the genesis block has been committed. + // TODO: + // - Move durations to constants + // - Add logs + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + }; + + let (tip_height, tip_hash) = if let Some(tip) = non_finalized_state.best_tip() { + tip + } else { + let db = db.clone(); + tokio::task::spawn_blocking(move || db.tip()) + .wait_for_panics() + .await + .expect("checked for genesis block above") + }; + + // TODO: + // - Tail the db as well for new blocks to update the latest chain channels + // - Read config for mandatory checkpoint height - poll block height more occasionally until Zebra passes mandatory checkpoint height? + + if node_tip_hash != tip_hash { + break SyncCursor::new(tip_height, tip_hash, node_tip_hash); + } + + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// Sends the new chain tip and non-finalized state to the latest chain channels. +fn update_channels( + best_tip: impl Into, + non_finalized_state: &NonFinalizedState, + non_finalized_state_sender: &mut tokio::sync::watch::Sender, + chain_tip_sender: &mut ChainTipSender, +) { + // If the final receiver was just dropped, ignore the error. + let _ = non_finalized_state_sender.send(non_finalized_state.clone()); + chain_tip_sender.set_best_non_finalized_tip(best_tip.into()); +} From f94bd417a118e3ab776f6c32888f25fe82d6b4c7 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 20:54:23 -0400 Subject: [PATCH 05/30] moves, removes, updates, or addresses TODOs --- zebra-rpc/src/sync.rs | 44 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index bc71316096f..82a99ecc259 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -2,6 +2,7 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tokio::task::JoinHandle; use tower::BoxError; use zebra_chain::{ block::{self, Block, Height}, @@ -19,6 +20,10 @@ use zebra_chain::diagnostic::task::WaitForPanics; use crate::methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHash}; +/// How long to wait between calls to `getbestblockhash` when it returns an error or the block hash +/// of the current chain tip in the process that's syncing blocks from Zebra. +const POLL_DELAY: Duration = Duration::from_millis(100); + /// Syncs non-finalized blocks in the best chain from a trusted Zebra node's RPC methods. struct TrustedChainSync { /// RPC client for calling Zebra's RPC methods. @@ -60,11 +65,10 @@ impl TrustedChainSync { rpc_address: SocketAddr, db: ZebraDb, non_finalized_state_sender: tokio::sync::watch::Sender, - ) -> (LatestChainTip, ChainTipChange, tokio::task::JoinHandle<()>) { - // TODO: Spawn a tokio task to do this in and return a `JoinHandle`. + ) -> (LatestChainTip, ChainTipChange, JoinHandle<()>) { let rpc_client = RpcRequestClient::new(rpc_address); let non_finalized_state = NonFinalizedState::new(&db.network()); - let cursor = wait_for_new_blocks(&rpc_client, &non_finalized_state, &db).await; + let cursor = wait_for_chain_tip_change(&rpc_client, &non_finalized_state, &db).await; let tip = { let db = db.clone(); tokio::task::spawn_blocking(move || db.tip_block()) @@ -104,13 +108,8 @@ impl TrustedChainSync { .send(self.non_finalized_state.clone()); } - // TODO: Move the loop in `wait_for_new_blocks()` here, return the latest tip height every iteration, - // track the previous finalized tip height from the previous iteration, and if the latest tip height - // is higher and the chain has grown, despite there being no blocks to sync, read the latest block - // from the finalized state and send it to `chain_tip_sender`. - // Wait until the best block hash in Zebra is different from the tip hash in this read state - self.cursor = self.wait_for_new_blocks().await; + self.cursor = self.wait_for_chain_tip_change().await; } } @@ -121,7 +120,6 @@ impl TrustedChainSync { // If we fail to get a block for any reason, we assume the block is missing and the chain hasn't grown, so there must have // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block past the finalized tip. // It should always deserialize successfully, but this resets the non-finalized state if it somehow fails. - // TODO: Check for the MISSING_BLOCK_ERROR_CODE? .get_block(self.cursor.tip_height) .await .map(SemanticallyVerifiedBlock::from) @@ -190,8 +188,8 @@ impl TrustedChainSync { } /// Polls `getbestblockhash` RPC method until there are new blocks in the Zebra node's non-finalized state. - async fn wait_for_new_blocks(&self) -> SyncCursor { - wait_for_new_blocks(&self.rpc_client, &self.non_finalized_state, &self.db).await + async fn wait_for_chain_tip_change(&self) -> SyncCursor { + wait_for_chain_tip_change(&self.rpc_client, &self.non_finalized_state, &self.db).await } } @@ -218,9 +216,12 @@ pub fn init_read_state_with_syncer( BoxError, >, > { - // TODO: Return an error or panic `if config.ephemeral == true`? (It'll panic anyway but it could be useful to say it's because the state is ephemeral). let network = network.clone(); tokio::spawn(async move { + if config.ephemeral { + return Err("standalone read state service cannot be used with ephemeral state".into()); + } + let (read_state, db, non_finalized_state_sender) = spawn_init_read_only(config, &network).await?; let (latest_chain_tip, chain_tip_change, sync_task) = @@ -245,27 +246,23 @@ impl SyncerRpcMethods for RpcRequestClient { async fn get_block(&self, Height(height): Height) -> Option> { self.json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) .await + // TODO: Check for the MISSING_BLOCK_ERROR_CODE and return a `Result>`? .ok() .and_then(|HexData(raw_block)| raw_block.zcash_deserialize_into::().ok()) .map(Arc::new) } } -async fn wait_for_new_blocks( +async fn wait_for_chain_tip_change( rpc_client: &RpcRequestClient, non_finalized_state: &NonFinalizedState, db: &ZebraDb, ) -> SyncCursor { // Wait until the best block hash in Zebra is different from the tip hash in this read state loop { - // TODO: Use `getblockchaininfo` RPC instead to get the height and only break if the node tip height is above the - // tip height visible in this read state service? Or at least above the finalized tip height. let Some(node_tip_hash) = rpc_client.get_best_block_hash().await else { // Wait until the genesis block has been committed. - // TODO: - // - Move durations to constants - // - Add logs - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(POLL_DELAY).await; continue; }; @@ -279,15 +276,12 @@ async fn wait_for_new_blocks( .expect("checked for genesis block above") }; - // TODO: - // - Tail the db as well for new blocks to update the latest chain channels - // - Read config for mandatory checkpoint height - poll block height more occasionally until Zebra passes mandatory checkpoint height? - + // TODO: Return when the finalized tip has changed as well. if node_tip_hash != tip_hash { break SyncCursor::new(tip_height, tip_hash, node_tip_hash); } - tokio::time::sleep(Duration::from_millis(200)).await; + tokio::time::sleep(POLL_DELAY).await; } } From d57c50aab178009d3e16c3a3cf8bad55d65cacd1 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 21:37:31 -0400 Subject: [PATCH 06/30] reduces disk IO while waiting for the a new chain tip & updates the chain tip sender when the finalized tip has changed. --- zebra-rpc/src/sync.rs | 180 +++++++++++++++++++++++------------------- 1 file changed, 100 insertions(+), 80 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 82a99ecc259..dab48b4fad3 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -11,8 +11,8 @@ use zebra_chain::{ }; use zebra_node_services::rpc_client::RpcRequestClient; use zebra_state::{ - spawn_init_read_only, ChainTipBlock, ChainTipChange, ChainTipSender, CheckpointVerifiedBlock, - LatestChainTip, NonFinalizedState, ReadStateService, SemanticallyVerifiedBlock, ZebraDb, + spawn_init_read_only, ChainTipBlock, ChainTipChange, ChainTipSender, LatestChainTip, + NonFinalizedState, ReadStateService, SemanticallyVerifiedBlock, ZebraDb, MAX_BLOCK_REORG_HEIGHT, }; @@ -28,8 +28,6 @@ const POLL_DELAY: Duration = Duration::from_millis(100); struct TrustedChainSync { /// RPC client for calling Zebra's RPC methods. rpc_client: RpcRequestClient, - /// Information about the next block height to request and how it should be processed. - cursor: SyncCursor, /// The read state service db: ZebraDb, /// The non-finalized state - currently only contains the best chain. @@ -40,21 +38,28 @@ struct TrustedChainSync { non_finalized_state_sender: tokio::sync::watch::Sender, } +enum NewChainTip { + /// Information about the next block height to request and how it should be processed. + Cursor(SyncCursor), + /// The latest finalized tip. + Block(Arc), +} + struct SyncCursor { /// The best chain tip height in this process. tip_height: Height, /// The best chain tip hash in this process. tip_hash: block::Hash, /// The best chain tip hash in the Zebra node. - node_tip_hash: block::Hash, + target_tip_hash: block::Hash, } impl SyncCursor { - fn new(tip_height: Height, tip_hash: block::Hash, node_tip_hash: block::Hash) -> Self { + fn new(tip_height: Height, tip_hash: block::Hash, target_tip_hash: block::Hash) -> Self { Self { tip_height, tip_hash, - node_tip_hash, + target_tip_hash, } } } @@ -68,21 +73,11 @@ impl TrustedChainSync { ) -> (LatestChainTip, ChainTipChange, JoinHandle<()>) { let rpc_client = RpcRequestClient::new(rpc_address); let non_finalized_state = NonFinalizedState::new(&db.network()); - let cursor = wait_for_chain_tip_change(&rpc_client, &non_finalized_state, &db).await; - let tip = { - let db = db.clone(); - tokio::task::spawn_blocking(move || db.tip_block()) - .wait_for_panics() - .await - .expect("checked for genesis block above") - }; - let chain_tip: ChainTipBlock = CheckpointVerifiedBlock::from(tip).into(); let (chain_tip_sender, latest_chain_tip, chain_tip_change) = - ChainTipSender::new(chain_tip, &db.network()); + ChainTipSender::new(None, &db.network()); let mut syncer = Self { rpc_client, - cursor, db, non_finalized_state, chain_tip_sender, @@ -99,40 +94,42 @@ impl TrustedChainSync { /// Starts syncing blocks from the node's non-finalized best chain. async fn sync(&mut self) { loop { - let has_found_new_best_chain = self.sync_new_blocks().await; - - if has_found_new_best_chain { - self.non_finalized_state = NonFinalizedState::new(&self.db.network()); - let _ = self - .non_finalized_state_sender - .send(self.non_finalized_state.clone()); - } - // Wait until the best block hash in Zebra is different from the tip hash in this read state - self.cursor = self.wait_for_chain_tip_change().await; + match self.wait_for_chain_tip_change().await { + NewChainTip::Cursor(cursor) => { + self.sync_new_blocks(cursor).await; + } + + NewChainTip::Block(block) => update_channels( + block, + &self.non_finalized_state, + &mut self.non_finalized_state_sender, + &mut self.chain_tip_sender, + ), + } } } - async fn sync_new_blocks(&mut self) -> bool { - loop { + async fn sync_new_blocks(&mut self, mut cursor: SyncCursor) { + let has_found_new_best_chain = loop { let Some(block) = self .rpc_client // If we fail to get a block for any reason, we assume the block is missing and the chain hasn't grown, so there must have // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block past the finalized tip. // It should always deserialize successfully, but this resets the non-finalized state if it somehow fails. - .get_block(self.cursor.tip_height) + .get_block(cursor.tip_height) .await .map(SemanticallyVerifiedBlock::from) // If the next block's previous block hash doesn't match the expected hash, there must have // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block // past the finalized tip. - .filter(|block| block.block.header.previous_block_hash == self.cursor.tip_hash) + .filter(|block| block.block.header.previous_block_hash == cursor.tip_hash) else { break true; }; let parent_hash = block.block.header.previous_block_hash; - if parent_hash != self.cursor.tip_hash { + if parent_hash != cursor.tip_hash { break true; } @@ -155,13 +152,6 @@ impl TrustedChainSync { continue; } - update_channels( - block, - &self.non_finalized_state, - &mut self.non_finalized_state_sender, - &mut self.chain_tip_sender, - ); - while self .non_finalized_state .best_chain_len() @@ -172,24 +162,86 @@ impl TrustedChainSync { self.non_finalized_state.finalize(); } - let _ = self - .non_finalized_state_sender - .send(self.non_finalized_state.clone()); + update_channels( + block, + &self.non_finalized_state, + &mut self.non_finalized_state_sender, + &mut self.chain_tip_sender, + ); - self.cursor.tip_height = block_height; - self.cursor.tip_hash = block_hash; + cursor.tip_height = block_height; + cursor.tip_hash = block_hash; // If the block hash matches the output from the `getbestblockhash` RPC method, we can wait until // the best block hash changes to get the next block. - if block_hash == self.cursor.node_tip_hash { + if block_hash == cursor.target_tip_hash { break false; } + }; + + if has_found_new_best_chain { + self.non_finalized_state = NonFinalizedState::new(&self.db.network()); + let db = self.db.clone(); + let finalized_tip = tokio::task::spawn_blocking(move || { + db.tip_block().expect("should have genesis block") + }) + .wait_for_panics() + .await; + + update_channels( + finalized_tip, + &self.non_finalized_state, + &mut self.non_finalized_state_sender, + &mut self.chain_tip_sender, + ); } } /// Polls `getbestblockhash` RPC method until there are new blocks in the Zebra node's non-finalized state. - async fn wait_for_chain_tip_change(&self) -> SyncCursor { - wait_for_chain_tip_change(&self.rpc_client, &self.non_finalized_state, &self.db).await + async fn wait_for_chain_tip_change(&self) -> NewChainTip { + let (tip_height, tip_hash) = if let Some(tip) = self.non_finalized_state.best_tip() { + tip + } else { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || db.tip()) + .wait_for_panics() + .await + .expect("checked for genesis block above") + }; + + // Wait until the best block hash in Zebra is different from the tip hash in this read state + loop { + let Some(target_tip_hash) = self.rpc_client.get_best_block_hash().await else { + // Wait until the genesis block has been committed. + tokio::time::sleep(POLL_DELAY).await; + continue; + }; + + if target_tip_hash != tip_hash { + let cursor = + NewChainTip::Cursor(SyncCursor::new(tip_height, tip_hash, target_tip_hash)); + + // Check if there's are blocks in the non-finalized state, or that + // the node tip hash is different from our finalized tip hash before returning + // a cursor for syncing blocks via the `getblock` RPC. + if self.non_finalized_state.chain_count() != 0 { + break cursor; + } + + let db = self.db.clone(); + break tokio::task::spawn_blocking(move || { + if db.finalized_tip_hash() != target_tip_hash { + cursor + } else { + NewChainTip::Block(db.tip_block().unwrap()) + } + }) + .wait_for_panics() + .await; + } + + tokio::time::sleep(POLL_DELAY).await; + } } } @@ -253,38 +305,6 @@ impl SyncerRpcMethods for RpcRequestClient { } } -async fn wait_for_chain_tip_change( - rpc_client: &RpcRequestClient, - non_finalized_state: &NonFinalizedState, - db: &ZebraDb, -) -> SyncCursor { - // Wait until the best block hash in Zebra is different from the tip hash in this read state - loop { - let Some(node_tip_hash) = rpc_client.get_best_block_hash().await else { - // Wait until the genesis block has been committed. - tokio::time::sleep(POLL_DELAY).await; - continue; - }; - - let (tip_height, tip_hash) = if let Some(tip) = non_finalized_state.best_tip() { - tip - } else { - let db = db.clone(); - tokio::task::spawn_blocking(move || db.tip()) - .wait_for_panics() - .await - .expect("checked for genesis block above") - }; - - // TODO: Return when the finalized tip has changed as well. - if node_tip_hash != tip_hash { - break SyncCursor::new(tip_height, tip_hash, node_tip_hash); - } - - tokio::time::sleep(POLL_DELAY).await; - } -} - /// Sends the new chain tip and non-finalized state to the latest chain channels. fn update_channels( best_tip: impl Into, From 8d246c320794034180a53d5fed0b20566d4e6934 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 22:35:14 -0400 Subject: [PATCH 07/30] Returns boxed errors from RpcRequestClient methods instead of color_eyre type --- zebra-node-services/src/rpc_client.rs | 8 ++++---- .../tests/common/get_block_template_rpcs/get_peer_info.rs | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/zebra-node-services/src/rpc_client.rs b/zebra-node-services/src/rpc_client.rs index 350b373aa72..52c4a923d69 100644 --- a/zebra-node-services/src/rpc_client.rs +++ b/zebra-node-services/src/rpc_client.rs @@ -6,7 +6,7 @@ use std::net::SocketAddr; use reqwest::Client; -use color_eyre::{eyre::eyre, Result}; +use crate::BoxError; /// An HTTP client for making JSON-RPC requests. #[derive(Clone, Debug)] @@ -99,7 +99,7 @@ impl RpcRequestClient { &self, method: impl AsRef, params: impl AsRef, - ) -> Result { + ) -> std::result::Result { Self::json_result_from_response_text(&self.text_from_call(method, params).await?) } @@ -107,13 +107,13 @@ impl RpcRequestClient { /// Returns `Ok` with a deserialized `result` value in the expected type, or an error report. fn json_result_from_response_text( response_text: &str, - ) -> Result { + ) -> std::result::Result { use jsonrpc_core::Output; let output: Output = serde_json::from_str(response_text)?; match output { Output::Success(success) => Ok(serde_json::from_value(success.result)?), - Output::Failure(failure) => Err(eyre!("RPC call failed with: {failure:?}")), + Output::Failure(failure) => Err(failure.error.into()), } } } diff --git a/zebrad/tests/common/get_block_template_rpcs/get_peer_info.rs b/zebrad/tests/common/get_block_template_rpcs/get_peer_info.rs index 5dd0fd81604..4ca0bc797ad 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_peer_info.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_peer_info.rs @@ -1,6 +1,6 @@ //! Tests that `getpeerinfo` RPC method responds with info about at least 1 peer. -use color_eyre::eyre::{Context, Result}; +use color_eyre::eyre::{eyre, Context, Result}; use zebra_chain::parameters::Network; use zebra_node_services::rpc_client::RpcRequestClient; @@ -41,7 +41,8 @@ pub(crate) async fn run() -> Result<()> { // call `getpeerinfo` RPC method let peer_info_result: Vec = RpcRequestClient::new(rpc_address) .json_result_from_call("getpeerinfo", "[]".to_string()) - .await?; + .await + .map_err(|err| eyre!(err))?; assert!( !peer_info_result.is_empty(), From 5470979bce4dfd448c8985ae358ad9973f80a7ba Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 22:36:11 -0400 Subject: [PATCH 08/30] Avoids resetting the non-finalized state when there's an error getting a block unless it has the missing block error code. --- zebra-rpc/src/sync.rs | 87 +++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 27 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index dab48b4fad3..480755e98f8 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -1,4 +1,4 @@ -//! Syncer task for maintaining a non-finalized state in Zebra's ReadStateService via RPCs +//! Syncer task for maintaining a non-finalized state in Zebra's ReadStateService and updating `ChainTipSender` via RPCs use std::{net::SocketAddr, sync::Arc, time::Duration}; @@ -18,13 +18,17 @@ use zebra_state::{ use zebra_chain::diagnostic::task::WaitForPanics; -use crate::methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHash}; +use crate::{ + constants::MISSING_BLOCK_ERROR_CODE, + methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHash}, +}; /// How long to wait between calls to `getbestblockhash` when it returns an error or the block hash /// of the current chain tip in the process that's syncing blocks from Zebra. const POLL_DELAY: Duration = Duration::from_millis(100); /// Syncs non-finalized blocks in the best chain from a trusted Zebra node's RPC methods. +#[derive(Debug)] struct TrustedChainSync { /// RPC client for calling Zebra's RPC methods. rpc_client: RpcRequestClient, @@ -38,6 +42,7 @@ struct TrustedChainSync { non_finalized_state_sender: tokio::sync::watch::Sender, } +#[derive(Debug)] enum NewChainTip { /// Information about the next block height to request and how it should be processed. Cursor(SyncCursor), @@ -45,6 +50,7 @@ enum NewChainTip { Block(Arc), } +#[derive(Debug)] struct SyncCursor { /// The best chain tip height in this process. tip_height: Height, @@ -110,28 +116,43 @@ impl TrustedChainSync { } } + /// Accepts a [`SyncCursor`] and gets blocks after the current chain tip until reaching the target block hash in the cursor. + /// + /// Clears the non-finalized state and sets the finalized tip as the current chain tip if + /// the `getblock` RPC method returns the `MISSING_BLOCK_ERROR_CODE` before this function reaches + /// the target block hash. async fn sync_new_blocks(&mut self, mut cursor: SyncCursor) { + let mut consecutive_unexpected_error_count = 0; + let has_found_new_best_chain = loop { - let Some(block) = self - .rpc_client - // If we fail to get a block for any reason, we assume the block is missing and the chain hasn't grown, so there must have - // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block past the finalized tip. - // It should always deserialize successfully, but this resets the non-finalized state if it somehow fails. - .get_block(cursor.tip_height) - .await - .map(SemanticallyVerifiedBlock::from) - // If the next block's previous block hash doesn't match the expected hash, there must have - // been a chain re-org/fork, and we can clear the non-finalized state and re-fetch every block - // past the finalized tip. - .filter(|block| block.block.header.previous_block_hash == cursor.tip_hash) - else { - break true; + let block = match self.rpc_client.get_block(cursor.tip_height).await { + Ok(Some(block)) if block.header.previous_block_hash == cursor.tip_hash => { + SemanticallyVerifiedBlock::from(block) + } + // If the next block's previous block hash doesn't match the expected hash, or if the block is missing + // before this function reaches the target block hash, there was likely a chain re-org/fork, and + // we can clear the non-finalized state and re-fetch every block past the finalized tip. + Ok(_) => break true, + // If there was an unexpected error, retry 4 more times before returning early. + // TODO: Propagate this error up and exit the spawned sync task with the error if the RPC server is down for too long. + Err(err) => { + tracing::warn!( + ?consecutive_unexpected_error_count, + ?cursor, + ?err, + "encountered an unexpected error while calling getblock method" + ); + + if consecutive_unexpected_error_count >= 5 { + break false; + } else { + consecutive_unexpected_error_count += 1; + continue; + }; + } }; - let parent_hash = block.block.header.previous_block_hash; - if parent_hash != cursor.tip_hash { - break true; - } + consecutive_unexpected_error_count = 0; let block_hash = block.hash; let block_height = block.height; @@ -284,7 +305,7 @@ pub fn init_read_state_with_syncer( trait SyncerRpcMethods { async fn get_best_block_hash(&self) -> Option; - async fn get_block(&self, height: block::Height) -> Option>; + async fn get_block(&self, height: block::Height) -> Result>, BoxError>; } impl SyncerRpcMethods for RpcRequestClient { @@ -295,13 +316,25 @@ impl SyncerRpcMethods for RpcRequestClient { .ok() } - async fn get_block(&self, Height(height): Height) -> Option> { - self.json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) + async fn get_block(&self, Height(height): Height) -> Result>, BoxError> { + match self + .json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) .await - // TODO: Check for the MISSING_BLOCK_ERROR_CODE and return a `Result>`? - .ok() - .and_then(|HexData(raw_block)| raw_block.zcash_deserialize_into::().ok()) - .map(Arc::new) + { + Ok(HexData(raw_block)) => { + let block = raw_block.zcash_deserialize_into::()?; + Ok(Some(Arc::new(block))) + } + Err(err) + if err + .downcast_ref::() + .is_some_and(|err| err.code == MISSING_BLOCK_ERROR_CODE) => + { + Ok(None) + } + Err(err) => Err(err), + } + // TODO: Check for the MISSING_BLOCK_ERROR_CODE and return a `Result>`? } } From 0d35fc32590e807beb3f8ddc26f1150c59d45e94 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 22:51:11 -0400 Subject: [PATCH 09/30] Adds stub for acceptance test(s) and removes outdated TODO --- zebra-rpc/src/sync.rs | 1 - zebrad/Cargo.toml | 2 ++ zebrad/tests/acceptance.rs | 40 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 480755e98f8..96b31d49c00 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -334,7 +334,6 @@ impl SyncerRpcMethods for RpcRequestClient { } Err(err) => Err(err), } - // TODO: Check for the MISSING_BLOCK_ERROR_CODE and return a `Result>`? } } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index a86d2afe8a2..04a21dfcfc9 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -100,6 +100,8 @@ progress-bar = [ prometheus = ["metrics-exporter-prometheus"] +rpc-syncer = ["zebra-rpc/syncer"] + # Production features that modify dependency behaviour # Enable additional error debugging in release builds diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 2c8c692d4b9..6994b3bf05d 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3170,3 +3170,43 @@ async fn regtest_submit_blocks() -> Result<()> { common::regtest::submit_blocks_test().await?; Ok(()) } + +// TODO: +// - Test that chain forks are handled correctly and that `ChainTipChange` sees a `Reset` +// - Test that `ChainTipChange` is updated when the non-finalized best chain grows +// - Test that the `ChainTipChange` updated by the RPC syncer is updated when the finalized tip changes +#[cfg(feature = "rpc-syncer")] +#[tokio::test] +async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { + let _init_guard = zebra_test::init(); + let mut config = random_known_rpc_port_config(false, &Mainnet)?; + config.state.ephemeral = false; + config.network.network = Network::new_regtest(None); + + let testdir = testdir()?.with_config(&mut config)?; + let mut child = testdir.spawn_child(args!["start"])?; + + std::thread::sleep(LAUNCH_DELAY); + + // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra + let (_read_state, _latest_chain_tip, _chain_tip_change, _sync_task) = + zebra_rpc::sync::init_read_state_with_syncer( + config.state, + &config.network.network, + config.rpc.listen_addr.unwrap(), + ) + .await? + .map_err(|err| eyre!(err))?; + + // Submit some blocks on the best chain + + child.kill(false)?; + let output = child.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + + output.assert_failure()?; + + Ok(()) +} From 0d1f434bdf3e09768334eb6af6e25512a3ba03ac Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 6 Jun 2024 23:38:56 -0400 Subject: [PATCH 10/30] adds TODOs for testing --- zebra-rpc/src/sync.rs | 2 +- zebrad/tests/acceptance.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 96b31d49c00..8c7c6c5a4e3 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -25,7 +25,7 @@ use crate::{ /// How long to wait between calls to `getbestblockhash` when it returns an error or the block hash /// of the current chain tip in the process that's syncing blocks from Zebra. -const POLL_DELAY: Duration = Duration::from_millis(100); +const POLL_DELAY: Duration = Duration::from_millis(200); /// Syncs non-finalized blocks in the best chain from a trusted Zebra node's RPC methods. #[derive(Debug)] diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 6994b3bf05d..594b35d1646 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3198,7 +3198,14 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { .await? .map_err(|err| eyre!(err))?; - // Submit some blocks on the best chain + // TODO: + // - Submit blocks while checking that `ChainTipChange` shows the chain growing before submitting the next block + // - Submit several blocks before checking that `ChainTipChange` has the latest block hash and is a `TipAction::Reset` + // - Check that `getblock` RPC returns the same block as the read state for every height + // - Submit more blocks with an older block template (and a different nonce so the hash is different) to trigger a chain reorg + // - Check that `ChainTipChange` isn't being updated until the best chain changes + // - Check that the first `ChainTipChange` `TipAction` is a `TipAction::Reset` + // - Check that `getblock` RPC returns the same block as the read state for every height child.kill(false)?; let output = child.wait_with_output()?; @@ -3208,5 +3215,10 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { output.assert_failure()?; + // TODO: + // - Start another Zebra testchild on Testnet or Mainnet + // - Wait for it to start syncing blocks from the network + // - Check that `LatestChainTip` is being updated with the newly synced finalized blocks + Ok(()) } From dc514b82e2d37afd0a0d9b13375704034318e4c8 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 10 Jun 2024 21:22:03 -0400 Subject: [PATCH 11/30] Tests that `ChainTipChange` is updated when the non-finalized best chain grows --- .../types/submit_block.rs | 4 +- zebrad/tests/acceptance.rs | 36 ++++++-- zebrad/tests/common/regtest.rs | 91 +++++++++++-------- 3 files changed, 85 insertions(+), 46 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/submit_block.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/submit_block.rs index e135b0a5563..bf3b22d88c2 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/submit_block.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/submit_block.rs @@ -29,7 +29,7 @@ pub struct JsonParameters { /// Response to a `submitblock` RPC request. /// /// Zebra never returns "duplicate-invalid", because it does not store invalid blocks. -#[derive(Debug, PartialEq, Eq, serde::Serialize)] +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub enum ErrorResponse { /// Block was already committed to the non-finalized or finalized state @@ -45,7 +45,7 @@ pub enum ErrorResponse { /// Response to a `submitblock` RPC request. /// /// Zebra never returns "duplicate-invalid", because it does not store invalid blocks. -#[derive(Debug, PartialEq, Eq, serde::Serialize)] +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(untagged)] pub enum Response { /// Block was not successfully submitted, return error diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 594b35d1646..f924dc51b5e 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3173,15 +3173,18 @@ async fn regtest_submit_blocks() -> Result<()> { // TODO: // - Test that chain forks are handled correctly and that `ChainTipChange` sees a `Reset` -// - Test that `ChainTipChange` is updated when the non-finalized best chain grows // - Test that the `ChainTipChange` updated by the RPC syncer is updated when the finalized tip changes #[cfg(feature = "rpc-syncer")] #[tokio::test] async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { + use common::regtest::MiningRpcMethods; + use tokio::time::timeout; + use zebra_chain::parameters::testnet::REGTEST_NU5_ACTIVATION_HEIGHT; + let _init_guard = zebra_test::init(); - let mut config = random_known_rpc_port_config(false, &Mainnet)?; + let mut config = random_known_rpc_port_config(false, &Network::new_regtest(None))?; config.state.ephemeral = false; - config.network.network = Network::new_regtest(None); + let rpc_address = config.rpc.listen_addr.unwrap(); let testdir = testdir()?.with_config(&mut config)?; let mut child = testdir.spawn_child(args!["start"])?; @@ -3189,17 +3192,38 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { std::thread::sleep(LAUNCH_DELAY); // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra - let (_read_state, _latest_chain_tip, _chain_tip_change, _sync_task) = + let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = zebra_rpc::sync::init_read_state_with_syncer( config.state, &config.network.network, - config.rpc.listen_addr.unwrap(), + rpc_address, ) .await? .map_err(|err| eyre!(err))?; + let rpc_client = RpcRequestClient::new(rpc_address); + + for _ in 0..100 { + let (block, height) = rpc_client + .block_from_template(Height(REGTEST_NU5_ACTIVATION_HEIGHT)) + .await?; + + rpc_client.submit_block(block).await?; + + let tip_action = timeout( + Duration::from_secs(1), + chain_tip_change.wait_for_tip_change(), + ) + .await??; + + assert_eq!( + tip_action.best_tip_height(), + height, + "tip action height should match block submission" + ); + } + // TODO: - // - Submit blocks while checking that `ChainTipChange` shows the chain growing before submitting the next block // - Submit several blocks before checking that `ChainTipChange` has the latest block hash and is a `TipAction::Reset` // - Check that `getblock` RPC returns the same block as the read state for every height // - Submit more blocks with an older block template (and a different nonce so the hash is different) to trigger a chain reorg diff --git a/zebrad/tests/common/regtest.rs b/zebrad/tests/common/regtest.rs index b089cb8e842..b031df68af9 100644 --- a/zebrad/tests/common/regtest.rs +++ b/zebrad/tests/common/regtest.rs @@ -5,17 +5,19 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; -use color_eyre::eyre::{Context, Result}; +use color_eyre::eyre::{eyre, Context, Result}; use tracing::*; use zebra_chain::{ + block::{Block, Height}, parameters::{testnet::REGTEST_NU5_ACTIVATION_HEIGHT, Network, NetworkUpgrade}, primitives::byte_array::increment_big_endian, serialization::ZcashSerialize, }; use zebra_node_services::rpc_client::RpcRequestClient; -use zebra_rpc::methods::get_block_template_rpcs::get_block_template::{ - proposal::TimeSource, proposal_block_from_template, GetBlockTemplate, +use zebra_rpc::methods::get_block_template_rpcs::{ + get_block_template::{proposal::TimeSource, proposal_block_from_template, GetBlockTemplate}, + types::submit_block, }; use zebra_test::args; @@ -62,54 +64,67 @@ pub(crate) async fn submit_blocks_test() -> Result<()> { async fn submit_blocks(network: Network, rpc_address: SocketAddr) -> Result<()> { let client = RpcRequestClient::new(rpc_address); - for height in 1..=NUM_BLOCKS_TO_SUBMIT { - let block_template: GetBlockTemplate = client + for _ in 1..=NUM_BLOCKS_TO_SUBMIT { + let (mut block, height) = client + .block_from_template(Height(REGTEST_NU5_ACTIVATION_HEIGHT)) + .await?; + + while !network.disable_pow() + && zebra_consensus::difficulty_is_valid(&block.header, &network, &height, &block.hash()) + .is_err() + { + increment_big_endian(Arc::make_mut(&mut block.header).nonce.as_mut()); + } + + if height.0 % 40 == 0 { + info!(?block, ?height, "submitting block"); + } + + client.submit_block(block).await?; + } + + Ok(()) +} + +pub trait MiningRpcMethods { + async fn block_from_template(&self, nu5_activation_height: Height) -> Result<(Block, Height)>; + async fn submit_block(&self, block: Block) -> Result<()>; +} + +impl MiningRpcMethods for RpcRequestClient { + async fn block_from_template(&self, nu5_activation_height: Height) -> Result<(Block, Height)> { + let block_template: GetBlockTemplate = self .json_result_from_call("getblocktemplate", "[]".to_string()) .await .expect("response should be success output with a serialized `GetBlockTemplate`"); - let network_upgrade = if height < REGTEST_NU5_ACTIVATION_HEIGHT.try_into().unwrap() { + let height = Height(block_template.height); + + let network_upgrade = if height < nu5_activation_height { NetworkUpgrade::Canopy } else { NetworkUpgrade::Nu5 }; - let mut block = - proposal_block_from_template(&block_template, TimeSource::default(), network_upgrade)?; - let height = block - .coinbase_height() - .expect("should have a coinbase height"); - - while !network.disable_pow() - && zebra_consensus::difficulty_is_valid(&block.header, &network, &height, &block.hash()) - .is_err() - { - increment_big_endian(Arc::make_mut(&mut block.header).nonce.as_mut()); - } + Ok(( + proposal_block_from_template(&block_template, TimeSource::default(), network_upgrade)?, + height, + )) + } + async fn submit_block(&self, block: Block) -> Result<()> { let block_data = hex::encode(block.zcash_serialize_to_vec()?); - let submit_block_response = client - .text_from_call("submitblock", format!(r#"["{block_data}"]"#)) - .await?; - - let was_submission_successful = submit_block_response.contains(r#""result":null"#); + let submit_block_response: submit_block::Response = self + .json_result_from_call("submitblock", format!(r#"["{block_data}"]"#)) + .await + .map_err(|err| eyre!(err))?; - if height.0 % 40 == 0 { - info!( - was_submission_successful, - ?block_template, - ?network_upgrade, - "submitted block" - ); + match submit_block_response { + submit_block::Response::Accepted => Ok(()), + submit_block::Response::ErrorResponse(err) => { + Err(eyre!("block submission failed: {err:?}")) + } } - - // Check that the block was validated and committed. - assert!( - submit_block_response.contains(r#""result":null"#), - "unexpected response from submitblock RPC, should be null, was: {submit_block_response}" - ); } - - Ok(()) } From 302facf78ef251d52e8c8705ee8e13b5aae37b3e Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 12 Jun 2024 00:17:05 -0400 Subject: [PATCH 12/30] adds a last_chain_tip_hash and uses a FuturesOrdered for getblock requests --- zebra-rpc/Cargo.toml | 2 +- zebra-rpc/src/methods.rs | 22 ++- zebra-rpc/src/sync.rs | 342 +++++++++++++++++-------------------- zebrad/Cargo.toml | 1 + zebrad/tests/acceptance.rs | 22 ++- 5 files changed, 200 insertions(+), 189 deletions(-) diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index 6b151825da1..b0d59bb123d 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -15,7 +15,7 @@ keywords = ["zebra", "zcash"] categories = ["asynchronous", "cryptography::cryptocurrencies", "encoding", "network-programming"] [features] -default = [] +default = ["syncer"] # Production features that activate extra dependencies, or extra features in dependencies diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 0ff129a643f..fc12d2ed19d 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -171,6 +171,10 @@ pub trait Rpc { #[rpc(name = "getbestblockhash")] fn get_best_block_hash(&self) -> Result; + /// Returns the height and hash of the current best blockchain tip block, as a [`GetBlockHeightAndHash`] JSON struct. + #[rpc(name = "getbestblockheightandhash")] + fn get_best_block_height_and_hash(&self) -> Result; + /// Returns all transaction ids in the memory pool, as a JSON array. /// /// zcashd reference: [`getrawmempool`](https://zcash.github.io/rpc/getrawmempool.html) @@ -867,7 +871,6 @@ where .boxed() } - // TODO: use a generic error constructor (#5548) fn get_best_block_hash(&self) -> Result { self.latest_chain_tip .best_tip_hash() @@ -875,7 +878,13 @@ where .ok_or_server_error("No blocks in state") } - // TODO: use a generic error constructor (#5548) + fn get_best_block_height_and_hash(&self) -> Result { + self.latest_chain_tip + .best_tip_height_and_hash() + .map(|(height, hash)| GetBlockHeightAndHash { height, hash }) + .ok_or_server_error("No blocks in state") + } + fn get_raw_mempool(&self) -> BoxFuture>> { #[cfg(feature = "getblocktemplate-rpcs")] use zebra_chain::block::MAX_BLOCK_BYTES; @@ -1541,6 +1550,15 @@ impl Default for GetBlock { #[serde(transparent)] pub struct GetBlockHash(#[serde(with = "hex")] pub block::Hash); +/// Response to a `getbestblockheightandhash` RPC request. +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct GetBlockHeightAndHash { + /// The best chain tip block height + pub height: block::Height, + /// The best chain tip block hash + pub hash: block::Hash, +} + impl Default for GetBlockHash { fn default() -> Self { GetBlockHash(block::Hash([0; 32])) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 8c7c6c5a4e3..7530fc0c159 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -1,7 +1,8 @@ //! Syncer task for maintaining a non-finalized state in Zebra's ReadStateService and updating `ChainTipSender` via RPCs -use std::{net::SocketAddr, sync::Arc, time::Duration}; +use std::{net::SocketAddr, ops::RangeInclusive, sync::Arc, time::Duration}; +use futures::{stream::FuturesOrdered, StreamExt}; use tokio::task::JoinHandle; use tower::BoxError; use zebra_chain::{ @@ -11,8 +12,8 @@ use zebra_chain::{ }; use zebra_node_services::rpc_client::RpcRequestClient; use zebra_state::{ - spawn_init_read_only, ChainTipBlock, ChainTipChange, ChainTipSender, LatestChainTip, - NonFinalizedState, ReadStateService, SemanticallyVerifiedBlock, ZebraDb, + spawn_init_read_only, ChainTipBlock, ChainTipChange, ChainTipSender, CheckpointVerifiedBlock, + LatestChainTip, NonFinalizedState, ReadStateService, SemanticallyVerifiedBlock, ZebraDb, MAX_BLOCK_REORG_HEIGHT, }; @@ -20,7 +21,7 @@ use zebra_chain::diagnostic::task::WaitForPanics; use crate::{ constants::MISSING_BLOCK_ERROR_CODE, - methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHash}, + methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHeightAndHash}, }; /// How long to wait between calls to `getbestblockhash` when it returns an error or the block hash @@ -42,34 +43,6 @@ struct TrustedChainSync { non_finalized_state_sender: tokio::sync::watch::Sender, } -#[derive(Debug)] -enum NewChainTip { - /// Information about the next block height to request and how it should be processed. - Cursor(SyncCursor), - /// The latest finalized tip. - Block(Arc), -} - -#[derive(Debug)] -struct SyncCursor { - /// The best chain tip height in this process. - tip_height: Height, - /// The best chain tip hash in this process. - tip_hash: block::Hash, - /// The best chain tip hash in the Zebra node. - target_tip_hash: block::Hash, -} - -impl SyncCursor { - fn new(tip_height: Height, tip_hash: block::Hash, target_tip_hash: block::Hash) -> Self { - Self { - tip_height, - tip_hash, - target_tip_hash, - } - } -} - impl TrustedChainSync { /// Creates a new [`TrustedChainSync`] and starts syncing blocks from the node's non-finalized best chain. pub async fn spawn( @@ -99,171 +72,169 @@ impl TrustedChainSync { /// Starts syncing blocks from the node's non-finalized best chain. async fn sync(&mut self) { + let mut last_chain_tip_hash = self.db.finalized_tip_hash(); + loop { - // Wait until the best block hash in Zebra is different from the tip hash in this read state - match self.wait_for_chain_tip_change().await { - NewChainTip::Cursor(cursor) => { - self.sync_new_blocks(cursor).await; - } + let (target_tip_height, target_tip_hash) = + self.wait_for_chain_tip_change(last_chain_tip_hash).await; + + // TODO: do in spawn_blocking + if self.non_finalized_state.chain_count() == 0 && self.db.contains_hash(target_tip_hash) + { + let block = self.finalized_chain_tip_block().await.expect( + "should have genesis block after successful bestblockheightandhash response", + ); - NewChainTip::Block(block) => update_channels( - block, - &self.non_finalized_state, - &mut self.non_finalized_state_sender, - &mut self.chain_tip_sender, - ), + last_chain_tip_hash = block.hash; + self.chain_tip_sender.set_finalized_tip(block); + continue; } - } - } - /// Accepts a [`SyncCursor`] and gets blocks after the current chain tip until reaching the target block hash in the cursor. - /// - /// Clears the non-finalized state and sets the finalized tip as the current chain tip if - /// the `getblock` RPC method returns the `MISSING_BLOCK_ERROR_CODE` before this function reaches - /// the target block hash. - async fn sync_new_blocks(&mut self, mut cursor: SyncCursor) { - let mut consecutive_unexpected_error_count = 0; - - let has_found_new_best_chain = loop { - let block = match self.rpc_client.get_block(cursor.tip_height).await { - Ok(Some(block)) if block.header.previous_block_hash == cursor.tip_hash => { - SemanticallyVerifiedBlock::from(block) - } - // If the next block's previous block hash doesn't match the expected hash, or if the block is missing - // before this function reaches the target block hash, there was likely a chain re-org/fork, and - // we can clear the non-finalized state and re-fetch every block past the finalized tip. - Ok(_) => break true, - // If there was an unexpected error, retry 4 more times before returning early. - // TODO: Propagate this error up and exit the spawned sync task with the error if the RPC server is down for too long. - Err(err) => { + let (current_tip_height, mut current_tip_hash) = self.current_tip().await.expect( + "should have genesis block after successful bestblockheightandhash response", + ); + + last_chain_tip_hash = current_tip_hash; + + let next_tip_height = current_tip_height.next().expect("should be valid height"); + let rpc_client = self.rpc_client.clone(); + let mut block_futs = + rpc_client.block_range_ordered(next_tip_height..=target_tip_height); + + let should_reset_non_finalized_state = loop { + let block = match block_futs.next().await { + Some(Ok(Some(block))) + if block.header.previous_block_hash == current_tip_hash => + { + SemanticallyVerifiedBlock::from(block) + } + // Clear the non-finalized state and re-fetch every block past the finalized tip if: + // - the next block's previous block hash doesn't match the expected hash, + // - the next block is missing + // - the target tip hash is missing from the blocks in `block_futs` + // because there was likely a chain re-org/fork + Some(Ok(_)) | None => break true, + // If calling the `getblock` RPC method fails with an unexpected error, wait for the next chain tip change + // without resetting the non-finalized state. + Some(Err(err)) => { + tracing::warn!( + ?err, + "encountered an unexpected error while calling getblock method" + ); + + break false; + } + }; + + let block_hash = block.hash; + let commit_result = if self.non_finalized_state.chain_count() == 0 { + self.non_finalized_state + .commit_new_chain(block.clone(), &self.db) + } else { + self.non_finalized_state + .commit_block(block.clone(), &self.db) + }; + + // The previous block hash is checked above, if committing the block fails for some reason, try again. + if let Err(error) = commit_result { tracing::warn!( - ?consecutive_unexpected_error_count, - ?cursor, - ?err, - "encountered an unexpected error while calling getblock method" + ?error, + ?block_hash, + "failed to commit block to non-finalized state" ); - if consecutive_unexpected_error_count >= 5 { - break false; - } else { - consecutive_unexpected_error_count += 1; - continue; - }; + break false; } - }; - consecutive_unexpected_error_count = 0; + while self + .non_finalized_state + .best_chain_len() + .expect("just successfully inserted a non-finalized block above") + > MAX_BLOCK_REORG_HEIGHT + { + tracing::trace!("finalizing block past the reorg limit"); + self.non_finalized_state.finalize(); + } + + self.update_channels(block); + current_tip_hash = block_hash; + last_chain_tip_hash = current_tip_hash; - let block_hash = block.hash; - let block_height = block.height; - let commit_result = if self.non_finalized_state.chain_count() == 0 { - self.non_finalized_state - .commit_new_chain(block.clone(), &self.db) - } else { - self.non_finalized_state - .commit_block(block.clone(), &self.db) + // If the block hash matches the output from the `getbestblockhash` RPC method, we can wait until + // the best block hash changes to get the next block. + if block_hash == target_tip_hash { + break false; + } }; - if let Err(error) = commit_result { - tracing::warn!( - ?error, - ?block_hash, - "failed to commit block to non-finalized state" + if should_reset_non_finalized_state { + let block = self.finalized_chain_tip_block().await.expect( + "should have genesis block after successful bestblockheightandhash response", ); - continue; - } - while self - .non_finalized_state - .best_chain_len() - .expect("just successfully inserted a non-finalized block above") - > MAX_BLOCK_REORG_HEIGHT - { - tracing::trace!("finalizing block past the reorg limit"); - self.non_finalized_state.finalize(); + last_chain_tip_hash = block.hash; + self.update_channels(block); } - - update_channels( - block, - &self.non_finalized_state, - &mut self.non_finalized_state_sender, - &mut self.chain_tip_sender, - ); - - cursor.tip_height = block_height; - cursor.tip_hash = block_hash; - - // If the block hash matches the output from the `getbestblockhash` RPC method, we can wait until - // the best block hash changes to get the next block. - if block_hash == cursor.target_tip_hash { - break false; - } - }; - - if has_found_new_best_chain { - self.non_finalized_state = NonFinalizedState::new(&self.db.network()); - let db = self.db.clone(); - let finalized_tip = tokio::task::spawn_blocking(move || { - db.tip_block().expect("should have genesis block") - }) - .wait_for_panics() - .await; - - update_channels( - finalized_tip, - &self.non_finalized_state, - &mut self.non_finalized_state_sender, - &mut self.chain_tip_sender, - ); } } - /// Polls `getbestblockhash` RPC method until there are new blocks in the Zebra node's non-finalized state. - async fn wait_for_chain_tip_change(&self) -> NewChainTip { - let (tip_height, tip_hash) = if let Some(tip) = self.non_finalized_state.best_tip() { - tip + /// Returns the current tip height and hash + async fn current_tip(&self) -> Option<(block::Height, block::Hash)> { + if let Some(tip) = self.non_finalized_state.best_tip() { + Some(tip) } else { let db = self.db.clone(); tokio::task::spawn_blocking(move || db.tip()) .wait_for_panics() .await - .expect("checked for genesis block above") - }; + } + } + + async fn finalized_chain_tip_block(&self) -> Option { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let (height, hash) = db.tip()?; + db.block(height.into()) + .map(|block| CheckpointVerifiedBlock::with_hash(block, hash)) + .map(ChainTipBlock::from) + }) + .wait_for_panics() + .await + } - // Wait until the best block hash in Zebra is different from the tip hash in this read state + /// Accepts a block hash. + /// + /// Polls `getbestblockhash` RPC method until it successfully returns a different hash from the last chain tip hash. + /// + /// Returns the node's best block hash + async fn wait_for_chain_tip_change( + &self, + last_chain_tip_hash: block::Hash, + ) -> (block::Height, block::Hash) { loop { - let Some(target_tip_hash) = self.rpc_client.get_best_block_hash().await else { - // Wait until the genesis block has been committed. + let Some(target_height_and_hash) = self + .rpc_client + .get_best_block_height_and_hash() + .await + .filter(|&(_height, hash)| hash != last_chain_tip_hash) + else { tokio::time::sleep(POLL_DELAY).await; continue; }; - if target_tip_hash != tip_hash { - let cursor = - NewChainTip::Cursor(SyncCursor::new(tip_height, tip_hash, target_tip_hash)); - - // Check if there's are blocks in the non-finalized state, or that - // the node tip hash is different from our finalized tip hash before returning - // a cursor for syncing blocks via the `getblock` RPC. - if self.non_finalized_state.chain_count() != 0 { - break cursor; - } - - let db = self.db.clone(); - break tokio::task::spawn_blocking(move || { - if db.finalized_tip_hash() != target_tip_hash { - cursor - } else { - NewChainTip::Block(db.tip_block().unwrap()) - } - }) - .wait_for_panics() - .await; - } - - tokio::time::sleep(POLL_DELAY).await; + break target_height_and_hash; } } + + /// Sends the new chain tip and non-finalized state to the latest chain channels. + fn update_channels(&mut self, best_tip: impl Into) { + // If the final receiver was just dropped, ignore the error. + let _ = self + .non_finalized_state_sender + .send(self.non_finalized_state.clone()); + self.chain_tip_sender + .set_best_non_finalized_tip(Some(best_tip.into())); + } } /// Accepts a [zebra-state configuration](zebra_state::Config), a [`Network`], and @@ -304,19 +275,34 @@ pub fn init_read_state_with_syncer( } trait SyncerRpcMethods { - async fn get_best_block_hash(&self) -> Option; - async fn get_block(&self, height: block::Height) -> Result>, BoxError>; + async fn get_best_block_height_and_hash(&self) -> Option<(block::Height, block::Hash)>; + async fn get_block(&self, height: u32) -> Result>, BoxError>; + fn block_range_ordered( + &self, + height_range: RangeInclusive, + ) -> FuturesOrdered>, BoxError>>> + { + let &Height(start_height) = height_range.start(); + let &Height(end_height) = height_range.end(); + let mut futs = FuturesOrdered::new(); + + for height in start_height..=end_height { + futs.push_back(self.get_block(height)); + } + + futs + } } impl SyncerRpcMethods for RpcRequestClient { - async fn get_best_block_hash(&self) -> Option { - self.json_result_from_call("getbestblockhash", "[]") + async fn get_best_block_height_and_hash(&self) -> Option<(block::Height, block::Hash)> { + self.json_result_from_call("getbestblockheightandhash", "[]") .await - .map(|GetBlockHash(hash)| hash) + .map(|GetBlockHeightAndHash { height, hash }| (height, hash)) .ok() } - async fn get_block(&self, Height(height): Height) -> Result>, BoxError> { + async fn get_block(&self, height: u32) -> Result>, BoxError> { match self .json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) .await @@ -336,15 +322,3 @@ impl SyncerRpcMethods for RpcRequestClient { } } } - -/// Sends the new chain tip and non-finalized state to the latest chain channels. -fn update_channels( - best_tip: impl Into, - non_finalized_state: &NonFinalizedState, - non_finalized_state_sender: &mut tokio::sync::watch::Sender, - chain_tip_sender: &mut ChainTipSender, -) { - // If the final receiver was just dropped, ignore the error. - let _ = non_finalized_state_sender.send(non_finalized_state.clone()); - chain_tip_sender.set_best_non_finalized_tip(best_tip.into()); -} diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 04a21dfcfc9..d8baebea92a 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -287,6 +287,7 @@ zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.37", feat zebra-network = { path = "../zebra-network", version = "1.0.0-beta.37", features = ["proptest-impl"] } zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.6", features = ["proptest-impl"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.37", features = ["proptest-impl"] } +zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.37", features = ["syncer"] } zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.37", features = ["rpc-client"] } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index f924dc51b5e..e2135cfe269 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3189,7 +3189,8 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { let testdir = testdir()?.with_config(&mut config)?; let mut child = testdir.spawn_child(args!["start"])?; - std::thread::sleep(LAUNCH_DELAY); + // Wait for Zebrad to start up + tokio::time::sleep(LAUNCH_DELAY).await; // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = @@ -3201,15 +3202,32 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { .await? .map_err(|err| eyre!(err))?; + // Wait for sync task to catch up to the best tip (genesis block) + tokio::time::sleep(Duration::from_secs(3)).await; + + let tip_action = timeout( + Duration::from_secs(15), + chain_tip_change.wait_for_tip_change(), + ) + .await??; + + assert!( + tip_action.is_reset(), + "first tip action should be a reset for the genesis block" + ); + let rpc_client = RpcRequestClient::new(rpc_address); - for _ in 0..100 { + for _ in 0..10 { let (block, height) = rpc_client .block_from_template(Height(REGTEST_NU5_ACTIVATION_HEIGHT)) .await?; rpc_client.submit_block(block).await?; + // Wait for sync task to catch up to the best tip + tokio::time::sleep(Duration::from_secs(3)).await; + let tip_action = timeout( Duration::from_secs(1), chain_tip_change.wait_for_tip_change(), From a1ad80f75bfe62d3343018c469122739f6c6a8af Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 12 Jun 2024 01:58:43 -0400 Subject: [PATCH 13/30] Fixes a pre-flush sync issue by using a secondary db instead of a read-only db --- zebra-rpc/src/sync.rs | 39 +++++++++++++++---- .../src/service/finalized_state/disk_db.rs | 14 ++++++- zebrad/tests/acceptance.rs | 13 ++++--- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 7530fc0c159..bebd38ecfec 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -5,9 +5,10 @@ use std::{net::SocketAddr, ops::RangeInclusive, sync::Arc, time::Duration}; use futures::{stream::FuturesOrdered, StreamExt}; use tokio::task::JoinHandle; use tower::BoxError; +use tracing::info; use zebra_chain::{ block::{self, Block, Height}, - parameters::Network, + parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH}, serialization::ZcashDeserializeInto, }; use zebra_node_services::rpc_client::RpcRequestClient; @@ -72,12 +73,29 @@ impl TrustedChainSync { /// Starts syncing blocks from the node's non-finalized best chain. async fn sync(&mut self) { - let mut last_chain_tip_hash = self.db.finalized_tip_hash(); + let mut last_chain_tip_hash = + if let Some(finalized_tip_block) = self.finalized_chain_tip_block().await { + let last_chain_tip_hash = finalized_tip_block.hash; + self.chain_tip_sender.set_finalized_tip(finalized_tip_block); + last_chain_tip_hash + } else { + GENESIS_PREVIOUS_BLOCK_HASH + }; loop { let (target_tip_height, target_tip_hash) = self.wait_for_chain_tip_change(last_chain_tip_hash).await; + info!( + ?target_tip_height, + ?target_tip_hash, + "got a chain tip change" + ); + + let catch_up_result = self.db.try_catch_up_with_primary(); + + info!(?catch_up_result, "trying to catch up to primary"); + // TODO: do in spawn_blocking if self.non_finalized_state.chain_count() == 0 && self.db.contains_hash(target_tip_hash) { @@ -90,16 +108,14 @@ impl TrustedChainSync { continue; } - let (current_tip_height, mut current_tip_hash) = self.current_tip().await.expect( - "should have genesis block after successful bestblockheightandhash response", - ); + let (next_block_height, mut current_tip_hash) = + self.next_block_height_and_prev_hash().await; last_chain_tip_hash = current_tip_hash; - let next_tip_height = current_tip_height.next().expect("should be valid height"); let rpc_client = self.rpc_client.clone(); let mut block_futs = - rpc_client.block_range_ordered(next_tip_height..=target_tip_height); + rpc_client.block_range_ordered(next_block_height..=target_tip_height); let should_reset_non_finalized_state = loop { let block = match block_futs.next().await { @@ -179,7 +195,7 @@ impl TrustedChainSync { } /// Returns the current tip height and hash - async fn current_tip(&self) -> Option<(block::Height, block::Hash)> { + async fn next_block_height_and_prev_hash(&self) -> (block::Height, block::Hash) { if let Some(tip) = self.non_finalized_state.best_tip() { Some(tip) } else { @@ -188,6 +204,13 @@ impl TrustedChainSync { .wait_for_panics() .await } + .map(|(current_tip_height, current_tip_hash)| { + ( + current_tip_height.next().expect("should be valid height"), + current_tip_hash, + ) + }) + .unwrap_or((Height::MIN, GENESIS_PREVIOUS_BLOCK_HASH)) } async fn finalized_chain_tip_block(&self) -> Option { diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index ac0da2795c1..642240d75f3 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -834,7 +834,19 @@ impl DiskDb { .map(|cf_name| rocksdb::ColumnFamilyDescriptor::new(cf_name, db_options.clone())); let db_result = if read_only { - DB::open_cf_descriptors_read_only(&db_options, &path, column_families, false) + // TODO: Make this path configurable? + let secondary_path = + config.db_path("secondary_state", format_version_in_code.major, network); + let create_dir_result = std::fs::create_dir_all(&secondary_path); + + info!(?create_dir_result, "creating secondary db directory"); + + DB::open_cf_descriptors_as_secondary( + &db_options, + &path, + &secondary_path, + column_families, + ) } else { DB::open_cf_descriptors(&db_options, &path, column_families) }; diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index e2135cfe269..ea54cc17513 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3187,10 +3187,13 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { let rpc_address = config.rpc.listen_addr.unwrap(); let testdir = testdir()?.with_config(&mut config)?; + let mut child = testdir.spawn_child(args!["start"])?; - // Wait for Zebrad to start up - tokio::time::sleep(LAUNCH_DELAY).await; + child.expect_stdout_line_matches(format!( + "Opened Zebra state cache at {}", + config.state.cache_dir.to_str().unwrap() + ))?; // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = @@ -3202,11 +3205,11 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { .await? .map_err(|err| eyre!(err))?; - // Wait for sync task to catch up to the best tip (genesis block) - tokio::time::sleep(Duration::from_secs(3)).await; + // Wait for Zebrad to start up + tokio::time::sleep(LAUNCH_DELAY).await; let tip_action = timeout( - Duration::from_secs(15), + Duration::from_secs(10), chain_tip_change.wait_for_tip_change(), ) .await??; From c9319e0f51b9c866703d1403cc590909c6178135 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 18:09:19 -0400 Subject: [PATCH 14/30] Moves disk IO to blocking tasks --- zebra-rpc/src/sync.rs | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index bebd38ecfec..49b5c0f70a9 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -73,6 +73,7 @@ impl TrustedChainSync { /// Starts syncing blocks from the node's non-finalized best chain. async fn sync(&mut self) { + self.try_catch_up_with_primary().await; let mut last_chain_tip_hash = if let Some(finalized_tip_block) = self.finalized_chain_tip_block().await { let last_chain_tip_hash = finalized_tip_block.hash; @@ -92,13 +93,7 @@ impl TrustedChainSync { "got a chain tip change" ); - let catch_up_result = self.db.try_catch_up_with_primary(); - - info!(?catch_up_result, "trying to catch up to primary"); - - // TODO: do in spawn_blocking - if self.non_finalized_state.chain_count() == 0 && self.db.contains_hash(target_tip_hash) - { + if self.is_finalized_tip_change(target_tip_hash).await { let block = self.finalized_chain_tip_block().await.expect( "should have genesis block after successful bestblockheightandhash response", ); @@ -194,6 +189,29 @@ impl TrustedChainSync { } } + async fn try_catch_up_with_primary(&self) { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let catch_up_result = db.try_catch_up_with_primary(); + tracing::debug!(?catch_up_result, "trying to catch up to primary"); + }) + .wait_for_panics() + .await + } + + async fn is_finalized_tip_change(&self, target_tip_hash: block::Hash) -> bool { + self.non_finalized_state.chain_count() == 0 && { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let catch_up_result = db.try_catch_up_with_primary(); + tracing::debug!(?catch_up_result, "trying to catch up to primary"); + db.contains_hash(target_tip_hash) + }) + .wait_for_panics() + .await + } + } + /// Returns the current tip height and hash async fn next_block_height_and_prev_hash(&self) -> (block::Height, block::Hash) { if let Some(tip) = self.non_finalized_state.best_tip() { From 9d2d1df6b2daa1f78aa91c7575f1de8b8bbb71b1 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 22:03:56 -0400 Subject: [PATCH 15/30] Updates acceptance test to how forks are handled --- zebra-rpc/src/sync.rs | 8 +++ zebrad/tests/acceptance.rs | 124 +++++++++++++++++++++++++++------ zebrad/tests/common/regtest.rs | 11 +++ 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 49b5c0f70a9..0d670d2bbed 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -179,16 +179,20 @@ impl TrustedChainSync { }; if should_reset_non_finalized_state { + self.try_catch_up_with_primary().await; let block = self.finalized_chain_tip_block().await.expect( "should have genesis block after successful bestblockheightandhash response", ); last_chain_tip_hash = block.hash; + self.non_finalized_state = + NonFinalizedState::new(&self.non_finalized_state.network); self.update_channels(block); } } } + /// Tries to catch up to the primary db instance for an up-to-date view of finalized blocks. async fn try_catch_up_with_primary(&self) { let db = self.db.clone(); tokio::task::spawn_blocking(move || { @@ -199,6 +203,10 @@ impl TrustedChainSync { .await } + /// If the non-finalized state is empty, tries to catch up to the primary db instance for + /// an up-to-date view of finalized blocks. + /// + /// Returns true if the non-finalized state is empty and the target hash is in the finalized state. async fn is_finalized_tip_change(&self, target_tip_hash: block::Hash) -> bool { self.non_finalized_state.chain_count() == 0 && { let db = self.db.clone(); diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index ea54cc17513..443be4d4bc4 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3172,29 +3172,39 @@ async fn regtest_submit_blocks() -> Result<()> { } // TODO: -// - Test that chain forks are handled correctly and that `ChainTipChange` sees a `Reset` // - Test that the `ChainTipChange` updated by the RPC syncer is updated when the finalized tip changes #[cfg(feature = "rpc-syncer")] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { + use std::sync::Arc; + use common::regtest::MiningRpcMethods; use tokio::time::timeout; - use zebra_chain::parameters::testnet::REGTEST_NU5_ACTIVATION_HEIGHT; + use zebra_chain::{ + chain_tip::ChainTip, parameters::NetworkUpgrade, + primitives::byte_array::increment_big_endian, + }; + use zebra_rpc::methods::GetBlockHash; + use zebra_state::{ReadResponse, Response}; let _init_guard = zebra_test::init(); let mut config = random_known_rpc_port_config(false, &Network::new_regtest(None))?; config.state.ephemeral = false; + let network = config.network.network.clone(); let rpc_address = config.rpc.listen_addr.unwrap(); let testdir = testdir()?.with_config(&mut config)?; let mut child = testdir.spawn_child(args!["start"])?; + tracing::info!("waiting for Zebra state cache to be opened"); + child.expect_stdout_line_matches(format!( "Opened Zebra state cache at {}", config.state.cache_dir.to_str().unwrap() ))?; + tracing::info!("starting read state with syncer"); // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = zebra_rpc::sync::init_read_state_with_syncer( @@ -3205,32 +3215,22 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { .await? .map_err(|err| eyre!(err))?; - // Wait for Zebrad to start up - tokio::time::sleep(LAUNCH_DELAY).await; - - let tip_action = timeout( - Duration::from_secs(10), - chain_tip_change.wait_for_tip_change(), - ) - .await??; + tracing::info!("waiting for first chain tip change"); + // Wait for Zebrad to start up + let tip_action = timeout(LAUNCH_DELAY, chain_tip_change.wait_for_tip_change()).await??; assert!( tip_action.is_reset(), "first tip action should be a reset for the genesis block" ); - let rpc_client = RpcRequestClient::new(rpc_address); + tracing::info!("got genesis chain tip change, submitting more blocks .."); + let rpc_client = RpcRequestClient::new(rpc_address); + let mut blocks = Vec::new(); for _ in 0..10 { - let (block, height) = rpc_client - .block_from_template(Height(REGTEST_NU5_ACTIVATION_HEIGHT)) - .await?; - - rpc_client.submit_block(block).await?; - - // Wait for sync task to catch up to the best tip - tokio::time::sleep(Duration::from_secs(3)).await; - + let (block, height) = rpc_client.submit_block_from_template().await?; + blocks.push(block); let tip_action = timeout( Duration::from_secs(1), chain_tip_change.wait_for_tip_change(), @@ -3244,8 +3244,88 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { ); } + tracing::info!("getting next block template"); + let (block_11, _) = rpc_client.block_from_template(Height(100)).await?; + let next_blocks: Vec<_> = blocks + .split_off(5) + .into_iter() + .chain(std::iter::once(block_11)) + .collect(); + + tracing::info!("creating populated state"); + let genesis_block = regtest_genesis_block(); + let (state2, read_state2, latest_chain_tip2, _chain_tip_change2) = + zebra_state::populated_state( + std::iter::once(genesis_block).chain(blocks.into_iter().map(Arc::new)), + &network, + ) + .await; + + tracing::info!("attempting to trigger a best chain change"); + for mut block in next_blocks { + let is_chain_history_activation_height = NetworkUpgrade::Heartwood + .activation_height(&network) + == Some(block.coinbase_height().unwrap()); + let header = Arc::make_mut(&mut block.header); + increment_big_endian(header.nonce.as_mut()); + let ReadResponse::ChainInfo(chain_info) = read_state2 + .clone() + .oneshot(zebra_state::ReadRequest::ChainInfo) + .await + .map_err(|err| eyre!(err))? + else { + unreachable!("wrong response variant"); + }; + + header.previous_block_hash = chain_info.tip_hash; + header.commitment_bytes = chain_info + .history_tree + .hash() + .or(is_chain_history_activation_height.then_some([0; 32].into())) + .expect("history tree can't be empty") + .bytes_in_serialized_order() + .into(); + + let Response::Committed(block_hash) = state2 + .clone() + .oneshot(zebra_state::Request::CommitSemanticallyVerifiedBlock( + Arc::new(block.clone()).into(), + )) + .await + .map_err(|err| eyre!(err))? + else { + unreachable!("wrong response variant"); + }; + + rpc_client.submit_block(block).await?; + let GetBlockHash(best_block_hash) = rpc_client + .json_result_from_call("getbestblockhash", "[]") + .await + .map_err(|err| eyre!(err))?; + + if block_hash == best_block_hash { + break; + } + } + + tracing::info!("newly submitted blocks are in the best chain, checking for reset"); + tokio::time::sleep(Duration::from_secs(3)).await; + let tip_action = timeout( + Duration::from_secs(1), + chain_tip_change.wait_for_tip_change(), + ) + .await??; + let (expected_height, expected_hash) = latest_chain_tip2 + .best_tip_height_and_hash() + .expect("should have a chain tip"); + assert!(tip_action.is_reset(), "tip action should be reset"); + assert_eq!( + tip_action.best_tip_hash_and_height(), + (expected_hash, expected_height), + "tip action hashes and heights should match" + ); + // TODO: - // - Submit several blocks before checking that `ChainTipChange` has the latest block hash and is a `TipAction::Reset` // - Check that `getblock` RPC returns the same block as the read state for every height // - Submit more blocks with an older block template (and a different nonce so the hash is different) to trigger a chain reorg // - Check that `ChainTipChange` isn't being updated until the best chain changes diff --git a/zebrad/tests/common/regtest.rs b/zebrad/tests/common/regtest.rs index b031df68af9..eb1c416299e 100644 --- a/zebrad/tests/common/regtest.rs +++ b/zebrad/tests/common/regtest.rs @@ -89,6 +89,7 @@ async fn submit_blocks(network: Network, rpc_address: SocketAddr) -> Result<()> pub trait MiningRpcMethods { async fn block_from_template(&self, nu5_activation_height: Height) -> Result<(Block, Height)>; async fn submit_block(&self, block: Block) -> Result<()>; + async fn submit_block_from_template(&self) -> Result<(Block, Height)>; } impl MiningRpcMethods for RpcRequestClient { @@ -127,4 +128,14 @@ impl MiningRpcMethods for RpcRequestClient { } } } + + async fn submit_block_from_template(&self) -> Result<(Block, Height)> { + let (block, height) = self + .block_from_template(Height(REGTEST_NU5_ACTIVATION_HEIGHT)) + .await?; + + self.submit_block(block.clone()).await?; + + Ok((block, height)) + } } From d31526e0b35ba6bed1cd24172e9c268b6e20b211 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 22:17:54 -0400 Subject: [PATCH 16/30] Checks synced read state for all of the expected blocks --- zebrad/tests/acceptance.rs | 68 ++++++++++++++++++++++++++++++++-- zebrad/tests/common/regtest.rs | 35 +++++++++++++++-- 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 443be4d4bc4..60e75adee09 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3206,7 +3206,7 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { tracing::info!("starting read state with syncer"); // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra - let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = + let (read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = zebra_rpc::sync::init_read_state_with_syncer( config.state, &config.network.network, @@ -3244,6 +3244,36 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { ); } + for expected_block in blocks.clone() { + let height = expected_block.coinbase_height().unwrap(); + let zebra_block = rpc_client + .get_block(height.0) + .await + .map_err(|err| eyre!(err))? + .expect("Zebra test child should have the expected block"); + + assert_eq!( + zebra_block, + Arc::new(expected_block), + "Zebra should have the same block" + ); + + let ReadResponse::Block(read_state_block) = read_state + .clone() + .oneshot(zebra_state::ReadRequest::Block(height.into())) + .await + .map_err(|err| eyre!(err))? + else { + unreachable!("unexpected read response to a block request") + }; + + assert_eq!( + zebra_block, + read_state_block.expect("read state should have the block"), + "read state should have the same block" + ); + } + tracing::info!("getting next block template"); let (block_11, _) = rpc_client.block_from_template(Height(100)).await?; let next_blocks: Vec<_> = blocks @@ -3256,7 +3286,7 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { let genesis_block = regtest_genesis_block(); let (state2, read_state2, latest_chain_tip2, _chain_tip_change2) = zebra_state::populated_state( - std::iter::once(genesis_block).chain(blocks.into_iter().map(Arc::new)), + std::iter::once(genesis_block).chain(blocks.iter().cloned().map(Arc::new)), &network, ) .await; @@ -3297,7 +3327,8 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { unreachable!("wrong response variant"); }; - rpc_client.submit_block(block).await?; + rpc_client.submit_block(block.clone()).await?; + blocks.push(block); let GetBlockHash(best_block_hash) = rpc_client .json_result_from_call("getbestblockhash", "[]") .await @@ -3325,8 +3356,37 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { "tip action hashes and heights should match" ); + for expected_block in blocks { + let height = expected_block.coinbase_height().unwrap(); + let zebra_block = rpc_client + .get_block(height.0) + .await + .map_err(|err| eyre!(err))? + .expect("Zebra test child should have the expected block"); + + assert_eq!( + zebra_block, + Arc::new(expected_block), + "Zebra should have the same block" + ); + + let ReadResponse::Block(read_state_block) = read_state + .clone() + .oneshot(zebra_state::ReadRequest::Block(height.into())) + .await + .map_err(|err| eyre!(err))? + else { + unreachable!("unexpected read response to a block request") + }; + + assert_eq!( + zebra_block, + read_state_block.expect("read state should have the block"), + "read state should have the same block" + ); + } + // TODO: - // - Check that `getblock` RPC returns the same block as the read state for every height // - Submit more blocks with an older block template (and a different nonce so the hash is different) to trigger a chain reorg // - Check that `ChainTipChange` isn't being updated until the best chain changes // - Check that the first `ChainTipChange` `TipAction` is a `TipAction::Reset` diff --git a/zebrad/tests/common/regtest.rs b/zebrad/tests/common/regtest.rs index eb1c416299e..1b36cba80e5 100644 --- a/zebrad/tests/common/regtest.rs +++ b/zebrad/tests/common/regtest.rs @@ -6,18 +6,24 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; use color_eyre::eyre::{eyre, Context, Result}; +use tower::BoxError; use tracing::*; use zebra_chain::{ block::{Block, Height}, parameters::{testnet::REGTEST_NU5_ACTIVATION_HEIGHT, Network, NetworkUpgrade}, primitives::byte_array::increment_big_endian, - serialization::ZcashSerialize, + serialization::{ZcashDeserializeInto, ZcashSerialize}, }; use zebra_node_services::rpc_client::RpcRequestClient; -use zebra_rpc::methods::get_block_template_rpcs::{ - get_block_template::{proposal::TimeSource, proposal_block_from_template, GetBlockTemplate}, - types::submit_block, +use zebra_rpc::{ + constants::MISSING_BLOCK_ERROR_CODE, + methods::get_block_template_rpcs::{ + get_block_template::{ + proposal::TimeSource, proposal_block_from_template, GetBlockTemplate, + }, + types::{hex_data::HexData, submit_block}, + }, }; use zebra_test::args; @@ -90,6 +96,7 @@ pub trait MiningRpcMethods { async fn block_from_template(&self, nu5_activation_height: Height) -> Result<(Block, Height)>; async fn submit_block(&self, block: Block) -> Result<()>; async fn submit_block_from_template(&self) -> Result<(Block, Height)>; + async fn get_block(&self, height: u32) -> Result>, BoxError>; } impl MiningRpcMethods for RpcRequestClient { @@ -138,4 +145,24 @@ impl MiningRpcMethods for RpcRequestClient { Ok((block, height)) } + + async fn get_block(&self, height: u32) -> Result>, BoxError> { + match self + .json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) + .await + { + Ok(HexData(raw_block)) => { + let block = raw_block.zcash_deserialize_into::()?; + Ok(Some(Arc::new(block))) + } + Err(err) + if err + .downcast_ref::() + .is_some_and(|err| err.code == MISSING_BLOCK_ERROR_CODE) => + { + Ok(None) + } + Err(err) => Err(err), + } + } } From f18344ff79babfa0521cf9bd01ee907cc6361ed8 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 22:24:57 -0400 Subject: [PATCH 17/30] checks that there isn't a tip change until the best chain changes --- zebrad/tests/acceptance.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 60e75adee09..fff60003e50 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3172,7 +3172,7 @@ async fn regtest_submit_blocks() -> Result<()> { } // TODO: -// - Test that the `ChainTipChange` updated by the RPC syncer is updated when the finalized tip changes +// - Test that the `ChainTipChange` updated by the RPC syncer is updated when the finalized tip changes (outside Regtest) #[cfg(feature = "rpc-syncer")] #[tokio::test(flavor = "multi_thread")] async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { @@ -3327,6 +3327,11 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { unreachable!("wrong response variant"); }; + assert!( + chain_tip_change.last_tip_change().is_none(), + "there should be no tip change until the last block is submitted" + ); + rpc_client.submit_block(block.clone()).await?; blocks.push(block); let GetBlockHash(best_block_hash) = rpc_client @@ -3386,12 +3391,6 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { ); } - // TODO: - // - Submit more blocks with an older block template (and a different nonce so the hash is different) to trigger a chain reorg - // - Check that `ChainTipChange` isn't being updated until the best chain changes - // - Check that the first `ChainTipChange` `TipAction` is a `TipAction::Reset` - // - Check that `getblock` RPC returns the same block as the read state for every height - child.kill(false)?; let output = child.wait_with_output()?; From 112294f1436540d6b5ebfce5908bc3ccbaa43527 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 23:05:43 -0400 Subject: [PATCH 18/30] checks for chain tip changes in test --- zebra-rpc/src/sync.rs | 11 +++--- zebrad/tests/acceptance.rs | 61 ++++++++++++++++++++++++++++------ zebrad/tests/common/regtest.rs | 4 +-- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 0d670d2bbed..93045b183a0 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -193,11 +193,13 @@ impl TrustedChainSync { } /// Tries to catch up to the primary db instance for an up-to-date view of finalized blocks. + // TODO: Use getblock RPC if it fails to catch up to primary? async fn try_catch_up_with_primary(&self) { let db = self.db.clone(); tokio::task::spawn_blocking(move || { - let catch_up_result = db.try_catch_up_with_primary(); - tracing::debug!(?catch_up_result, "trying to catch up to primary"); + if let Err(catch_up_error) = db.try_catch_up_with_primary() { + tracing::warn!(?catch_up_error, "failed to catch up to primary"); + } }) .wait_for_panics() .await @@ -211,8 +213,9 @@ impl TrustedChainSync { self.non_finalized_state.chain_count() == 0 && { let db = self.db.clone(); tokio::task::spawn_blocking(move || { - let catch_up_result = db.try_catch_up_with_primary(); - tracing::debug!(?catch_up_result, "trying to catch up to primary"); + if let Err(catch_up_error) = db.try_catch_up_with_primary() { + tracing::warn!(?catch_up_error, "failed to catch up to primary"); + } db.contains_hash(target_tip_hash) }) .wait_for_panics() diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index fff60003e50..672b7cd9470 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3171,14 +3171,13 @@ async fn regtest_submit_blocks() -> Result<()> { Ok(()) } -// TODO: -// - Test that the `ChainTipChange` updated by the RPC syncer is updated when the finalized tip changes (outside Regtest) #[cfg(feature = "rpc-syncer")] #[tokio::test(flavor = "multi_thread")] async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { use std::sync::Arc; use common::regtest::MiningRpcMethods; + use eyre::Error; use tokio::time::timeout; use zebra_chain::{ chain_tip::ChainTip, parameters::NetworkUpgrade, @@ -3193,9 +3192,9 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { let network = config.network.network.clone(); let rpc_address = config.rpc.listen_addr.unwrap(); - let testdir = testdir()?.with_config(&mut config)?; + let test_dir = testdir()?.with_config(&mut config)?; - let mut child = testdir.spawn_child(args!["start"])?; + let mut child = test_dir.spawn_child(args!["start"])?; tracing::info!("waiting for Zebra state cache to be opened"); @@ -3244,10 +3243,11 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { ); } + tracing::info!("checking that read state has the new non-finalized best chain blocks"); for expected_block in blocks.clone() { let height = expected_block.coinbase_height().unwrap(); let zebra_block = rpc_client - .get_block(height.0) + .get_block(height.0 as i32) .await .map_err(|err| eyre!(err))? .expect("Zebra test child should have the expected block"); @@ -3361,10 +3361,11 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { "tip action hashes and heights should match" ); + tracing::info!("checking that read state has the new non-finalized best chain blocks"); for expected_block in blocks { let height = expected_block.coinbase_height().unwrap(); let zebra_block = rpc_client - .get_block(height.0) + .get_block(height.0 as i32) .await .map_err(|err| eyre!(err))? .expect("Zebra test child should have the expected block"); @@ -3391,6 +3392,8 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { ); } + tracing::info!("restarting Zebra on Mainnet"); + child.kill(false)?; let output = child.wait_with_output()?; @@ -3399,10 +3402,48 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { output.assert_failure()?; - // TODO: - // - Start another Zebra testchild on Testnet or Mainnet - // - Wait for it to start syncing blocks from the network - // - Check that `LatestChainTip` is being updated with the newly synced finalized blocks + let mut config = random_known_rpc_port_config(false, &Network::Mainnet)?; + config.state.ephemeral = false; + let rpc_address = config.rpc.listen_addr.unwrap(); + + let test_dir = testdir()?.with_config(&mut config)?; + + let mut child = test_dir.spawn_child(args!["start"])?; + + tracing::info!("waiting for Zebra state cache to be opened"); + + child.expect_stdout_line_matches(format!( + "Opened Zebra state cache at {}", + config.state.cache_dir.to_str().unwrap() + ))?; + + tracing::info!("starting read state with syncer"); + // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra + let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = + zebra_rpc::sync::init_read_state_with_syncer( + config.state, + &config.network.network, + rpc_address, + ) + .await? + .map_err(|err| eyre!(err))?; + + tracing::info!("waiting for finalized chain tip changes"); + + timeout( + Duration::from_secs(100), + tokio::spawn(async move { + for _ in 0..2 { + chain_tip_change + .wait_for_tip_change() + .await + .map_err(|err| eyre!(err))?; + } + + Ok::<(), Error>(()) + }), + ) + .await???; Ok(()) } diff --git a/zebrad/tests/common/regtest.rs b/zebrad/tests/common/regtest.rs index 1b36cba80e5..5140314b04b 100644 --- a/zebrad/tests/common/regtest.rs +++ b/zebrad/tests/common/regtest.rs @@ -96,7 +96,7 @@ pub trait MiningRpcMethods { async fn block_from_template(&self, nu5_activation_height: Height) -> Result<(Block, Height)>; async fn submit_block(&self, block: Block) -> Result<()>; async fn submit_block_from_template(&self) -> Result<(Block, Height)>; - async fn get_block(&self, height: u32) -> Result>, BoxError>; + async fn get_block(&self, height: i32) -> Result>, BoxError>; } impl MiningRpcMethods for RpcRequestClient { @@ -146,7 +146,7 @@ impl MiningRpcMethods for RpcRequestClient { Ok((block, height)) } - async fn get_block(&self, height: u32) -> Result>, BoxError> { + async fn get_block(&self, height: i32) -> Result>, BoxError> { match self .json_result_from_call("getblock", format!(r#"["{}", 0]"#, height)) .await From ce5f7edc09d2e1bb6357a0cd8d7780d68a9633b9 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 23:07:37 -0400 Subject: [PATCH 19/30] run test without feature --- zebrad/tests/acceptance.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 446baac8a2c..e224b3e9d09 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3174,7 +3174,6 @@ async fn regtest_submit_blocks() -> Result<()> { Ok(()) } -#[cfg(feature = "rpc-syncer")] #[tokio::test(flavor = "multi_thread")] async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { use std::sync::Arc; From e13202db442ebd334c267b0f370fd11ec8bd0f03 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 13 Jun 2024 23:13:22 -0400 Subject: [PATCH 20/30] fixes lint --- zebra-chain/src/sapling/keys.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-chain/src/sapling/keys.rs b/zebra-chain/src/sapling/keys.rs index 7b6e783b8c3..0e395c59eda 100644 --- a/zebra-chain/src/sapling/keys.rs +++ b/zebra-chain/src/sapling/keys.rs @@ -91,7 +91,7 @@ pub(super) fn find_group_hash(d: [u8; 8], m: &[u8]) -> jubjub::ExtendedPoint { let gh = jubjub_group_hash(d, &tag[..]); // We don't want to overflow and start reusing generators - assert!(tag[i] != u8::max_value()); + assert!(tag[i] != u8::MAX); tag[i] += 1; if let Some(gh) = gh { From bc378071d45d93b34229cc47c1e369860ad520e0 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 17 Jun 2024 16:38:55 -0400 Subject: [PATCH 21/30] Fixes compilation/test issues --- zebra-rpc/src/sync.rs | 2 +- zebra-state/src/lib.rs | 10 ++++------ zebra-state/src/service/finalized_state/disk_db.rs | 5 +++++ zebra-state/src/service/finalized_state/zebra_db.rs | 5 +++++ zebrad/tests/acceptance.rs | 6 +++++- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 93045b183a0..c679a01c665 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -296,7 +296,7 @@ impl TrustedChainSync { /// non-finalized best chain and the latest chain tip. /// /// Returns a [`ReadStateService`], [`LatestChainTip`], [`ChainTipChange`], and -/// a [`JoinHandle`](tokio::task::JoinHandle) for the sync task. +/// a [`JoinHandle`] for the sync task. pub fn init_read_state_with_syncer( config: zebra_state::Config, network: &Network, diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 58a83c9e15d..e93a3b8f905 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -63,14 +63,12 @@ pub use service::finalized_state::{ // Allow use in the scanner and external tests #[cfg(any(test, feature = "proptest-impl", feature = "shielded-scan"))] -pub use service::{ - finalized_state::{ - DiskWriteBatch, FromDisk, IntoDisk, ReadDisk, TypedColumnFamily, WriteDisk, - WriteTypedBatch, ZebraDb, - }, - ReadStateService, +pub use service::finalized_state::{ + DiskWriteBatch, FromDisk, IntoDisk, ReadDisk, TypedColumnFamily, WriteDisk, WriteTypedBatch, }; +pub use service::{finalized_state::ZebraDb, ReadStateService}; + #[cfg(feature = "getblocktemplate-rpcs")] pub use response::GetBlockTemplateChainInfo; diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 642240d75f3..902c624885d 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -567,6 +567,11 @@ impl DiskDb { ); } + /// When called with a secondary DB instance, tries to catch up with the primary DB instance + pub fn try_catch_up_with_primary(&self) -> Result<(), rocksdb::Error> { + self.db.try_catch_up_with_primary() + } + /// Returns a forward iterator over the items in `cf` in `range`. /// /// Holding this iterator open might delay block commit transactions. diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index 17891e38f39..56145b1d4e2 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -233,6 +233,11 @@ impl ZebraDb { } } + /// When called with a secondary DB instance, tries to catch up with the primary DB instance + pub fn try_catch_up_with_primary(&self) -> Result<(), rocksdb::Error> { + self.db.try_catch_up_with_primary() + } + /// Shut down the database, cleaning up background tasks and ephemeral data. /// /// If `force` is true, clean up regardless of any shared references. diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index e224b3e9d09..ec0d35b76ab 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3200,11 +3200,15 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { tracing::info!("waiting for Zebra state cache to be opened"); + #[cfg(not(target_os = "windows"))] child.expect_stdout_line_matches(format!( "Opened Zebra state cache at {}", - config.state.cache_dir.to_str().unwrap() + config.state.cache_dir.to_str().expect("should convert") ))?; + #[cfg(target_os = "windows")] + tokio::time::sleep(Duration::from_secs(LAUNCH_DELAY)).await; + tracing::info!("starting read state with syncer"); // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra let (read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = From fd1ab3b51e49820f91aac15cb457b33db54f50ba Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 17 Jun 2024 17:07:01 -0400 Subject: [PATCH 22/30] Adds docs / comments, moves HexData out from behind the getblocktemplate-rpcs feature flag, moves test behind the mining feature flag. --- zebra-rpc/src/methods.rs | 1 + zebra-rpc/src/methods/get_block_template_rpcs.rs | 5 +++-- .../src/methods/get_block_template_rpcs/types.rs | 1 - .../types/get_block_template/parameters.rs | 2 +- .../types => }/hex_data.rs | 0 .../tests/snapshot/get_block_template_rpcs.rs | 2 +- zebra-rpc/src/methods/tests/vectors.rs | 5 +++-- zebra-rpc/src/sync.rs | 15 ++++++++++++--- zebrad/tests/acceptance.rs | 1 + 9 files changed, 22 insertions(+), 10 deletions(-) rename zebra-rpc/src/methods/{get_block_template_rpcs/types => }/hex_data.rs (100%) diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index fc12d2ed19d..b0f55867ee6 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -38,6 +38,7 @@ use crate::{ }; mod errors; +pub mod hex_data; use errors::{MapServerError, OkOrServerError}; diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 77267d006f0..a1c368ac4e0 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -46,7 +46,6 @@ use crate::methods::{ types::{ get_block_template::GetBlockTemplate, get_mining_info, - hex_data::HexData, long_poll::LongPollInput, peer_info::PeerInfo, submit_block, @@ -54,7 +53,9 @@ use crate::methods::{ unified_address, validate_address, z_validate_address, }, }, - height_from_signed_int, GetBlockHash, MISSING_BLOCK_ERROR_CODE, + height_from_signed_int, + hex_data::HexData, + GetBlockHash, MISSING_BLOCK_ERROR_CODE, }; pub mod constants; diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types.rs index fc3b94cee81..a2f5ccc265e 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types.rs @@ -3,7 +3,6 @@ pub mod default_roots; pub mod get_block_template; pub mod get_mining_info; -pub mod hex_data; pub mod long_poll; pub mod peer_info; pub mod submit_block; diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs index 9329bb65c6f..73e1ed820ba 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs @@ -1,6 +1,6 @@ //! Parameter types for the `getblocktemplate` RPC. -use crate::methods::get_block_template_rpcs::types::{hex_data::HexData, long_poll::LongPollId}; +use crate::methods::{get_block_template_rpcs::types::long_poll::LongPollId, hex_data::HexData}; /// Defines whether the RPC method should generate a block template or attempt to validate a block proposal. #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)] diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/hex_data.rs b/zebra-rpc/src/methods/hex_data.rs similarity index 100% rename from zebra-rpc/src/methods/get_block_template_rpcs/types/hex_data.rs rename to zebra-rpc/src/methods/hex_data.rs diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index 04b3139913f..93343d6059f 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -39,13 +39,13 @@ use crate::methods::{ get_block_template_rpcs::types::{ get_block_template::{self, GetBlockTemplateRequestMode}, get_mining_info, - hex_data::HexData, long_poll::{LongPollId, LONG_POLL_ID_LENGTH}, peer_info::PeerInfo, submit_block, subsidy::BlockSubsidy, unified_address, validate_address, z_validate_address, }, + hex_data::HexData, tests::{snapshot::EXCESSIVE_BLOCK_HEIGHT, utils::fake_history_tree}, GetBlockHash, GetBlockTemplateRpc, GetBlockTemplateRpcImpl, }; diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 4d830c51e93..5b5a21e23d0 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1259,8 +1259,9 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { GET_BLOCK_TEMPLATE_NONCE_RANGE_FIELD, }, get_block_template::{self, GetBlockTemplateRequestMode}, - types::{hex_data::HexData, long_poll::LONG_POLL_ID_LENGTH}, + types::long_poll::LONG_POLL_ID_LENGTH, }, + hex_data::HexData, tests::utils::fake_history_tree, }; @@ -1547,7 +1548,7 @@ async fn rpc_submitblock_errors() { use zebra_chain::chain_sync_status::MockSyncStatus; use zebra_network::address_book_peers::MockAddressBookPeers; - use crate::methods::get_block_template_rpcs::types::{hex_data::HexData, submit_block}; + use crate::methods::{get_block_template_rpcs::types::submit_block, hex_data::HexData}; let _init_guard = zebra_test::init(); diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index c679a01c665..db7d7c30faf 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -22,7 +22,7 @@ use zebra_chain::diagnostic::task::WaitForPanics; use crate::{ constants::MISSING_BLOCK_ERROR_CODE, - methods::{get_block_template_rpcs::types::hex_data::HexData, GetBlockHeightAndHash}, + methods::{hex_data::HexData, GetBlockHeightAndHash}, }; /// How long to wait between calls to `getbestblockhash` when it returns an error or the block hash @@ -45,7 +45,10 @@ struct TrustedChainSync { } impl TrustedChainSync { - /// Creates a new [`TrustedChainSync`] and starts syncing blocks from the node's non-finalized best chain. + /// Creates a new [`TrustedChainSync`] with a [`ChainTipSender`], then spawns a task to sync blocks + /// from the node's non-finalized best chain. + /// + /// Returns the [`LatestChainTip`], [`ChainTipChange`], and a [`JoinHandle`] for the sync task. pub async fn spawn( rpc_address: SocketAddr, db: ZebraDb, @@ -71,7 +74,11 @@ impl TrustedChainSync { (latest_chain_tip, chain_tip_change, sync_task) } - /// Starts syncing blocks from the node's non-finalized best chain. + /// Starts syncing blocks from the node's non-finalized best chain and checking for chain tip changes in the finalized state. + /// + /// When the best chain tip in Zebra is not available in the finalized state or the local non-finalized state, + /// gets any unavailable blocks in Zebra's best chain from the RPC server, adds them to the local non-finalized state, then + /// sends the updated chain tip block and non-finalized state to the [`ChainTipSender`] and non-finalized state sender. async fn sync(&mut self) { self.try_catch_up_with_primary().await; let mut last_chain_tip_hash = @@ -103,6 +110,8 @@ impl TrustedChainSync { continue; } + // If the new best chain tip is unavailable in the finalized state, start syncing non-finalized blocks from + // the non-finalized best chain tip height or finalized tip height. let (next_block_height, mut current_tip_hash) = self.next_block_height_and_prev_hash().await; diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index ec0d35b76ab..e5480f3f483 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3175,6 +3175,7 @@ async fn regtest_submit_blocks() -> Result<()> { } #[tokio::test(flavor = "multi_thread")] +#[cfg(feature = "getblocktemplate-rpcs")] async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { use std::sync::Arc; From 737c2e5044df095b59710cfbc56f0a4ebcc97f91 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 17 Jun 2024 17:13:34 -0400 Subject: [PATCH 23/30] Fixes lints --- zebrad/src/components/miner.rs | 10 ++++------ zebrad/tests/common/regtest.rs | 11 +++++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/zebrad/src/components/miner.rs b/zebrad/src/components/miner.rs index 193f30e8c5b..cb32cc91981 100644 --- a/zebrad/src/components/miner.rs +++ b/zebrad/src/components/miner.rs @@ -30,13 +30,11 @@ use zebra_node_services::mempool; use zebra_rpc::{ config::mining::Config, methods::{ - get_block_template_rpcs::{ - get_block_template::{ - self, proposal::TimeSource, proposal_block_from_template, - GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, - }, - types::hex_data::HexData, + get_block_template_rpcs::get_block_template::{ + self, proposal::TimeSource, proposal_block_from_template, + GetBlockTemplateCapability::*, GetBlockTemplateRequestMode::*, }, + hex_data::HexData, GetBlockTemplateRpc, GetBlockTemplateRpcImpl, }, }; diff --git a/zebrad/tests/common/regtest.rs b/zebrad/tests/common/regtest.rs index 5140314b04b..6d83b69a881 100644 --- a/zebrad/tests/common/regtest.rs +++ b/zebrad/tests/common/regtest.rs @@ -18,11 +18,14 @@ use zebra_chain::{ use zebra_node_services::rpc_client::RpcRequestClient; use zebra_rpc::{ constants::MISSING_BLOCK_ERROR_CODE, - methods::get_block_template_rpcs::{ - get_block_template::{ - proposal::TimeSource, proposal_block_from_template, GetBlockTemplate, + methods::{ + get_block_template_rpcs::{ + get_block_template::{ + proposal::TimeSource, proposal_block_from_template, GetBlockTemplate, + }, + types::submit_block, }, - types::{hex_data::HexData, submit_block}, + hex_data::HexData, }, }; use zebra_test::args; From 28902fa397b28fe4790b5a4e754ab11f2085ead7 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 17 Jun 2024 22:35:33 -0400 Subject: [PATCH 24/30] removes syncer and rpc-syncer features --- zebra-rpc/Cargo.toml | 5 +---- zebrad/Cargo.toml | 4 +--- zebrad/tests/acceptance.rs | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index b0d59bb123d..c9a6167287b 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -15,7 +15,6 @@ keywords = ["zebra", "zcash"] categories = ["asynchronous", "cryptography::cryptocurrencies", "encoding", "network-programming"] [features] -default = ["syncer"] # Production features that activate extra dependencies, or extra features in dependencies @@ -32,8 +31,6 @@ getblocktemplate-rpcs = [ # Experimental internal miner support internal-miner = [] -syncer = ["zebra-node-services/rpc-client"] - # Test-only features proptest-impl = [ "proptest", @@ -79,7 +76,7 @@ proptest = { version = "1.4.0", optional = true } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.37", features = ["json-conversion"] } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.37" } zebra-network = { path = "../zebra-network", version = "1.0.0-beta.37" } -zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.37" } +zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.37", features = ["rpc-client"]} zebra-script = { path = "../zebra-script", version = "1.0.0-beta.37" } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.37" } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 9c1f732166b..399e367d196 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -100,8 +100,6 @@ progress-bar = [ prometheus = ["metrics-exporter-prometheus"] -rpc-syncer = ["zebra-rpc/syncer"] - # Production features that modify dependency behaviour # Enable additional error debugging in release builds @@ -287,7 +285,7 @@ zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.37", feat zebra-network = { path = "../zebra-network", version = "1.0.0-beta.37", features = ["proptest-impl"] } zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.6", features = ["proptest-impl"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.37", features = ["proptest-impl"] } -zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.37", features = ["syncer"] } +zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.37" } zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.37", features = ["rpc-client"] } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index e5480f3f483..ff743031966 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3208,7 +3208,7 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { ))?; #[cfg(target_os = "windows")] - tokio::time::sleep(Duration::from_secs(LAUNCH_DELAY)).await; + tokio::time::sleep(LAUNCH_DELAY).await; tracing::info!("starting read state with syncer"); // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra From 15feed173daced582340fdcdb4d4bb8cc9593a39 Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 19 Jun 2024 19:28:21 -0400 Subject: [PATCH 25/30] Fixes test on Windows, applies suggestions from code review --- zebra-rpc/src/methods.rs | 13 +++++++++++++ zebra-rpc/src/sync.rs | 6 ++++-- zebra-state/src/service/finalized_state/disk_db.rs | 8 ++++++-- zebrad/tests/acceptance.rs | 4 ++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index b0f55867ee6..556b482a29c 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -173,6 +173,10 @@ pub trait Rpc { fn get_best_block_hash(&self) -> Result; /// Returns the height and hash of the current best blockchain tip block, as a [`GetBlockHeightAndHash`] JSON struct. + /// + /// zcashd reference: none + /// method: post + /// tags: blockchain #[rpc(name = "getbestblockheightandhash")] fn get_best_block_height_and_hash(&self) -> Result; @@ -1560,6 +1564,15 @@ pub struct GetBlockHeightAndHash { pub hash: block::Hash, } +impl Default for GetBlockHeightAndHash { + fn default() -> Self { + Self { + height: block::Height::MIN, + hash: block::Hash([0; 32]), + } + } +} + impl Default for GetBlockHash { fn default() -> Self { GetBlockHash(block::Hash([0; 32])) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index db7d7c30faf..9f394bdea32 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -166,6 +166,8 @@ impl TrustedChainSync { break false; } + // TODO: Check the finalized tip height and finalize blocks from the non-finalized state until + // all non-finalized state chain root previous block hashes match the finalized tip hash while self .non_finalized_state .best_chain_len() @@ -202,7 +204,6 @@ impl TrustedChainSync { } /// Tries to catch up to the primary db instance for an up-to-date view of finalized blocks. - // TODO: Use getblock RPC if it fails to catch up to primary? async fn try_catch_up_with_primary(&self) { let db = self.db.clone(); tokio::task::spawn_blocking(move || { @@ -232,7 +233,7 @@ impl TrustedChainSync { } } - /// Returns the current tip height and hash + /// Returns the current tip hash and the next height immediately after the current tip height async fn next_block_height_and_prev_hash(&self) -> (block::Height, block::Hash) { if let Some(tip) = self.non_finalized_state.best_tip() { Some(tip) @@ -251,6 +252,7 @@ impl TrustedChainSync { .unwrap_or((Height::MIN, GENESIS_PREVIOUS_BLOCK_HASH)) } + /// Reads the finalized tip block from the secondary db instance and converts it to a [`ChainTipBlock`]. async fn finalized_chain_tip_block(&self) -> Option { let db = self.db.clone(); tokio::task::spawn_blocking(move || { diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 902c624885d..0c25f6a273c 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -839,9 +839,13 @@ impl DiskDb { .map(|cf_name| rocksdb::ColumnFamilyDescriptor::new(cf_name, db_options.clone())); let db_result = if read_only { - // TODO: Make this path configurable? + // Use a tempfile for the secondary instance cache directory + let secondary_config = Config { + ephemeral: true, + ..config.clone() + }; let secondary_path = - config.db_path("secondary_state", format_version_in_code.major, network); + secondary_config.db_path("secondary_state", format_version_in_code.major, network); let create_dir_result = std::fs::create_dir_all(&secondary_path); info!(?create_dir_result, "creating secondary db directory"); diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index ff743031966..f5119f32c17 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3419,11 +3419,15 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { tracing::info!("waiting for Zebra state cache to be opened"); + #[cfg(not(target_os = "windows"))] child.expect_stdout_line_matches(format!( "Opened Zebra state cache at {}", config.state.cache_dir.to_str().unwrap() ))?; + #[cfg(target_os = "windows")] + tokio::time::sleep(LAUNCH_DELAY).await; + tracing::info!("starting read state with syncer"); // Spawn a read state with the RPC syncer to check that it has the same best chain as Zebra let (_read_state, _latest_chain_tip, mut chain_tip_change, _sync_task) = From 8fb798b2c005fa5c35088c9a6783bdde1fae5d0e Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 20 Jun 2024 10:15:10 -0400 Subject: [PATCH 26/30] Updates `POLL_DELAY` documentation --- zebra-rpc/src/sync.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 9f394bdea32..5bb3eac5ff6 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -25,8 +25,11 @@ use crate::{ methods::{hex_data::HexData, GetBlockHeightAndHash}, }; -/// How long to wait between calls to `getbestblockhash` when it returns an error or the block hash -/// of the current chain tip in the process that's syncing blocks from Zebra. +/// How long to wait between calls to `getbestblockheightandhash` when it: +/// - Returns an error, or +/// - Returns the block hash of a block that the read state already contains, +/// (so that there's nothing for the syncer to do except wait for the next chain tip change). +/// See the [`TrustedChainSync::wait_for_chain_tip_change()`] method documentation for more information. const POLL_DELAY: Duration = Duration::from_millis(200); /// Syncs non-finalized blocks in the best chain from a trusted Zebra node's RPC methods. @@ -34,11 +37,11 @@ const POLL_DELAY: Duration = Duration::from_millis(200); struct TrustedChainSync { /// RPC client for calling Zebra's RPC methods. rpc_client: RpcRequestClient, - /// The read state service + /// The read state service. db: ZebraDb, /// The non-finalized state - currently only contains the best chain. non_finalized_state: NonFinalizedState, - /// The chain tip sender for updating [`LatestChainTip`] and [`ChainTipChange`] + /// The chain tip sender for updating [`LatestChainTip`] and [`ChainTipChange`]. chain_tip_sender: ChainTipSender, /// The non-finalized state sender, for updating the [`ReadStateService`] when the non-finalized best chain changes. non_finalized_state_sender: tokio::sync::watch::Sender, @@ -132,7 +135,7 @@ impl TrustedChainSync { // - the next block's previous block hash doesn't match the expected hash, // - the next block is missing // - the target tip hash is missing from the blocks in `block_futs` - // because there was likely a chain re-org/fork + // because there was likely a chain re-org/fork. Some(Ok(_)) | None => break true, // If calling the `getblock` RPC method fails with an unexpected error, wait for the next chain tip change // without resetting the non-finalized state. @@ -167,7 +170,7 @@ impl TrustedChainSync { } // TODO: Check the finalized tip height and finalize blocks from the non-finalized state until - // all non-finalized state chain root previous block hashes match the finalized tip hash + // all non-finalized state chain root previous block hashes match the finalized tip hash. while self .non_finalized_state .best_chain_len() @@ -233,7 +236,7 @@ impl TrustedChainSync { } } - /// Returns the current tip hash and the next height immediately after the current tip height + /// Returns the current tip hash and the next height immediately after the current tip height. async fn next_block_height_and_prev_hash(&self) -> (block::Height, block::Hash) { if let Some(tip) = self.non_finalized_state.best_tip() { Some(tip) @@ -267,9 +270,9 @@ impl TrustedChainSync { /// Accepts a block hash. /// - /// Polls `getbestblockhash` RPC method until it successfully returns a different hash from the last chain tip hash. + /// Polls `getbestblockhash` RPC method until it successfully returns a block hash that is different from the last chain tip hash. /// - /// Returns the node's best block hash + /// Returns the node's best block hash. async fn wait_for_chain_tip_change( &self, last_chain_tip_hash: block::Hash, @@ -281,6 +284,8 @@ impl TrustedChainSync { .await .filter(|&(_height, hash)| hash != last_chain_tip_hash) else { + // If `get_best_block_height_and_hash()` returns an error, or returns + // the current chain tip hash, wait [`POLL_DELAY`], then try again. tokio::time::sleep(POLL_DELAY).await; continue; }; From f0b5ee52488d0f1e3dbc834d0e20a1b4a41b1df7 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 20 Jun 2024 10:16:59 -0400 Subject: [PATCH 27/30] Updates method docs --- zebra-rpc/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-rpc/src/sync.rs b/zebra-rpc/src/sync.rs index 5bb3eac5ff6..53eed4072f0 100644 --- a/zebra-rpc/src/sync.rs +++ b/zebra-rpc/src/sync.rs @@ -270,7 +270,7 @@ impl TrustedChainSync { /// Accepts a block hash. /// - /// Polls `getbestblockhash` RPC method until it successfully returns a block hash that is different from the last chain tip hash. + /// Polls `getbestblockheightandhash` RPC method until it successfully returns a block hash that is different from the last chain tip hash. /// /// Returns the node's best block hash. async fn wait_for_chain_tip_change( From 274305163026d3f3145a4ef50eddc4819a8d22d6 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 20 Jun 2024 12:32:47 -0400 Subject: [PATCH 28/30] Fixes a test bug --- zebrad/tests/acceptance.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index f5119f32c17..0d1af3fb78a 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3202,10 +3202,7 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { tracing::info!("waiting for Zebra state cache to be opened"); #[cfg(not(target_os = "windows"))] - child.expect_stdout_line_matches(format!( - "Opened Zebra state cache at {}", - config.state.cache_dir.to_str().expect("should convert") - ))?; + child.expect_stdout_line_matches("marked database format as newly created")?; #[cfg(target_os = "windows")] tokio::time::sleep(LAUNCH_DELAY).await; @@ -3420,10 +3417,7 @@ async fn trusted_chain_sync_handles_forks_correctly() -> Result<()> { tracing::info!("waiting for Zebra state cache to be opened"); #[cfg(not(target_os = "windows"))] - child.expect_stdout_line_matches(format!( - "Opened Zebra state cache at {}", - config.state.cache_dir.to_str().unwrap() - ))?; + child.expect_stdout_line_matches("marked database format as newly created")?; #[cfg(target_os = "windows")] tokio::time::sleep(LAUNCH_DELAY).await; From 8391ae73c0d1362afd6fd00be73647c7f8388114 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 8 Jul 2024 15:07:40 -0300 Subject: [PATCH 29/30] use rpc-client feature in zebrad production code --- zebra-node-services/src/rpc_client.rs | 2 +- zebrad/Cargo.toml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/zebra-node-services/src/rpc_client.rs b/zebra-node-services/src/rpc_client.rs index 52c4a923d69..7f5ffbf192e 100644 --- a/zebra-node-services/src/rpc_client.rs +++ b/zebra-node-services/src/rpc_client.rs @@ -1,6 +1,6 @@ //! A client for calling Zebra's JSON-RPC methods. //! -//! Only used in tests and tools. +//! Used in the rpc sync scanning functionality and in various tests and tools. use std::net::SocketAddr; diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index df2a481f228..b9b19846a36 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -161,7 +161,7 @@ test_sync_past_mandatory_checkpoint_testnet = [] zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38" } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38" } zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38" } -zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" } +zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38", features = ["rpc-client"] } zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38" } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" } @@ -286,8 +286,6 @@ zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38", features zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", features = ["proptest-impl"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] } -zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38", features = ["rpc-client"] } - zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" } zebra-grpc = { path = "../zebra-grpc", version = "0.1.0-alpha.5" } From d996cc68b0d006ceefa104cb7c90e53d6aed5844 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 8 Jul 2024 15:41:23 -0300 Subject: [PATCH 30/30] use rpc-client feature in zebra-node-services for building zebra-rpc crate --- zebra-rpc/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index b696922588c..a97f879ef13 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -76,7 +76,7 @@ proptest = { version = "1.4.0", optional = true } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38", features = ["json-conversion"] } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38" } zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38" } -zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" } +zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38", features = ["rpc-client"] } zebra-script = { path = "../zebra-script", version = "1.0.0-beta.38" } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" }