From 1223873401fead28c06fc77d21b6e7efb041e214 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 28 Jun 2024 14:52:47 -0300 Subject: [PATCH 01/19] readd the scanner tests in the new binary context --- Cargo.lock | 4 + zebra-scan/Cargo.toml | 5 + zebra-scan/tests/scanner.rs | 257 ++++++++++++++++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 zebra-scan/tests/scanner.rs diff --git a/Cargo.lock b/Cargo.lock index 4741d56a66d..f5bcd64debe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6055,7 +6055,10 @@ dependencies = [ "serde", "serde_json", "structopt", + "tempfile", "tokio", + "toml 0.8.13", + "tonic 0.11.0", "tower", "tracing", "tracing-subscriber", @@ -6070,6 +6073,7 @@ dependencies = [ "zebra-rpc", "zebra-state", "zebra-test", + "zebrad", ] [[package]] diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index 314fd73ca57..315e6688172 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -102,7 +102,12 @@ ff = "0.13.0" group = "0.13.0" jubjub = "0.10.0" rand = "0.8.5" +tempfile = "3.10.1" zcash_note_encryption = "0.4.0" +toml = "0.8.13" +tonic = "0.11.0" zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" } +zebrad = { path = "../zebrad", version = "1.8.0" } + diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs new file mode 100644 index 00000000000..cc9d67dc894 --- /dev/null +++ b/zebra-scan/tests/scanner.rs @@ -0,0 +1,257 @@ +//! `zebra-scanner` binary tests. +use tempfile::TempDir; + +use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty}; +use zebra_test::{ + args, + command::{Arguments, TestDirExt}, + prelude::*, +}; + +use std::{io::Write, path::Path}; + +/// Test the scanner binary with the `--help` flag. +#[test] +fn scanner_help() -> eyre::Result<()> { + let _init_guard = zebra_test::init(); + + let testdir = testdir()?; + + let child = testdir.spawn_child(args!["--help"], false)?; + let output = child.wait_with_output()?; + let output = output.assert_success()?; + + output.stdout_line_contains("USAGE:")?; + + Ok(()) +} + +/// Test that the scanner binary starts the scan task. +/// +/// This test creates a new zebrad cache dir by using a `zebrad` binary specified in the `CARGO_BIN_EXE_zebrad` env variable. +/// +/// To run it locally, one way is: +/// ``` +/// export CARGO_BIN_EXE_zebrad="/path/to/zebrad_binary" +/// cargo test scan_binary_starts -- --ignored --nocapture +/// ``` +#[ignore] +#[tokio::test] +#[cfg(not(target_os = "windows"))] +async fn scan_binary_starts() -> Result<()> { + use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; + + let _init_guard = zebra_test::init(); + + // Create a directory to dump test data into it. + let test_dir = testdir()?; + + // Create a config for zebrad with a cache directory inside the test directory. + let mut config = zebrad::config::ZebradConfig::default(); + config.state.cache_dir = test_dir.path().join("zebra").to_path_buf(); + let config_file = test_dir.path().join("zebrad.toml"); + std::fs::File::create(config_file.clone())?.write_all(toml::to_string(&config)?.as_bytes())?; + + // Start the node just to make sure a cache is created. + let mut zebrad = test_dir.spawn_child( + args!["-c", config_file.clone().to_str().unwrap(), "start"], + true, + )?; + zebrad.expect_stdout_line_matches("Opened Zebra state cache at .*")?; + + // Kill the node now that we have a valid zebra node cache. + zebrad.kill(false)?; + let output = zebrad.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + + // Create a new directory for the scanner + let test_dir = testdir()?; + + // Create arguments for the scanner + let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string(); + let listen_addr = "127.0.0.1:8231"; + let rpc_listen_addr = "127.0.0.1:8232"; + let zebrad_cache_dir = config.state.cache_dir.clone(); + let scanning_cache_dir = test_dir.path().join("scanner").to_path_buf(); + let args = args![ + "--zebrad-cache-dir", + zebrad_cache_dir.to_str().unwrap(), + "--scanning-cache-dir", + scanning_cache_dir.to_str().unwrap(), + "--sapling-keys-to-scan", + key, + "--listen-addr", + listen_addr, + "--zebra-rpc-listen-addr", + rpc_listen_addr + ]; + + // Start the scaner using another test directory. + let mut zebra_scanner = test_dir.spawn_child(args, false)?; + + // Check scanner was started. + zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + + // Check if the grpc server is up + let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?; + let request = tonic::Request::new(Empty {}); + client.get_info(request).await?; + + // Look for 2 scanner notices indicating we are below sapling activation. + zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; + zebra_scanner.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; + + // Kill the scanner. + zebra_scanner.kill(true)?; + + // Check that scan task started and the scanning is done. + let output = zebra_scanner.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + output.assert_failure()?; + + Ok(()) +} + +/// Test that the scanner can continue scanning where it was left when zebrad restarts. +/// +/// Needs a cache state close to the tip. A possible way to run it locally is: +/// +/// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" +/// cargo test scan_start_where_left -- --ignored --nocapture +/// +/// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops. +/// Then it will restart zebrad and check that it resumes scanning where it was left. +#[ignore] +#[tokio::test] +#[cfg(not(target_os = "windows"))] +async fn scan_start_where_left() -> Result<()> { + use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; + + let _init_guard = zebra_test::init(); + + let zebrad_cachedir = std::env::var("ZEBRA_CACHED_STATE_DIR") + .expect("please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path"); + let scanning_cache_dir = testdir()?.path().join("scanner").to_path_buf(); + + // Create arguments for the scanner + let key = serde_json::json!({"key": ZECPAGES_SAPLING_VIEWING_KEY}).to_string(); + let listen_addr = "127.0.0.1:18231"; + let rpc_listen_addr = "127.0.0.1:18232"; + let args = args![ + "--zebrad-cache-dir", + zebrad_cachedir, + "--scanning-cache-dir", + scanning_cache_dir.to_str().unwrap(), + "--sapling-keys-to-scan", + key, + "--listen-addr", + listen_addr, + "--zebra-rpc-listen-addr", + rpc_listen_addr + ]; + + // Start the scaner using another test directory. + let mut zebra_scanner: TestChild = testdir()?.spawn_child(args.clone(), false)?; + + // Check scanner was started. + zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + + // The first time + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 419200, now at block 420000", + )?; + + // Make sure scanner scans a few blocks. + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 419200, now at block 430000", + )?; + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 419200, now at block 440000", + )?; + + // Kill the scanner. + zebra_scanner.kill(true)?; + + // Check that scan task started and the first scanning is done. + let output = zebra_scanner.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + output.assert_failure()?; + + // Start the node again with the same arguments. + let mut zebra_scanner: TestChild = testdir()?.spawn_child(args, false)?; + + // Resuming message. + zebra_scanner.expect_stdout_line_matches( + "Last scanned height for key number 0 is 439000, resuming at 439001", + )?; + zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + + // Start scanning where it was left. + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 439001, now at block 440000", + )?; + zebra_scanner.expect_stdout_line_matches( + r"Scanning the blockchain for key 0, started at block 439001, now at block 450000", + )?; + + // Kill the scanner. + zebra_scanner.kill(true)?; + + // Check that scan task started and the second scanning is done. + let output = zebra_scanner.wait_with_output()?; + + // Make sure the command was killed + output.assert_was_killed()?; + output.assert_failure()?; + + Ok(()) +} + +/// Create a temporary directory for testing `zebra-scanner`. +pub fn testdir() -> eyre::Result { + tempfile::Builder::new() + .prefix("zebra_scanner_tests") + .tempdir() + .map_err(Into::into) +} + +/// Extension trait for methods on `tempfile::TempDir` for using it as a test +/// directory for `zebra-scanner`. +pub trait ZebradTestDirExt +where + Self: AsRef + Sized, +{ + // Zebra-scanner methods + + /// Spawn `zebra-scanner` (`zebrad` flag = `false`) or `zebrad` (`zebrad` flag = `true`) + /// with `args` as a child process in this test directory. + fn spawn_child(self, args: Arguments, zebrad: bool) -> Result>; +} + +impl ZebradTestDirExt for T +where + Self: TestDirExt + AsRef + Sized, +{ + #[allow(clippy::unwrap_in_result)] + fn spawn_child(self, extra_args: Arguments, zebrad: bool) -> Result> { + let mut args = Arguments::new(); + + args.merge_with(extra_args); + + if zebrad { + self.spawn_child_with_command( + &std::env::var("CARGO_BIN_EXE_zebrad") + .expect("please set CARGO_BIN_EXE_zebrad env var to a zebrad path"), + args, + ) + } else { + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args) + } + } +} From a3ec2d704b94327eaf41e15007a69aeffa274e66 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 28 Jun 2024 15:38:37 -0300 Subject: [PATCH 02/19] remove commented out tests from zebrad --- zebrad/tests/acceptance.rs | 193 ------------------------------------- 1 file changed, 193 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index eee5be6e016..ce73fd43b42 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -2891,199 +2891,6 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> { Ok(()) } -// TODO: Move this test once we have the new scanner binary. -// /// Test that the scanner task gets started when the node starts. -// #[test] -// #[cfg(feature = "shielded-scan")] -// fn scan_task_starts() -> Result<()> { -// use indexmap::IndexMap; -// use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; - -// let _init_guard = zebra_test::init(); - -// let test_type = TestType::LaunchWithEmptyState { -// launches_lightwalletd: false, -// }; -// let mut config = default_test_config(&Mainnet)?; -// let mut keys = IndexMap::new(); -// keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1); -// config.shielded_scan.sapling_keys_to_scan = keys; - -// // Start zebra with the config. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Check scanner was started. -// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?; - -// // Look for 2 scanner notices indicating we are below sapling activation. -// zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; -// zebrad.expect_stdout_line_matches("scanner is waiting for Sapling activation. Current tip: [0-9]{1,4}, Sapling activation: 419200")?; - -// // Kill the node. -// zebrad.kill(false)?; - -// // Check that scan task started and the first scanning is done. -// let output = zebrad.wait_with_output()?; - -// // Make sure the command was killed -// output.assert_was_killed()?; -// output.assert_failure()?; - -// Ok(()) -// } - -// TODO: Move this test once we have the new scanner binary. -// /// Test that the scanner gRPC server starts when the node starts. -// #[tokio::test] -// #[cfg(all(feature = "shielded-scan", not(target_os = "windows")))] -// async fn scan_rpc_server_starts() -> Result<()> { -// use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty}; - -// let _init_guard = zebra_test::init(); - -// let test_type = TestType::LaunchWithEmptyState { -// launches_lightwalletd: false, -// }; - -// let port = random_known_port(); -// let listen_addr = format!("127.0.0.1:{port}"); -// let mut config = default_test_config(&Mainnet)?; -// config.shielded_scan.listen_addr = Some(listen_addr.parse()?); - -// // Start zebra with the config. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Wait until gRPC server is starting. -// tokio::time::sleep(LAUNCH_DELAY).await; -// zebrad.expect_stdout_line_matches("starting scan gRPC server")?; -// tokio::time::sleep(Duration::from_secs(1)).await; - -// let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?; - -// let request = tonic::Request::new(Empty {}); - -// client.get_info(request).await?; - -// // Kill the node. -// zebrad.kill(false)?; - -// // Check that scan task started and the first scanning is done. -// let output = zebrad.wait_with_output()?; - -// // Make sure the command was killed -// output.assert_was_killed()?; -// output.assert_failure()?; - -// Ok(()) -// } - -// TODO: Move this test once we have the new scanner binary. -// /// Test that the scanner can continue scanning where it was left when zebrad restarts. -// /// -// /// Needs a cache state close to the tip. A possible way to run it locally is: -// /// -// /// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" -// /// cargo test scan_start_where_left --features="shielded-scan" -- --ignored --nocapture -// /// -// /// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops. -// /// Then it will restart zebrad and check that it resumes scanning where it was left. -// /// -// /// Note: This test will remove all the contents you may have in the ZEBRA_CACHED_STATE_DIR/private-scan directory -// /// so it can start with an empty scanning state. -// #[ignore] -// #[test] -// #[cfg(feature = "shielded-scan")] -// fn scan_start_where_left() -> Result<()> { -// use indexmap::IndexMap; -// use zebra_scan::{storage::db::SCANNER_DATABASE_KIND, tests::ZECPAGES_SAPLING_VIEWING_KEY}; - -// let _init_guard = zebra_test::init(); - -// // use `UpdateZebraCachedStateNoRpc` as the test type to make sure a zebrad cache state is available. -// let test_type = TestType::UpdateZebraCachedStateNoRpc; -// if let Some(cache_dir) = test_type.zebrad_state_path("scan test") { -// // Add a key to the config -// let mut config = default_test_config(&Mainnet)?; -// let mut keys = IndexMap::new(); -// keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1); -// config.shielded_scan.sapling_keys_to_scan = keys; - -// // Add the cache dir to shielded scan, make it the same as the zebrad cache state. -// config -// .shielded_scan -// .db_config_mut() -// .cache_dir -// .clone_from(&cache_dir); -// config.shielded_scan.db_config_mut().ephemeral = false; - -// // Add the cache dir to state. -// config.state.cache_dir.clone_from(&cache_dir); -// config.state.ephemeral = false; - -// // Remove the scan directory before starting. -// let scan_db_path = cache_dir.join(SCANNER_DATABASE_KIND); -// fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok(); - -// // Start zebra with the config. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Check scanner was started. -// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?; - -// // The first time -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 419200, now at block 420000", -// )?; - -// // Make sure scanner scans a few blocks. -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 419200, now at block 430000", -// )?; -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 419200, now at block 440000", -// )?; - -// // Kill the node. -// zebrad.kill(false)?; -// let output = zebrad.wait_with_output()?; - -// // Make sure the command was killed -// output.assert_was_killed()?; -// output.assert_failure()?; - -// // Start the node again. -// let mut zebrad = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .with_timeout(test_type.zebrad_timeout()); - -// // Resuming message. -// zebrad.expect_stdout_line_matches( -// "Last scanned height for key number 0 is 439000, resuming at 439001", -// )?; -// zebrad.expect_stdout_line_matches("loaded Zebra scanner cache")?; - -// // Start scanning where it was left. -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 439001, now at block 440000", -// )?; -// zebrad.expect_stdout_line_matches( -// r"Scanning the blockchain for key 0, started at block 439001, now at block 450000", -// )?; -// } - -// Ok(()) -// } - // TODO: Add this test to CI (#8236) /// Tests successful: /// - Registration of a new key, From 749e23ebdccb75d6c176b0da8054ef9ed678764b Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 9 Jul 2024 17:22:46 -0300 Subject: [PATCH 03/19] add Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f5bcd64debe..1a98a440328 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6057,7 +6057,7 @@ dependencies = [ "structopt", "tempfile", "tokio", - "toml 0.8.13", + "toml 0.8.14", "tonic 0.11.0", "tower", "tracing", From ec5e685be5b4e4300caa75e5f507c2f8aa365bbb Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 9 Jul 2024 21:02:22 -0300 Subject: [PATCH 04/19] add local copy of `ZECPAGES_SAPLING_VIEWING_KEY` to binary scanning tests --- zebra-scan/tests/scanner.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index cc9d67dc894..0751b326b6d 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -10,6 +10,9 @@ use zebra_test::{ use std::{io::Write, path::Path}; +/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) +const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; + /// Test the scanner binary with the `--help` flag. #[test] fn scanner_help() -> eyre::Result<()> { @@ -39,8 +42,6 @@ fn scanner_help() -> eyre::Result<()> { #[tokio::test] #[cfg(not(target_os = "windows"))] async fn scan_binary_starts() -> Result<()> { - use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; - let _init_guard = zebra_test::init(); // Create a directory to dump test data into it. From f8bfd9dff1fd61e1d3c6d68769fa2ca3269660af Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 10 Jul 2024 09:33:11 -0400 Subject: [PATCH 05/19] moves scanner-results-reader to zebra-scan, adds zebrad bin to zebra-scan, adds a short sleep before trying to connect to gRPC server in test --- Cargo.lock | 4 ++-- zebra-scan/Cargo.toml | 22 +++++++++++++++++-- .../src/bin/scanning-results-reader/main.rs | 0 zebra-scan/src/bin/zebrad/main.rs | 8 +++++++ zebra-scan/tests/scanner.rs | 12 +++++----- zebra-utils/Cargo.toml | 7 ------ zebrad/Cargo.toml | 8 ------- 7 files changed, 36 insertions(+), 25 deletions(-) rename {zebra-utils => zebra-scan}/src/bin/scanning-results-reader/main.rs (100%) create mode 100644 zebra-scan/src/bin/zebrad/main.rs diff --git a/Cargo.lock b/Cargo.lock index 1a98a440328..258b8103c4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6042,9 +6042,11 @@ dependencies = [ "ff", "futures", "group", + "hex", "indexmap 2.2.6", "insta", "itertools 0.13.0", + "jsonrpc", "jubjub", "lazy_static", "proptest", @@ -6189,7 +6191,6 @@ dependencies = [ "zebra-chain", "zebra-node-services", "zebra-rpc", - "zebra-scan", ] [[package]] @@ -6256,7 +6257,6 @@ dependencies = [ "zebra-network", "zebra-node-services", "zebra-rpc", - "zebra-scan", "zebra-state", "zebra-test", "zebra-utils", diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index 315e6688172..dbefbe9924a 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -23,6 +23,16 @@ required-features = ["proptest-impl"] name = "zebra-scanner" path = "src/bin/scanner/main.rs" +[[bin]] +name = "scanning-results-reader" +path = "src/bin/scanning-results-reader/main.rs" +required-features = ["results-reader"] + +[[bin]] # Bin to run zebrad, used in tests +name = "zebrad" +path = "src/bin/zebrad/main.rs" +required-features = ["proptest-impl"] + [features] # Production features that activate extra dependencies, or extra features in dependencies @@ -41,10 +51,14 @@ proptest-impl = [ "jubjub", "rand", "zcash_note_encryption", + "zebrad" ] # Needed for the zebra-scanner binary. -shielded-scan = [] +results-reader = [ + "jsonrpc", + "hex" +] [dependencies] @@ -91,6 +105,11 @@ structopt = "0.3.26" lazy_static = "1.4.0" serde_json = "1.0.117" +jsonrpc = { version = "0.18.0", optional = true } +hex = { version = "0.4.3", optional = true } + +zebrad = { path = "../zebrad", version = "1.8.0", optional = true } + [dev-dependencies] insta = { version = "1.39.0", features = ["ron", "redactions"] } tokio = { version = "1.37.0", features = ["test-util"] } @@ -109,5 +128,4 @@ tonic = "0.11.0" zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" } -zebrad = { path = "../zebrad", version = "1.8.0" } diff --git a/zebra-utils/src/bin/scanning-results-reader/main.rs b/zebra-scan/src/bin/scanning-results-reader/main.rs similarity index 100% rename from zebra-utils/src/bin/scanning-results-reader/main.rs rename to zebra-scan/src/bin/scanning-results-reader/main.rs diff --git a/zebra-scan/src/bin/zebrad/main.rs b/zebra-scan/src/bin/zebrad/main.rs new file mode 100644 index 00000000000..0eaa81677ce --- /dev/null +++ b/zebra-scan/src/bin/zebrad/main.rs @@ -0,0 +1,8 @@ +//! Main entry point for Zebrad, to be used in zebra-scan tests + +use zebrad::application::{boot, APPLICATION}; + +/// Process entry point for `zebrad` +fn main() { + boot(&APPLICATION); +} diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index 0751b326b6d..48b60db5ece 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -38,10 +38,11 @@ fn scanner_help() -> eyre::Result<()> { /// export CARGO_BIN_EXE_zebrad="/path/to/zebrad_binary" /// cargo test scan_binary_starts -- --ignored --nocapture /// ``` -#[ignore] #[tokio::test] #[cfg(not(target_os = "windows"))] async fn scan_binary_starts() -> Result<()> { + use std::time::Duration; + let _init_guard = zebra_test::init(); // Create a directory to dump test data into it. @@ -95,6 +96,9 @@ async fn scan_binary_starts() -> Result<()> { // Check scanner was started. zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; + // Wait for the scanner's gRPC server to start up + tokio::time::sleep(Duration::from_secs(1)).await; + // Check if the grpc server is up let mut client = ScannerClient::connect(format!("http://{listen_addr}")).await?; let request = tonic::Request::new(Empty {}); @@ -246,11 +250,7 @@ where args.merge_with(extra_args); if zebrad { - self.spawn_child_with_command( - &std::env::var("CARGO_BIN_EXE_zebrad") - .expect("please set CARGO_BIN_EXE_zebrad env var to a zebrad path"), - args, - ) + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad"), args) } else { self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args) } diff --git a/zebra-utils/Cargo.toml b/zebra-utils/Cargo.toml index 52883258fd7..15cfd4bbca1 100644 --- a/zebra-utils/Cargo.toml +++ b/zebra-utils/Cargo.toml @@ -36,11 +36,6 @@ name = "block-template-to-proposal" path = "src/bin/block-template-to-proposal/main.rs" required-features = ["getblocktemplate-rpcs"] -[[bin]] -name = "scanning-results-reader" -path = "src/bin/scanning-results-reader/main.rs" -required-features = ["shielded-scan"] - [[bin]] name = "openapi-generator" path = "src/bin/openapi-generator/main.rs" @@ -76,7 +71,6 @@ shielded-scan = [ "jsonrpc", "zcash_primitives", "zcash_client_backend", - "zebra-scan" ] openapi-generator = [ @@ -102,7 +96,6 @@ thiserror = "1.0.61" zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38" } -zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true } # These crates are needed for the block-template-to-proposal binary zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38", optional = true } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index b9b19846a36..dbb2640c281 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -77,9 +77,6 @@ internal-miner = [ "zebra-rpc/getblocktemplate-rpcs", ] -# Experimental shielded blockchain scanning -shielded-scan = ["zebra-scan"] - # Experimental elasticsearch indexing elasticsearch = [ "zebra-state/elasticsearch", @@ -127,7 +124,6 @@ proptest-impl = [ "zebra-state/proptest-impl", "zebra-network/proptest-impl", "zebra-chain/proptest-impl", - "zebra-scan?/proptest-impl", ] # Build the zebra-checkpoints utility for checkpoint generation tests @@ -165,9 +161,6 @@ zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.3 zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.38" } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" } -# Experimental shielded-scan feature -zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", optional = true } - # Required for crates.io publishing, but it's only used in tests zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.38", optional = true } @@ -283,7 +276,6 @@ color-eyre = { version = "0.6.3" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38", features = ["proptest-impl"] } -zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.7", features = ["proptest-impl"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.38" } From 213496d6fb169a9fb3178e32f588adcfd98b8db2 Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 10 Jul 2024 10:06:24 -0400 Subject: [PATCH 06/19] always include zebrad dependency in zebra-scan --- zebra-scan/Cargo.toml | 4 +--- zebra-scan/tests/scanner.rs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index dbefbe9924a..b6ef2eb5ad3 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -31,7 +31,6 @@ required-features = ["results-reader"] [[bin]] # Bin to run zebrad, used in tests name = "zebrad" path = "src/bin/zebrad/main.rs" -required-features = ["proptest-impl"] [features] @@ -51,7 +50,6 @@ proptest-impl = [ "jubjub", "rand", "zcash_note_encryption", - "zebrad" ] # Needed for the zebra-scanner binary. @@ -108,7 +106,7 @@ serde_json = "1.0.117" jsonrpc = { version = "0.18.0", optional = true } hex = { version = "0.4.3", optional = true } -zebrad = { path = "../zebrad", version = "1.8.0", optional = true } +zebrad = { path = "../zebrad", version = "1.8.0" } [dev-dependencies] insta = { version = "1.39.0", features = ["ron", "redactions"] } diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index 48b60db5ece..16e91853fa1 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -134,7 +134,7 @@ async fn scan_binary_starts() -> Result<()> { #[tokio::test] #[cfg(not(target_os = "windows"))] async fn scan_start_where_left() -> Result<()> { - use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; + use ZECPAGES_SAPLING_VIEWING_KEY; let _init_guard = zebra_test::init(); From 0e3b832b0e917534f967ebfe1036cf0582d870ab Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 10 Jul 2024 11:23:54 -0400 Subject: [PATCH 07/19] renames zebrad bin in zebra-scan to zebrad-for-scanner --- zebra-scan/Cargo.toml | 6 +++--- zebra-scan/src/bin/{zebrad => zebrad-for-scanner}/main.rs | 0 zebra-scan/tests/scanner.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename zebra-scan/src/bin/{zebrad => zebrad-for-scanner}/main.rs (100%) diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index b6ef2eb5ad3..61bf2f7e9f8 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -28,9 +28,9 @@ name = "scanning-results-reader" path = "src/bin/scanning-results-reader/main.rs" required-features = ["results-reader"] -[[bin]] # Bin to run zebrad, used in tests -name = "zebrad" -path = "src/bin/zebrad/main.rs" +[[bin]] # Bin to run zebrad, used in scanner tests +name = "zebrad-for-scanner" +path = "src/bin/zebrad-for-scanner/main.rs" [features] diff --git a/zebra-scan/src/bin/zebrad/main.rs b/zebra-scan/src/bin/zebrad-for-scanner/main.rs similarity index 100% rename from zebra-scan/src/bin/zebrad/main.rs rename to zebra-scan/src/bin/zebrad-for-scanner/main.rs diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index 16e91853fa1..89ae39a0333 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -250,7 +250,7 @@ where args.merge_with(extra_args); if zebrad { - self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad"), args) + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad-for-scanner"), args) } else { self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args) } From 314c4afedcab1df24dd4172668b827199f466fa3 Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 10 Jul 2024 11:27:05 -0400 Subject: [PATCH 08/19] skip test when a cached state is missing --- zebra-scan/tests/scanner.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index 89ae39a0333..0b7ec334117 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -138,8 +138,12 @@ async fn scan_start_where_left() -> Result<()> { let _init_guard = zebra_test::init(); - let zebrad_cachedir = std::env::var("ZEBRA_CACHED_STATE_DIR") - .expect("please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path"); + let Ok(zebrad_cachedir) = std::env::var("ZEBRA_CACHED_STATE_DIR") else { + tracing::info!("skipping scan_start_where_left test due to missing cached state, \ + please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path to run this test"); + return Ok(()); + }; + let scanning_cache_dir = testdir()?.path().join("scanner").to_path_buf(); // Create arguments for the scanner From 54bc897abc46a75c8c709a20eecc5623d9bc7dce Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 10 Jul 2024 15:09:00 -0300 Subject: [PATCH 09/19] remove scanning config from zebrad --- zebrad/src/config.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index c0b22479c36..a6174599ef8 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -54,8 +54,4 @@ pub struct ZebradConfig { #[serde(skip_serializing_if = "zebra_rpc::config::mining::Config::skip_getblocktemplate")] /// Mining configuration pub mining: zebra_rpc::config::mining::Config, - - #[cfg(feature = "shielded-scan")] - /// Scanner configuration - pub shielded_scan: zebra_scan::config::Config, } From 508e1b0808227f2ead704e5c089b35510f657eab Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jul 2024 16:46:11 -0300 Subject: [PATCH 10/19] move `scan_task_commands` out of zebrad acceptance tests --- .../tests}/scan_task_commands.rs | 72 +++++++++++-------- zebra-scan/tests/scanner.rs | 23 ++++++ zebrad/src/config.rs | 4 ++ zebrad/tests/acceptance.rs | 22 ------ zebrad/tests/common/mod.rs | 3 - zebrad/tests/common/shielded_scan.rs | 3 - 6 files changed, 70 insertions(+), 57 deletions(-) rename {zebrad/tests/common/shielded_scan => zebra-scan/tests}/scan_task_commands.rs (76%) delete mode 100644 zebrad/tests/common/shielded_scan.rs diff --git a/zebrad/tests/common/shielded_scan/scan_task_commands.rs b/zebra-scan/tests/scan_task_commands.rs similarity index 76% rename from zebrad/tests/common/shielded_scan/scan_task_commands.rs rename to zebra-scan/tests/scan_task_commands.rs index a7693b8c812..914710fbb41 100644 --- a/zebrad/tests/common/shielded_scan/scan_task_commands.rs +++ b/zebra-scan/tests/scan_task_commands.rs @@ -5,12 +5,14 @@ //! //! export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" //! cargo test scan_task_commands --features="shielded-scan" -- --ignored --nocapture +#![allow(dead_code)] use std::{fs, time::Duration}; use color_eyre::{eyre::eyre, Result}; - use tokio::sync::mpsc::error::TryRecvError; +use tower::{util::BoxService, Service}; + use zebra_chain::{ block::Height, chain_tip::ChainTip, @@ -20,13 +22,12 @@ use zebra_chain::{ use zebra_scan::{ service::ScanTask, storage::{db::SCANNER_DATABASE_KIND, Storage}, - tests::ZECPAGES_SAPLING_VIEWING_KEY, }; -use crate::common::{ - cached_state::start_state_service_with_cache_dir, launch::can_spawn_zebrad_for_test_type, - test_type::TestType, -}; +use zebra_state::{ChainTipChange, LatestChainTip}; + +pub type BoxStateService = + BoxService; /// The minimum required tip height for the cached state in this test. const REQUIRED_MIN_TIP_HEIGHT: Height = Height(1_000_000); @@ -38,6 +39,9 @@ const WAIT_FOR_RESULTS_DURATION: Duration = Duration::from_secs(60); /// A block height where a scan result can be found with the [`ZECPAGES_SAPLING_VIEWING_KEY`] const EXPECTED_RESULT_HEIGHT: Height = Height(780_532); +/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) +const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; + /// Initialize Zebra's state service with a cached state, then: /// - Start the scan task, /// - Add a new key, @@ -48,24 +52,20 @@ const EXPECTED_RESULT_HEIGHT: Height = Height(780_532); pub(crate) async fn run() -> Result<()> { let _init_guard = zebra_test::init(); - let test_type = TestType::UpdateZebraCachedStateNoRpc; - let test_name = "scan_task_commands"; let network = Network::Mainnet; - // Skip the test unless the user specifically asked for it and there is a zebrad_state_path - if !can_spawn_zebrad_for_test_type(test_name, test_type, true) { - return Ok(()); - } + // Logs the network as zebrad would as part of the metadata when starting up. + // This is currently needed for the 'Check startup logs' step in CI to pass. + tracing::info!("Zcash network: {network}"); - tracing::info!( - ?network, - ?test_type, - "running scan_subscribe_results test using zebra state service", + let zebrad_state_path = std::path::PathBuf::from( + std::env::var("ZEBRA_CACHED_STATE_DIR") + .map_err(|_| eyre!("ZEBRA_CACHED_STATE_DIR env var not set"))?, ); - let zebrad_state_path = test_type - .zebrad_state_path(test_name) - .expect("already checked that there is a cached state path"); + // Remove the scan directory before starting. + let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND); + fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok(); let mut scan_config = zebra_scan::Config::default(); scan_config @@ -73,14 +73,6 @@ pub(crate) async fn run() -> Result<()> { .cache_dir .clone_from(&zebrad_state_path); - // Logs the network as zebrad would as part of the metadata when starting up. - // This is currently needed for the 'Check startup logs' step in CI to pass. - tracing::info!("Zcash network: {network}"); - - // Remove the scan directory before starting. - let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND); - fs::remove_dir_all(std::path::Path::new(&scan_db_path)).ok(); - let (_state_service, _read_state_service, latest_chain_tip, chain_tip_change) = start_state_service_with_cache_dir(&network, zebrad_state_path.clone()).await?; @@ -88,7 +80,8 @@ pub(crate) async fn run() -> Result<()> { cache_dir: zebrad_state_path.clone(), ..zebra_state::Config::default() }; - let (read_state, _db, _) = zebra_state::init_read_only(state_config, &network); + + let (read_state, _, _) = zebra_state::init_read_only(state_config.clone(), &network); let chain_tip_height = latest_chain_tip .best_tip_height() @@ -158,7 +151,6 @@ pub(crate) async fn run() -> Result<()> { TryRecvError::Disconnected, "any result senders should have been dropped" ); - break; } } @@ -166,3 +158,25 @@ pub(crate) async fn run() -> Result<()> { Ok(()) } + +pub async fn start_state_service_with_cache_dir( + network: &Network, + cache_dir: impl Into, +) -> Result<( + BoxStateService, + impl Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + >, + LatestChainTip, + ChainTipChange, +)> { + let config = zebra_state::Config { + cache_dir: cache_dir.into(), + ..zebra_state::Config::default() + }; + + // These tests don't need UTXOs to be verified efficiently, because they use cached states. + Ok(zebra_state::init(config, network, Height::MAX, 0)) +} diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index 0b7ec334117..dda5b638098 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -1,4 +1,12 @@ //! `zebra-scanner` binary tests. +//! +//! +//! Example of how to run the scan_task_commands test: +//! +//! ```console +//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands --features shielded-scan --release -- --ignored --nocapture +//! ``` +//! use tempfile::TempDir; use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty}; @@ -10,6 +18,8 @@ use zebra_test::{ use std::{io::Write, path::Path}; +mod scan_task_commands; + /// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; @@ -260,3 +270,16 @@ where } } } + +// TODO: Add this test to CI (#8236) +/// Tests successful: +/// - Registration of a new key, +/// - Subscription to scan results of new key, and +/// - Deletion of keys +/// in the scan task +/// See [`common::shielded_scan::scan_task_commands`] for more information. +#[tokio::test] +#[ignore] +async fn scan_task_commands() -> Result<()> { + scan_task_commands::run().await +} diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index a6174599ef8..c0b22479c36 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -54,4 +54,8 @@ pub struct ZebradConfig { #[serde(skip_serializing_if = "zebra_rpc::config::mining::Config::skip_getblocktemplate")] /// Mining configuration pub mining: zebra_rpc::config::mining::Config, + + #[cfg(feature = "shielded-scan")] + /// Scanner configuration + pub shielded_scan: zebra_scan::config::Config, } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index ce73fd43b42..810cf100f31 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -120,14 +120,6 @@ //! //! Please refer to the documentation of each test for more information. //! -//! ## Shielded scanning tests -//! -//! Example of how to run the scan_task_commands test: -//! -//! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands --features shielded-scan --release -- --ignored --nocapture -//! ``` -//! //! ## Checkpoint Generation Tests //! //! Generate checkpoints on mainnet and testnet using a cached state: @@ -2891,20 +2883,6 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> { Ok(()) } -// TODO: Add this test to CI (#8236) -/// Tests successful: -/// - Registration of a new key, -/// - Subscription to scan results of new key, and -/// - Deletion of keys -/// in the scan task -/// See [`common::shielded_scan::scan_task_commands`] for more information. -#[tokio::test] -#[ignore] -#[cfg(feature = "shielded-scan")] -async fn scan_task_commands() -> Result<()> { - common::shielded_scan::scan_task_commands::run().await -} - /// Checks that the Regtest genesis block can be validated. #[tokio::test] async fn validate_regtest_genesis_block() { diff --git a/zebrad/tests/common/mod.rs b/zebrad/tests/common/mod.rs index 3ff3f63db3a..6fe1b10f034 100644 --- a/zebrad/tests/common/mod.rs +++ b/zebrad/tests/common/mod.rs @@ -26,6 +26,3 @@ pub mod get_block_template_rpcs; #[cfg(feature = "getblocktemplate-rpcs")] pub mod regtest; - -#[cfg(feature = "shielded-scan")] -pub mod shielded_scan; diff --git a/zebrad/tests/common/shielded_scan.rs b/zebrad/tests/common/shielded_scan.rs deleted file mode 100644 index 106d6a16af3..00000000000 --- a/zebrad/tests/common/shielded_scan.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Acceptance tests for `shielded-scan`` feature in zebrad. - -pub(crate) mod scan_task_commands; From 06d61446ee5b45813e90145bb6f429c7736874af Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jul 2024 16:56:20 -0300 Subject: [PATCH 11/19] remove scanner config from zebrad --- zebrad/src/config.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index c0b22479c36..a6174599ef8 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -54,8 +54,4 @@ pub struct ZebradConfig { #[serde(skip_serializing_if = "zebra_rpc::config::mining::Config::skip_getblocktemplate")] /// Mining configuration pub mining: zebra_rpc::config::mining::Config, - - #[cfg(feature = "shielded-scan")] - /// Scanner configuration - pub shielded_scan: zebra_scan::config::Config, } From 15accc13a3e3c4b819faea7667f047d058df3251 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jul 2024 19:56:00 -0300 Subject: [PATCH 12/19] do not panic if env var is not there --- zebra-scan/tests/scan_task_commands.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/zebra-scan/tests/scan_task_commands.rs b/zebra-scan/tests/scan_task_commands.rs index 914710fbb41..20bbcca190d 100644 --- a/zebra-scan/tests/scan_task_commands.rs +++ b/zebra-scan/tests/scan_task_commands.rs @@ -58,10 +58,13 @@ pub(crate) async fn run() -> Result<()> { // This is currently needed for the 'Check startup logs' step in CI to pass. tracing::info!("Zcash network: {network}"); - let zebrad_state_path = std::path::PathBuf::from( - std::env::var("ZEBRA_CACHED_STATE_DIR") - .map_err(|_| eyre!("ZEBRA_CACHED_STATE_DIR env var not set"))?, - ); + let zebrad_state_path = match std::env::var_os("ZEBRA_CACHED_STATE_DIR") { + None => { + tracing::error!("ZEBRA_CACHED_STATE_DIR is not set"); + return Ok(()); + } + Some(path) => std::path::PathBuf::from(path), + }; // Remove the scan directory before starting. let scan_db_path = zebrad_state_path.join(SCANNER_DATABASE_KIND); From 25df03f9fd8f1f03243c72b07ba336d99cd84983 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jul 2024 20:54:44 -0300 Subject: [PATCH 13/19] re enable start_scan_where left, remove shielded-scan feature foir scanning tests in CI --- .../sub-ci-integration-tests-gcp.yml | 51 +++++++++---------- docker/entrypoint.sh | 8 +-- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/.github/workflows/sub-ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml index fabd90da2a4..ea507629bf3 100644 --- a/.github/workflows/sub-ci-integration-tests-gcp.yml +++ b/.github/workflows/sub-ci-integration-tests-gcp.yml @@ -516,32 +516,31 @@ jobs: zebra_state_dir: "zebrad-cache" secrets: inherit - # TODO: Move this test once we have the new scanner binary. - # # Test that the scanner can continue scanning where it was left when zebrad restarts. - # # - # # Runs: - # # - after every PR is merged to `main` - # # - on every PR update - # # - # # If the state version has changed, waits for the new cached states to be created. - # # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - # scan-start-where-left-test: - # name: Scan starts where left - # needs: [test-full-sync, get-available-disks] - # uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml - # if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - # with: - # app_name: zebrad - # test_id: scan-start-where-left - # test_description: Test that the scanner can continue scanning where it was left when zebrad restarts. - # test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" - # needs_zebra_state: true - # needs_lwd_state: false - # saves_to_disk: true - # disk_suffix: tip - # root_state_path: "/var/cache" - # zebra_state_dir: "zebrad-cache" - # secrets: inherit + # Test that the scanner can continue scanning where it was left when zebrad restarts. + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + scan-start-where-left-test: + name: Scan starts where left + needs: [test-full-sync, get-available-disks] + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: scan-start-where-left + test_description: Test that the scanner can continue scanning where it was left when zebrad restarts. + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: true + disk_suffix: tip + root_state_path: "/var/cache" + zebra_state_dir: "zebrad-cache" + secrets: inherit # Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running. # diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8cd61a49bb3..925e89bf7c8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -347,14 +347,14 @@ case "$1" in run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block" elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then - # Test that the scanner can continue scanning where it was left when zebrad restarts. + # Test that the scanner can continue scanning where it was left when zebra-scanner restarts. check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "shielded-scan" "scan_start_where_left" + run_cargo_test "${ENTRYPOINT_FEATURES}" "scan_start_where_left" elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then - # Test that the scanner can continue scanning where it was left when zebrad restarts. + # Test that the scan task commands are working. check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "shielded-scan" "scan_task_commands" + run_cargo_test "${ENTRYPOINT_FEATURES}" "scan_task_commands" else exec "$@" From af26b6a26a469994c5b575c3aca46ea9ddb1979c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jul 2024 21:26:22 -0300 Subject: [PATCH 14/19] change app names --- .github/workflows/sub-ci-integration-tests-gcp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sub-ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml index ea507629bf3..64ffc0ae4f5 100644 --- a/.github/workflows/sub-ci-integration-tests-gcp.yml +++ b/.github/workflows/sub-ci-integration-tests-gcp.yml @@ -530,7 +530,7 @@ jobs: uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: - app_name: zebrad + app_name: zebra-scan test_id: scan-start-where-left test_description: Test that the scanner can continue scanning where it was left when zebrad restarts. test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_START_WHERE_LEFT=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" @@ -556,7 +556,7 @@ jobs: uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: - app_name: zebrad + app_name: zebra-scan test_id: scan-task-commands test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running. test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" From b8deb3d396010c1c73752bb3a4262e496719c039 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jul 2024 21:28:23 -0300 Subject: [PATCH 15/19] uncomment test --- .github/workflows/sub-ci-integration-tests-gcp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sub-ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml index 64ffc0ae4f5..76cb168feb9 100644 --- a/.github/workflows/sub-ci-integration-tests-gcp.yml +++ b/.github/workflows/sub-ci-integration-tests-gcp.yml @@ -588,7 +588,7 @@ jobs: lightwalletd-grpc-test, get-block-template-test, submit-block-test, - # scan-start-where-left-test, + scan-start-where-left-test, scan-task-commands-test, ] # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. From 1b860dc3c5ed2b194cc4f23518787e64c1a77d98 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Sat, 13 Jul 2024 10:18:16 -0300 Subject: [PATCH 16/19] fix the scanner tests in CI by changing the test command --- docker/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 925e89bf7c8..cc4fb5ac82e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -349,12 +349,12 @@ case "$1" in elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then # Test that the scanner can continue scanning where it was left when zebra-scanner restarts. check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "scan_start_where_left" - + exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_start_where_left + elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then # Test that the scan task commands are working. check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "scan_task_commands" + exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands else exec "$@" From aac832ebc64d9e5087e99c7cf67a718d17b65611 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Sat, 13 Jul 2024 11:25:53 -0300 Subject: [PATCH 17/19] add log needed by CI? --- zebra-scan/tests/scanner.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index dda5b638098..d479bcd900e 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -154,6 +154,11 @@ async fn scan_start_where_left() -> Result<()> { return Ok(()); }; + // Logs the network as zebrad would as part of the metadata when starting up. + // This is currently needed for the 'Check startup logs' step in CI to pass. + let network = zebra_chain::parameters::Network::Mainnet; + tracing::info!("Zcash network: {network}"); + let scanning_cache_dir = testdir()?.path().join("scanner").to_path_buf(); // Create arguments for the scanner From 4383d1d30a5ec29051e84b58544420f170f7dd35 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Sat, 13 Jul 2024 17:08:29 -0300 Subject: [PATCH 18/19] move test --- zebra-scan/tests/scanner.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index d479bcd900e..de1c3476a96 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -237,6 +237,18 @@ async fn scan_start_where_left() -> Result<()> { Ok(()) } +/// Tests successful: +/// - Registration of a new key, +/// - Subscription to scan results of new key, and +/// - Deletion of keys +/// in the scan task +/// See [`common::shielded_scan::scan_task_commands`] for more information. +#[tokio::test] +#[ignore] +async fn scan_task_commands() -> Result<()> { + scan_task_commands::run().await +} + /// Create a temporary directory for testing `zebra-scanner`. pub fn testdir() -> eyre::Result { tempfile::Builder::new() @@ -275,16 +287,3 @@ where } } } - -// TODO: Add this test to CI (#8236) -/// Tests successful: -/// - Registration of a new key, -/// - Subscription to scan results of new key, and -/// - Deletion of keys -/// in the scan task -/// See [`common::shielded_scan::scan_task_commands`] for more information. -#[tokio::test] -#[ignore] -async fn scan_task_commands() -> Result<()> { - scan_task_commands::run().await -} From d70d6735d401229e75b24b9c06b971d668b7f5ba Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Sat, 13 Jul 2024 18:24:43 -0300 Subject: [PATCH 19/19] rename trait, docs fix --- zebra-scan/tests/scanner.rs | 62 +++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index de1c3476a96..3cd8573d342 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -1,12 +1,4 @@ //! `zebra-scanner` binary tests. -//! -//! -//! Example of how to run the scan_task_commands test: -//! -//! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands --features shielded-scan --release -- --ignored --nocapture -//! ``` -//! use tempfile::TempDir; use zebra_grpc::scanner::{scanner_client::ScannerClient, Empty}; @@ -30,7 +22,7 @@ fn scanner_help() -> eyre::Result<()> { let testdir = testdir()?; - let child = testdir.spawn_child(args!["--help"], false)?; + let child = testdir.spawn_scanner_child(args!["--help"])?; let output = child.wait_with_output()?; let output = output.assert_success()?; @@ -44,9 +36,9 @@ fn scanner_help() -> eyre::Result<()> { /// This test creates a new zebrad cache dir by using a `zebrad` binary specified in the `CARGO_BIN_EXE_zebrad` env variable. /// /// To run it locally, one way is: +/// /// ``` -/// export CARGO_BIN_EXE_zebrad="/path/to/zebrad_binary" -/// cargo test scan_binary_starts -- --ignored --nocapture +/// RUST_LOG=info cargo test scan_binary_starts -- --include-ignored --nocapture /// ``` #[tokio::test] #[cfg(not(target_os = "windows"))] @@ -65,10 +57,8 @@ async fn scan_binary_starts() -> Result<()> { std::fs::File::create(config_file.clone())?.write_all(toml::to_string(&config)?.as_bytes())?; // Start the node just to make sure a cache is created. - let mut zebrad = test_dir.spawn_child( - args!["-c", config_file.clone().to_str().unwrap(), "start"], - true, - )?; + let mut zebrad = + test_dir.spawn_zebrad_child(args!["-c", config_file.clone().to_str().unwrap(), "start"])?; zebrad.expect_stdout_line_matches("Opened Zebra state cache at .*")?; // Kill the node now that we have a valid zebra node cache. @@ -101,7 +91,7 @@ async fn scan_binary_starts() -> Result<()> { ]; // Start the scaner using another test directory. - let mut zebra_scanner = test_dir.spawn_child(args, false)?; + let mut zebra_scanner = test_dir.spawn_scanner_child(args)?; // Check scanner was started. zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; @@ -136,7 +126,7 @@ async fn scan_binary_starts() -> Result<()> { /// Needs a cache state close to the tip. A possible way to run it locally is: /// /// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" -/// cargo test scan_start_where_left -- --ignored --nocapture +/// RUST_LOG=info cargo test scan_start_where_left -- --ignored --nocapture /// /// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops. /// Then it will restart zebrad and check that it resumes scanning where it was left. @@ -179,7 +169,7 @@ async fn scan_start_where_left() -> Result<()> { ]; // Start the scaner using another test directory. - let mut zebra_scanner: TestChild = testdir()?.spawn_child(args.clone(), false)?; + let mut zebra_scanner: TestChild = testdir()?.spawn_scanner_child(args.clone())?; // Check scanner was started. zebra_scanner.expect_stdout_line_matches("loaded Zebra scanner cache")?; @@ -208,7 +198,7 @@ async fn scan_start_where_left() -> Result<()> { output.assert_failure()?; // Start the node again with the same arguments. - let mut zebra_scanner: TestChild = testdir()?.spawn_child(args, false)?; + let mut zebra_scanner: TestChild = testdir()?.spawn_scanner_child(args)?; // Resuming message. zebra_scanner.expect_stdout_line_matches( @@ -243,6 +233,12 @@ async fn scan_start_where_left() -> Result<()> { /// - Deletion of keys /// in the scan task /// See [`common::shielded_scan::scan_task_commands`] for more information. +/// +/// Example of how to run the scan_task_commands test locally: +/// +/// ```console +/// RUST_LOG=info ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands -- --include-ignored --nocapture +/// ``` #[tokio::test] #[ignore] async fn scan_task_commands() -> Result<()> { @@ -259,31 +255,29 @@ pub fn testdir() -> eyre::Result { /// Extension trait for methods on `tempfile::TempDir` for using it as a test /// directory for `zebra-scanner`. -pub trait ZebradTestDirExt +pub trait ScannerTestDirExt where Self: AsRef + Sized, { - // Zebra-scanner methods + /// Spawn `zebra-scanner` with `args` as a child process in this test directory. + fn spawn_scanner_child(self, args: Arguments) -> Result>; - /// Spawn `zebra-scanner` (`zebrad` flag = `false`) or `zebrad` (`zebrad` flag = `true`) - /// with `args` as a child process in this test directory. - fn spawn_child(self, args: Arguments, zebrad: bool) -> Result>; + /// Spawn `zebrad` with `args` as a child process in this test directory. + fn spawn_zebrad_child(self, args: Arguments) -> Result>; } -impl ZebradTestDirExt for T +impl ScannerTestDirExt for T where Self: TestDirExt + AsRef + Sized, { - #[allow(clippy::unwrap_in_result)] - fn spawn_child(self, extra_args: Arguments, zebrad: bool) -> Result> { + fn spawn_scanner_child(self, extra_args: Arguments) -> Result> { let mut args = Arguments::new(); - args.merge_with(extra_args); - - if zebrad { - self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad-for-scanner"), args) - } else { - self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args) - } + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebra-scanner"), args) + } + fn spawn_zebrad_child(self, extra_args: Arguments) -> Result> { + let mut args = Arguments::new(); + args.merge_with(extra_args); + self.spawn_child_with_command(env!("CARGO_BIN_EXE_zebrad-for-scanner"), args) } }