From 6663ca983b447c72c4452848d4cdd689743b00ea Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 11 Jun 2024 00:06:34 +1000 Subject: [PATCH 01/66] Add a snapshot command for creating snapshots of ledger state --- Cargo.lock | 4 + cmd/soroban-cli/Cargo.toml | 4 + cmd/soroban-cli/src/commands/mod.rs | 6 + cmd/soroban-cli/src/commands/snapshot.rs | 314 +++++++++++++++++++++++ 4 files changed, 328 insertions(+) create mode 100644 cmd/soroban-cli/src/commands/snapshot.rs diff --git a/Cargo.lock b/Cargo.lock index 2172171c5..c667da5c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4650,6 +4650,8 @@ dependencies = [ "dotenvy", "ed25519-dalek 2.0.0", "ethnum", + "flate2", + "futures", "futures-util", "gix", "heck 0.5.0", @@ -4694,6 +4696,7 @@ dependencies = [ "termcolor_output", "thiserror", "tokio", + "tokio-util", "toml 0.5.11", "toml_edit 0.21.1", "tracing", @@ -5508,6 +5511,7 @@ checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index 0950aa94d..20999329e 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -66,6 +66,7 @@ serde-aux = { workspace = true } hex = { workspace = true } num-bigint = "0.4" tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7.11", features = ["io", "io-util", "compat"] } termcolor = { workspace = true } termcolor_output = { workspace = true } rand = "0.8.5" @@ -113,7 +114,10 @@ toml_edit = "0.21.0" rust-embed = { version = "8.2.0", features = ["debug-embed"] } bollard = { workspace=true } futures-util = "0.3.30" +futures = "0.3.30" home = "0.5.9" +flate2 = "1.0.30" + # For hyper-tls [target.'cfg(unix)'.dependencies] openssl = { version = "=0.10.55", features = ["vendored"] } diff --git a/cmd/soroban-cli/src/commands/mod.rs b/cmd/soroban-cli/src/commands/mod.rs index 4552f93b3..df4ed72e1 100644 --- a/cmd/soroban-cli/src/commands/mod.rs +++ b/cmd/soroban-cli/src/commands/mod.rs @@ -12,6 +12,7 @@ pub mod global; pub mod keys; pub mod network; pub mod plugin; +pub mod snapshot; pub mod version; pub mod txn_result; @@ -98,6 +99,7 @@ impl Root { Cmd::Events(events) => events.run().await?, Cmd::Xdr(xdr) => xdr.run()?, Cmd::Network(network) => network.run().await?, + Cmd::Snapshot(snapshot) => snapshot.run().await?, Cmd::Version(version) => version.run(), Cmd::Keys(id) => id.run().await?, Cmd::Cache(data) => data.run()?, @@ -132,6 +134,8 @@ pub enum Cmd { /// Start and configure networks #[command(subcommand)] Network(network::Cmd), + /// Download a snapshot of a ledger. + Snapshot(snapshot::Cmd), /// Print version information Version(version::Cmd), /// Cache for tranasctions and contract specs @@ -157,6 +161,8 @@ pub enum Error { #[error(transparent)] Network(#[from] network::Error), #[error(transparent)] + Snapshot(#[from] snapshot::Error), + #[error(transparent)] Cache(#[from] cache::Error), } diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs new file mode 100644 index 000000000..d7b7ba6d3 --- /dev/null +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -0,0 +1,314 @@ +use clap::{arg, Parser}; +use flate2::read::GzDecoder; +use futures::TryStreamExt; +use http::Uri; +use soroban_ledger_snapshot::LedgerSnapshot; +use std::{ + collections::HashSet, + io::{self, Read}, + str::FromStr, +}; +use stellar_xdr::curr::{ + BucketEntry, ConfigSettingEntry, ConfigSettingId, Frame, LedgerEntry, LedgerEntryData, + LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, LedgerKeyConfigSetting, + LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, + LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, +}; +use tokio_util::compat::FuturesAsyncReadCompatExt as _; + +use soroban_env_host::xdr::{self}; + +use super::{ + config::{self, locator}, + network, +}; +use crate::rpc; + +#[derive(Parser, Debug, Clone)] +#[group(skip)] +pub struct Cmd { + /// The ledger sequence number to snapshot. + #[arg(long)] + ledger: u32, + /// Account IDs to filter by. + #[arg(long = "account-id", help_heading = "FILTERS")] + account_ids: Vec, + /// Contract IDs to filter by. + #[arg(long = "contract-id", help_heading = "FILTERS")] + contract_ids: Vec, + // #[command(flatten)] + // locator: locator::Args, + // #[command(flatten)] + // network: network::Args, +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("cursor is not valid")] + InvalidCursor, + #[error("filepath does not exist: {path}")] + InvalidFile { path: String }, + #[error("filepath ({path}) cannot be read: {error}")] + CannotReadFile { path: String, error: String }, + #[error("cannot parse topic filter {topic} into 1-4 segments")] + InvalidTopicFilter { topic: String }, + #[error("invalid segment ({segment}) in topic filter ({topic}): {error}")] + InvalidSegment { + topic: String, + segment: String, + error: xdr::Error, + }, + #[error("cannot parse contract ID {contract_id}: {error}")] + InvalidContractId { + contract_id: String, + error: stellar_strkey::DecodeError, + }, + #[error("invalid JSON string: {error} ({debug})")] + InvalidJson { + debug: String, + error: serde_json::Error, + }, + #[error("invalid timestamp in event: {ts}")] + InvalidTimestamp { ts: String }, + #[error("missing start_ledger and cursor")] + MissingStartLedgerAndCursor, + #[error("missing target")] + MissingTarget, + #[error(transparent)] + Rpc(#[from] rpc::Error), + #[error(transparent)] + Generic(#[from] Box), + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + Xdr(#[from] xdr::Error), + #[error(transparent)] + Serde(#[from] serde_json::Error), + #[error(transparent)] + Network(#[from] network::Error), + #[error(transparent)] + Locator(#[from] locator::Error), + #[error(transparent)] + Config(#[from] config::Error), +} + +impl Cmd { + pub async fn run(&self) -> Result<(), Error> { + const BASE_URL: &str = "http://history.stellar.org/prd/core-live/core_live_001"; + let ledger = self.ledger; + + let ledger_hex = format!("{ledger:08x}"); + let ledger_hex_0 = &ledger_hex[0..=1]; + let ledger_hex_1 = &ledger_hex[2..=3]; + let ledger_hex_2 = &ledger_hex[4..=5]; + let history_url = format!("{BASE_URL}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json"); + tracing::debug!(?history_url); + let history_url = Uri::from_str(&history_url).unwrap(); + + let https = hyper_tls::HttpsConnector::new(); + let response = hyper::Client::builder() + .build::<_, hyper::Body>(https) + .get(history_url) + .await + .unwrap(); + let body = hyper::body::to_bytes(response.into_body()).await.unwrap(); + + let history = serde_json::from_slice::(&body).unwrap(); + + let buckets = history + .current_buckets + .iter() + .flat_map(|h| [h.curr.clone(), h.snap.clone()]) + .filter(|b| b != "0000000000000000000000000000000000000000000000000000000000000000") + .collect::>(); + + let mut seen = HashSet::::new(); + let mut snapshot = LedgerSnapshot { + protocol_version: 20, + sequence_number: ledger, + timestamp: 0, + network_id: [0u8; 32], + base_reserve: 1, + min_persistent_entry_ttl: 0, + min_temp_entry_ttl: 0, + max_entry_ttl: 0, + ledger_entries: Vec::new(), + }; + + for (i, bucket) in buckets.iter().enumerate() { + let bucket_0 = &bucket[0..=1]; + let bucket_1 = &bucket[2..=3]; + let bucket_2 = &bucket[4..=5]; + let bucket_url = format!( + "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" + ); + println!("bucket {i}: {} {}", &bucket[0..8], bucket_url); + tracing::debug!(?bucket_url); + let bucket_url = Uri::from_str(&bucket_url).unwrap(); + + let https = hyper_tls::HttpsConnector::new(); + let response = hyper::Client::builder() + .build::<_, hyper::Body>(https) + .get(bucket_url) + .await + .unwrap(); + let read = tokio_util::io::SyncIoBridge::new( + response + .into_body() + .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e)) + .into_async_read() + .compat(), + ); + (seen, snapshot) = tokio::task::spawn_blocking(move || { + let mut counter = ReadCount::new(read); + { + let gz = GzDecoder::new(&mut counter); + let lz = &mut Limited::new(gz, Limits::none()); + let sz = Frame::::read_xdr_iter(lz); + for entry in sz { + let Frame(entry) = entry.unwrap(); + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + (data_into_key(&l), Some(l)) + } + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(_) => continue, + }; + if seen.contains(&key) { + continue; + } + seen.insert(key.clone()); + if let Some(val) = val { + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), None))); + } + } + } + println!("size {}", counter.count()); + (seen, snapshot) + }) + .await + .unwrap(); + } + + snapshot + .write_file(format!("snapshot-{ledger}.json")) + .unwrap(); + + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct History { + current_buckets: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct HistoryBucket { + curr: String, + snap: String, +} + +struct ReadCount { + inner: R, + count: usize, +} + +impl ReadCount { + fn new(r: R) -> Self { + ReadCount { inner: r, count: 0 } + } + + pub fn count(&self) -> usize { + self.count + } +} + +impl Read for ReadCount { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf).map(|n| { + self.count += n; + n + }) + } +} + +fn data_into_key(d: &LedgerEntry) -> LedgerKey { + // TODO: Move this function into stellar-xdr. + match &d.data { + LedgerEntryData::Account(e) => LedgerKey::Account(LedgerKeyAccount { + account_id: e.account_id.clone(), + }), + LedgerEntryData::Trustline(e) => LedgerKey::Trustline(LedgerKeyTrustLine { + account_id: e.account_id.clone(), + asset: e.asset.clone(), + }), + LedgerEntryData::Offer(e) => LedgerKey::Offer(LedgerKeyOffer { + seller_id: e.seller_id.clone(), + offer_id: e.offer_id, + }), + LedgerEntryData::Data(e) => LedgerKey::Data(LedgerKeyData { + account_id: e.account_id.clone(), + data_name: e.data_name.clone(), + }), + LedgerEntryData::ClaimableBalance(e) => { + LedgerKey::ClaimableBalance(LedgerKeyClaimableBalance { + balance_id: e.balance_id.clone(), + }) + } + LedgerEntryData::LiquidityPool(e) => LedgerKey::LiquidityPool(LedgerKeyLiquidityPool { + liquidity_pool_id: e.liquidity_pool_id.clone(), + }), + LedgerEntryData::ContractData(e) => LedgerKey::ContractData(LedgerKeyContractData { + contract: e.contract.clone(), + key: e.key.clone(), + durability: e.durability, + }), + LedgerEntryData::ContractCode(e) => LedgerKey::ContractCode(LedgerKeyContractCode { + hash: e.hash.clone(), + }), + LedgerEntryData::ConfigSetting(e) => LedgerKey::ConfigSetting(LedgerKeyConfigSetting { + config_setting_id: match e { + ConfigSettingEntry::ContractMaxSizeBytes(_) => { + ConfigSettingId::ContractMaxSizeBytes + } + ConfigSettingEntry::ContractComputeV0(_) => ConfigSettingId::ContractComputeV0, + ConfigSettingEntry::ContractLedgerCostV0(_) => { + ConfigSettingId::ContractLedgerCostV0 + } + ConfigSettingEntry::ContractHistoricalDataV0(_) => { + ConfigSettingId::ContractHistoricalDataV0 + } + ConfigSettingEntry::ContractEventsV0(_) => ConfigSettingId::ContractEventsV0, + ConfigSettingEntry::ContractBandwidthV0(_) => ConfigSettingId::ContractBandwidthV0, + ConfigSettingEntry::ContractCostParamsCpuInstructions(_) => { + ConfigSettingId::ContractCostParamsCpuInstructions + } + ConfigSettingEntry::ContractCostParamsMemoryBytes(_) => { + ConfigSettingId::ContractCostParamsMemoryBytes + } + ConfigSettingEntry::ContractDataKeySizeBytes(_) => { + ConfigSettingId::ContractDataKeySizeBytes + } + ConfigSettingEntry::ContractDataEntrySizeBytes(_) => { + ConfigSettingId::ContractDataEntrySizeBytes + } + ConfigSettingEntry::StateArchival(_) => ConfigSettingId::StateArchival, + ConfigSettingEntry::ContractExecutionLanes(_) => { + ConfigSettingId::ContractExecutionLanes + } + ConfigSettingEntry::BucketlistSizeWindow(_) => { + ConfigSettingId::BucketlistSizeWindow + } + ConfigSettingEntry::EvictionIterator(_) => ConfigSettingId::EvictionIterator, + }, + }), + LedgerEntryData::Ttl(e) => LedgerKey::Ttl(LedgerKeyTtl { + key_hash: e.key_hash.clone(), + }), + } +} From 9b8c53bf9e1630538d999a634f764abb5bb03353 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 11 Jun 2024 22:45:39 +1000 Subject: [PATCH 02/66] Add filtering --- Cargo.lock | 7 ++ Makefile | 2 +- cmd/soroban-cli/Cargo.toml | 1 + cmd/soroban-cli/src/commands/snapshot.rs | 125 ++++++++++++++--------- 4 files changed, 87 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c667da5c4..1ed407e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,6 +610,12 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" + [[package]] name = "camino" version = "1.1.7" @@ -4638,6 +4644,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "bollard", + "bytesize", "cargo_metadata", "chrono", "clap", diff --git a/Makefile b/Makefile index 7e307b16c..49caa15b8 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ endif install_rust: install install: - cargo install --force --locked --path ./cmd/stellar-cli --debug + cargo install --force --locked --path ./cmd/stellar-cli cargo install --force --locked --path ./cmd/crates/soroban-test/tests/fixtures/hello --root ./target --debug --quiet # regenerate the example lib in `cmd/crates/soroban-spec-typsecript/fixtures/ts` diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index 20999329e..7874cb2f5 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -117,6 +117,7 @@ futures-util = "0.3.30" futures = "0.3.30" home = "0.5.9" flate2 = "1.0.30" +bytesize = "1.3.0" # For hyper-tls [target.'cfg(unix)'.dependencies] diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index d7b7ba6d3..6ec47d8ab 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -1,11 +1,12 @@ +use bytesize::ByteSize; use clap::{arg, Parser}; -use flate2::read::GzDecoder; +use flate2::bufread::GzDecoder; use futures::TryStreamExt; use http::Uri; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, - io::{self, Read}, + io::{self, BufReader}, str::FromStr, }; use stellar_xdr::curr::{ @@ -36,6 +37,9 @@ pub struct Cmd { /// Contract IDs to filter by. #[arg(long = "contract-id", help_heading = "FILTERS")] contract_ids: Vec, + /// Contract IDs to filter by. + #[arg(long = "wasm-hash", help_heading = "FILTERS")] + wasm_hashes: Vec, // #[command(flatten)] // locator: locator::Args, // #[command(flatten)] @@ -97,13 +101,23 @@ impl Cmd { const BASE_URL: &str = "http://history.stellar.org/prd/core-live/core_live_001"; let ledger = self.ledger; + let ledger_offset = (ledger + 1) % 64; + if ledger_offset != 0 { + println!( + "ledger {ledger} not a checkpoint ledger, use {} or {}", + ledger - ledger_offset, + ledger + (64 - ledger_offset), + ); + return Ok(()); + } + let ledger_hex = format!("{ledger:08x}"); let ledger_hex_0 = &ledger_hex[0..=1]; let ledger_hex_1 = &ledger_hex[2..=3]; let ledger_hex_2 = &ledger_hex[4..=5]; let history_url = format!("{BASE_URL}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json"); - tracing::debug!(?history_url); let history_url = Uri::from_str(&history_url).unwrap(); + println!("🌎 Downloading history {history_url}"); let https = hyper_tls::HttpsConnector::new(); let response = hyper::Client::builder() @@ -142,8 +156,7 @@ impl Cmd { let bucket_url = format!( "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" ); - println!("bucket {i}: {} {}", &bucket[0..8], bucket_url); - tracing::debug!(?bucket_url); + print!("🪣 Downloading bucket {i} {bucket_url}"); let bucket_url = Uri::from_str(&bucket_url).unwrap(); let https = hyper_tls::HttpsConnector::new(); @@ -152,40 +165,81 @@ impl Cmd { .get(bucket_url) .await .unwrap(); + + if let Some(val) = response.headers().get("Content-Length") { + if let Ok(str) = val.to_str() { + if let Ok(len) = str.parse::() { + print!(" ({})", ByteSize(len)); + } + } + } + println!(); + let read = tokio_util::io::SyncIoBridge::new( response .into_body() - .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e)) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) .into_async_read() .compat(), ); + + let account_ids = self.account_ids.clone(); + let contract_ids = self.contract_ids.clone(); + let wasm_hashes = self.wasm_hashes.clone(); (seen, snapshot) = tokio::task::spawn_blocking(move || { - let mut counter = ReadCount::new(read); - { - let gz = GzDecoder::new(&mut counter); - let lz = &mut Limited::new(gz, Limits::none()); - let sz = Frame::::read_xdr_iter(lz); - for entry in sz { - let Frame(entry) = entry.unwrap(); - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - (data_into_key(&l), Some(l)) + let buf = BufReader::new(read); + let gz = GzDecoder::new(buf); + let buf = BufReader::new(gz); + let limited = &mut Limited::new(buf, Limits::none()); + let sz = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in sz { + let Frame(entry) = entry.unwrap(); + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) + } + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(_) => continue, + }; + if seen.contains(&key) { + continue; + } + if let Some(val) = val { + let keep = match &val.data { + LedgerEntryData::Account(e) => { + account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::Trustline(e) => { + account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::ContractData(e) => { + contract_ids.contains(&e.contract.to_string()) } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(_) => continue, + LedgerEntryData::ContractCode(e) => { + let hash = hex::encode(e.hash.0); + wasm_hashes.contains(&hash) + } + LedgerEntryData::Offer(_) + | LedgerEntryData::Data(_) + | LedgerEntryData::ClaimableBalance(_) + | LedgerEntryData::LiquidityPool(_) + | LedgerEntryData::ConfigSetting(_) + | LedgerEntryData::Ttl(_) => false, }; - if seen.contains(&key) { - continue; - } seen.insert(key.clone()); - if let Some(val) = val { + if keep { snapshot .ledger_entries .push((Box::new(key), (Box::new(val), None))); + count_saved += 1; } } } - println!("size {}", counter.count()); + if count_saved > 0 { + println!("🔎 Found {count_saved} entries"); + } (seen, snapshot) }) .await @@ -195,6 +249,7 @@ impl Cmd { snapshot .write_file(format!("snapshot-{ledger}.json")) .unwrap(); + println!("💾 Saved {} entries", snapshot.ledger_entries.len()); Ok(()) } @@ -213,30 +268,6 @@ struct HistoryBucket { snap: String, } -struct ReadCount { - inner: R, - count: usize, -} - -impl ReadCount { - fn new(r: R) -> Self { - ReadCount { inner: r, count: 0 } - } - - pub fn count(&self) -> usize { - self.count - } -} - -impl Read for ReadCount { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.inner.read(buf).map(|n| { - self.count += n; - n - }) - } -} - fn data_into_key(d: &LedgerEntry) -> LedgerKey { // TODO: Move this function into stellar-xdr. match &d.data { From 27df088aaec7688da02b7b666a5b470c079f1b04 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 00:22:57 +1000 Subject: [PATCH 03/66] Configurable out file and protocol version from meta --- cmd/soroban-cli/src/commands/snapshot.rs | 25 ++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 6ec47d8ab..4019e871f 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -7,6 +7,7 @@ use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, io::{self, BufReader}, + path::PathBuf, str::FromStr, }; use stellar_xdr::curr::{ @@ -25,12 +26,19 @@ use super::{ }; use crate::rpc; +fn default_out_path() -> PathBuf { + PathBuf::new().join("snapshot.json") +} + #[derive(Parser, Debug, Clone)] #[group(skip)] pub struct Cmd { /// The ledger sequence number to snapshot. #[arg(long)] ledger: u32, + /// The out path that the snapshot is written to. + #[arg(long, default_value=default_out_path().into_os_string())] + out: PathBuf, /// Account IDs to filter by. #[arg(long = "account-id", help_heading = "FILTERS")] account_ids: Vec, @@ -138,7 +146,7 @@ impl Cmd { let mut seen = HashSet::::new(); let mut snapshot = LedgerSnapshot { - protocol_version: 20, + protocol_version: 0, sequence_number: ledger, timestamp: 0, network_id: [0u8; 32], @@ -201,7 +209,10 @@ impl Cmd { (k, Some(l)) } BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(_) => continue, + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; + continue; + } }; if seen.contains(&key) { continue; @@ -246,10 +257,12 @@ impl Cmd { .unwrap(); } - snapshot - .write_file(format!("snapshot-{ledger}.json")) - .unwrap(); - println!("💾 Saved {} entries", snapshot.ledger_entries.len()); + snapshot.write_file(&self.out).unwrap(); + println!( + "💾 Saved {} entries to {:?}", + snapshot.ledger_entries.len(), + self.out + ); Ok(()) } From 4072f18fbecc919549d35cabbb1f94897ce2fc69 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 00:52:04 +1000 Subject: [PATCH 04/66] Cache downloaded buckets --- Cargo.lock | 7 ++ cmd/soroban-cli/Cargo.toml | 1 + cmd/soroban-cli/src/commands/config/data.rs | 6 ++ cmd/soroban-cli/src/commands/snapshot.rs | 94 +++++++++++++-------- 4 files changed, 74 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ed407e83..dae11dd35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2948,6 +2948,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "io_tee" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" + [[package]] name = "ipnet" version = "2.9.0" @@ -4667,6 +4673,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.28", "hyper-tls", + "io_tee", "itertools 0.10.5", "jsonrpsee-core", "jsonrpsee-http-client", diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index 7874cb2f5..27689c3f5 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -118,6 +118,7 @@ futures = "0.3.30" home = "0.5.9" flate2 = "1.0.30" bytesize = "1.3.0" +io_tee = "0.1.1" # For hyper-tls [target.'cfg(unix)'.dependencies] diff --git a/cmd/soroban-cli/src/commands/config/data.rs b/cmd/soroban-cli/src/commands/config/data.rs index 409ef2227..23dedc619 100644 --- a/cmd/soroban-cli/src/commands/config/data.rs +++ b/cmd/soroban-cli/src/commands/config/data.rs @@ -50,6 +50,12 @@ pub fn spec_dir() -> Result { Ok(dir) } +pub fn bucket_dir() -> Result { + let dir = data_local_dir()?.join("bucket"); + std::fs::create_dir_all(&dir)?; + Ok(dir) +} + pub fn write(action: Action, rpc_url: &Uri) -> Result { let data = Data { action, diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 4019e871f..a17c5a4dd 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -3,10 +3,12 @@ use clap::{arg, Parser}; use flate2::bufread::GzDecoder; use futures::TryStreamExt; use http::Uri; +use io_tee::TeeReader; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, - io::{self, BufReader}, + fs::OpenOptions, + io::{self, BufReader, Read}, path::PathBuf, str::FromStr, }; @@ -24,7 +26,7 @@ use super::{ config::{self, locator}, network, }; -use crate::rpc; +use crate::{commands::config::data, rpc}; fn default_out_path() -> PathBuf { PathBuf::new().join("snapshot.json") @@ -48,8 +50,8 @@ pub struct Cmd { /// Contract IDs to filter by. #[arg(long = "wasm-hash", help_heading = "FILTERS")] wasm_hashes: Vec, - // #[command(flatten)] - // locator: locator::Args, + #[command(flatten)] + locator: locator::Args, // #[command(flatten)] // network: network::Args, } @@ -158,47 +160,71 @@ impl Cmd { }; for (i, bucket) in buckets.iter().enumerate() { - let bucket_0 = &bucket[0..=1]; - let bucket_1 = &bucket[2..=3]; - let bucket_2 = &bucket[4..=5]; - let bucket_url = format!( - "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" - ); - print!("🪣 Downloading bucket {i} {bucket_url}"); - let bucket_url = Uri::from_str(&bucket_url).unwrap(); + let cache_path = data::bucket_dir() + .unwrap() + .join(format!("bucket-{bucket}.xdr")); + + let (read, gz): (Box, bool) = if cache_path.exists() { + println!("🪣 Loading cached bucket {i} {bucket}"); + let file = OpenOptions::new().read(true).open(&cache_path).unwrap(); + (Box::new(file), false) + } else { + let bucket_0 = &bucket[0..=1]; + let bucket_1 = &bucket[2..=3]; + let bucket_2 = &bucket[4..=5]; + let bucket_url = format!( + "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" + ); + print!("🪣 Downloading bucket {i} {bucket_url}"); + let bucket_url = Uri::from_str(&bucket_url).unwrap(); - let https = hyper_tls::HttpsConnector::new(); - let response = hyper::Client::builder() - .build::<_, hyper::Body>(https) - .get(bucket_url) - .await - .unwrap(); + let https = hyper_tls::HttpsConnector::new(); + let response = hyper::Client::builder() + .build::<_, hyper::Body>(https) + .get(bucket_url) + .await + .unwrap(); - if let Some(val) = response.headers().get("Content-Length") { - if let Ok(str) = val.to_str() { - if let Ok(len) = str.parse::() { - print!(" ({})", ByteSize(len)); + if let Some(val) = response.headers().get("Content-Length") { + if let Ok(str) = val.to_str() { + if let Ok(len) = str.parse::() { + print!(" ({})", ByteSize(len)); + } } } - } - println!(); + println!(); - let read = tokio_util::io::SyncIoBridge::new( - response - .into_body() - .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) - .into_async_read() - .compat(), - ); + let read = tokio_util::io::SyncIoBridge::new( + response + .into_body() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .into_async_read() + .compat(), + ); + (Box::new(read), true) + }; + let cache_path = cache_path.clone(); let account_ids = self.account_ids.clone(); let contract_ids = self.contract_ids.clone(); let wasm_hashes = self.wasm_hashes.clone(); (seen, snapshot) = tokio::task::spawn_blocking(move || { let buf = BufReader::new(read); - let gz = GzDecoder::new(buf); - let buf = BufReader::new(gz); - let limited = &mut Limited::new(buf, Limits::none()); + let read: Box = if gz { + let gz = GzDecoder::new(buf); + let buf = BufReader::new(gz); + let file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&cache_path) + .unwrap(); + let tee = TeeReader::new(buf, file); + Box::new(tee) + } else { + Box::new(buf) + }; + let limited = &mut Limited::new(read, Limits::none()); let sz = Frame::::read_xdr_iter(limited); let mut count_saved = 0; for entry in sz { From 9825893cebebf63aefef1a0b43ffbf0a997436ab Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 00:59:24 +1000 Subject: [PATCH 05/66] Make bucket cache resilient to partial downloads --- cmd/soroban-cli/src/commands/snapshot.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index a17c5a4dd..11bf0bc8f 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -7,7 +7,7 @@ use io_tee::TeeReader; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, - fs::OpenOptions, + fs::{self, OpenOptions}, io::{self, BufReader, Read}, path::PathBuf, str::FromStr, @@ -209,6 +209,7 @@ impl Cmd { let contract_ids = self.contract_ids.clone(); let wasm_hashes = self.wasm_hashes.clone(); (seen, snapshot) = tokio::task::spawn_blocking(move || { + let dl_path = cache_path.with_extension("dl"); let buf = BufReader::new(read); let read: Box = if gz { let gz = GzDecoder::new(buf); @@ -217,7 +218,7 @@ impl Cmd { .create(true) .truncate(true) .write(true) - .open(&cache_path) + .open(&dl_path) .unwrap(); let tee = TeeReader::new(buf, file); Box::new(tee) @@ -274,6 +275,9 @@ impl Cmd { } } } + if gz { + fs::rename(&dl_path, &cache_path).unwrap(); + } if count_saved > 0 { println!("🔎 Found {count_saved} entries"); } From 80058e74be8339fc32d2d75f13ae942a210d717f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 09:18:39 +1000 Subject: [PATCH 06/66] Add duration --- Cargo.lock | 1 + cmd/soroban-cli/Cargo.toml | 1 + cmd/soroban-cli/src/commands/snapshot.rs | 43 +++++++++++++++++------- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dae11dd35..286bf71c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4671,6 +4671,7 @@ dependencies = [ "hex", "home", "http 0.2.12", + "humantime", "hyper 0.14.28", "hyper-tls", "io_tee", diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index 27689c3f5..5f7a113a5 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -119,6 +119,7 @@ home = "0.5.9" flate2 = "1.0.30" bytesize = "1.3.0" io_tee = "0.1.1" +humantime = "2.1.0" # For hyper-tls [target.'cfg(unix)'.dependencies] diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 11bf0bc8f..755c57565 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -3,6 +3,7 @@ use clap::{arg, Parser}; use flate2::bufread::GzDecoder; use futures::TryStreamExt; use http::Uri; +use humantime::format_duration; use io_tee::TeeReader; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ @@ -11,6 +12,7 @@ use std::{ io::{self, BufReader, Read}, path::PathBuf, str::FromStr, + time::{Duration, Instant}, }; use stellar_xdr::curr::{ BucketEntry, ConfigSettingEntry, ConfigSettingId, Frame, LedgerEntry, LedgerEntryData, @@ -106,21 +108,27 @@ pub enum Error { Config(#[from] config::Error), } +const CHECKPOINT_FREQUENCY: u32 = 64; + impl Cmd { pub async fn run(&self) -> Result<(), Error> { const BASE_URL: &str = "http://history.stellar.org/prd/core-live/core_live_001"; let ledger = self.ledger; - let ledger_offset = (ledger + 1) % 64; + let start = Instant::now(); + + // Check ledger is a checkpoint ledger and available in archives. + let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; if ledger_offset != 0 { println!( "ledger {ledger} not a checkpoint ledger, use {} or {}", ledger - ledger_offset, - ledger + (64 - ledger_offset), + ledger + (CHECKPOINT_FREQUENCY - ledger_offset), ); return Ok(()); } + // Download history JSON file. let ledger_hex = format!("{ledger:08x}"); let ledger_hex_0 = &ledger_hex[0..=1]; let ledger_hex_1 = &ledger_hex[2..=3]; @@ -128,7 +136,6 @@ impl Cmd { let history_url = format!("{BASE_URL}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json"); let history_url = Uri::from_str(&history_url).unwrap(); println!("🌎 Downloading history {history_url}"); - let https = hyper_tls::HttpsConnector::new(); let response = hyper::Client::builder() .build::<_, hyper::Body>(https) @@ -136,9 +143,10 @@ impl Cmd { .await .unwrap(); let body = hyper::body::to_bytes(response.into_body()).await.unwrap(); - let history = serde_json::from_slice::(&body).unwrap(); + // Prepare a flat list of buckets to read. They'll be ordered by their + // level so that they can iterated higher level to lower level. let buckets = history .current_buckets .iter() @@ -146,7 +154,15 @@ impl Cmd { .filter(|b| b != "0000000000000000000000000000000000000000000000000000000000000000") .collect::>(); + // Track ledger keys seen, so that we can ignore old versions of + // entries. Entries can appear in both higher level and lower level + // buckets, and to get the latest version of the entry the version in + // the higher level bucket should be used. let mut seen = HashSet::::new(); + + // The snapshot is what will be written to file at the end. Fields will + // be updated while parsing the history archive. + // TODO: Update more of the fields. let mut snapshot = LedgerSnapshot { protocol_version: 0, sequence_number: ledger, @@ -160,11 +176,12 @@ impl Cmd { }; for (i, bucket) in buckets.iter().enumerate() { + // Defined where the bucket will be read from, either from cache on + // disk, or streamed from the archive. let cache_path = data::bucket_dir() .unwrap() .join(format!("bucket-{bucket}.xdr")); - - let (read, gz): (Box, bool) = if cache_path.exists() { + let (read, stream): (Box, bool) = if cache_path.exists() { println!("🪣 Loading cached bucket {i} {bucket}"); let file = OpenOptions::new().read(true).open(&cache_path).unwrap(); (Box::new(file), false) @@ -175,16 +192,14 @@ impl Cmd { let bucket_url = format!( "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" ); - print!("🪣 Downloading bucket {i} {bucket_url}"); + print!("🪣 Downloading bucket {i} {bucket}"); let bucket_url = Uri::from_str(&bucket_url).unwrap(); - let https = hyper_tls::HttpsConnector::new(); let response = hyper::Client::builder() .build::<_, hyper::Body>(https) .get(bucket_url) .await .unwrap(); - if let Some(val) = response.headers().get("Content-Length") { if let Ok(str) = val.to_str() { if let Ok(len) = str.parse::() { @@ -193,7 +208,6 @@ impl Cmd { } } println!(); - let read = tokio_util::io::SyncIoBridge::new( response .into_body() @@ -211,7 +225,9 @@ impl Cmd { (seen, snapshot) = tokio::task::spawn_blocking(move || { let dl_path = cache_path.with_extension("dl"); let buf = BufReader::new(read); - let read: Box = if gz { + let read: Box = if stream { + // When streamed from the archive the bucket will be + // uncompressed, and also be streamed to cache. let gz = GzDecoder::new(buf); let buf = BufReader::new(gz); let file = OpenOptions::new() @@ -275,7 +291,7 @@ impl Cmd { } } } - if gz { + if stream { fs::rename(&dl_path, &cache_path).unwrap(); } if count_saved > 0 { @@ -294,6 +310,9 @@ impl Cmd { self.out ); + let duration = Duration::from_secs(start.elapsed().as_secs()); + println!("✅ Completed in {}", format_duration(duration)); + Ok(()) } } From bb67a7f70f00580404e5fd10741c1d84040f372f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 09:21:10 +1000 Subject: [PATCH 07/66] Docs --- cmd/soroban-cli/src/commands/snapshot.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 755c57565..6de46b0ed 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -241,6 +241,9 @@ impl Cmd { } else { Box::new(buf) }; + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. let limited = &mut Limited::new(read, Limits::none()); let sz = Frame::::read_xdr_iter(limited); let mut count_saved = 0; From 379cf5d28f084dad811e6e5aa3117b03841d329f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 23:32:16 +1000 Subject: [PATCH 08/66] Remove unused error --- cmd/soroban-cli/src/commands/snapshot.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 6de46b0ed..c1893e2e4 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -28,7 +28,7 @@ use super::{ config::{self, locator}, network, }; -use crate::{commands::config::data, rpc}; +use crate::commands::config::data; fn default_out_path() -> PathBuf { PathBuf::new().join("snapshot.json") @@ -91,8 +91,6 @@ pub enum Error { #[error("missing target")] MissingTarget, #[error(transparent)] - Rpc(#[from] rpc::Error), - #[error(transparent)] Generic(#[from] Box), #[error(transparent)] Io(#[from] io::Error), From 449036004d02e408a3eda555ddf28c692ca9ffb1 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 12 Jun 2024 23:33:24 +1000 Subject: [PATCH 09/66] Collect wasm with contract id and make ledger optional --- cmd/soroban-cli/src/commands/snapshot.rs | 244 +++++++++++++---------- 1 file changed, 141 insertions(+), 103 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index c1893e2e4..b8e2fc38b 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -5,6 +5,7 @@ use futures::TryStreamExt; use http::Uri; use humantime::format_duration; use io_tee::TeeReader; +use sha2::{Digest, Sha256}; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, @@ -15,10 +16,11 @@ use std::{ time::{Duration, Instant}, }; use stellar_xdr::curr::{ - BucketEntry, ConfigSettingEntry, ConfigSettingId, Frame, LedgerEntry, LedgerEntryData, - LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, LedgerKeyConfigSetting, - LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, - LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, + BucketEntry, ConfigSettingEntry, ConfigSettingId, ContractExecutable, Frame, Hash, LedgerEntry, + LedgerEntryData, LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, + LedgerKeyConfigSetting, LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, + LedgerKeyLiquidityPool, LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, + ReadXdr, ScContractInstance, ScVal, }; use tokio_util::compat::FuturesAsyncReadCompatExt as _; @@ -37,9 +39,9 @@ fn default_out_path() -> PathBuf { #[derive(Parser, Debug, Clone)] #[group(skip)] pub struct Cmd { - /// The ledger sequence number to snapshot. + /// The ledger sequence number to snapshot. Defaults to latest history archived ledger. #[arg(long)] - ledger: u32, + ledger: Option, /// The out path that the snapshot is written to. #[arg(long, default_value=default_out_path().into_os_string())] out: PathBuf, @@ -111,27 +113,31 @@ const CHECKPOINT_FREQUENCY: u32 = 64; impl Cmd { pub async fn run(&self) -> Result<(), Error> { const BASE_URL: &str = "http://history.stellar.org/prd/core-live/core_live_001"; - let ledger = self.ledger; let start = Instant::now(); - // Check ledger is a checkpoint ledger and available in archives. - let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; - if ledger_offset != 0 { - println!( - "ledger {ledger} not a checkpoint ledger, use {} or {}", - ledger - ledger_offset, - ledger + (CHECKPOINT_FREQUENCY - ledger_offset), - ); - return Ok(()); - } + let history_url = if let Some(ledger) = self.ledger { + // Check ledger is a checkpoint ledger and available in archives. + let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; + if ledger_offset != 0 { + println!( + "ledger {ledger} not a checkpoint ledger, use {} or {}", + ledger - ledger_offset, + ledger + (CHECKPOINT_FREQUENCY - ledger_offset), + ); + return Ok(()); + } + + // Download history JSON file. + let ledger_hex = format!("{ledger:08x}"); + let ledger_hex_0 = &ledger_hex[0..=1]; + let ledger_hex_1 = &ledger_hex[2..=3]; + let ledger_hex_2 = &ledger_hex[4..=5]; + format!("{BASE_URL}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json") + } else { + format!("{BASE_URL}/.well-known/stellar-history.json") + }; - // Download history JSON file. - let ledger_hex = format!("{ledger:08x}"); - let ledger_hex_0 = &ledger_hex[0..=1]; - let ledger_hex_1 = &ledger_hex[2..=3]; - let ledger_hex_2 = &ledger_hex[4..=5]; - let history_url = format!("{BASE_URL}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json"); let history_url = Uri::from_str(&history_url).unwrap(); println!("🌎 Downloading history {history_url}"); let https = hyper_tls::HttpsConnector::new(); @@ -143,6 +149,13 @@ impl Cmd { let body = hyper::body::to_bytes(response.into_body()).await.unwrap(); let history = serde_json::from_slice::(&body).unwrap(); + let ledger = history.current_ledger; + let network_passphrase = &history.network_passphrase; + let network_id = Sha256::digest(network_passphrase); + println!("ℹ️ Ledger: {ledger}"); + println!("ℹ️ Network Passphrase: {network_passphrase}"); + println!("ℹ️ Network ID: {}", hex::encode(network_id)); + // Prepare a flat list of buckets to read. They'll be ordered by their // level so that they can iterated higher level to lower level. let buckets = history @@ -165,7 +178,7 @@ impl Cmd { protocol_version: 0, sequence_number: ledger, timestamp: 0, - network_id: [0u8; 32], + network_id: network_id.into(), base_reserve: 1, min_persistent_entry_ttl: 0, min_temp_entry_ttl: 0, @@ -173,6 +186,9 @@ impl Cmd { ledger_entries: Vec::new(), }; + let mut account_ids = self.account_ids.clone(); + let mut contract_ids = self.contract_ids.clone(); + let mut wasm_hashes = self.wasm_hashes.clone(); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. @@ -217,91 +233,111 @@ impl Cmd { }; let cache_path = cache_path.clone(); - let account_ids = self.account_ids.clone(); - let contract_ids = self.contract_ids.clone(); - let wasm_hashes = self.wasm_hashes.clone(); - (seen, snapshot) = tokio::task::spawn_blocking(move || { - let dl_path = cache_path.with_extension("dl"); - let buf = BufReader::new(read); - let read: Box = if stream { - // When streamed from the archive the bucket will be - // uncompressed, and also be streamed to cache. - let gz = GzDecoder::new(buf); - let buf = BufReader::new(gz); - let file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&dl_path) - .unwrap(); - let tee = TeeReader::new(buf, file); - Box::new(tee) - } else { - Box::new(buf) - }; - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(read, Limits::none()); - let sz = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in sz { - let Frame(entry) = entry.unwrap(); - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) - } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(m) => { - snapshot.protocol_version = m.ledger_version; - continue; - } + (seen, snapshot, account_ids, contract_ids, wasm_hashes) = + tokio::task::spawn_blocking(move || { + let dl_path = cache_path.with_extension("dl"); + let buf = BufReader::new(read); + let read: Box = if stream { + // When streamed from the archive the bucket will be + // uncompressed, and also be streamed to cache. + let gz = GzDecoder::new(buf); + let buf = BufReader::new(gz); + let file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&dl_path) + .unwrap(); + let tee = TeeReader::new(buf, file); + Box::new(tee) + } else { + Box::new(buf) }; - if seen.contains(&key) { - continue; - } - if let Some(val) = val { - let keep = match &val.data { - LedgerEntryData::Account(e) => { - account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::Trustline(e) => { - account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::ContractData(e) => { - contract_ids.contains(&e.contract.to_string()) + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(read, Limits::none()); + let sz = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in sz { + let Frame(entry) = entry.unwrap(); + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) } - LedgerEntryData::ContractCode(e) => { - let hash = hex::encode(e.hash.0); - wasm_hashes.contains(&hash) + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; + continue; } - LedgerEntryData::Offer(_) - | LedgerEntryData::Data(_) - | LedgerEntryData::ClaimableBalance(_) - | LedgerEntryData::LiquidityPool(_) - | LedgerEntryData::ConfigSetting(_) - | LedgerEntryData::Ttl(_) => false, }; - seen.insert(key.clone()); - if keep { - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), None))); - count_saved += 1; + if seen.contains(&key) { + continue; + } + if let Some(val) = val { + let keep = match &val.data { + LedgerEntryData::Account(e) => { + account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::Trustline(e) => { + account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::ContractData(e) => { + let keep = contract_ids.contains(&e.contract.to_string()); + // If a contract instance references + // contract executable stored in another + // ledger entry, add that ledger entry to + // the filter so that Wasm for any filtered + // contract is collected too. TODO: Change + // this to support Wasm ledger entries + // appearing in earlier buckets after state + // archival is rolled out. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(Hash(hash)), + .. + }) = e.val + { + let hash = hex::encode(hash); + wasm_hashes.push(hash); + } + } + keep + } + LedgerEntryData::ContractCode(e) => { + let hash = hex::encode(e.hash.0); + wasm_hashes.contains(&hash) + } + LedgerEntryData::Offer(_) + | LedgerEntryData::Data(_) + | LedgerEntryData::ClaimableBalance(_) + | LedgerEntryData::LiquidityPool(_) + | LedgerEntryData::ConfigSetting(_) + | LedgerEntryData::Ttl(_) => false, + }; + seen.insert(key.clone()); + if keep { + // Store the found ledger entry in the snapshot with + // a max u32 expiry. TODO: Change the expiry to come + // from the corresponding TTL ledger entry. + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; + } } } - } - if stream { - fs::rename(&dl_path, &cache_path).unwrap(); - } - if count_saved > 0 { - println!("🔎 Found {count_saved} entries"); - } - (seen, snapshot) - }) - .await - .unwrap(); + if stream { + fs::rename(&dl_path, &cache_path).unwrap(); + } + if count_saved > 0 { + println!("🔎 Found {count_saved} entries"); + } + (seen, snapshot, account_ids, contract_ids, wasm_hashes) + }) + .await + .unwrap(); } snapshot.write_file(&self.out).unwrap(); @@ -321,7 +357,9 @@ impl Cmd { #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] #[serde(rename_all = "camelCase")] struct History { + current_ledger: u32, current_buckets: Vec, + network_passphrase: String, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] From 7954ad3358d424607e40c43edd3120fee5430f4c Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:20:51 +1000 Subject: [PATCH 10/66] Reorder help fields --- cmd/soroban-cli/src/commands/snapshot.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index b8e2fc38b..9702be6bb 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -42,9 +42,6 @@ pub struct Cmd { /// The ledger sequence number to snapshot. Defaults to latest history archived ledger. #[arg(long)] ledger: Option, - /// The out path that the snapshot is written to. - #[arg(long, default_value=default_out_path().into_os_string())] - out: PathBuf, /// Account IDs to filter by. #[arg(long = "account-id", help_heading = "FILTERS")] account_ids: Vec, @@ -54,6 +51,9 @@ pub struct Cmd { /// Contract IDs to filter by. #[arg(long = "wasm-hash", help_heading = "FILTERS")] wasm_hashes: Vec, + /// The out path that the snapshot is written to. + #[arg(long, default_value=default_out_path().into_os_string())] + out: PathBuf, #[command(flatten)] locator: locator::Args, // #[command(flatten)] From bc846dd3f7470d9e7976a920180c2b93614d3135 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:31:01 +1000 Subject: [PATCH 11/66] Display help when no args submitted --- cmd/soroban-cli/src/commands/snapshot.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 9702be6bb..a5f8a4139 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -38,6 +38,7 @@ fn default_out_path() -> PathBuf { #[derive(Parser, Debug, Clone)] #[group(skip)] +#[command(arg_required_else_help = true)] pub struct Cmd { /// The ledger sequence number to snapshot. Defaults to latest history archived ledger. #[arg(long)] From f6f44b3a32c27490bcd47d5a081b7ec4ffb11de5 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:24:24 +1000 Subject: [PATCH 12/66] Unwrap -> ? --- cmd/soroban-cli/src/commands/snapshot.rs | 74 ++++++++---------------- 1 file changed, 23 insertions(+), 51 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index a5f8a4139..1783a8fe9 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -63,38 +63,16 @@ pub struct Cmd { #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("cursor is not valid")] - InvalidCursor, - #[error("filepath does not exist: {path}")] - InvalidFile { path: String }, - #[error("filepath ({path}) cannot be read: {error}")] - CannotReadFile { path: String, error: String }, - #[error("cannot parse topic filter {topic} into 1-4 segments")] - InvalidTopicFilter { topic: String }, - #[error("invalid segment ({segment}) in topic filter ({topic}): {error}")] - InvalidSegment { - topic: String, - segment: String, - error: xdr::Error, - }, - #[error("cannot parse contract ID {contract_id}: {error}")] - InvalidContractId { - contract_id: String, - error: stellar_strkey::DecodeError, - }, - #[error("invalid JSON string: {error} ({debug})")] - InvalidJson { - debug: String, - error: serde_json::Error, - }, - #[error("invalid timestamp in event: {ts}")] - InvalidTimestamp { ts: String }, - #[error("missing start_ledger and cursor")] - MissingStartLedgerAndCursor, - #[error("missing target")] - MissingTarget, #[error(transparent)] - Generic(#[from] Box), + LedgerSnapshot(#[from] soroban_ledger_snapshot::Error), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error(transparent)] + InvalidUri(#[from] http::uri::InvalidUri), + #[error(transparent)] + Data(#[from] data::Error), + #[error(transparent)] + Hyper(#[from] hyper::Error), #[error(transparent)] Io(#[from] io::Error), #[error(transparent)] @@ -145,10 +123,9 @@ impl Cmd { let response = hyper::Client::builder() .build::<_, hyper::Body>(https) .get(history_url) - .await - .unwrap(); - let body = hyper::body::to_bytes(response.into_body()).await.unwrap(); - let history = serde_json::from_slice::(&body).unwrap(); + .await?; + let body = hyper::body::to_bytes(response.into_body()).await?; + let history = serde_json::from_slice::(&body)?; let ledger = history.current_ledger; let network_passphrase = &history.network_passphrase; @@ -193,12 +170,10 @@ impl Cmd { for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. - let cache_path = data::bucket_dir() - .unwrap() - .join(format!("bucket-{bucket}.xdr")); + let cache_path = data::bucket_dir()?.join(format!("bucket-{bucket}.xdr")); let (read, stream): (Box, bool) = if cache_path.exists() { println!("🪣 Loading cached bucket {i} {bucket}"); - let file = OpenOptions::new().read(true).open(&cache_path).unwrap(); + let file = OpenOptions::new().read(true).open(&cache_path)?; (Box::new(file), false) } else { let bucket_0 = &bucket[0..=1]; @@ -208,13 +183,12 @@ impl Cmd { "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" ); print!("🪣 Downloading bucket {i} {bucket}"); - let bucket_url = Uri::from_str(&bucket_url).unwrap(); + let bucket_url = Uri::from_str(&bucket_url)?; let https = hyper_tls::HttpsConnector::new(); let response = hyper::Client::builder() .build::<_, hyper::Body>(https) .get(bucket_url) - .await - .unwrap(); + .await?; if let Some(val) = response.headers().get("Content-Length") { if let Ok(str) = val.to_str() { if let Ok(len) = str.parse::() { @@ -235,7 +209,7 @@ impl Cmd { let cache_path = cache_path.clone(); (seen, snapshot, account_ids, contract_ids, wasm_hashes) = - tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || -> Result<_, Error> { let dl_path = cache_path.with_extension("dl"); let buf = BufReader::new(read); let read: Box = if stream { @@ -247,8 +221,7 @@ impl Cmd { .create(true) .truncate(true) .write(true) - .open(&dl_path) - .unwrap(); + .open(&dl_path)?; let tee = TeeReader::new(buf, file); Box::new(tee) } else { @@ -261,7 +234,7 @@ impl Cmd { let sz = Frame::::read_xdr_iter(limited); let mut count_saved = 0; for entry in sz { - let Frame(entry) = entry.unwrap(); + let Frame(entry) = entry?; let (key, val) = match entry { BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { let k = data_into_key(&l); @@ -330,18 +303,17 @@ impl Cmd { } } if stream { - fs::rename(&dl_path, &cache_path).unwrap(); + fs::rename(&dl_path, &cache_path)?; } if count_saved > 0 { println!("🔎 Found {count_saved} entries"); } - (seen, snapshot, account_ids, contract_ids, wasm_hashes) + Ok((seen, snapshot, account_ids, contract_ids, wasm_hashes)) }) - .await - .unwrap(); + .await??; } - snapshot.write_file(&self.out).unwrap(); + snapshot.write_file(&self.out)?; println!( "💾 Saved {} entries to {:?}", snapshot.ledger_entries.len(), From 16aed389b53011966768c0025b9ef678cad9479f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:47:39 +1000 Subject: [PATCH 13/66] Give errors names --- cmd/soroban-cli/src/commands/snapshot.rs | 70 +++++++++++++++--------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 1783a8fe9..92f77d4f0 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -63,23 +63,31 @@ pub struct Cmd { #[derive(thiserror::Error, Debug)] pub enum Error { - #[error(transparent)] - LedgerSnapshot(#[from] soroban_ledger_snapshot::Error), + #[error("downloading history: {0}")] + DownloadingHistory(hyper::Error), + #[error("json decoding history: {0}")] + JsonDecodingHistory(serde_json::Error), + #[error("opening cached bucket to read: {0}")] + ReadOpeningCachedBucket(io::Error), + #[error("parsing bucket url: {0}")] + ParsingBucketUrl(http::uri::InvalidUri), + #[error("getting bucket: {0}")] + GettingBucket(hyper::Error), + #[error("opening cached bucket to write: {0}")] + WriteOpeningCachedBucket(io::Error), + #[error("read XDR frame bucket entry: {0}")] + ReadXdrFrameBucketEntry(xdr::Error), + #[error("renaming temporary downloaded file to final destination: {0}")] + RenameDownloadFile(io::Error), + #[error("getting bucket directory: {0}")] + GetBucketDir(data::Error), + #[error("reading history http stream: {0}")] + ReadHistoryHttpStream(hyper::Error), + #[error("writing ledger snapshot: {0}")] + WriteLedgerSnapshot(soroban_ledger_snapshot::Error), #[error(transparent)] Join(#[from] tokio::task::JoinError), #[error(transparent)] - InvalidUri(#[from] http::uri::InvalidUri), - #[error(transparent)] - Data(#[from] data::Error), - #[error(transparent)] - Hyper(#[from] hyper::Error), - #[error(transparent)] - Io(#[from] io::Error), - #[error(transparent)] - Xdr(#[from] xdr::Error), - #[error(transparent)] - Serde(#[from] serde_json::Error), - #[error(transparent)] Network(#[from] network::Error), #[error(transparent)] Locator(#[from] locator::Error), @@ -123,9 +131,13 @@ impl Cmd { let response = hyper::Client::builder() .build::<_, hyper::Body>(https) .get(history_url) - .await?; - let body = hyper::body::to_bytes(response.into_body()).await?; - let history = serde_json::from_slice::(&body)?; + .await + .map_err(Error::DownloadingHistory)?; + let body = hyper::body::to_bytes(response.into_body()) + .await + .map_err(Error::ReadHistoryHttpStream)?; + let history = + serde_json::from_slice::(&body).map_err(Error::JsonDecodingHistory)?; let ledger = history.current_ledger; let network_passphrase = &history.network_passphrase; @@ -170,10 +182,14 @@ impl Cmd { for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. - let cache_path = data::bucket_dir()?.join(format!("bucket-{bucket}.xdr")); + let bucket_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; + let cache_path = bucket_dir.join(format!("bucket-{bucket}.xdr")); let (read, stream): (Box, bool) = if cache_path.exists() { println!("🪣 Loading cached bucket {i} {bucket}"); - let file = OpenOptions::new().read(true).open(&cache_path)?; + let file = OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; (Box::new(file), false) } else { let bucket_0 = &bucket[0..=1]; @@ -183,12 +199,13 @@ impl Cmd { "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" ); print!("🪣 Downloading bucket {i} {bucket}"); - let bucket_url = Uri::from_str(&bucket_url)?; + let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; let https = hyper_tls::HttpsConnector::new(); let response = hyper::Client::builder() .build::<_, hyper::Body>(https) .get(bucket_url) - .await?; + .await + .map_err(Error::GettingBucket)?; if let Some(val) = response.headers().get("Content-Length") { if let Ok(str) = val.to_str() { if let Ok(len) = str.parse::() { @@ -221,7 +238,8 @@ impl Cmd { .create(true) .truncate(true) .write(true) - .open(&dl_path)?; + .open(&dl_path) + .map_err(Error::WriteOpeningCachedBucket)?; let tee = TeeReader::new(buf, file); Box::new(tee) } else { @@ -234,7 +252,7 @@ impl Cmd { let sz = Frame::::read_xdr_iter(limited); let mut count_saved = 0; for entry in sz { - let Frame(entry) = entry?; + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; let (key, val) = match entry { BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { let k = data_into_key(&l); @@ -303,7 +321,7 @@ impl Cmd { } } if stream { - fs::rename(&dl_path, &cache_path)?; + fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; } if count_saved > 0 { println!("🔎 Found {count_saved} entries"); @@ -313,7 +331,9 @@ impl Cmd { .await??; } - snapshot.write_file(&self.out)?; + snapshot + .write_file(&self.out) + .map_err(Error::WriteLedgerSnapshot)?; println!( "💾 Saved {} entries to {:?}", snapshot.ledger_entries.len(), From cca26952fa1bc5679ab9c71ca83bd8c5fd53ef79 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 14 Jun 2024 00:40:29 +1000 Subject: [PATCH 14/66] Make archive configurable --- .../src/commands/config/locator.rs | 21 ++++- .../src/commands/contract/deploy/wasm.rs | 4 +- cmd/soroban-cli/src/commands/mod.rs | 4 +- cmd/soroban-cli/src/commands/network/mod.rs | 77 ++++++++++++++++--- cmd/soroban-cli/src/commands/snapshot.rs | 12 +-- cmd/soroban-cli/src/fee.rs | 12 +-- 6 files changed, 100 insertions(+), 30 deletions(-) diff --git a/cmd/soroban-cli/src/commands/config/locator.rs b/cmd/soroban-cli/src/commands/config/locator.rs index 680bcb536..79eb7f88d 100644 --- a/cmd/soroban-cli/src/commands/config/locator.rs +++ b/cmd/soroban-cli/src/commands/config/locator.rs @@ -199,10 +199,23 @@ impl Args { pub fn read_network(&self, name: &str) -> Result { let res = KeyType::Network.read_with_global(name, &self.local_config()?); if let Err(Error::ConfigMissing(_, _)) = &res { - if name == "futurenet" { - let network = Network::futurenet(); - self.write_network(name, &network)?; - return Ok(network); + match name { + "pubnet" => { + let network = Network::pubnet(); + self.write_network(name, &network)?; + return Ok(network); + } + "testnet" => { + let network = Network::testnet(); + self.write_network(name, &network)?; + return Ok(network); + } + "futurenet" => { + let network = Network::futurenet(); + self.write_network(name, &network)?; + return Ok(network); + } + _ => {} } } res diff --git a/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs b/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs index 0b944c78a..83d4ecc41 100644 --- a/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs +++ b/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs @@ -22,7 +22,7 @@ use crate::commands::{ NetworkRunnable, }; use crate::{ - commands::{config, contract::install, HEADING_RPC}, + commands::{config, contract::install, HEADING_NETWORK}, rpc::{self, Client}, utils, wasm, }; @@ -44,7 +44,7 @@ pub struct Cmd { /// Custom salt 32-byte salt for the token id #[arg( long, - help_heading = HEADING_RPC, + help_heading = HEADING_NETWORK, )] salt: Option, #[command(flatten)] diff --git a/cmd/soroban-cli/src/commands/mod.rs b/cmd/soroban-cli/src/commands/mod.rs index df4ed72e1..fd5caea9f 100644 --- a/cmd/soroban-cli/src/commands/mod.rs +++ b/cmd/soroban-cli/src/commands/mod.rs @@ -17,7 +17,7 @@ pub mod version; pub mod txn_result; -pub const HEADING_RPC: &str = "Options (RPC)"; +pub const HEADING_NETWORK: &str = "Options (Network)"; const ABOUT: &str = "Build, deploy, & interact with contracts; set identities to sign with; configure networks; generate keys; and more. Stellar Docs: https://developers.stellar.org @@ -134,7 +134,7 @@ pub enum Cmd { /// Start and configure networks #[command(subcommand)] Network(network::Cmd), - /// Download a snapshot of a ledger. + /// Download a snapshot of a ledger from an archive. Snapshot(snapshot::Cmd), /// Print version information Version(version::Cmd), diff --git a/cmd/soroban-cli/src/commands/network/mod.rs b/cmd/soroban-cli/src/commands/network/mod.rs index 581b965a0..c42d7f4a4 100644 --- a/cmd/soroban-cli/src/commands/network/mod.rs +++ b/cmd/soroban-cli/src/commands/network/mod.rs @@ -1,12 +1,13 @@ use std::str::FromStr; use clap::{arg, Parser}; +use http::Uri; use serde::{Deserialize, Serialize}; use serde_json::Value; use stellar_strkey::ed25519::PublicKey; use crate::{ - commands::HEADING_RPC, + commands::HEADING_NETWORK, rpc::{self, Client}, }; @@ -89,6 +90,8 @@ pub enum Error { InproperResponse(String), #[error("Currently not supported on windows. Please visit:\n{0}")] WindowsNotSupported(String), + #[error("Archive URL not configured")] + ArchiveUrlNotConfigured, } impl Cmd { @@ -123,7 +126,7 @@ pub struct Args { requires = "network_passphrase", required_unless_present = "network", env = "STELLAR_RPC_URL", - help_heading = HEADING_RPC, + help_heading = HEADING_NETWORK, )] pub rpc_url: Option, /// Network passphrase to sign the transaction sent to the rpc server @@ -132,15 +135,23 @@ pub struct Args { requires = "rpc_url", required_unless_present = "network", env = "STELLAR_NETWORK_PASSPHRASE", - help_heading = HEADING_RPC, + help_heading = HEADING_NETWORK, )] pub network_passphrase: Option, + /// Archive URL + #[arg( + long = "archive-url", + requires = "network_passphrase", + env = "STELLAR_ARCHIVE_URL", + help_heading = HEADING_NETWORK, + )] + pub archive_url: Option, /// Name of network to use from config #[arg( long, required_unless_present = "rpc_url", env = "STELLAR_NETWORK", - help_heading = HEADING_RPC, + help_heading = HEADING_NETWORK, )] pub network: Option, } @@ -158,6 +169,7 @@ impl Args { Ok(Network { rpc_url, network_passphrase, + archive_url: self.archive_url.clone(), }) } else { Err(Error::Network) @@ -172,21 +184,27 @@ pub struct Network { #[arg( long = "rpc-url", env = "STELLAR_RPC_URL", - help_heading = HEADING_RPC, + help_heading = HEADING_NETWORK, )] pub rpc_url: String, /// Network passphrase to sign the transaction sent to the rpc server #[arg( - long, - env = "STELLAR_NETWORK_PASSPHRASE", - help_heading = HEADING_RPC, - )] + long, + env = "STELLAR_NETWORK_PASSPHRASE", + help_heading = HEADING_NETWORK, + )] pub network_passphrase: String, + /// Archive URL + #[arg( + long = "archive-url", + env = "STELLAR_ARCHIVE_URL", + help_heading = HEADING_NETWORK, + )] + pub archive_url: Option, } impl Network { pub async fn helper_url(&self, addr: &str) -> Result { - use http::Uri; tracing::debug!("address {addr:?}"); let rpc_uri = Uri::from_str(&self.rpc_url) .map_err(|_| Error::InvalidUrl(self.rpc_url.to_string()))?; @@ -211,6 +229,30 @@ impl Network { } } + pub fn archive_url(&self) -> Result { + // Return the configured archive URL, or if one is not configured, guess + // at an appropriate archive URL given the network passphrase. + self.archive_url + .as_deref() + .or(match self.network_passphrase.as_str() { + "Public Global Stellar Network ; September 2015" => { + Some("https://history.stellar.org/prd/core-live/core_live_001") + } + "Test SDF Network ; September 2015" => { + Some("https://history.stellar.org/prd/core-testnet/core_testnet_001") + } + "Test SDF Future Network ; October 2022" => { + Some("https://history-futurenet.stellar.org") + } + _ => None, + }) + .ok_or(Error::ArchiveUrlNotConfigured) + .and_then(|archive_url| { + Uri::from_str(archive_url) + .map_err(|_| Error::InvalidUrl((*archive_url).to_string())) + }) + } + #[allow(clippy::similar_names)] pub async fn fund_address(&self, addr: &PublicKey) -> Result<(), Error> { let uri = self.helper_url(&addr.to_string()).await?; @@ -248,10 +290,25 @@ impl Network { } impl Network { + pub fn pubnet() -> Self { + Network { + rpc_url: String::new(), + network_passphrase: "Public Global Stellar Network ; September 2015".to_owned(), + archive_url: None, + } + } + pub fn testnet() -> Self { + Network { + rpc_url: "https://soroban-testnet.stellar.org:443".to_owned(), + network_passphrase: "Test SDF Network ; September 2015".to_owned(), + archive_url: None, + } + } pub fn futurenet() -> Self { Network { rpc_url: "https://rpc-futurenet.stellar.org:443".to_owned(), network_passphrase: "Test SDF Future Network ; October 2022".to_owned(), + archive_url: None, } } } diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 92f77d4f0..69c011db8 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -57,8 +57,8 @@ pub struct Cmd { out: PathBuf, #[command(flatten)] locator: locator::Args, - // #[command(flatten)] - // network: network::Args, + #[command(flatten)] + network: network::Args, } #[derive(thiserror::Error, Debug)] @@ -99,7 +99,7 @@ const CHECKPOINT_FREQUENCY: u32 = 64; impl Cmd { pub async fn run(&self) -> Result<(), Error> { - const BASE_URL: &str = "http://history.stellar.org/prd/core-live/core_live_001"; + let archive_url = self.network.get(&self.locator)?.archive_url()?.to_string(); let start = Instant::now(); @@ -120,9 +120,9 @@ impl Cmd { let ledger_hex_0 = &ledger_hex[0..=1]; let ledger_hex_1 = &ledger_hex[2..=3]; let ledger_hex_2 = &ledger_hex[4..=5]; - format!("{BASE_URL}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json") + format!("{archive_url}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json") } else { - format!("{BASE_URL}/.well-known/stellar-history.json") + format!("{archive_url}/.well-known/stellar-history.json") }; let history_url = Uri::from_str(&history_url).unwrap(); @@ -196,7 +196,7 @@ impl Cmd { let bucket_1 = &bucket[2..=3]; let bucket_2 = &bucket[4..=5]; let bucket_url = format!( - "{BASE_URL}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" + "{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" ); print!("🪣 Downloading bucket {i} {bucket}"); let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; diff --git a/cmd/soroban-cli/src/fee.rs b/cmd/soroban-cli/src/fee.rs index ab057a587..c56a96dc5 100644 --- a/cmd/soroban-cli/src/fee.rs +++ b/cmd/soroban-cli/src/fee.rs @@ -3,25 +3,25 @@ use clap::arg; use soroban_env_host::xdr; use soroban_rpc::Assembled; -use crate::commands::HEADING_RPC; +use crate::commands::HEADING_NETWORK; #[derive(Debug, clap::Args, Clone)] #[group(skip)] pub struct Args { /// fee amount for transaction, in stroops. 1 stroop = 0.0000001 xlm - #[arg(long, default_value = "100", env = "STELLAR_FEE", help_heading = HEADING_RPC)] + #[arg(long, default_value = "100", env = "STELLAR_FEE", help_heading = HEADING_NETWORK)] pub fee: u32, /// Output the cost execution to stderr - #[arg(long = "cost", help_heading = HEADING_RPC)] + #[arg(long = "cost", help_heading = HEADING_NETWORK)] pub cost: bool, /// Number of instructions to simulate - #[arg(long, help_heading = HEADING_RPC)] + #[arg(long, help_heading = HEADING_NETWORK)] pub instructions: Option, /// Build the transaction only write the base64 xdr to stdout - #[arg(long, help_heading = HEADING_RPC)] + #[arg(long, help_heading = HEADING_NETWORK)] pub build_only: bool, /// Simulation the transaction only write the base64 xdr to stdout - #[arg(long, help_heading = HEADING_RPC, conflicts_with = "build_only")] + #[arg(long, help_heading = HEADING_NETWORK, conflicts_with = "build_only")] pub sim_only: bool, } From 72b202ca21d417c37b5a833ef2b17042cc8130ee Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:08:27 +1000 Subject: [PATCH 15/66] comment --- cmd/soroban-cli/src/commands/snapshot.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 69c011db8..98d71dcb0 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -281,10 +281,12 @@ impl Cmd { // contract executable stored in another // ledger entry, add that ledger entry to // the filter so that Wasm for any filtered - // contract is collected too. TODO: Change + // contract is collected too. TODO: Change // this to support Wasm ledger entries - // appearing in earlier buckets after state - // archival is rolled out. + // appearing in earlier buckets. In some + // cases that won't be sufficient and a dev + // will need to specify the wasm hash + // manually until this todo is complete. if keep && e.key == ScVal::LedgerKeyContractInstance { if let ScVal::ContractInstance(ScContractInstance { executable: ContractExecutable::Wasm(Hash(hash)), From 1b1955e42646223ebf22d2b1a4bbdbe3c866030f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:23:25 +1000 Subject: [PATCH 16/66] clippy --- cmd/soroban-cli/src/commands/snapshot.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 98d71dcb0..980645474 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -98,6 +98,7 @@ pub enum Error { const CHECKPOINT_FREQUENCY: u32 = 64; impl Cmd { + #[allow(clippy::too-many-lines)] pub async fn run(&self) -> Result<(), Error> { let archive_url = self.network.get(&self.locator)?.archive_url()?.to_string(); From 0bd7eb8f0d60807f07a16e3531cec352cdf34cb9 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:48:09 +1000 Subject: [PATCH 17/66] fix help --- FULL_HELP_DOCS.md | 68 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index 689ce3036..61012acd3 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -4,7 +4,7 @@ This document contains the help content for the `stellar` command-line program. ## `stellar` -With the Stellar CLI you can: +Build, deploy, & interact with contracts; set identities to sign with; configure networks; generate keys; and more. - build, deploy and interact with contracts - set identities to sign with @@ -47,6 +47,7 @@ Anything after the `--` double dash (the "slop") is parsed as arguments to the c * `keys` — Create and manage identities including keys and addresses * `xdr` — Decode and encode XDR * `network` — Start and configure networks +* `snapshot` — Download a snapshot of a ledger from an archive * `version` — Print version information * `tx` — Sign, Simulate, and Send transactions * `cache` — Cache for transactions and contract specs @@ -135,6 +136,7 @@ Get Id of builtin Soroban Asset Contract. Deprecated, use `stellar contract id a * `--asset ` — ID of the Stellar classic asset to wrap, e.g. "USDC:G...5" * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -154,6 +156,7 @@ Deploy builtin Soroban Asset Contract * `--asset ` — ID of the Stellar classic asset to wrap, e.g. "USDC:G...5" * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -164,8 +167,8 @@ Deploy builtin Soroban Asset Contract Default value: `100` * `--cost` — Output the cost execution to stderr * `--instructions ` — Number of instructions to simulate -* `--build-only` — Build the transaction and only write the base64 xdr to stdout -* `--sim-only` — Simulate the transaction and only write the base64 xdr to stdout +* `--build-only` — Build the transaction only write the base64 xdr to stdout +* `--sim-only` — Simulation the transaction only write the base64 xdr to stdout @@ -223,6 +226,7 @@ Generate a TypeScript / JavaScript package * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -289,6 +293,7 @@ If no keys are specified the contract itself is extended. * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -299,8 +304,8 @@ If no keys are specified the contract itself is extended. Default value: `100` * `--cost` — Output the cost execution to stderr * `--instructions ` — Number of instructions to simulate -* `--build-only` — Build the transaction and only write the base64 xdr to stdout -* `--sim-only` — Simulate the transaction and only write the base64 xdr to stdout +* `--build-only` — Build the transaction only write the base64 xdr to stdout +* `--sim-only` — Simulation the transaction only write the base64 xdr to stdout @@ -317,6 +322,7 @@ Deploy a wasm contract * `--salt ` — Custom salt 32-byte salt for the token id * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -327,8 +333,8 @@ Deploy a wasm contract Default value: `100` * `--cost` — Output the cost execution to stderr * `--instructions ` — Number of instructions to simulate -* `--build-only` — Build the transaction and only write the base64 xdr to stdout -* `--sim-only` — Simulate the transaction and only write the base64 xdr to stdout +* `--build-only` — Build the transaction only write the base64 xdr to stdout +* `--sim-only` — Simulation the transaction only write the base64 xdr to stdout * `-i`, `--ignore-checks` — Whether to ignore safety checks when deploying contracts Default value: `false` @@ -350,6 +356,7 @@ Fetch a contract's Wasm binary * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -378,6 +385,7 @@ Deploy builtin Soroban Asset Contract * `--asset ` — ID of the Stellar classic asset to wrap, e.g. "USDC:G...5" * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -397,6 +405,7 @@ Deploy normal Wasm Contract * `--salt ` — ID of the Soroban contract * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -463,6 +472,7 @@ Install a WASM file to the ledger without creating a contract instance * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -473,8 +483,8 @@ Install a WASM file to the ledger without creating a contract instance Default value: `100` * `--cost` — Output the cost execution to stderr * `--instructions ` — Number of instructions to simulate -* `--build-only` — Build the transaction and only write the base64 xdr to stdout -* `--sim-only` — Simulate the transaction and only write the base64 xdr to stdout +* `--build-only` — Build the transaction only write the base64 xdr to stdout +* `--sim-only` — Simulation the transaction only write the base64 xdr to stdout * `--wasm ` — Path to wasm binary * `-i`, `--ignore-checks` — Whether to ignore safety checks when deploying contracts @@ -502,6 +512,7 @@ stellar contract invoke ... -- --help * `--is-view` — View the result simulating and do not sign and submit transaction * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -512,8 +523,8 @@ stellar contract invoke ... -- --help Default value: `100` * `--cost` — Output the cost execution to stderr * `--instructions ` — Number of instructions to simulate -* `--build-only` — Build the transaction and only write the base64 xdr to stdout -* `--sim-only` — Simulate the transaction and only write the base64 xdr to stdout +* `--build-only` — Build the transaction only write the base64 xdr to stdout +* `--sim-only` — Simulation the transaction only write the base64 xdr to stdout @@ -567,6 +578,7 @@ Print the current value of a contract-data ledger entry * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -604,6 +616,7 @@ If no keys are specificed the contract itself is restored. * `--ttl-ledger-only` — Only print the new Time To Live ledger * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -614,8 +627,8 @@ If no keys are specificed the contract itself is restored. Default value: `100` * `--cost` — Output the cost execution to stderr * `--instructions ` — Number of instructions to simulate -* `--build-only` — Build the transaction and only write the base64 xdr to stdout -* `--sim-only` — Simulate the transaction and only write the base64 xdr to stdout +* `--build-only` — Build the transaction only write the base64 xdr to stdout +* `--sim-only` — Simulation the transaction only write the base64 xdr to stdout @@ -664,6 +677,7 @@ Watch the network for contract events * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -737,6 +751,7 @@ Fund an identity on a test network * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--hd-path ` — If identity is a seed phrase use this hd path, default is 0 * `--global` — Use global config @@ -765,6 +780,7 @@ Generate a new identity with a seed phrase, currently 12 words * `-d`, `--default-seed` — Generate the default seed phrase. Useful for testing. Equivalent to --seed 0000000000000000 * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -1012,6 +1028,7 @@ Add a new network * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--global` — Use global config * `--config-dir ` — Location of config directory, default is "." @@ -1189,6 +1206,30 @@ Stop a network started with `network container start`. For example, if you ran ` +## `stellar snapshot` + +Download a snapshot of a ledger from an archive + +**Usage:** `stellar snapshot [OPTIONS]` + +###### **Options:** + +* `--ledger ` — The ledger sequence number to snapshot. Defaults to latest history archived ledger +* `--account-id ` — Account IDs to filter by +* `--contract-id ` — Contract IDs to filter by +* `--wasm-hash ` — Contract IDs to filter by +* `--out ` — The out path that the snapshot is written to + + Default value: `snapshot.json` +* `--global` — Use global config +* `--config-dir ` — Location of config directory, default is "." +* `--rpc-url ` — RPC server endpoint +* `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL +* `--network ` — Name of network to use from config + + + ## `stellar version` Print version information @@ -1219,6 +1260,7 @@ Simulate a transaction envelope from stdin * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server +* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` From dd7fcad111fbe69e7fb80d8d994e1f3c40647025 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jul 2024 21:17:35 +1000 Subject: [PATCH 18/66] undo --- Makefile | 2 +- .../src/commands/contract/deploy/wasm.rs | 4 ++-- cmd/soroban-cli/src/commands/mod.rs | 4 ++-- cmd/soroban-cli/src/commands/network/mod.rs | 16 ++++++++-------- cmd/soroban-cli/src/fee.rs | 16 ++++++++-------- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 49caa15b8..7e307b16c 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ endif install_rust: install install: - cargo install --force --locked --path ./cmd/stellar-cli + cargo install --force --locked --path ./cmd/stellar-cli --debug cargo install --force --locked --path ./cmd/crates/soroban-test/tests/fixtures/hello --root ./target --debug --quiet # regenerate the example lib in `cmd/crates/soroban-spec-typsecript/fixtures/ts` diff --git a/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs b/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs index 480d73892..b85b65bcc 100644 --- a/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs +++ b/cmd/soroban-cli/src/commands/contract/deploy/wasm.rs @@ -23,7 +23,7 @@ use crate::commands::{ NetworkRunnable, }; use crate::{ - commands::{config, contract::install, HEADING_NETWORK}, + commands::{config, contract::install, HEADING_RPC}, rpc::{self, Client}, utils, wasm, }; @@ -45,7 +45,7 @@ pub struct Cmd { /// Custom salt 32-byte salt for the token id #[arg( long, - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub salt: Option, #[command(flatten)] diff --git a/cmd/soroban-cli/src/commands/mod.rs b/cmd/soroban-cli/src/commands/mod.rs index b9fc3c31f..39dba186b 100644 --- a/cmd/soroban-cli/src/commands/mod.rs +++ b/cmd/soroban-cli/src/commands/mod.rs @@ -18,8 +18,8 @@ pub mod version; pub mod txn_result; -pub const HEADING_NETWORK: &str = "Options (Network)"; -const ABOUT: &str = "Build, deploy, & interact with contracts; set identities to sign with; configure networks; generate keys; and more. +pub const HEADING_RPC: &str = "Options (RPC)"; +const ABOUT: &str = "With the Stellar CLI you can: - build, deploy and interact with contracts - set identities to sign with diff --git a/cmd/soroban-cli/src/commands/network/mod.rs b/cmd/soroban-cli/src/commands/network/mod.rs index ceb325a11..e8eb4d6d7 100644 --- a/cmd/soroban-cli/src/commands/network/mod.rs +++ b/cmd/soroban-cli/src/commands/network/mod.rs @@ -8,7 +8,7 @@ use serde_json::Value; use stellar_strkey::ed25519::PublicKey; use crate::{ - commands::HEADING_NETWORK, + commands::HEADING_RPC, rpc::{self, Client}, }; @@ -128,7 +128,7 @@ pub struct Args { requires = "network_passphrase", required_unless_present = "network", env = "STELLAR_RPC_URL", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub rpc_url: Option, /// Network passphrase to sign the transaction sent to the rpc server @@ -137,7 +137,7 @@ pub struct Args { requires = "rpc_url", required_unless_present = "network", env = "STELLAR_NETWORK_PASSPHRASE", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub network_passphrase: Option, /// Archive URL @@ -145,7 +145,7 @@ pub struct Args { long = "archive-url", requires = "network_passphrase", env = "STELLAR_ARCHIVE_URL", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub archive_url: Option, /// Name of network to use from config @@ -153,7 +153,7 @@ pub struct Args { long, required_unless_present = "rpc_url", env = "STELLAR_NETWORK", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub network: Option, } @@ -186,21 +186,21 @@ pub struct Network { #[arg( long = "rpc-url", env = "STELLAR_RPC_URL", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub rpc_url: String, /// Network passphrase to sign the transaction sent to the rpc server #[arg( long, env = "STELLAR_NETWORK_PASSPHRASE", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub network_passphrase: String, /// Archive URL #[arg( long = "archive-url", env = "STELLAR_ARCHIVE_URL", - help_heading = HEADING_NETWORK, + help_heading = HEADING_RPC, )] pub archive_url: Option, } diff --git a/cmd/soroban-cli/src/fee.rs b/cmd/soroban-cli/src/fee.rs index c56a96dc5..698d66007 100644 --- a/cmd/soroban-cli/src/fee.rs +++ b/cmd/soroban-cli/src/fee.rs @@ -3,25 +3,25 @@ use clap::arg; use soroban_env_host::xdr; use soroban_rpc::Assembled; -use crate::commands::HEADING_NETWORK; +use crate::commands::HEADING_RPC; #[derive(Debug, clap::Args, Clone)] #[group(skip)] pub struct Args { /// fee amount for transaction, in stroops. 1 stroop = 0.0000001 xlm - #[arg(long, default_value = "100", env = "STELLAR_FEE", help_heading = HEADING_NETWORK)] + #[arg(long, default_value = "100", env = "STELLAR_FEE", help_heading = HEADING_RPC)] pub fee: u32, /// Output the cost execution to stderr - #[arg(long = "cost", help_heading = HEADING_NETWORK)] + #[arg(long = "cost", help_heading = HEADING_RPC)] pub cost: bool, /// Number of instructions to simulate - #[arg(long, help_heading = HEADING_NETWORK)] + #[arg(long, help_heading = HEADING_RPC)] pub instructions: Option, - /// Build the transaction only write the base64 xdr to stdout - #[arg(long, help_heading = HEADING_NETWORK)] + /// Build the transaction and only write the base64 xdr to stdout + #[arg(long, help_heading = HEADING_RPC)] pub build_only: bool, - /// Simulation the transaction only write the base64 xdr to stdout - #[arg(long, help_heading = HEADING_NETWORK, conflicts_with = "build_only")] + /// Simulate the transaction and only write the base64 xdr to stdout + #[arg(long, help_heading = HEADING_RPC, conflicts_with = "build_only")] pub sim_only: bool, } From f4414e0d7ad2d448d4a00539bbb230de82635d65 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 23 Jul 2024 16:02:08 +1000 Subject: [PATCH 19/66] get all wasm hashes for contracts --- cmd/soroban-cli/src/commands/snapshot.rs | 281 +++++++++++------------ 1 file changed, 138 insertions(+), 143 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 48558b17b..7ca1e1407 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -49,10 +49,10 @@ pub struct Cmd { /// Contract IDs to filter by. #[arg(long = "contract-id", help_heading = "FILTERS")] contract_ids: Vec, - /// Contract IDs to filter by. + /// WASM hashes to filter by. #[arg(long = "wasm-hash", help_heading = "FILTERS")] wasm_hashes: Vec, - /// The out path that the snapshot is written to. + /// Out path that the snapshot is written to. #[arg(long, default_value=default_out_path().into_os_string())] out: PathBuf, #[command(flatten)] @@ -180,158 +180,153 @@ impl Cmd { let mut account_ids = self.account_ids.clone(); let mut contract_ids = self.contract_ids.clone(); let mut wasm_hashes = self.wasm_hashes.clone(); - for (i, bucket) in buckets.iter().enumerate() { - // Defined where the bucket will be read from, either from cache on - // disk, or streamed from the archive. - let bucket_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; - let cache_path = bucket_dir.join(format!("bucket-{bucket}.xdr")); - let (read, stream): (Box, bool) = if cache_path.exists() { - println!("🪣 Loading cached bucket {i} {bucket}"); - let file = OpenOptions::new() - .read(true) - .open(&cache_path) - .map_err(Error::ReadOpeningCachedBucket)?; - (Box::new(file), false) - } else { - let bucket_0 = &bucket[0..=1]; - let bucket_1 = &bucket[2..=3]; - let bucket_2 = &bucket[4..=5]; - let bucket_url = format!( - "{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz" - ); - print!("🪣 Downloading bucket {i} {bucket}"); - let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; - let https = hyper_tls::HttpsConnector::new(); - let response = hyper::Client::builder() - .build::<_, hyper::Body>(https) - .get(bucket_url) - .await - .map_err(Error::GettingBucket)?; - if let Some(val) = response.headers().get("Content-Length") { - if let Ok(str) = val.to_str() { - if let Ok(len) = str.parse::() { - print!(" ({})", ByteSize(len)); + for _ in 0..2 { + for (i, bucket) in buckets.iter().enumerate() { + // Defined where the bucket will be read from, either from cache on + // disk, or streamed from the archive. + let bucket_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; + let cache_path = bucket_dir.join(format!("bucket-{bucket}.xdr")); + let (read, stream): (Box, bool) = if cache_path.exists() { + println!("🪣 Loading cached bucket {i} {bucket}"); + let file = OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; + (Box::new(file), false) + } else { + let bucket_0 = &bucket[0..=1]; + let bucket_1 = &bucket[2..=3]; + let bucket_2 = &bucket[4..=5]; + let bucket_url = format!("{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz"); + print!("🪣 Downloading bucket {i} {bucket}"); + let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; + let https = hyper_tls::HttpsConnector::new(); + let response = hyper::Client::builder() + .build::<_, hyper::Body>(https) + .get(bucket_url) + .await + .map_err(Error::GettingBucket)?; + if let Some(val) = response.headers().get("Content-Length") { + if let Ok(str) = val.to_str() { + if let Ok(len) = str.parse::() { + print!(" ({})", ByteSize(len)); + } } } - } - println!(); - let read = tokio_util::io::SyncIoBridge::new( - response - .into_body() - .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) - .into_async_read() - .compat(), - ); - (Box::new(read), true) - }; + println!(); + let read = tokio_util::io::SyncIoBridge::new( + response + .into_body() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .into_async_read() + .compat(), + ); + (Box::new(read), true) + }; - let cache_path = cache_path.clone(); - (seen, snapshot, account_ids, contract_ids, wasm_hashes) = - tokio::task::spawn_blocking(move || -> Result<_, Error> { - let dl_path = cache_path.with_extension("dl"); - let buf = BufReader::new(read); - let read: Box = if stream { - // When streamed from the archive the bucket will be - // uncompressed, and also be streamed to cache. - let gz = GzDecoder::new(buf); - let buf = BufReader::new(gz); - let file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&dl_path) - .map_err(Error::WriteOpeningCachedBucket)?; - let tee = TeeReader::new(buf, file); - Box::new(tee) - } else { - Box::new(buf) - }; - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(read, Limits::none()); - let sz = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in sz { - let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) - } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(m) => { - snapshot.protocol_version = m.ledger_version; - continue; - } + let cache_path = cache_path.clone(); + (seen, snapshot, account_ids, contract_ids, wasm_hashes) = + tokio::task::spawn_blocking(move || -> Result<_, Error> { + let dl_path = cache_path.with_extension("dl"); + let buf = BufReader::new(read); + let read: Box = if stream { + // When streamed from the archive the bucket will be + // uncompressed, and also be streamed to cache. + let gz = GzDecoder::new(buf); + let buf = BufReader::new(gz); + let file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&dl_path) + .map_err(Error::WriteOpeningCachedBucket)?; + let tee = TeeReader::new(buf, file); + Box::new(tee) + } else { + Box::new(buf) }; - if seen.contains(&key) { - continue; - } - if let Some(val) = val { - let keep = match &val.data { - LedgerEntryData::Account(e) => { - account_ids.contains(&e.account_id.to_string()) + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(read, Limits::none()); + let sz = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in sz { + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) } - LedgerEntryData::Trustline(e) => { - account_ids.contains(&e.account_id.to_string()) + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; + continue; } - LedgerEntryData::ContractData(e) => { - let keep = contract_ids.contains(&e.contract.to_string()); - // If a contract instance references - // contract executable stored in another - // ledger entry, add that ledger entry to - // the filter so that Wasm for any filtered - // contract is collected too. TODO: Change - // this to support Wasm ledger entries - // appearing in earlier buckets. In some - // cases that won't be sufficient and a dev - // will need to specify the wasm hash - // manually until this todo is complete. - if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(Hash(hash)), - .. - }) = e.val - { - let hash = hex::encode(hash); - wasm_hashes.push(hash); + }; + if seen.contains(&key) { + continue; + } + if let Some(val) = val { + let keep = match &val.data { + LedgerEntryData::Account(e) => { + account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::Trustline(e) => { + account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::ContractData(e) => { + let keep = contract_ids.contains(&e.contract.to_string()); + // If a contract instance references + // contract executable stored in another + // ledger entry, add that ledger entry to + // the filter so that Wasm for any filtered + // contract is collected too. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(Hash(hash)), + .. + }) = e.val + { + let hash = hex::encode(hash); + wasm_hashes.push(hash); + } } + keep } - keep - } - LedgerEntryData::ContractCode(e) => { - let hash = hex::encode(e.hash.0); - wasm_hashes.contains(&hash) + LedgerEntryData::ContractCode(e) => { + let hash = hex::encode(e.hash.0); + wasm_hashes.contains(&hash) + } + LedgerEntryData::Offer(_) + | LedgerEntryData::Data(_) + | LedgerEntryData::ClaimableBalance(_) + | LedgerEntryData::LiquidityPool(_) + | LedgerEntryData::ConfigSetting(_) + | LedgerEntryData::Ttl(_) => false, + }; + seen.insert(key.clone()); + if keep { + // Store the found ledger entry in the snapshot with + // a max u32 expiry. TODO: Change the expiry to come + // from the corresponding TTL ledger entry. + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; } - LedgerEntryData::Offer(_) - | LedgerEntryData::Data(_) - | LedgerEntryData::ClaimableBalance(_) - | LedgerEntryData::LiquidityPool(_) - | LedgerEntryData::ConfigSetting(_) - | LedgerEntryData::Ttl(_) => false, - }; - seen.insert(key.clone()); - if keep { - // Store the found ledger entry in the snapshot with - // a max u32 expiry. TODO: Change the expiry to come - // from the corresponding TTL ledger entry. - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; } } - } - if stream { - fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; - } - if count_saved > 0 { - println!("🔎 Found {count_saved} entries"); - } - Ok((seen, snapshot, account_ids, contract_ids, wasm_hashes)) - }) - .await??; + if stream { + fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; + } + if count_saved > 0 { + println!("🔎 Found {count_saved} entries"); + } + Ok((seen, snapshot, account_ids, contract_ids, wasm_hashes)) + }) + .await??; + } } snapshot From 2c8e512e88d9e29a6b66d63b23f60127675f3c46 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:52:04 +1000 Subject: [PATCH 20/66] add format rename filters --- cmd/soroban-cli/src/commands/snapshot.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 7ca1e1407..0efd3e7bc 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -1,5 +1,5 @@ use bytesize::ByteSize; -use clap::{arg, Parser}; +use clap::{arg, Parser, ValueEnum}; use flate2::bufread::GzDecoder; use futures::TryStreamExt; use http::Uri; @@ -32,6 +32,17 @@ use super::{ }; use crate::commands::config::data; +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] +pub enum Format { + Json, +} + +impl Default for Format { + fn default() -> Self { + Self::Json + } +} + fn default_out_path() -> PathBuf { PathBuf::new().join("snapshot.json") } @@ -44,14 +55,17 @@ pub struct Cmd { #[arg(long)] ledger: Option, /// Account IDs to filter by. - #[arg(long = "account-id", help_heading = "FILTERS")] + #[arg(long = "account-id", help_heading = "Filter Options")] account_ids: Vec, /// Contract IDs to filter by. - #[arg(long = "contract-id", help_heading = "FILTERS")] + #[arg(long = "contract-id", help_heading = "Filter Options")] contract_ids: Vec, /// WASM hashes to filter by. - #[arg(long = "wasm-hash", help_heading = "FILTERS")] + #[arg(long = "wasm-hash", help_heading = "Filter Options")] wasm_hashes: Vec, + /// Format of the out file. + #[arg(long)] + format: Format, /// Out path that the snapshot is written to. #[arg(long, default_value=default_out_path().into_os_string())] out: PathBuf, From 69ceffe445fcf3083128e006615de24231f33a42 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:03:17 +1000 Subject: [PATCH 21/66] better error handling with responses --- cmd/soroban-cli/src/commands/snapshot.rs | 39 ++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 0efd3e7bc..6a1b0488f 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -79,6 +79,8 @@ pub struct Cmd { pub enum Error { #[error("downloading history: {0}")] DownloadingHistory(hyper::Error), + #[error("downloading history: got status code {0}")] + DownloadingHistoryGotStatusCode(hyper::StatusCode), #[error("json decoding history: {0}")] JsonDecodingHistory(serde_json::Error), #[error("opening cached bucket to read: {0}")] @@ -87,6 +89,8 @@ pub enum Error { ParsingBucketUrl(http::uri::InvalidUri), #[error("getting bucket: {0}")] GettingBucket(hyper::Error), + #[error("getting bucket: got status code {0}")] + GettingBucketGotStatusCode(hyper::StatusCode), #[error("opening cached bucket to write: {0}")] WriteOpeningCachedBucket(io::Error), #[error("read XDR frame bucket entry: {0}")] @@ -109,6 +113,10 @@ pub enum Error { Config(#[from] config::Error), } +/// Checkpoint frequency is usually 64 ledgers, but in local test nets it'll +/// often by 8. There's no way to simply detect what frequency to expect ledgers +/// at, so it is hardcoded at 64, and this value is used only to help the user +/// select good ledger numbers when they select one that doesn't exist. const CHECKPOINT_FREQUENCY: u32 = 64; impl Cmd { @@ -119,17 +127,6 @@ impl Cmd { let start = Instant::now(); let history_url = if let Some(ledger) = self.ledger { - // Check ledger is a checkpoint ledger and available in archives. - let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; - if ledger_offset != 0 { - println!( - "ledger {ledger} not a checkpoint ledger, use {} or {}", - ledger - ledger_offset, - ledger + (CHECKPOINT_FREQUENCY - ledger_offset), - ); - return Ok(()); - } - // Download history JSON file. let ledger_hex = format!("{ledger:08x}"); let ledger_hex_0 = &ledger_hex[0..=1]; @@ -148,6 +145,20 @@ impl Cmd { .get(history_url) .await .map_err(Error::DownloadingHistory)?; + if !response.status().is_success() { + // Check ledger is a checkpoint ledger and available in archives. + if let Some(ledger) = self.ledger { + let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; + if ledger_offset != 0 { + println!( + "ℹ️ Ledger {ledger} may not be a checkpoint ledger, try {} or {}", + ledger - ledger_offset, + ledger + (CHECKPOINT_FREQUENCY - ledger_offset), + ); + } + } + return Err(Error::DownloadingHistoryGotStatusCode(response.status())); + } let body = hyper::body::to_bytes(response.into_body()) .await .map_err(Error::ReadHistoryHttpStream)?; @@ -194,7 +205,8 @@ impl Cmd { let mut account_ids = self.account_ids.clone(); let mut contract_ids = self.contract_ids.clone(); let mut wasm_hashes = self.wasm_hashes.clone(); - for _ in 0..2 { + for p in 1..=2 { + println!("ℹ️ Beginning parse {p}/2 of buckets..."); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. @@ -220,6 +232,9 @@ impl Cmd { .get(bucket_url) .await .map_err(Error::GettingBucket)?; + if !response.status().is_success() { + return Err(Error::GettingBucketGotStatusCode(response.status())); + } if let Some(val) = response.headers().get("Content-Length") { if let Ok(str) = val.to_str() { if let Ok(len) = str.parse::() { From 2ef0154d3d4e5c0b539be3ca76f5b1089aa30078 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:06:09 +1000 Subject: [PATCH 22/66] fix new line --- cmd/soroban-cli/src/commands/snapshot.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 6a1b0488f..4c3e966ea 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -233,6 +233,7 @@ impl Cmd { .await .map_err(Error::GettingBucket)?; if !response.status().is_success() { + println!(); return Err(Error::GettingBucketGotStatusCode(response.status())); } if let Some(val) = response.headers().get("Content-Length") { From e7b668cae0277401386961277b2c3881f977a49d Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 26 Jul 2024 21:13:27 +1000 Subject: [PATCH 23/66] reorg --- cmd/soroban-cli/src/commands/snapshot.rs | 231 +++++++++++++---------- 1 file changed, 130 insertions(+), 101 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 4c3e966ea..84faaf9b6 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -77,6 +77,8 @@ pub struct Cmd { #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("wasm hash invalid: {0}")] + WasmHashInvalid(String), #[error("downloading history: {0}")] DownloadingHistory(hyper::Error), #[error("downloading history: got status code {0}")] @@ -122,48 +124,10 @@ const CHECKPOINT_FREQUENCY: u32 = 64; impl Cmd { #[allow(clippy::too_many_lines)] pub async fn run(&self) -> Result<(), Error> { - let archive_url = self.network.get(&self.locator)?.archive_url()?.to_string(); - let start = Instant::now(); - let history_url = if let Some(ledger) = self.ledger { - // Download history JSON file. - let ledger_hex = format!("{ledger:08x}"); - let ledger_hex_0 = &ledger_hex[0..=1]; - let ledger_hex_1 = &ledger_hex[2..=3]; - let ledger_hex_2 = &ledger_hex[4..=5]; - format!("{archive_url}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json") - } else { - format!("{archive_url}/.well-known/stellar-history.json") - }; - - let history_url = Uri::from_str(&history_url).unwrap(); - println!("🌎 Downloading history {history_url}"); - let https = hyper_tls::HttpsConnector::new(); - let response = hyper::Client::builder() - .build::<_, hyper::Body>(https) - .get(history_url) - .await - .map_err(Error::DownloadingHistory)?; - if !response.status().is_success() { - // Check ledger is a checkpoint ledger and available in archives. - if let Some(ledger) = self.ledger { - let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; - if ledger_offset != 0 { - println!( - "ℹ️ Ledger {ledger} may not be a checkpoint ledger, try {} or {}", - ledger - ledger_offset, - ledger + (CHECKPOINT_FREQUENCY - ledger_offset), - ); - } - } - return Err(Error::DownloadingHistoryGotStatusCode(response.status())); - } - let body = hyper::body::to_bytes(response.into_body()) - .await - .map_err(Error::ReadHistoryHttpStream)?; - let history = - serde_json::from_slice::(&body).map_err(Error::JsonDecodingHistory)?; + let archive_url = self.network.get(&self.locator)?.archive_url()?; + let history = get_history(&archive_url, self.ledger).await?; let ledger = history.current_ledger; let network_passphrase = &history.network_passphrase; @@ -204,66 +168,41 @@ impl Cmd { let mut account_ids = self.account_ids.clone(); let mut contract_ids = self.contract_ids.clone(); - let mut wasm_hashes = self.wasm_hashes.clone(); + let mut wasm_hashes = self + .wasm_hashes + .iter() + .map(|h| { + hex::decode(h) + .map_err(|_| Error::WasmHashInvalid(h.clone())) + .and_then(|vec| { + vec.try_into() + .map_err(|_| Error::WasmHashInvalid("".to_string())) + }) + }) + .collect::, _>>()?; + // Parse the buckets twice, because during the first pass contracts + // and accounts will be found, along with explicitly provided wasm + // hashes. Contracts found will have their wasm hashes added to the + // filter because in most cases if someone wants a ledger snapshot + // of a contract they also want the contracts code entry (e.g. + // wasm). for p in 1..=2 { println!("ℹ️ Beginning parse {p}/2 of buckets..."); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. - let bucket_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; - let cache_path = bucket_dir.join(format!("bucket-{bucket}.xdr")); - let (read, stream): (Box, bool) = if cache_path.exists() { - println!("🪣 Loading cached bucket {i} {bucket}"); - let file = OpenOptions::new() - .read(true) - .open(&cache_path) - .map_err(Error::ReadOpeningCachedBucket)?; - (Box::new(file), false) - } else { - let bucket_0 = &bucket[0..=1]; - let bucket_1 = &bucket[2..=3]; - let bucket_2 = &bucket[4..=5]; - let bucket_url = format!("{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz"); - print!("🪣 Downloading bucket {i} {bucket}"); - let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; - let https = hyper_tls::HttpsConnector::new(); - let response = hyper::Client::builder() - .build::<_, hyper::Body>(https) - .get(bucket_url) - .await - .map_err(Error::GettingBucket)?; - if !response.status().is_success() { - println!(); - return Err(Error::GettingBucketGotStatusCode(response.status())); - } - if let Some(val) = response.headers().get("Content-Length") { - if let Ok(str) = val.to_str() { - if let Ok(len) = str.parse::() { - print!(" ({})", ByteSize(len)); - } - } - } - println!(); - let read = tokio_util::io::SyncIoBridge::new( - response - .into_body() - .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) - .into_async_read() - .compat(), - ); - (Box::new(read), true) - }; + let (read, cache_path, from_cache) = + get_bucket_stream(&archive_url, i, bucket).await?; - let cache_path = cache_path.clone(); (seen, snapshot, account_ids, contract_ids, wasm_hashes) = tokio::task::spawn_blocking(move || -> Result<_, Error> { let dl_path = cache_path.with_extension("dl"); let buf = BufReader::new(read); - let read: Box = if stream { - // When streamed from the archive the bucket will be - // uncompressed, and also be streamed to cache. - let gz = GzDecoder::new(buf); - let buf = BufReader::new(gz); + let read: Box = if from_cache { + Box::new(buf) + } else { + // When the stream is not from the cache, write it + // to the cache. let file = OpenOptions::new() .create(true) .truncate(true) @@ -272,16 +211,14 @@ impl Cmd { .map_err(Error::WriteOpeningCachedBucket)?; let tee = TeeReader::new(buf, file); Box::new(tee) - } else { - Box::new(buf) }; // Stream the bucket entries from the bucket, identifying // entries that match the filters, and including only the // entries that match in the snapshot. let limited = &mut Limited::new(read, Limits::none()); - let sz = Frame::::read_xdr_iter(limited); + let entries = Frame::::read_xdr_iter(limited); let mut count_saved = 0; - for entry in sz { + for entry in entries { let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; let (key, val) = match entry { BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { @@ -311,22 +248,20 @@ impl Cmd { // contract executable stored in another // ledger entry, add that ledger entry to // the filter so that Wasm for any filtered - // contract is collected too. + // contract is collected too in the second pass. if keep && e.key == ScVal::LedgerKeyContractInstance { if let ScVal::ContractInstance(ScContractInstance { executable: ContractExecutable::Wasm(Hash(hash)), .. }) = e.val { - let hash = hex::encode(hash); wasm_hashes.push(hash); } } keep } LedgerEntryData::ContractCode(e) => { - let hash = hex::encode(e.hash.0); - wasm_hashes.contains(&hash) + wasm_hashes.contains(&e.hash.0) } LedgerEntryData::Offer(_) | LedgerEntryData::Data(_) @@ -338,8 +273,9 @@ impl Cmd { seen.insert(key.clone()); if keep { // Store the found ledger entry in the snapshot with - // a max u32 expiry. TODO: Change the expiry to come - // from the corresponding TTL ledger entry. + // a max u32 expiry. + // TODO: Change the expiry to come from the + // corresponding TTL ledger entry. snapshot .ledger_entries .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); @@ -347,7 +283,7 @@ impl Cmd { } } } - if stream { + if !from_cache { fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; } if count_saved > 0 { @@ -375,6 +311,99 @@ impl Cmd { } } +async fn get_history(archive_url: &Uri, ledger: Option) -> Result { + let history_url = if let Some(ledger) = ledger { + let ledger_hex = format!("{ledger:08x}"); + let ledger_hex_0 = &ledger_hex[0..=1]; + let ledger_hex_1 = &ledger_hex[2..=3]; + let ledger_hex_2 = &ledger_hex[4..=5]; + format!("{archive_url}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json") + } else { + format!("{archive_url}/.well-known/stellar-history.json") + }; + let history_url = Uri::from_str(&history_url).unwrap(); + + println!("🌎 Downloading history {history_url}"); + let https = hyper_tls::HttpsConnector::new(); + let response = hyper::Client::builder() + .build::<_, hyper::Body>(https) + .get(history_url) + .await + .map_err(Error::DownloadingHistory)?; + if !response.status().is_success() { + // Check ledger is a checkpoint ledger and available in archives. + if let Some(ledger) = ledger { + let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; + if ledger_offset != 0 { + println!( + "ℹ️ Ledger {ledger} may not be a checkpoint ledger, try {} or {}", + ledger - ledger_offset, + ledger + (CHECKPOINT_FREQUENCY - ledger_offset), + ); + } + } + return Err(Error::DownloadingHistoryGotStatusCode(response.status())); + } + let body = hyper::body::to_bytes(response.into_body()) + .await + .map_err(Error::ReadHistoryHttpStream)?; + serde_json::from_slice::(&body).map_err(Error::JsonDecodingHistory) +} + +async fn get_bucket_stream( + archive_url: &Uri, + bucket_index: usize, + bucket: &str, +) -> Result<(Box, PathBuf, bool), Error> { + let bucket_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; + let cache_path = bucket_dir.join(format!("bucket-{bucket}.xdr")); + let (read, from_cache): (Box, bool) = if cache_path.exists() { + println!("🪣 Loading cached bucket {bucket_index} {bucket}"); + let file = OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; + (Box::new(file), true) + } else { + let bucket_0 = &bucket[0..=1]; + let bucket_1 = &bucket[2..=3]; + let bucket_2 = &bucket[4..=5]; + let bucket_url = + format!("{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz"); + print!("🪣 Downloading bucket {bucket_index} {bucket}"); + let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; + let https = hyper_tls::HttpsConnector::new(); + let response = hyper::Client::builder() + .build::<_, hyper::Body>(https) + .get(bucket_url) + .await + .map_err(Error::GettingBucket)?; + if !response.status().is_success() { + println!(); + return Err(Error::GettingBucketGotStatusCode(response.status())); + } + if let Some(val) = response.headers().get("Content-Length") { + if let Ok(str) = val.to_str() { + if let Ok(len) = str.parse::() { + print!(" ({})", ByteSize(len)); + } + } + } + println!(); + let read = tokio_util::io::SyncIoBridge::new( + response + .into_body() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .into_async_read() + .compat(), + ); + let read = GzDecoder::new(read); + let read = BufReader::new(read); + (Box::new(read), false) + }; + Ok((read, cache_path, from_cache)) +} + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] #[serde(rename_all = "camelCase")] struct History { From b4cbe80c3bc7b8f0a1732f9e0cef273c3c49f982 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:21:32 +1000 Subject: [PATCH 24/66] include instead of filter vocab for less ambiguity --- cmd/soroban-cli/src/commands/snapshot.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 54c153f6b..62b6fdf6e 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -51,13 +51,13 @@ pub struct Cmd { /// The ledger sequence number to snapshot. Defaults to latest history archived ledger. #[arg(long)] ledger: Option, - /// Account IDs to filter by. + /// Account IDs to include in the snapshot. #[arg(long = "account-id", help_heading = "Filter Options")] account_ids: Vec, - /// Contract IDs to filter by. + /// Contract IDs to include in the snapshot. #[arg(long = "contract-id", help_heading = "Filter Options")] contract_ids: Vec, - /// WASM hashes to filter by. + /// WASM hashes to include in the snapshot. #[arg(long = "wasm-hash", help_heading = "Filter Options")] wasm_hashes: Vec, /// Format of the out file. From e3fbd43e013d571b4aee30bad55f751730adae4c Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:35:25 +1000 Subject: [PATCH 25/66] fn --- cmd/soroban-cli/src/commands/snapshot.rs | 27 +++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 62b6fdf6e..5a44b8fa3 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -165,18 +165,7 @@ impl Cmd { let mut account_ids = self.account_ids.clone(); let mut contract_ids = self.contract_ids.clone(); - let mut wasm_hashes = self - .wasm_hashes - .iter() - .map(|h| { - hex::decode(h) - .map_err(|_| Error::WasmHashInvalid(h.clone())) - .and_then(|vec| { - vec.try_into() - .map_err(|_| Error::WasmHashInvalid("".to_string())) - }) - }) - .collect::, _>>()?; + let mut wasm_hashes = self.wasm_hashes()?; // Parse the buckets twice, because during the first pass contracts // and accounts will be found, along with explicitly provided wasm // hashes. Contracts found will have their wasm hashes added to the @@ -306,6 +295,20 @@ impl Cmd { Ok(()) } + + fn wasm_hashes(&self) -> Result, Error> { + self.wasm_hashes + .iter() + .map(|h| { + hex::decode(h) + .map_err(|_| Error::WasmHashInvalid(h.clone())) + .and_then(|vec| { + vec.try_into() + .map_err(|_| Error::WasmHashInvalid(h.clone())) + }) + }) + .collect::, _>>() + } } async fn get_history(archive_url: &Uri, ledger: Option) -> Result { From 1abb4306170e422246a26ff6f367918df879cba7 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:01:51 +1000 Subject: [PATCH 26/66] fix async escape --- cmd/soroban-cli/src/commands/snapshot.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 5a44b8fa3..7a5bce304 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -198,6 +198,8 @@ impl Cmd { let tee = TeeReader::new(buf, file); Box::new(tee) }; + let read = BufReader::new(read); + let read = GzDecoder::new(read); // Stream the bucket entries from the bucket, identifying // entries that match the filters, and including only the // entries that match in the snapshot. @@ -397,8 +399,6 @@ async fn get_bucket_stream( .into_async_read() .compat(), ); - let read = GzDecoder::new(read); - let read = BufReader::new(read); (Box::new(read), false) }; Ok((read, cache_path, from_cache)) From 61236fc33243d281abe54dbcbf0f76f5ab1244da Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 22:01:36 +1000 Subject: [PATCH 27/66] bugs --- cmd/soroban-cli/src/commands/snapshot.rs | 14 ++++++++------ cmd/soroban-cli/src/config/network.rs | 9 ++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 7a5bce304..4ba4ad5ae 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -10,7 +10,7 @@ use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, fs::{self, OpenOptions}, - io::{self, BufReader, Read}, + io::{self, BufReader, BufWriter, Read}, path::PathBuf, str::FromStr, time::{Duration, Instant}, @@ -183,10 +183,13 @@ impl Cmd { (seen, snapshot, account_ids, contract_ids, wasm_hashes) = tokio::task::spawn_blocking(move || -> Result<_, Error> { let dl_path = cache_path.with_extension("dl"); - let buf = BufReader::new(read); let read: Box = if from_cache { - Box::new(buf) + let read = BufReader::new(read); + Box::new(read) } else { + let read = BufReader::new(read); + let read = GzDecoder::new(read); + let read = BufReader::new(read); // When the stream is not from the cache, write it // to the cache. let file = OpenOptions::new() @@ -195,11 +198,10 @@ impl Cmd { .write(true) .open(&dl_path) .map_err(Error::WriteOpeningCachedBucket)?; - let tee = TeeReader::new(buf, file); + let write = BufWriter::new(file); + let tee = TeeReader::new(read, write); Box::new(tee) }; - let read = BufReader::new(read); - let read = GzDecoder::new(read); // Stream the bucket entries from the bucket, identifying // entries that match the filters, and including only the // entries that match in the snapshot. diff --git a/cmd/soroban-cli/src/config/network.rs b/cmd/soroban-cli/src/config/network.rs index d8fefc344..2d05f43f9 100644 --- a/cmd/soroban-cli/src/config/network.rs +++ b/cmd/soroban-cli/src/config/network.rs @@ -194,15 +194,18 @@ impl Network { self.archive_url .as_deref() .or(match self.network_passphrase.as_str() { - "Public Global Stellar Network ; September 2015" => { + passphrase::MAINNET => { Some("https://history.stellar.org/prd/core-live/core_live_001") } - "Test SDF Network ; September 2015" => { + passphrase::TESTNET => { Some("https://history.stellar.org/prd/core-testnet/core_testnet_001") } - "Test SDF Future Network ; October 2022" => { + passphrase::FUTURENET => { Some("https://history-futurenet.stellar.org") } + passphrase::LOCAL => { + Some("http://localhost:8000/archive") + } _ => None, }) .ok_or(Error::ArchiveUrlNotConfigured) From b53a17edbf900a8b29868e52cc6c2a1502ab1ed1 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 30 Jul 2024 23:19:28 +1000 Subject: [PATCH 28/66] fix --- FULL_HELP_DOCS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index 95675f951..ee468617d 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -1024,9 +1024,9 @@ Download a snapshot of a ledger from an archive ###### **Options:** * `--ledger ` — The ledger sequence number to snapshot. Defaults to latest history archived ledger -* `--account-id ` — Account IDs to filter by -* `--contract-id ` — Contract IDs to filter by -* `--wasm-hash ` — WASM hashes to filter by +* `--account-id ` — Account IDs to include in the snapshot +* `--contract-id ` — Contract IDs to include in the snapshot +* `--wasm-hash ` — WASM hashes to include in the snapshot * `--format ` — Format of the out file Possible values: `json` From 181f90175d09c5433b71f7d9c0c0a5b923054f33 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 30 Jul 2024 23:38:22 +1000 Subject: [PATCH 29/66] fix --- cmd/soroban-cli/src/config/network.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-cli/src/config/network.rs b/cmd/soroban-cli/src/config/network.rs index c2b52a7b9..959b95b7d 100644 --- a/cmd/soroban-cli/src/config/network.rs +++ b/cmd/soroban-cli/src/config/network.rs @@ -198,12 +198,8 @@ impl Network { passphrase::TESTNET => { Some("https://history.stellar.org/prd/core-testnet/core_testnet_001") } - passphrase::FUTURENET => { - Some("https://history-futurenet.stellar.org") - } - passphrase::LOCAL => { - Some("http://localhost:8000/archive") - } + passphrase::FUTURENET => Some("https://history-futurenet.stellar.org"), + passphrase::LOCAL => Some("http://localhost:8000/archive"), _ => None, }) .ok_or(Error::ArchiveUrlNotConfigured) From 77bbc12a2899f2e9a830a43e1227c99d74259cbc Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 00:05:37 +1000 Subject: [PATCH 30/66] fix --- cmd/crates/soroban-test/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/crates/soroban-test/src/lib.rs b/cmd/crates/soroban-test/src/lib.rs index 544e2d59e..f642b083f 100644 --- a/cmd/crates/soroban-test/src/lib.rs +++ b/cmd/crates/soroban-test/src/lib.rs @@ -226,6 +226,7 @@ impl TestEnv { rpc_url: Some(self.rpc_url.clone()), network_passphrase: Some(LOCAL_NETWORK_PASSPHRASE.to_string()), network: None, + archive_url: None, }, source_account: account.to_string(), locator: config::locator::Args { From 88630603b3a915991bbe63f258e244aa15c9a0e6 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:24:23 +1000 Subject: [PATCH 31/66] add test --- .../soroban-test/tests/it/integration.rs | 1 + .../tests/it/integration/snapshot.rs | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 cmd/crates/soroban-test/tests/it/integration/snapshot.rs diff --git a/cmd/crates/soroban-test/tests/it/integration.rs b/cmd/crates/soroban-test/tests/it/integration.rs index b25d47313..8e2bb89a4 100644 --- a/cmd/crates/soroban-test/tests/it/integration.rs +++ b/cmd/crates/soroban-test/tests/it/integration.rs @@ -3,6 +3,7 @@ mod custom_types; mod dotenv; mod fund; mod hello_world; +mod snapshot; mod tx; mod util; mod wrap; diff --git a/cmd/crates/soroban-test/tests/it/integration/snapshot.rs b/cmd/crates/soroban-test/tests/it/integration/snapshot.rs new file mode 100644 index 000000000..897728b6b --- /dev/null +++ b/cmd/crates/soroban-test/tests/it/integration/snapshot.rs @@ -0,0 +1,82 @@ +use assert_fs::prelude::*; +use predicates::prelude::*; +use soroban_test::{AssertExt, TestEnv}; + +#[test] +#[allow(clippy::too_many_lines)] +fn snapshot() { + let sandbox = &TestEnv::new(); + // Create a couple accounts and a couple contracts, which we'll filter on to + // make sure we only get the account and contract requested. + sandbox + .new_assert_cmd("keys") + .arg("generate") + .arg("a") + .assert() + .success(); + let account_a = sandbox + .new_assert_cmd("keys") + .arg("address") + .arg("a") + .assert() + .success() + .stdout_as_str(); + sandbox + .new_assert_cmd("keys") + .arg("generate") + .arg("b") + .assert() + .success(); + let account_b = sandbox + .new_assert_cmd("keys") + .arg("address") + .arg("b") + .assert() + .success() + .stdout_as_str(); + let contract_a = sandbox + .new_assert_cmd("contract") + .arg("asset") + .arg("deploy") + .arg(format!("--asset=A1:{account_a}")) + .assert() + .success() + .stdout_as_str(); + let contract_b = sandbox + .new_assert_cmd("contract") + .arg("asset") + .arg("deploy") + .arg(format!("--asset=A2:{account_a}")) + .assert() + .success() + .stdout_as_str(); + // Wait 8 ledgers for a checkpoint by submitting one tx per ledger, in this + // case a funding transaction. + for i in 1..=8 { + sandbox + .new_assert_cmd("keys") + .arg("generate") + .arg(format!("k{i}")) + .assert() + .success(); + } + // Create the snapshot. + sandbox + .new_assert_cmd("snapshot") + .arg("--format=json") + .arg("--account-id") + .arg(&account_a) + .arg("--contract-id") + .arg(&contract_b) + .assert() + .success(); + // Assert that the snapshot includes account a and contract b, but not + // account b and contract a. + sandbox + .dir() + .child("snapshot.json") + .assert(predicates::str::contains(&account_a)) + .assert(predicates::str::contains(&account_b).not()) + .assert(predicates::str::contains(&contract_b)) + .assert(predicates::str::contains(&contract_a).not()); +} From c299525fcac374dc5e7ec8b904878ad0f16367f2 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:48:54 +1000 Subject: [PATCH 32/66] fix --- cmd/soroban-cli/src/commands/snapshot.rs | 212 +++++++++++++---------- 1 file changed, 120 insertions(+), 92 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 4ba4ad5ae..7242b9a7e 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -166,6 +166,7 @@ impl Cmd { let mut account_ids = self.account_ids.clone(); let mut contract_ids = self.contract_ids.clone(); let mut wasm_hashes = self.wasm_hashes()?; + let mut wasm_hashes_2nd_pass = Vec::<[u8; 32]>::new(); // Parse the buckets twice, because during the first pass contracts // and accounts will be found, along with explicitly provided wasm // hashes. Contracts found will have their wasm hashes added to the @@ -173,6 +174,10 @@ impl Cmd { // of a contract they also want the contracts code entry (e.g. // wasm). for p in 1..=2 { + if p == 2 && wasm_hashes_2nd_pass.is_empty() { + println!("ℹ️ Skipping parse {p}/2 of buckets"); + break; + } println!("ℹ️ Beginning parse {p}/2 of buckets..."); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on @@ -180,108 +185,131 @@ impl Cmd { let (read, cache_path, from_cache) = get_bucket_stream(&archive_url, i, bucket).await?; - (seen, snapshot, account_ids, contract_ids, wasm_hashes) = - tokio::task::spawn_blocking(move || -> Result<_, Error> { - let dl_path = cache_path.with_extension("dl"); - let read: Box = if from_cache { - let read = BufReader::new(read); - Box::new(read) - } else { - let read = BufReader::new(read); - let read = GzDecoder::new(read); - let read = BufReader::new(read); - // When the stream is not from the cache, write it - // to the cache. - let file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&dl_path) - .map_err(Error::WriteOpeningCachedBucket)?; - let write = BufWriter::new(file); - let tee = TeeReader::new(read, write); - Box::new(tee) + ( + seen, + snapshot, + account_ids, + contract_ids, + wasm_hashes, + wasm_hashes_2nd_pass, + ) = tokio::task::spawn_blocking(move || -> Result<_, Error> { + let dl_path = cache_path.with_extension("dl"); + let read: Box = if from_cache { + let read = BufReader::new(read); + Box::new(read) + } else { + let read = BufReader::new(read); + let read = GzDecoder::new(read); + let read = BufReader::new(read); + // When the stream is not from the cache, write it + // to the cache. + let file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&dl_path) + .map_err(Error::WriteOpeningCachedBucket)?; + let write = BufWriter::new(file); + let tee = TeeReader::new(read, write); + Box::new(tee) + }; + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(read, Limits::none()); + let entries = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in entries { + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) + } + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; + continue; + } }; - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(read, Limits::none()); - let entries = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in entries { - let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) + if seen.contains(&key) { + continue; + } + if let Some(val) = val { + let keep = match &val.data { + LedgerEntryData::Account(e) => { + account_ids.contains(&e.account_id.to_string()) } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(m) => { - snapshot.protocol_version = m.ledger_version; - continue; + LedgerEntryData::Trustline(e) => { + account_ids.contains(&e.account_id.to_string()) } - }; - if seen.contains(&key) { - continue; - } - if let Some(val) = val { - let keep = match &val.data { - LedgerEntryData::Account(e) => { - account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::Trustline(e) => { - account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::ContractData(e) => { - let keep = contract_ids.contains(&e.contract.to_string()); - // If a contract instance references - // contract executable stored in another - // ledger entry, add that ledger entry to - // the filter so that Wasm for any filtered - // contract is collected too in the second pass. - if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(Hash(hash)), - .. - }) = e.val - { - wasm_hashes.push(hash); - } + LedgerEntryData::ContractData(e) => { + let keep = contract_ids.contains(&e.contract.to_string()); + // If a contract instance references + // contract executable stored in another + // ledger entry, add that ledger entry to + // the filter so that Wasm for any filtered + // contract is collected too in the second pass. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(Hash(hash)), + .. + }) = e.val + { + wasm_hashes_2nd_pass.push(hash); + println!( + "ℹ️ Adding wasm hash {} to find in second pass.", + hex::encode(hash) + ); } - keep } - LedgerEntryData::ContractCode(e) => { + keep + } + LedgerEntryData::ContractCode(e) => { + if p == 1 { wasm_hashes.contains(&e.hash.0) + } else if p == 2 { + wasm_hashes_2nd_pass.contains(&e.hash.0) + } else { + false } - LedgerEntryData::Offer(_) - | LedgerEntryData::Data(_) - | LedgerEntryData::ClaimableBalance(_) - | LedgerEntryData::LiquidityPool(_) - | LedgerEntryData::ConfigSetting(_) - | LedgerEntryData::Ttl(_) => false, - }; - seen.insert(key.clone()); - if keep { - // Store the found ledger entry in the snapshot with - // a max u32 expiry. - // TODO: Change the expiry to come from the - // corresponding TTL ledger entry. - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; } + LedgerEntryData::Offer(_) + | LedgerEntryData::Data(_) + | LedgerEntryData::ClaimableBalance(_) + | LedgerEntryData::LiquidityPool(_) + | LedgerEntryData::ConfigSetting(_) + | LedgerEntryData::Ttl(_) => false, + }; + seen.insert(key.clone()); + if keep { + // Store the found ledger entry in the snapshot with + // a max u32 expiry. + // TODO: Change the expiry to come from the + // corresponding TTL ledger entry. + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; } } - if !from_cache { - fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; - } - if count_saved > 0 { - println!("🔎 Found {count_saved} entries"); - } - Ok((seen, snapshot, account_ids, contract_ids, wasm_hashes)) - }) - .await??; + } + if !from_cache { + fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; + } + if count_saved > 0 { + println!("🔎 Found {count_saved} entries"); + } + Ok(( + seen, + snapshot, + account_ids, + contract_ids, + wasm_hashes, + wasm_hashes_2nd_pass, + )) + }) + .await??; } } From c2d49c611c8fd3aa0297e1aafc0a82fbb149d6f4 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:06:01 +1000 Subject: [PATCH 33/66] organize state --- cmd/soroban-cli/src/commands/snapshot.rs | 103 +++++++++++------------ 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 7242b9a7e..db2aa36e0 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -142,31 +142,42 @@ impl Cmd { .filter(|b| b != "0000000000000000000000000000000000000000000000000000000000000000") .collect::>(); - // Track ledger keys seen, so that we can ignore old versions of - // entries. Entries can appear in both higher level and lower level - // buckets, and to get the latest version of the entry the version in - // the higher level bucket should be used. - let mut seen = HashSet::::new(); - - // The snapshot is what will be written to file at the end. Fields will - // be updated while parsing the history archive. - // TODO: Update more of the fields. - let mut snapshot = LedgerSnapshot { - protocol_version: 0, - sequence_number: ledger, - timestamp: 0, - network_id: network_id.into(), - base_reserve: 1, - min_persistent_entry_ttl: 0, - min_temp_entry_ttl: 0, - max_entry_ttl: 0, - ledger_entries: Vec::new(), + #[allow(clippy::items_after_statements)] + struct State { + // The snapshot is what will be written to file at the end. Fields will + // be updated while parsing the history archive. + snapshot: LedgerSnapshot, + // Track ledger keys seen, so that we can ignore old versions of + // entries. Entries can appear in both higher level and lower level + // buckets, and to get the latest version of the entry the version in + // the higher level bucket should be used. + seen: HashSet, + // Tracking items to search by. + account_ids: Vec, + contract_ids: Vec, + wasm_hashes: Vec<[u8; 32]>, + wasm_hashes_2nd_pass: Vec<[u8; 32]>, + } + let mut state = State { + seen: HashSet::new(), + snapshot: LedgerSnapshot { + // TODO: Update more of the fields. + protocol_version: 0, + sequence_number: ledger, + timestamp: 0, + network_id: network_id.into(), + base_reserve: 1, + min_persistent_entry_ttl: 0, + min_temp_entry_ttl: 0, + max_entry_ttl: 0, + ledger_entries: Vec::new(), + }, + account_ids: self.account_ids.clone(), + contract_ids: self.contract_ids.clone(), + wasm_hashes: self.wasm_hashes()?, + wasm_hashes_2nd_pass: Vec::new(), }; - let mut account_ids = self.account_ids.clone(); - let mut contract_ids = self.contract_ids.clone(); - let mut wasm_hashes = self.wasm_hashes()?; - let mut wasm_hashes_2nd_pass = Vec::<[u8; 32]>::new(); // Parse the buckets twice, because during the first pass contracts // and accounts will be found, along with explicitly provided wasm // hashes. Contracts found will have their wasm hashes added to the @@ -174,7 +185,7 @@ impl Cmd { // of a contract they also want the contracts code entry (e.g. // wasm). for p in 1..=2 { - if p == 2 && wasm_hashes_2nd_pass.is_empty() { + if p == 2 && state.wasm_hashes_2nd_pass.is_empty() { println!("ℹ️ Skipping parse {p}/2 of buckets"); break; } @@ -185,14 +196,7 @@ impl Cmd { let (read, cache_path, from_cache) = get_bucket_stream(&archive_url, i, bucket).await?; - ( - seen, - snapshot, - account_ids, - contract_ids, - wasm_hashes, - wasm_hashes_2nd_pass, - ) = tokio::task::spawn_blocking(move || -> Result<_, Error> { + state = tokio::task::spawn_blocking(move || -> Result<_, Error> { let dl_path = cache_path.with_extension("dl"); let read: Box = if from_cache { let read = BufReader::new(read); @@ -228,23 +232,23 @@ impl Cmd { } BucketEntry::Deadentry(k) => (k, None), BucketEntry::Metaentry(m) => { - snapshot.protocol_version = m.ledger_version; + state.snapshot.protocol_version = m.ledger_version; continue; } }; - if seen.contains(&key) { + if state.seen.contains(&key) { continue; } if let Some(val) = val { let keep = match &val.data { LedgerEntryData::Account(e) => { - account_ids.contains(&e.account_id.to_string()) + state.account_ids.contains(&e.account_id.to_string()) } LedgerEntryData::Trustline(e) => { - account_ids.contains(&e.account_id.to_string()) + state.account_ids.contains(&e.account_id.to_string()) } LedgerEntryData::ContractData(e) => { - let keep = contract_ids.contains(&e.contract.to_string()); + let keep = state.contract_ids.contains(&e.contract.to_string()); // If a contract instance references // contract executable stored in another // ledger entry, add that ledger entry to @@ -256,7 +260,7 @@ impl Cmd { .. }) = e.val { - wasm_hashes_2nd_pass.push(hash); + state.wasm_hashes_2nd_pass.push(hash); println!( "ℹ️ Adding wasm hash {} to find in second pass.", hex::encode(hash) @@ -267,9 +271,9 @@ impl Cmd { } LedgerEntryData::ContractCode(e) => { if p == 1 { - wasm_hashes.contains(&e.hash.0) + state.wasm_hashes.contains(&e.hash.0) } else if p == 2 { - wasm_hashes_2nd_pass.contains(&e.hash.0) + state.wasm_hashes_2nd_pass.contains(&e.hash.0) } else { false } @@ -281,13 +285,14 @@ impl Cmd { | LedgerEntryData::ConfigSetting(_) | LedgerEntryData::Ttl(_) => false, }; - seen.insert(key.clone()); + state.seen.insert(key.clone()); if keep { // Store the found ledger entry in the snapshot with // a max u32 expiry. // TODO: Change the expiry to come from the // corresponding TTL ledger entry. - snapshot + state + .snapshot .ledger_entries .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); count_saved += 1; @@ -300,25 +305,19 @@ impl Cmd { if count_saved > 0 { println!("🔎 Found {count_saved} entries"); } - Ok(( - seen, - snapshot, - account_ids, - contract_ids, - wasm_hashes, - wasm_hashes_2nd_pass, - )) + Ok(state) }) .await??; } } - snapshot + state + .snapshot .write_file(&self.out) .map_err(Error::WriteLedgerSnapshot)?; println!( "💾 Saved {} entries to {:?}", - snapshot.ledger_entries.len(), + state.snapshot.ledger_entries.len(), self.out ); From 94d65d6b0e9b9589145ac535862b735fbef82264 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:36:53 +1000 Subject: [PATCH 34/66] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 74abdb3f1..b59c63720 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ target/ captive-core/ .soroban/ +snapshot.json !test.toml *.sqlite test_snapshots From c12adf8c58461c3baacf65f5239b869f0e33115e Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 23:32:22 +1000 Subject: [PATCH 35/66] simplify --- Cargo.lock | 14 + Makefile | 2 +- cmd/soroban-cli/Cargo.toml | 1 + cmd/soroban-cli/src/commands/snapshot.rs | 311 +++++++++++------------ 4 files changed, 161 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 598517bd1..00e14356f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,6 +199,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-executor" version = "1.12.0" @@ -4700,6 +4713,7 @@ version = "21.2.0" dependencies = [ "assert_cmd", "assert_fs", + "async-compression", "async-trait", "base64 0.21.7", "bollard", diff --git a/Makefile b/Makefile index 7e307b16c..9d02f93df 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ endif install_rust: install install: - cargo install --force --locked --path ./cmd/stellar-cli --debug + cargo install --force --locked --path ./cmd/stellar-cli #--debug cargo install --force --locked --path ./cmd/crates/soroban-test/tests/fixtures/hello --root ./target --debug --quiet # regenerate the example lib in `cmd/crates/soroban-spec-typsecript/fixtures/ts` diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index 080dfb45c..cbdce8616 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -108,6 +108,7 @@ gix = { version = "0.58.0", default-features = false, features = [ "worktree-mutation", ] } ureq = { version = "2.9.1", features = ["json"] } +async-compression = { version = "0.4.12", features = [ "tokio", "gzip" ] } tempfile = "3.8.1" toml_edit = "0.21.0" diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index db2aa36e0..b757201c0 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -1,16 +1,15 @@ +use async_compression::tokio::bufread::GzipDecoder; use bytesize::ByteSize; use clap::{arg, Parser, ValueEnum}; -use flate2::bufread::GzDecoder; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use http::Uri; use humantime::format_duration; -use io_tee::TeeReader; use sha2::{Digest, Sha256}; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ collections::HashSet, - fs::{self, OpenOptions}, - io::{self, BufReader, BufWriter, Read}, + fs::{self}, + io::{self}, path::PathBuf, str::FromStr, time::{Duration, Instant}, @@ -22,7 +21,7 @@ use stellar_xdr::curr::{ LedgerKeyLiquidityPool, LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, ScContractInstance, ScVal, }; -use tokio_util::compat::FuturesAsyncReadCompatExt as _; +use tokio::fs::OpenOptions; use soroban_env_host::xdr::{self}; @@ -142,41 +141,45 @@ impl Cmd { .filter(|b| b != "0000000000000000000000000000000000000000000000000000000000000000") .collect::>(); + // Pre-cache the buckets. + for (i, bucket) in buckets.iter().enumerate() { + cache_bucket(&archive_url, i, bucket).await?; + } + + // The snapshot is what will be written to file at the end. Fields will + // be updated while parsing the history archive. + let mut snapshot = LedgerSnapshot { + // TODO: Update more of the fields. + protocol_version: 0, + sequence_number: ledger, + timestamp: 0, + network_id: network_id.into(), + base_reserve: 1, + min_persistent_entry_ttl: 0, + min_temp_entry_ttl: 0, + max_entry_ttl: 0, + ledger_entries: Vec::new(), + }; + + // Track ledger keys seen, so that we can ignore old versions of + // entries. Entries can appear in both higher level and lower level + // buckets, and to get the latest version of the entry the version in + // the higher level bucket should be used. + let mut seen = HashSet::new(); + #[allow(clippy::items_after_statements)] - struct State { - // The snapshot is what will be written to file at the end. Fields will - // be updated while parsing the history archive. - snapshot: LedgerSnapshot, - // Track ledger keys seen, so that we can ignore old versions of - // entries. Entries can appear in both higher level and lower level - // buckets, and to get the latest version of the entry the version in - // the higher level bucket should be used. - seen: HashSet, - // Tracking items to search by. + #[derive(Default)] + struct SearchInputs { account_ids: Vec, contract_ids: Vec, wasm_hashes: Vec<[u8; 32]>, - wasm_hashes_2nd_pass: Vec<[u8; 32]>, } - let mut state = State { - seen: HashSet::new(), - snapshot: LedgerSnapshot { - // TODO: Update more of the fields. - protocol_version: 0, - sequence_number: ledger, - timestamp: 0, - network_id: network_id.into(), - base_reserve: 1, - min_persistent_entry_ttl: 0, - min_temp_entry_ttl: 0, - max_entry_ttl: 0, - ledger_entries: Vec::new(), - }, + let mut current = SearchInputs { account_ids: self.account_ids.clone(), contract_ids: self.contract_ids.clone(), wasm_hashes: self.wasm_hashes()?, - wasm_hashes_2nd_pass: Vec::new(), }; + let mut next = SearchInputs::default(); // Parse the buckets twice, because during the first pass contracts // and accounts will be found, along with explicitly provided wasm @@ -184,140 +187,115 @@ impl Cmd { // filter because in most cases if someone wants a ledger snapshot // of a contract they also want the contracts code entry (e.g. // wasm). - for p in 1..=2 { - if p == 2 && state.wasm_hashes_2nd_pass.is_empty() { - println!("ℹ️ Skipping parse {p}/2 of buckets"); + loop { + if current.account_ids.is_empty() + && current.contract_ids.is_empty() + && current.wasm_hashes.is_empty() + { break; } - println!("ℹ️ Beginning parse {p}/2 of buckets..."); + println!( + "ℹ️ Searching for {} accounts, {} contracts, {} wasms", + current.account_ids.len(), + current.contract_ids.len(), + current.wasm_hashes.len() + ); for (i, bucket) in buckets.iter().enumerate() { + println!("🔎 Looking in bucket {i} {bucket}"); // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. - let (read, cache_path, from_cache) = - get_bucket_stream(&archive_url, i, bucket).await?; - - state = tokio::task::spawn_blocking(move || -> Result<_, Error> { - let dl_path = cache_path.with_extension("dl"); - let read: Box = if from_cache { - let read = BufReader::new(read); - Box::new(read) - } else { - let read = BufReader::new(read); - let read = GzDecoder::new(read); - let read = BufReader::new(read); - // When the stream is not from the cache, write it - // to the cache. - let file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&dl_path) - .map_err(Error::WriteOpeningCachedBucket)?; - let write = BufWriter::new(file); - let tee = TeeReader::new(read, write); - Box::new(tee) + let cache_path = cache_bucket(&archive_url, i, bucket).await?; + let file = std::fs::OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(file, Limits::none()); + let entries = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in entries { + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) + } + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; + continue; + } }; - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(read, Limits::none()); - let entries = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in entries { - let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) + if seen.contains(&key) { + continue; + } + if let Some(val) = val { + let keep = match &val.data { + LedgerEntryData::Account(e) => { + current.account_ids.contains(&e.account_id.to_string()) } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(m) => { - state.snapshot.protocol_version = m.ledger_version; - continue; + LedgerEntryData::Trustline(e) => { + current.account_ids.contains(&e.account_id.to_string()) } - }; - if state.seen.contains(&key) { - continue; - } - if let Some(val) = val { - let keep = match &val.data { - LedgerEntryData::Account(e) => { - state.account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::Trustline(e) => { - state.account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::ContractData(e) => { - let keep = state.contract_ids.contains(&e.contract.to_string()); - // If a contract instance references - // contract executable stored in another - // ledger entry, add that ledger entry to - // the filter so that Wasm for any filtered - // contract is collected too in the second pass. - if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(Hash(hash)), - .. - }) = e.val - { - state.wasm_hashes_2nd_pass.push(hash); - println!( - "ℹ️ Adding wasm hash {} to find in second pass.", - hex::encode(hash) - ); - } - } - keep - } - LedgerEntryData::ContractCode(e) => { - if p == 1 { - state.wasm_hashes.contains(&e.hash.0) - } else if p == 2 { - state.wasm_hashes_2nd_pass.contains(&e.hash.0) - } else { - false + LedgerEntryData::ContractData(e) => { + let keep = current.contract_ids.contains(&e.contract.to_string()); + // If a contract instance references + // contract executable stored in another + // ledger entry, add that ledger entry to + // the filter so that Wasm for any filtered + // contract is collected too in the second pass. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(Hash(hash)), + .. + }) = e.val + { + next.wasm_hashes.push(hash); + println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); } } - LedgerEntryData::Offer(_) - | LedgerEntryData::Data(_) - | LedgerEntryData::ClaimableBalance(_) - | LedgerEntryData::LiquidityPool(_) - | LedgerEntryData::ConfigSetting(_) - | LedgerEntryData::Ttl(_) => false, - }; - state.seen.insert(key.clone()); - if keep { - // Store the found ledger entry in the snapshot with - // a max u32 expiry. - // TODO: Change the expiry to come from the - // corresponding TTL ledger entry. - state - .snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; + keep + } + LedgerEntryData::ContractCode(e) => { + current.wasm_hashes.contains(&e.hash.0) } + LedgerEntryData::Offer(_) + | LedgerEntryData::Data(_) + | LedgerEntryData::ClaimableBalance(_) + | LedgerEntryData::LiquidityPool(_) + | LedgerEntryData::ConfigSetting(_) + | LedgerEntryData::Ttl(_) => false, + }; + seen.insert(key.clone()); + if keep { + // Store the found ledger entry in the snapshot with + // a max u32 expiry. + // TODO: Change the expiry to come from the + // corresponding TTL ledger entry. + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; } } - if !from_cache { - fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; - } - if count_saved > 0 { - println!("🔎 Found {count_saved} entries"); - } - Ok(state) - }) - .await??; + } + if count_saved > 0 { + println!("ℹ️ Found {count_saved} entries"); + } } + seen.clear(); + current = next; + next = SearchInputs::default(); } - state - .snapshot + snapshot .write_file(&self.out) .map_err(Error::WriteLedgerSnapshot)?; println!( "💾 Saved {} entries to {:?}", - state.snapshot.ledger_entries.len(), + snapshot.ledger_entries.len(), self.out ); @@ -381,21 +359,14 @@ async fn get_history(archive_url: &Uri, ledger: Option) -> Result(&body).map_err(Error::JsonDecodingHistory) } -async fn get_bucket_stream( +async fn cache_bucket( archive_url: &Uri, bucket_index: usize, bucket: &str, -) -> Result<(Box, PathBuf, bool), Error> { +) -> Result { let bucket_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; let cache_path = bucket_dir.join(format!("bucket-{bucket}.xdr")); - let (read, from_cache): (Box, bool) = if cache_path.exists() { - println!("🪣 Loading cached bucket {bucket_index} {bucket}"); - let file = OpenOptions::new() - .read(true) - .open(&cache_path) - .map_err(Error::ReadOpeningCachedBucket)?; - (Box::new(file), true) - } else { + if !cache_path.exists() { let bucket_0 = &bucket[0..=1]; let bucket_1 = &bucket[2..=3]; let bucket_2 = &bucket[4..=5]; @@ -421,16 +392,24 @@ async fn get_bucket_stream( } } println!(); - let read = tokio_util::io::SyncIoBridge::new( - response - .into_body() - .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) - .into_async_read() - .compat(), - ); - (Box::new(read), false) - }; - Ok((read, cache_path, from_cache)) + let read = response + .into_body() + .map(|result| result.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))) + .into_async_read(); + let read = tokio_util::compat::FuturesAsyncReadCompatExt::compat(read); + let mut read = GzipDecoder::new(read); + let dl_path = cache_path.with_extension("dl"); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&dl_path) + .await + .map_err(Error::WriteOpeningCachedBucket)?; + tokio::io::copy(&mut read, &mut file).await.unwrap(); + fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; + } + Ok(cache_path) } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] From 1e251673614da5a06c41e96ae6a5a04e0f02fd7a Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 31 Jul 2024 23:34:24 +1000 Subject: [PATCH 36/66] fix unwrap --- cmd/soroban-cli/src/commands/snapshot.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index b757201c0..66b60ab43 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -91,6 +91,8 @@ pub enum Error { GettingBucketGotStatusCode(hyper::StatusCode), #[error("opening cached bucket to write: {0}")] WriteOpeningCachedBucket(io::Error), + #[error("streaming bucket: {0}")] + StreamingBucket(io::Error), #[error("read XDR frame bucket entry: {0}")] ReadXdrFrameBucketEntry(xdr::Error), #[error("renaming temporary downloaded file to final destination: {0}")] @@ -406,7 +408,9 @@ async fn cache_bucket( .open(&dl_path) .await .map_err(Error::WriteOpeningCachedBucket)?; - tokio::io::copy(&mut read, &mut file).await.unwrap(); + tokio::io::copy(&mut read, &mut file) + .await + .map_err(Error::StreamingBucket)?; fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; } Ok(cache_path) From 3dc7d00be6747f8cfac88ec8baa3a8884fa11e21 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:48:10 +1000 Subject: [PATCH 37/66] move archive url in --- cmd/crates/soroban-test/src/lib.rs | 1 - cmd/soroban-cli/src/commands/snapshot.rs | 33 +++++++++++++++++-- cmd/soroban-cli/src/config/network.rs | 42 ------------------------ 3 files changed, 31 insertions(+), 45 deletions(-) diff --git a/cmd/crates/soroban-test/src/lib.rs b/cmd/crates/soroban-test/src/lib.rs index f642b083f..544e2d59e 100644 --- a/cmd/crates/soroban-test/src/lib.rs +++ b/cmd/crates/soroban-test/src/lib.rs @@ -226,7 +226,6 @@ impl TestEnv { rpc_url: Some(self.rpc_url.clone()), network_passphrase: Some(LOCAL_NETWORK_PASSPHRASE.to_string()), network: None, - archive_url: None, }, source_account: account.to_string(), locator: config::locator::Args { diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot.rs index 66b60ab43..66e3715a8 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot.rs @@ -26,7 +26,7 @@ use tokio::fs::OpenOptions; use soroban_env_host::xdr::{self}; use super::config::{self, locator}; -use crate::commands::config::data; +use crate::{commands::config::data, config::network::passphrase}; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] pub enum Format { @@ -69,6 +69,9 @@ pub struct Cmd { locator: locator::Args, #[command(flatten)] network: config::network::Args, + /// Archive URL + #[arg(long)] + archive_url: Option, } #[derive(thiserror::Error, Debug)] @@ -111,6 +114,8 @@ pub enum Error { Locator(#[from] locator::Error), #[error(transparent)] Config(#[from] config::Error), + #[error("archive url not configured")] + ArchiveUrlNotConfigured, } /// Checkpoint frequency is usually 64 ledgers, but in local test nets it'll @@ -124,7 +129,7 @@ impl Cmd { pub async fn run(&self) -> Result<(), Error> { let start = Instant::now(); - let archive_url = self.network.get(&self.locator)?.archive_url()?; + let archive_url = self.archive_url()?; let history = get_history(&archive_url, self.ledger).await?; let ledger = history.current_ledger; @@ -320,6 +325,30 @@ impl Cmd { }) .collect::, _>>() } + + fn archive_url(&self) -> Result { + // Return the configured archive URL, or if one is not configured, guess + // at an appropriate archive URL given the network passphrase. + self.archive_url + .clone() + .or_else(|| { + self.network.get(&self.locator).ok().and_then(|network| { + match network.network_passphrase.as_str() { + passphrase::MAINNET => { + Some("https://history.stellar.org/prd/core-live/core_live_001") + } + passphrase::TESTNET => { + Some("https://history.stellar.org/prd/core-testnet/core_testnet_001") + } + passphrase::FUTURENET => Some("https://history-futurenet.stellar.org"), + passphrase::LOCAL => Some("http://localhost:8000/archive"), + _ => None, + } + .map(|s| Uri::from_str(s).expect("archive url valid")) + }) + }) + .ok_or(Error::ArchiveUrlNotConfigured) + } } async fn get_history(archive_url: &Uri, ledger: Option) -> Result { diff --git a/cmd/soroban-cli/src/config/network.rs b/cmd/soroban-cli/src/config/network.rs index ef634b57f..5ba313f83 100644 --- a/cmd/soroban-cli/src/config/network.rs +++ b/cmd/soroban-cli/src/config/network.rs @@ -34,8 +34,6 @@ pub enum Error { InvalidUrl(String), #[error("funding failed: {0}")] FundingFailed(String), - #[error("Archive URL not configured")] - ArchiveUrlNotConfigured, } #[derive(Debug, clap::Args, Clone, Default)] @@ -59,14 +57,6 @@ pub struct Args { help_heading = HEADING_RPC, )] pub network_passphrase: Option, - /// Archive URL - #[arg( - long = "archive-url", - requires = "network_passphrase", - env = "STELLAR_ARCHIVE_URL", - help_heading = HEADING_RPC, - )] - pub archive_url: Option, /// Name of network to use from config #[arg( long, @@ -90,7 +80,6 @@ impl Args { Ok(Network { rpc_url, network_passphrase, - archive_url: None, }) } else { Err(Error::Network) @@ -115,13 +104,6 @@ pub struct Network { help_heading = HEADING_RPC, )] pub network_passphrase: String, - /// Archive URL - #[arg( - long = "archive-url", - env = "STELLAR_ARCHIVE_URL", - help_heading = HEADING_RPC, - )] - pub archive_url: Option, } impl Network { @@ -194,29 +176,6 @@ impl Network { pub fn rpc_uri(&self) -> Result { http::Uri::from_str(&self.rpc_url).map_err(|_| Error::InvalidUrl(self.rpc_url.to_string())) } - - pub fn archive_url(&self) -> Result { - // Return the configured archive URL, or if one is not configured, guess - // at an appropriate archive URL given the network passphrase. - self.archive_url - .as_deref() - .or(match self.network_passphrase.as_str() { - passphrase::MAINNET => { - Some("https://history.stellar.org/prd/core-live/core_live_001") - } - passphrase::TESTNET => { - Some("https://history.stellar.org/prd/core-testnet/core_testnet_001") - } - passphrase::FUTURENET => Some("https://history-futurenet.stellar.org"), - passphrase::LOCAL => Some("http://localhost:8000/archive"), - _ => None, - }) - .ok_or(Error::ArchiveUrlNotConfigured) - .and_then(|archive_url| { - Uri::from_str(archive_url) - .map_err(|_| Error::InvalidUrl((*archive_url).to_string())) - }) - } } pub static DEFAULTS: phf::Map<&'static str, (&'static str, &'static str)> = phf_map! { @@ -244,7 +203,6 @@ impl From<&(&str, &str)> for Network { Self { rpc_url: n.0.to_string(), network_passphrase: n.1.to_string(), - archive_url: None, } } } From 1eab6cea73401bcd2b8b7196d70f6527e461464d Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:03:42 +1000 Subject: [PATCH 38/66] fix --- cmd/soroban-cli/src/config/network.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/soroban-cli/src/config/network.rs b/cmd/soroban-cli/src/config/network.rs index 5ba313f83..8599c0a3d 100644 --- a/cmd/soroban-cli/src/config/network.rs +++ b/cmd/soroban-cli/src/config/network.rs @@ -1,7 +1,6 @@ use std::str::FromStr; use clap::arg; -use http::Uri; use phf::phf_map; use serde::{Deserialize, Serialize}; use serde_json::Value; From 0f26a404c6c7ea9b179dee8f7eed27cfa2adeabd Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:08:40 +1000 Subject: [PATCH 39/66] upd help --- FULL_HELP_DOCS.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index ee468617d..d5d15b350 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -116,7 +116,6 @@ Get Id of builtin Soroban Asset Contract. Deprecated, use `stellar contract id a * `--asset ` — ID of the Stellar classic asset to wrap, e.g. "USDC:G...5" * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -136,7 +135,6 @@ Deploy builtin Soroban Asset Contract * `--asset ` — ID of the Stellar classic asset to wrap, e.g. "USDC:G...5" * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -206,7 +204,6 @@ Generate a TypeScript / JavaScript package * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -273,7 +270,6 @@ If no keys are specified the contract itself is extended. * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -302,7 +298,6 @@ Deploy a wasm contract * `--salt ` — Custom salt 32-byte salt for the token id * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -336,7 +331,6 @@ Fetch a contract's Wasm binary * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -365,7 +359,6 @@ Deploy builtin Soroban Asset Contract * `--asset ` — ID of the Stellar classic asset to wrap, e.g. "USDC:G...5" * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -385,7 +378,6 @@ Deploy normal Wasm Contract * `--salt ` — ID of the Soroban contract * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -453,7 +445,6 @@ Install a WASM file to the ledger without creating a contract instance * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -493,7 +484,6 @@ stellar contract invoke ... -- --help * `--is-view` — View the result simulating and do not sign and submit transaction * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -559,7 +549,6 @@ Print the current value of a contract-data ledger entry * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -597,7 +586,6 @@ If no keys are specificed the contract itself is restored. * `--ttl-ledger-only` — Only print the new Time To Live ledger * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -658,7 +646,6 @@ Watch the network for contract events * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -732,7 +719,6 @@ Fund an identity on a test network * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--hd-path ` — If identity is a seed phrase use this hd path, default is 0 * `--global` — Use global config @@ -761,7 +747,6 @@ Generate a new identity with a seed phrase, currently 12 words * `-d`, `--default-seed` — Generate the default seed phrase. Useful for testing. Equivalent to --seed 0000000000000000 * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config @@ -846,7 +831,6 @@ Add a new network * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--global` — Use global config * `--config-dir ` — Location of config directory, default is "." @@ -1038,8 +1022,8 @@ Download a snapshot of a ledger from an archive * `--config-dir ` — Location of config directory, default is "." * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config +* `--archive-url ` — Archive URL @@ -1066,7 +1050,6 @@ Simulate a transaction envelope from stdin * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config * `--source-account ` — Account that signs the final transaction. Alias `source`. Can be an identity (--source alice), a secret key (--source SC36…), or a seed phrase (--source "kite urban…") * `--hd-path ` — If using a seed phrase, which hierarchical deterministic path to use, e.g. `m/44'/148'/{hd_path}`. Example: `--hd-path 1`. Default: `0` @@ -1085,7 +1068,6 @@ Calculate the hash of a transaction envelope from stdin * `--rpc-url ` — RPC server endpoint * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server -* `--archive-url ` — Archive URL * `--network ` — Name of network to use from config From cdcf48ce2b7f1f48b3425ae2e140d4cb8aa07852 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:20:57 +1000 Subject: [PATCH 40/66] move into subcommand --- cmd/soroban-cli/src/commands/mod.rs | 1 + .../{snapshot.rs => snapshot/create.rs} | 8 ++++--- cmd/soroban-cli/src/commands/snapshot/mod.rs | 24 +++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) rename cmd/soroban-cli/src/commands/{snapshot.rs => snapshot/create.rs} (99%) create mode 100644 cmd/soroban-cli/src/commands/snapshot/mod.rs diff --git a/cmd/soroban-cli/src/commands/mod.rs b/cmd/soroban-cli/src/commands/mod.rs index 0affaed59..bab7ff391 100644 --- a/cmd/soroban-cli/src/commands/mod.rs +++ b/cmd/soroban-cli/src/commands/mod.rs @@ -146,6 +146,7 @@ pub enum Cmd { Network(network::Cmd), /// Download a snapshot of a ledger from an archive. + #[command(subcommand)] Snapshot(snapshot::Cmd), /// Sign, Simulate, and Send transactions diff --git a/cmd/soroban-cli/src/commands/snapshot.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs similarity index 99% rename from cmd/soroban-cli/src/commands/snapshot.rs rename to cmd/soroban-cli/src/commands/snapshot/create.rs index 66e3715a8..affdf3a0c 100644 --- a/cmd/soroban-cli/src/commands/snapshot.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -25,8 +25,10 @@ use tokio::fs::OpenOptions; use soroban_env_host::xdr::{self}; -use super::config::{self, locator}; -use crate::{commands::config::data, config::network::passphrase}; +use crate::{ + commands::{config::data, HEADING_RPC}, + config::{self, locator, network::passphrase}, +}; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] pub enum Format { @@ -70,7 +72,7 @@ pub struct Cmd { #[command(flatten)] network: config::network::Args, /// Archive URL - #[arg(long)] + #[arg(long, help_heading = HEADING_RPC, env = "STELLAR_ARCHIVE_URL")] archive_url: Option, } diff --git a/cmd/soroban-cli/src/commands/snapshot/mod.rs b/cmd/soroban-cli/src/commands/snapshot/mod.rs new file mode 100644 index 000000000..97ce414ae --- /dev/null +++ b/cmd/soroban-cli/src/commands/snapshot/mod.rs @@ -0,0 +1,24 @@ +use clap::Parser; + +pub mod create; + +#[derive(Debug, Parser)] +pub enum Cmd { + /// Create a snapshot using the archive + Create(create::Cmd), +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Create(#[from] create::Error), +} + +impl Cmd { + pub async fn run(&self) -> Result<(), Error> { + match self { + Cmd::Create(cmd) => cmd.run().await?, + }; + Ok(()) + } +} From b5e836ef41b3a4ba95720241fb0ecb5b95769527 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:21:51 +1000 Subject: [PATCH 41/66] rename format to output --- FULL_HELP_DOCS.md | 16 ++++++++++++++-- cmd/soroban-cli/src/commands/snapshot/create.rs | 6 +++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index d5d15b350..1bc3a996e 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -1003,7 +1003,19 @@ Stop a network container started with `network container start` Download a snapshot of a ledger from an archive -**Usage:** `stellar snapshot [OPTIONS] --format ` +**Usage:** `stellar snapshot ` + +###### **Subcommands:** + +* `create` — Create a snapshot using the archive + + + +## `stellar snapshot create` + +Create a snapshot using the archive + +**Usage:** `stellar snapshot create [OPTIONS] --output ` ###### **Options:** @@ -1011,7 +1023,7 @@ Download a snapshot of a ledger from an archive * `--account-id ` — Account IDs to include in the snapshot * `--contract-id ` — Contract IDs to include in the snapshot * `--wasm-hash ` — WASM hashes to include in the snapshot -* `--format ` — Format of the out file +* `--output ` — Format of the out file Possible values: `json` diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index affdf3a0c..55da27601 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -31,11 +31,11 @@ use crate::{ }; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] -pub enum Format { +pub enum Output { Json, } -impl Default for Format { +impl Default for Output { fn default() -> Self { Self::Json } @@ -63,7 +63,7 @@ pub struct Cmd { wasm_hashes: Vec, /// Format of the out file. #[arg(long)] - format: Format, + output: Output, /// Out path that the snapshot is written to. #[arg(long, default_value=default_out_path().into_os_string())] out: PathBuf, From fd176b691dd6ffbc286fe1a04614b06bd84832e1 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:07:06 +1000 Subject: [PATCH 42/66] print size --- cmd/soroban-cli/src/commands/snapshot/create.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 55da27601..17c572692 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -210,7 +210,6 @@ impl Cmd { current.wasm_hashes.len() ); for (i, bucket) in buckets.iter().enumerate() { - println!("🔎 Looking in bucket {i} {bucket}"); // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. let cache_path = cache_bucket(&archive_url, i, bucket).await?; @@ -218,6 +217,12 @@ impl Cmd { .read(true) .open(&cache_path) .map_err(Error::ReadOpeningCachedBucket)?; + print!("🔎 Searching bucket {i} {bucket}"); + if let Ok(metadata) = file.metadata() { + print!(" ({})", ByteSize(metadata.len())); + } + println!(); + // Stream the bucket entries from the bucket, identifying // entries that match the filters, and including only the // entries that match in the snapshot. From 8164620469d7a30569913fe9cf2308f1522db31c Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:41:56 +1000 Subject: [PATCH 43/66] fix multi-slash --- cmd/soroban-cli/src/commands/snapshot/create.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 17c572692..86eecdc8d 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -359,6 +359,8 @@ impl Cmd { } async fn get_history(archive_url: &Uri, ledger: Option) -> Result { + let archive_url = archive_url.to_string(); + let archive_url = archive_url.strip_suffix("/").unwrap_or(&archive_url); let history_url = if let Some(ledger) = ledger { let ledger_hex = format!("{ledger:08x}"); let ledger_hex_0 = &ledger_hex[0..=1]; From 4dc24a63dbed719667c869c21cd77dd9e334a616 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:53:27 +1000 Subject: [PATCH 44/66] fmt --- .../src/commands/snapshot/create.rs | 87 +++++++++---------- 1 file changed, 42 insertions(+), 45 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 86eecdc8d..47eb367e0 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -245,54 +245,51 @@ impl Cmd { if seen.contains(&key) { continue; } - if let Some(val) = val { - let keep = match &val.data { - LedgerEntryData::Account(e) => { - current.account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::Trustline(e) => { - current.account_ids.contains(&e.account_id.to_string()) - } - LedgerEntryData::ContractData(e) => { - let keep = current.contract_ids.contains(&e.contract.to_string()); - // If a contract instance references - // contract executable stored in another - // ledger entry, add that ledger entry to - // the filter so that Wasm for any filtered - // contract is collected too in the second pass. - if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(Hash(hash)), - .. - }) = e.val - { - next.wasm_hashes.push(hash); - println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); - } + let Some(val) = val else { continue }; + let keep = match &val.data { + LedgerEntryData::Account(e) => { + current.account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::Trustline(e) => { + current.account_ids.contains(&e.account_id.to_string()) + } + LedgerEntryData::ContractData(e) => { + let keep = current.contract_ids.contains(&e.contract.to_string()); + // If a contract instance references + // contract executable stored in another + // ledger entry, add that ledger entry to + // the filter so that Wasm for any filtered + // contract is collected too in the second pass. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(Hash(hash)), + .. + }) = e.val + { + next.wasm_hashes.push(hash); + println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); } - keep - } - LedgerEntryData::ContractCode(e) => { - current.wasm_hashes.contains(&e.hash.0) } - LedgerEntryData::Offer(_) - | LedgerEntryData::Data(_) - | LedgerEntryData::ClaimableBalance(_) - | LedgerEntryData::LiquidityPool(_) - | LedgerEntryData::ConfigSetting(_) - | LedgerEntryData::Ttl(_) => false, - }; - seen.insert(key.clone()); - if keep { - // Store the found ledger entry in the snapshot with - // a max u32 expiry. - // TODO: Change the expiry to come from the - // corresponding TTL ledger entry. - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; + keep } + LedgerEntryData::ContractCode(e) => current.wasm_hashes.contains(&e.hash.0), + LedgerEntryData::Offer(_) + | LedgerEntryData::Data(_) + | LedgerEntryData::ClaimableBalance(_) + | LedgerEntryData::LiquidityPool(_) + | LedgerEntryData::ConfigSetting(_) + | LedgerEntryData::Ttl(_) => false, + }; + seen.insert(key.clone()); + if keep { + // Store the found ledger entry in the snapshot with + // a max u32 expiry. + // TODO: Change the expiry to come from the + // corresponding TTL ledger entry. + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; } } if count_saved > 0 { From 93a04e47e741cfdd482e88f30784ffcd744ea048 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:42:34 +1000 Subject: [PATCH 45/66] fix --- .../src/commands/snapshot/create.rs | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 47eb367e0..17252070c 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -245,21 +245,31 @@ impl Cmd { if seen.contains(&key) { continue; } - let Some(val) = val else { continue }; - let keep = match &val.data { - LedgerEntryData::Account(e) => { - current.account_ids.contains(&e.account_id.to_string()) + let keep = match &key { + LedgerKey::Account(k) => { + current.account_ids.contains(&k.account_id.to_string()) + } + LedgerKey::Trustline(k) => { + current.account_ids.contains(&k.account_id.to_string()) } - LedgerEntryData::Trustline(e) => { - current.account_ids.contains(&e.account_id.to_string()) + LedgerKey::ContractData(k) => { + current.contract_ids.contains(&k.contract.to_string()) } + LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash.0), + _ => false, + }; + if !keep { + continue; + } + seen.insert(key.clone()); + let Some(val) = val else { continue }; + match &val.data { LedgerEntryData::ContractData(e) => { - let keep = current.contract_ids.contains(&e.contract.to_string()); - // If a contract instance references - // contract executable stored in another - // ledger entry, add that ledger entry to - // the filter so that Wasm for any filtered - // contract is collected too in the second pass. + // If a contract instance references contract + // executable stored in another ledger entry, add + // that ledger entry to the filter so that Wasm for + // any filtered contract is collected too in the + // second pass. if keep && e.key == ScVal::LedgerKeyContractInstance { if let ScVal::ContractInstance(ScContractInstance { executable: ContractExecutable::Wasm(Hash(hash)), @@ -272,25 +282,16 @@ impl Cmd { } keep } - LedgerEntryData::ContractCode(e) => current.wasm_hashes.contains(&e.hash.0), - LedgerEntryData::Offer(_) - | LedgerEntryData::Data(_) - | LedgerEntryData::ClaimableBalance(_) - | LedgerEntryData::LiquidityPool(_) - | LedgerEntryData::ConfigSetting(_) - | LedgerEntryData::Ttl(_) => false, + _ => false, }; - seen.insert(key.clone()); - if keep { - // Store the found ledger entry in the snapshot with - // a max u32 expiry. - // TODO: Change the expiry to come from the - // corresponding TTL ledger entry. - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; - } + // Store the found ledger entry in the snapshot with a max + // u32 expiry. + // TODO: Change the expiry to come from the corresponding + // TTL ledger entry. + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; } if count_saved > 0 { println!("ℹ️ Found {count_saved} entries"); From d2bb6c02206de9708aa7c8f054f34cf90044a5b4 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:56:54 +1000 Subject: [PATCH 46/66] simplify --- .../src/commands/snapshot/create.rs | 246 ++++++++++-------- 1 file changed, 144 insertions(+), 102 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 17252070c..c212934cc 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -183,125 +183,167 @@ impl Cmd { contract_ids: Vec, wasm_hashes: Vec<[u8; 32]>, } - let mut current = SearchInputs { + let current = SearchInputs { account_ids: self.account_ids.clone(), contract_ids: self.contract_ids.clone(), wasm_hashes: self.wasm_hashes()?, }; - let mut next = SearchInputs::default(); + let mut next_wasm_hashes = HashSet::<[u8; 32]>::new(); - // Parse the buckets twice, because during the first pass contracts - // and accounts will be found, along with explicitly provided wasm - // hashes. Contracts found will have their wasm hashes added to the - // filter because in most cases if someone wants a ledger snapshot - // of a contract they also want the contracts code entry (e.g. - // wasm). - loop { - if current.account_ids.is_empty() - && current.contract_ids.is_empty() - && current.wasm_hashes.is_empty() - { - break; + // Search the buckets. + println!( + "ℹ️ Searching for {} accounts, {} contracts, {} wasms", + current.account_ids.len(), + current.contract_ids.len(), + current.wasm_hashes.len() + ); + for (i, bucket) in buckets.iter().enumerate() { + // Defined where the bucket will be read from, either from cache on + // disk, or streamed from the archive. + let cache_path = cache_bucket(&archive_url, i, bucket).await?; + let file = std::fs::OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; + print!("🔎 Searching bucket {i} {bucket}"); + if let Ok(metadata) = file.metadata() { + print!(" ({})", ByteSize(metadata.len())); } - println!( - "ℹ️ Searching for {} accounts, {} contracts, {} wasms", - current.account_ids.len(), - current.contract_ids.len(), - current.wasm_hashes.len() - ); - for (i, bucket) in buckets.iter().enumerate() { - // Defined where the bucket will be read from, either from cache on - // disk, or streamed from the archive. - let cache_path = cache_bucket(&archive_url, i, bucket).await?; - let file = std::fs::OpenOptions::new() - .read(true) - .open(&cache_path) - .map_err(Error::ReadOpeningCachedBucket)?; - print!("🔎 Searching bucket {i} {bucket}"); - if let Ok(metadata) = file.metadata() { - print!(" ({})", ByteSize(metadata.len())); - } - println!(); + println!(); - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(file, Limits::none()); - let entries = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in entries { - let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) - } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(m) => { - snapshot.protocol_version = m.ledger_version; - continue; - } - }; - if seen.contains(&key) { - continue; + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(file, Limits::none()); + let entries = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in entries { + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) } - let keep = match &key { - LedgerKey::Account(k) => { - current.account_ids.contains(&k.account_id.to_string()) - } - LedgerKey::Trustline(k) => { - current.account_ids.contains(&k.account_id.to_string()) - } - LedgerKey::ContractData(k) => { - current.contract_ids.contains(&k.contract.to_string()) - } - LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash.0), - _ => false, - }; - if !keep { + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; continue; } - seen.insert(key.clone()); - let Some(val) = val else { continue }; - match &val.data { - LedgerEntryData::ContractData(e) => { - // If a contract instance references contract - // executable stored in another ledger entry, add - // that ledger entry to the filter so that Wasm for - // any filtered contract is collected too in the - // second pass. - if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(Hash(hash)), - .. - }) = e.val - { - next.wasm_hashes.push(hash); - println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); - } + }; + if seen.contains(&key) { + continue; + } + let keep = match &key { + LedgerKey::Account(k) => { + current.account_ids.contains(&k.account_id.to_string()) + } + LedgerKey::Trustline(k) => { + current.account_ids.contains(&k.account_id.to_string()) + } + LedgerKey::ContractData(k) => { + current.contract_ids.contains(&k.contract.to_string()) + } + LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash.0), + _ => false, + }; + if !keep { + continue; + } + seen.insert(key.clone()); + let Some(val) = val else { continue }; + match &val.data { + LedgerEntryData::ContractData(e) => { + // If a contract instance references contract + // executable stored in another ledger entry, add + // that ledger entry to the filter so that Wasm for + // any filtered contract is collected too in the + // second pass. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(Hash(hash)), + .. + }) = e.val + { + next_wasm_hashes.insert(hash); + println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); } - keep } - _ => false, - }; - // Store the found ledger entry in the snapshot with a max - // u32 expiry. - // TODO: Change the expiry to come from the corresponding - // TTL ledger entry. - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; + keep + } + _ => false, + }; + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; + } + if count_saved > 0 { + println!("ℹ️ Found {count_saved} entries"); + } + } + seen.clear(); + + // Parse the buckets a second time if we found wasms in the first pass + // that should be included. + println!( + "ℹ️ Searching for {} additional wasms", + next_wasm_hashes.len() + ); + for (i, bucket) in buckets.iter().enumerate() { + if next_wasm_hashes.is_empty() { + break; + } + // Defined where the bucket will be read from, either from cache on + // disk, or streamed from the archive. + let cache_path = cache_bucket(&archive_url, i, bucket).await?; + let file = std::fs::OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; + print!("🔎 Searching bucket {i} {bucket}"); + if let Ok(metadata) = file.metadata() { + print!(" ({})", ByteSize(metadata.len())); + } + println!(); + + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(file, Limits::none()); + let entries = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in entries { + if next_wasm_hashes.is_empty() { + break; } - if count_saved > 0 { - println!("ℹ️ Found {count_saved} entries"); + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) + } + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(_) => continue, + }; + let keep = match &key { + LedgerKey::ContractCode(e) => next_wasm_hashes.remove(&e.hash.0), + _ => false, + }; + if !keep { + continue; } + let Some(val) = val else { continue }; + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; + } + if count_saved > 0 { + println!("ℹ️ Found {count_saved} entries"); } - seen.clear(); - current = next; - next = SearchInputs::default(); } + // Write the snapshot to file. snapshot .write_file(&self.out) .map_err(Error::WriteLedgerSnapshot)?; From 981b10665efe8f08f9dc2739e68c84395c178b19 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:03:07 +1000 Subject: [PATCH 47/66] use hashset --- .../src/commands/snapshot/create.rs | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index c212934cc..977e79324 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -176,26 +176,17 @@ impl Cmd { // the higher level bucket should be used. let mut seen = HashSet::new(); - #[allow(clippy::items_after_statements)] - #[derive(Default)] - struct SearchInputs { - account_ids: Vec, - contract_ids: Vec, - wasm_hashes: Vec<[u8; 32]>, - } - let current = SearchInputs { - account_ids: self.account_ids.clone(), - contract_ids: self.contract_ids.clone(), - wasm_hashes: self.wasm_hashes()?, - }; + let account_ids = HashSet::<&String>::from_iter(&self.account_ids); + let contract_ids = HashSet::<&String>::from_iter(&self.contract_ids); + let wasm_hashes = self.wasm_hashes()?; let mut next_wasm_hashes = HashSet::<[u8; 32]>::new(); // Search the buckets. println!( "ℹ️ Searching for {} accounts, {} contracts, {} wasms", - current.account_ids.len(), - current.contract_ids.len(), - current.wasm_hashes.len() + account_ids.len(), + contract_ids.len(), + wasm_hashes.len() ); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on @@ -234,16 +225,10 @@ impl Cmd { continue; } let keep = match &key { - LedgerKey::Account(k) => { - current.account_ids.contains(&k.account_id.to_string()) - } - LedgerKey::Trustline(k) => { - current.account_ids.contains(&k.account_id.to_string()) - } - LedgerKey::ContractData(k) => { - current.contract_ids.contains(&k.contract.to_string()) - } - LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash.0), + LedgerKey::Account(k) => account_ids.contains(&k.account_id.to_string()), + LedgerKey::Trustline(k) => account_ids.contains(&k.account_id.to_string()), + LedgerKey::ContractData(k) => contract_ids.contains(&k.contract.to_string()), + LedgerKey::ContractCode(e) => wasm_hashes.contains(&e.hash.0), _ => false, }; if !keep { @@ -359,7 +344,7 @@ impl Cmd { Ok(()) } - fn wasm_hashes(&self) -> Result, Error> { + fn wasm_hashes(&self) -> Result, Error> { self.wasm_hashes .iter() .map(|h| { @@ -370,7 +355,7 @@ impl Cmd { .map_err(|_| Error::WasmHashInvalid(h.clone())) }) }) - .collect::, _>>() + .collect::, _>>() } fn archive_url(&self) -> Result { From e785bc4f705f538719a1581bf5cecd655c1f84e7 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:15:19 +1000 Subject: [PATCH 48/66] optimise make it fast! --- .../src/commands/snapshot/create.rs | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 977e79324..661ed92e6 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -4,6 +4,7 @@ use clap::{arg, Parser, ValueEnum}; use futures::{StreamExt, TryStreamExt}; use http::Uri; use humantime::format_duration; +use itertools::{Either, Itertools}; use sha2::{Digest, Sha256}; use soroban_ledger_snapshot::LedgerSnapshot; use std::{ @@ -15,11 +16,11 @@ use std::{ time::{Duration, Instant}, }; use stellar_xdr::curr::{ - BucketEntry, ConfigSettingEntry, ConfigSettingId, ContractExecutable, Frame, Hash, LedgerEntry, - LedgerEntryData, LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, + AccountId, BucketEntry, ConfigSettingEntry, ConfigSettingId, ContractExecutable, Frame, Hash, + LedgerEntry, LedgerEntryData, LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, LedgerKeyConfigSetting, LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, - ReadXdr, ScContractInstance, ScVal, + ReadXdr, ScAddress, ScContractInstance, ScVal, }; use tokio::fs::OpenOptions; @@ -52,12 +53,9 @@ pub struct Cmd { /// The ledger sequence number to snapshot. Defaults to latest history archived ledger. #[arg(long)] ledger: Option, - /// Account IDs to include in the snapshot. - #[arg(long = "account-id", help_heading = "Filter Options")] - account_ids: Vec, - /// Contract IDs to include in the snapshot. - #[arg(long = "contract-id", help_heading = "Filter Options")] - contract_ids: Vec, + /// Account or contract address to include in the snapshot. + #[arg(long = "address", help_heading = "Filter Options")] + address: Vec, /// WASM hashes to include in the snapshot. #[arg(long = "wasm-hash", help_heading = "Filter Options")] wasm_hashes: Vec, @@ -176,8 +174,7 @@ impl Cmd { // the higher level bucket should be used. let mut seen = HashSet::new(); - let account_ids = HashSet::<&String>::from_iter(&self.account_ids); - let contract_ids = HashSet::<&String>::from_iter(&self.contract_ids); + let (account_ids, contract_ids) = self.addresses(); let wasm_hashes = self.wasm_hashes()?; let mut next_wasm_hashes = HashSet::<[u8; 32]>::new(); @@ -225,9 +222,9 @@ impl Cmd { continue; } let keep = match &key { - LedgerKey::Account(k) => account_ids.contains(&k.account_id.to_string()), - LedgerKey::Trustline(k) => account_ids.contains(&k.account_id.to_string()), - LedgerKey::ContractData(k) => contract_ids.contains(&k.contract.to_string()), + LedgerKey::Account(k) => account_ids.contains(&k.account_id), + LedgerKey::Trustline(k) => account_ids.contains(&k.account_id), + LedgerKey::ContractData(k) => contract_ids.contains(&k.contract), LedgerKey::ContractCode(e) => wasm_hashes.contains(&e.hash.0), _ => false, }; @@ -344,6 +341,13 @@ impl Cmd { Ok(()) } + fn addresses(&self) -> (HashSet, HashSet) { + self.address.iter().cloned().partition_map(|a| match a { + ScAddress::Account(account_id) => Either::Left(account_id), + ScAddress::Contract(_) => Either::Right(a), + }) + } + fn wasm_hashes(&self) -> Result, Error> { self.wasm_hashes .iter() From 5df72b02446c7adfe7d29dff116b3de8845083be Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:31:00 +1000 Subject: [PATCH 49/66] avoid conversions --- .../src/commands/snapshot/create.rs | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 661ed92e6..7bb2956b1 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -58,7 +58,7 @@ pub struct Cmd { address: Vec, /// WASM hashes to include in the snapshot. #[arg(long = "wasm-hash", help_heading = "Filter Options")] - wasm_hashes: Vec, + wasm_hashes: Vec, /// Format of the out file. #[arg(long)] output: Output, @@ -175,8 +175,8 @@ impl Cmd { let mut seen = HashSet::new(); let (account_ids, contract_ids) = self.addresses(); - let wasm_hashes = self.wasm_hashes()?; - let mut next_wasm_hashes = HashSet::<[u8; 32]>::new(); + let wasm_hashes = HashSet::<&Hash>::from_iter(&self.wasm_hashes); + let mut next_wasm_hashes = HashSet::::new(); // Search the buckets. println!( @@ -225,7 +225,7 @@ impl Cmd { LedgerKey::Account(k) => account_ids.contains(&k.account_id), LedgerKey::Trustline(k) => account_ids.contains(&k.account_id), LedgerKey::ContractData(k) => contract_ids.contains(&k.contract), - LedgerKey::ContractCode(e) => wasm_hashes.contains(&e.hash.0), + LedgerKey::ContractCode(e) => wasm_hashes.contains(&e.hash), _ => false, }; if !keep { @@ -242,11 +242,11 @@ impl Cmd { // second pass. if keep && e.key == ScVal::LedgerKeyContractInstance { if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(Hash(hash)), + executable: ContractExecutable::Wasm(hash), .. - }) = e.val + }) = &e.val { - next_wasm_hashes.insert(hash); + next_wasm_hashes.insert(hash.clone()); println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); } } @@ -308,7 +308,7 @@ impl Cmd { BucketEntry::Metaentry(_) => continue, }; let keep = match &key { - LedgerKey::ContractCode(e) => next_wasm_hashes.remove(&e.hash.0), + LedgerKey::ContractCode(e) => next_wasm_hashes.remove(&e.hash), _ => false, }; if !keep { @@ -348,20 +348,6 @@ impl Cmd { }) } - fn wasm_hashes(&self) -> Result, Error> { - self.wasm_hashes - .iter() - .map(|h| { - hex::decode(h) - .map_err(|_| Error::WasmHashInvalid(h.clone())) - .and_then(|vec| { - vec.try_into() - .map_err(|_| Error::WasmHashInvalid(h.clone())) - }) - }) - .collect::, _>>() - } - fn archive_url(&self) -> Result { // Return the configured archive URL, or if one is not configured, guess // at an appropriate archive URL given the network passphrase. From 4cd8e72ebfb25913bc14786ea66c2ee8bc90e7e2 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:33:57 +1000 Subject: [PATCH 50/66] fix --- Cargo.lock | 7 ------- cmd/crates/soroban-test/tests/it/integration/snapshot.rs | 7 ++++--- cmd/soroban-cli/Cargo.toml | 1 - 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00e14356f..0fd9ea18e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2968,12 +2968,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "io_tee" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" - [[package]] name = "ipnet" version = "2.9.0" @@ -4741,7 +4735,6 @@ dependencies = [ "humantime", "hyper 0.14.28", "hyper-tls", - "io_tee", "itertools 0.10.5", "jsonrpsee-core", "jsonrpsee-http-client", diff --git a/cmd/crates/soroban-test/tests/it/integration/snapshot.rs b/cmd/crates/soroban-test/tests/it/integration/snapshot.rs index 897728b6b..ceb52985f 100644 --- a/cmd/crates/soroban-test/tests/it/integration/snapshot.rs +++ b/cmd/crates/soroban-test/tests/it/integration/snapshot.rs @@ -63,10 +63,11 @@ fn snapshot() { // Create the snapshot. sandbox .new_assert_cmd("snapshot") - .arg("--format=json") - .arg("--account-id") + .arg("create") + .arg("--output=json") + .arg("--address") .arg(&account_a) - .arg("--contract-id") + .arg("--address") .arg(&contract_b) .assert() .success(); diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index cbdce8616..ca6d01a22 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -119,7 +119,6 @@ futures = "0.3.30" home = "0.5.9" flate2 = "1.0.30" bytesize = "1.3.0" -io_tee = "0.1.1" humantime = "2.1.0" phf = { version = "0.11.2", features = ["macros"] } From f84f788ab2634aea5e265ba534cc75e463980f66 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:39:44 +1000 Subject: [PATCH 51/66] fix --- FULL_HELP_DOCS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index 1bc3a996e..46342161b 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -1020,8 +1020,7 @@ Create a snapshot using the archive ###### **Options:** * `--ledger ` — The ledger sequence number to snapshot. Defaults to latest history archived ledger -* `--account-id ` — Account IDs to include in the snapshot -* `--contract-id ` — Contract IDs to include in the snapshot +* `--address
` — Account or contract address to include in the snapshot * `--wasm-hash ` — WASM hashes to include in the snapshot * `--output ` — Format of the out file From 99e187a6115d197b7afa619bb2022efd8861ca40 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:20:50 +1000 Subject: [PATCH 52/66] fix dupe --- cmd/soroban-cli/src/commands/snapshot/create.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 7bb2956b1..f64aaae13 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -246,8 +246,10 @@ impl Cmd { .. }) = &e.val { - next_wasm_hashes.insert(hash.clone()); - println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); + if !wasm_hashes.contains(hash) { + next_wasm_hashes.insert(hash.clone()); + println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); + } } } keep From b182ab191d44db64dcbcf1ce265f62037412922d Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:51:35 +1000 Subject: [PATCH 53/66] remove early exit optimisation --- .../src/commands/snapshot/create.rs | 252 ++++++++---------- 1 file changed, 108 insertions(+), 144 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index f64aaae13..76ae44724 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -168,163 +168,127 @@ impl Cmd { ledger_entries: Vec::new(), }; - // Track ledger keys seen, so that we can ignore old versions of - // entries. Entries can appear in both higher level and lower level - // buckets, and to get the latest version of the entry the version in - // the higher level bucket should be used. - let mut seen = HashSet::new(); - - let (account_ids, contract_ids) = self.addresses(); - let wasm_hashes = HashSet::<&Hash>::from_iter(&self.wasm_hashes); - let mut next_wasm_hashes = HashSet::::new(); + #[allow(clippy::items_after_statements)] + #[derive(Default)] + struct SearchInputs { + account_ids: HashSet, + contract_ids: HashSet, + wasm_hashes: HashSet, + } + impl SearchInputs { + pub fn is_empty(&self) -> bool { + self.account_ids.is_empty() + && self.contract_ids.is_empty() + && self.wasm_hashes.is_empty() + } + } // Search the buckets. - println!( - "ℹ️ Searching for {} accounts, {} contracts, {} wasms", - account_ids.len(), - contract_ids.len(), - wasm_hashes.len() - ); - for (i, bucket) in buckets.iter().enumerate() { - // Defined where the bucket will be read from, either from cache on - // disk, or streamed from the archive. - let cache_path = cache_bucket(&archive_url, i, bucket).await?; - let file = std::fs::OpenOptions::new() - .read(true) - .open(&cache_path) - .map_err(Error::ReadOpeningCachedBucket)?; - print!("🔎 Searching bucket {i} {bucket}"); - if let Ok(metadata) = file.metadata() { - print!(" ({})", ByteSize(metadata.len())); + let (account_ids, contract_ids) = self.addresses(); + let mut current = SearchInputs { + account_ids, + contract_ids, + wasm_hashes: self.wasm_hashes.iter().cloned().collect(), + }; + let mut next = SearchInputs::default(); + loop { + if current.is_empty() { + break; } - println!(); + println!( + "ℹ️ Searching for {} accounts, {} contracts, {} wasms", + current.account_ids.len(), + current.contract_ids.len(), + current.wasm_hashes.len() + ); + // Track ledger keys seen, so that we can ignore old versions of + // entries. Entries can appear in both higher level and lower level + // buckets, and to get the latest version of the entry the version in + // the higher level bucket should be used. + let mut seen = HashSet::new(); + for (i, bucket) in buckets.iter().enumerate() { + // Defined where the bucket will be read from, either from cache on + // disk, or streamed from the archive. + let cache_path = cache_bucket(&archive_url, i, bucket).await?; + let file = std::fs::OpenOptions::new() + .read(true) + .open(&cache_path) + .map_err(Error::ReadOpeningCachedBucket)?; + print!("🔎 Searching bucket {i} {bucket}"); + if let Ok(metadata) = file.metadata() { + print!(" ({})", ByteSize(metadata.len())); + } + println!(); - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(file, Limits::none()); - let entries = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in entries { - let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) + // Stream the bucket entries from the bucket, identifying + // entries that match the filters, and including only the + // entries that match in the snapshot. + let limited = &mut Limited::new(file, Limits::none()); + let entries = Frame::::read_xdr_iter(limited); + let mut count_saved = 0; + for entry in entries { + let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; + let (key, val) = match entry { + BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { + let k = data_into_key(&l); + (k, Some(l)) + } + BucketEntry::Deadentry(k) => (k, None), + BucketEntry::Metaentry(m) => { + snapshot.protocol_version = m.ledger_version; + continue; + } + }; + if seen.contains(&key) { + continue; } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(m) => { - snapshot.protocol_version = m.ledger_version; + let keep = match &key { + LedgerKey::Account(k) => current.account_ids.contains(&k.account_id), + LedgerKey::Trustline(k) => current.account_ids.contains(&k.account_id), + LedgerKey::ContractData(k) => current.contract_ids.contains(&k.contract), + LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash), + _ => false, + }; + if !keep { continue; } - }; - if seen.contains(&key) { - continue; - } - let keep = match &key { - LedgerKey::Account(k) => account_ids.contains(&k.account_id), - LedgerKey::Trustline(k) => account_ids.contains(&k.account_id), - LedgerKey::ContractData(k) => contract_ids.contains(&k.contract), - LedgerKey::ContractCode(e) => wasm_hashes.contains(&e.hash), - _ => false, - }; - if !keep { - continue; - } - seen.insert(key.clone()); - let Some(val) = val else { continue }; - match &val.data { - LedgerEntryData::ContractData(e) => { - // If a contract instance references contract - // executable stored in another ledger entry, add - // that ledger entry to the filter so that Wasm for - // any filtered contract is collected too in the - // second pass. - if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(hash), - .. - }) = &e.val - { - if !wasm_hashes.contains(hash) { - next_wasm_hashes.insert(hash.clone()); - println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); + seen.insert(key.clone()); + let Some(val) = val else { continue }; + match &val.data { + LedgerEntryData::ContractData(e) => { + // If a contract instance references contract + // executable stored in another ledger entry, add + // that ledger entry to the filter so that Wasm for + // any filtered contract is collected too in the + // second pass. + if keep && e.key == ScVal::LedgerKeyContractInstance { + if let ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(hash), + .. + }) = &e.val + { + if !current.wasm_hashes.contains(hash) { + next.wasm_hashes.insert(hash.clone()); + println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); + } } } + keep } - keep - } - _ => false, - }; - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; - } - if count_saved > 0 { - println!("ℹ️ Found {count_saved} entries"); - } - } - seen.clear(); - - // Parse the buckets a second time if we found wasms in the first pass - // that should be included. - println!( - "ℹ️ Searching for {} additional wasms", - next_wasm_hashes.len() - ); - for (i, bucket) in buckets.iter().enumerate() { - if next_wasm_hashes.is_empty() { - break; - } - // Defined where the bucket will be read from, either from cache on - // disk, or streamed from the archive. - let cache_path = cache_bucket(&archive_url, i, bucket).await?; - let file = std::fs::OpenOptions::new() - .read(true) - .open(&cache_path) - .map_err(Error::ReadOpeningCachedBucket)?; - print!("🔎 Searching bucket {i} {bucket}"); - if let Ok(metadata) = file.metadata() { - print!(" ({})", ByteSize(metadata.len())); - } - println!(); - - // Stream the bucket entries from the bucket, identifying - // entries that match the filters, and including only the - // entries that match in the snapshot. - let limited = &mut Limited::new(file, Limits::none()); - let entries = Frame::::read_xdr_iter(limited); - let mut count_saved = 0; - for entry in entries { - if next_wasm_hashes.is_empty() { - break; + _ => false, + }; + snapshot + .ledger_entries + .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); + count_saved += 1; } - let Frame(entry) = entry.map_err(Error::ReadXdrFrameBucketEntry)?; - let (key, val) = match entry { - BucketEntry::Liveentry(l) | BucketEntry::Initentry(l) => { - let k = data_into_key(&l); - (k, Some(l)) - } - BucketEntry::Deadentry(k) => (k, None), - BucketEntry::Metaentry(_) => continue, - }; - let keep = match &key { - LedgerKey::ContractCode(e) => next_wasm_hashes.remove(&e.hash), - _ => false, - }; - if !keep { - continue; + if count_saved > 0 { + println!("ℹ️ Found {count_saved} entries"); } - let Some(val) = val else { continue }; - snapshot - .ledger_entries - .push((Box::new(key), (Box::new(val), Some(u32::MAX)))); - count_saved += 1; - } - if count_saved > 0 { - println!("ℹ️ Found {count_saved} entries"); } + seen.clear(); + current = next; + next = SearchInputs::default(); } // Write the snapshot to file. From f83d4f3a975b588d9b65fc76019b2c212b5f00c8 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:54:07 +1000 Subject: [PATCH 54/66] across all search seen --- cmd/soroban-cli/src/commands/snapshot/create.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 76ae44724..096901a94 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -168,6 +168,12 @@ impl Cmd { ledger_entries: Vec::new(), }; + // Track ledger keys seen, so that we can ignore old versions of + // entries. Entries can appear in both higher level and lower level + // buckets, and to get the latest version of the entry the version in + // the higher level bucket should be used. + let mut seen = HashSet::new(); + #[allow(clippy::items_after_statements)] #[derive(Default)] struct SearchInputs { @@ -201,11 +207,6 @@ impl Cmd { current.contract_ids.len(), current.wasm_hashes.len() ); - // Track ledger keys seen, so that we can ignore old versions of - // entries. Entries can appear in both higher level and lower level - // buckets, and to get the latest version of the entry the version in - // the higher level bucket should be used. - let mut seen = HashSet::new(); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on // disk, or streamed from the archive. @@ -286,7 +287,6 @@ impl Cmd { println!("ℹ️ Found {count_saved} entries"); } } - seen.clear(); current = next; next = SearchInputs::default(); } From 9db6917b6d9817ece97d53b5fd3d82127583cea8 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:55:09 +1000 Subject: [PATCH 55/66] add assets --- .../soroban-test/tests/it/integration/wrap.rs | 2 +- .../src/commands/contract/deploy/asset.rs | 2 +- .../src/commands/contract/id/asset.rs | 2 +- .../src/commands/snapshot/create.rs | 114 +++++++++++++----- cmd/soroban-cli/src/utils.rs | 11 +- 5 files changed, 94 insertions(+), 37 deletions(-) diff --git a/cmd/crates/soroban-test/tests/it/integration/wrap.rs b/cmd/crates/soroban-test/tests/it/integration/wrap.rs index aa356ce99..c43f5b007 100644 --- a/cmd/crates/soroban-test/tests/it/integration/wrap.rs +++ b/cmd/crates/soroban-test/tests/it/integration/wrap.rs @@ -24,7 +24,7 @@ async fn burn() { .success(); // wrap_cmd(&asset).run().await.unwrap(); let asset = soroban_cli::utils::parsing::parse_asset(&asset).unwrap(); - let hash = contract_id_hash_from_asset(&asset, &network_passphrase).unwrap(); + let hash = contract_id_hash_from_asset(&asset, &network_passphrase); let id = stellar_strkey::Contract(hash.0).to_string(); println!("{id}, {address}"); sandbox diff --git a/cmd/soroban-cli/src/commands/contract/deploy/asset.rs b/cmd/soroban-cli/src/commands/contract/deploy/asset.rs index 98276a86d..51baa55e9 100644 --- a/cmd/soroban-cli/src/commands/contract/deploy/asset.rs +++ b/cmd/soroban-cli/src/commands/contract/deploy/asset.rs @@ -106,7 +106,7 @@ impl NetworkRunnable for Cmd { let account_details = client.get_account(&public_strkey).await?; let sequence: i64 = account_details.seq_num.into(); let network_passphrase = &network.network_passphrase; - let contract_id = contract_id_hash_from_asset(&asset, network_passphrase)?; + let contract_id = contract_id_hash_from_asset(&asset, network_passphrase); let tx = build_wrap_token_tx( &asset, &contract_id, diff --git a/cmd/soroban-cli/src/commands/contract/id/asset.rs b/cmd/soroban-cli/src/commands/contract/id/asset.rs index 31b55f8d1..e57385859 100644 --- a/cmd/soroban-cli/src/commands/contract/id/asset.rs +++ b/cmd/soroban-cli/src/commands/contract/id/asset.rs @@ -33,7 +33,7 @@ impl Cmd { pub fn contract_address(&self) -> Result { let asset = parse_asset(&self.asset)?; let network = self.config.get_network()?; - let contract_id = contract_id_hash_from_asset(&asset, &network.network_passphrase)?; + let contract_id = contract_id_hash_from_asset(&asset, &network.network_passphrase); Ok(stellar_strkey::Contract(contract_id.0)) } } diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 096901a94..476ffb122 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -16,19 +16,19 @@ use std::{ time::{Duration, Instant}, }; use stellar_xdr::curr::{ - AccountId, BucketEntry, ConfigSettingEntry, ConfigSettingId, ContractExecutable, Frame, Hash, - LedgerEntry, LedgerEntryData, LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, - LedgerKeyConfigSetting, LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, - LedgerKeyLiquidityPool, LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, - ReadXdr, ScAddress, ScContractInstance, ScVal, + self as xdr, AccountId, Asset, BucketEntry, ConfigSettingEntry, ConfigSettingId, + ContractExecutable, Frame, Hash, LedgerEntry, LedgerEntryData, LedgerKey, LedgerKeyAccount, + LedgerKeyClaimableBalance, LedgerKeyConfigSetting, LedgerKeyContractCode, + LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, LedgerKeyOffer, + LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, ScAddress, ScContractInstance, + ScMapEntry, ScString, ScVal, TrustLineAsset, }; use tokio::fs::OpenOptions; -use soroban_env_host::xdr::{self}; - use crate::{ commands::{config::data, HEADING_RPC}, config::{self, locator, network::passphrase}, + utils::{contract_id_hash_from_asset, parsing::parse_asset}, }; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] @@ -59,6 +59,9 @@ pub struct Cmd { /// WASM hashes to include in the snapshot. #[arg(long = "wasm-hash", help_heading = "Filter Options")] wasm_hashes: Vec, + /// WASM hashes to include in the snapshot. + #[arg(long = "asset", help_heading = "Filter Options", value_parser=parse_asset)] + assets: Vec, /// Format of the out file. #[arg(long)] output: Output, @@ -116,6 +119,8 @@ pub enum Error { Config(#[from] config::Error), #[error("archive url not configured")] ArchiveUrlNotConfigured, + #[error("parsing asset name: {0}")] + ParseAssetName(String), } /// Checkpoint frequency is usually 64 ledgers, but in local test nets it'll @@ -180,21 +185,40 @@ impl Cmd { account_ids: HashSet, contract_ids: HashSet, wasm_hashes: HashSet, + assets: HashSet, } impl SearchInputs { pub fn is_empty(&self) -> bool { self.account_ids.is_empty() && self.contract_ids.is_empty() && self.wasm_hashes.is_empty() + && self.assets.is_empty() } } - // Search the buckets. - let (account_ids, contract_ids) = self.addresses(); + // Search the buckets using the user inputs as the starting inputs. + let (mut account_ids, mut contract_ids): (HashSet, HashSet) = + self.address.iter().cloned().partition_map(|a| match a { + ScAddress::Account(account_id) => Either::Left(account_id), + ScAddress::Contract(_) => Either::Right(a), + }); + // Include accounts of issuers of any asset requested. + account_ids.extend(self.assets.iter().filter_map(|a| match a { + Asset::Native => None, + Asset::CreditAlphanum4(a4) => Some(a4.issuer.clone()), + Asset::CreditAlphanum12(a12) => Some(a12.issuer.clone()), + })); + // Include contracts of any asset requested. + contract_ids.extend( + self.assets + .iter() + .map(|a| ScAddress::Contract(contract_id_hash_from_asset(a, network_passphrase))), + ); let mut current = SearchInputs { account_ids, contract_ids, wasm_hashes: self.wasm_hashes.iter().cloned().collect(), + assets: self.assets.iter().cloned().collect(), }; let mut next = SearchInputs::default(); loop { @@ -202,10 +226,11 @@ impl Cmd { break; } println!( - "ℹ️ Searching for {} accounts, {} contracts, {} wasms", + "ℹ️ Searching for {} accounts, {} contracts, {} wasms, {} assets", current.account_ids.len(), current.contract_ids.len(), - current.wasm_hashes.len() + current.wasm_hashes.len(), + current.assets.len(), ); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on @@ -244,8 +269,26 @@ impl Cmd { continue; } let keep = match &key { - LedgerKey::Account(k) => current.account_ids.contains(&k.account_id), - LedgerKey::Trustline(k) => current.account_ids.contains(&k.account_id), + LedgerKey::Account(k) => { + current.account_ids.contains(&k.account_id) + || current.assets.contains(&Asset::Native) + } + LedgerKey::Trustline(LedgerKeyTrustLine { + account_id, + asset: TrustLineAsset::CreditAlphanum4(a4), + }) => { + current.account_ids.contains(account_id) + || current.assets.contains(&Asset::CreditAlphanum4(a4.clone())) + } + LedgerKey::Trustline(LedgerKeyTrustLine { + account_id, + asset: TrustLineAsset::CreditAlphanum12(a12), + }) => { + current.account_ids.contains(account_id) + || current + .assets + .contains(&Asset::CreditAlphanum12(a12.clone())) + } LedgerKey::ContractData(k) => current.contract_ids.contains(&k.contract), LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash), _ => false, @@ -263,15 +306,37 @@ impl Cmd { // any filtered contract is collected too in the // second pass. if keep && e.key == ScVal::LedgerKeyContractInstance { - if let ScVal::ContractInstance(ScContractInstance { - executable: ContractExecutable::Wasm(hash), - .. - }) = &e.val - { - if !current.wasm_hashes.contains(hash) { - next.wasm_hashes.insert(hash.clone()); - println!("ℹ️ Adding wasm {} to search", hex::encode(hash)); + match &e.val { + ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::Wasm(hash), + .. + }) => { + if !current.wasm_hashes.contains(hash) { + next.wasm_hashes.insert(hash.clone()); + println!( + "ℹ️ Adding wasm {} to search", + hex::encode(hash) + ); + } + } + ScVal::ContractInstance(ScContractInstance { + executable: ContractExecutable::StellarAsset, + storage: Some(storage), + }) => { + if let Some(ScMapEntry { + val: ScVal::String(ScString(name)), + .. + }) = storage.iter().find(|ScMapEntry { key, .. }| { + key == &ScVal::Symbol("name".try_into().unwrap()) + }) { + let name = name.to_string(); + println!("ℹ️ Adding asset {name} to search"); + let asset = parse_asset(&name) + .map_err(|_| Error::ParseAssetName(name))?; + next.assets.insert(asset); + } } + _ => {} } } keep @@ -307,13 +372,6 @@ impl Cmd { Ok(()) } - fn addresses(&self) -> (HashSet, HashSet) { - self.address.iter().cloned().partition_map(|a| match a { - ScAddress::Account(account_id) => Either::Left(account_id), - ScAddress::Contract(_) => Either::Right(a), - }) - } - fn archive_url(&self) -> Result { // Return the configured archive URL, or if one is not configured, guess // at an appropriate archive URL given the network passphrase. diff --git a/cmd/soroban-cli/src/utils.rs b/cmd/soroban-cli/src/utils.rs index 88daa9a80..23f9184b6 100644 --- a/cmd/soroban-cli/src/utils.rs +++ b/cmd/soroban-cli/src/utils.rs @@ -116,17 +116,16 @@ pub fn is_hex_string(s: &str) -> bool { s.chars().all(|s| s.is_ascii_hexdigit()) } -pub fn contract_id_hash_from_asset( - asset: &Asset, - network_passphrase: &str, -) -> Result { +pub fn contract_id_hash_from_asset(asset: &Asset, network_passphrase: &str) -> Hash { let network_id = Hash(Sha256::digest(network_passphrase.as_bytes()).into()); let preimage = HashIdPreimage::ContractId(HashIdPreimageContractId { network_id, contract_id_preimage: ContractIdPreimage::Asset(asset.clone()), }); - let preimage_xdr = preimage.to_xdr(Limits::none())?; - Ok(Hash(Sha256::digest(preimage_xdr).into())) + let preimage_xdr = preimage + .to_xdr(Limits::none()) + .expect("HashIdPreimage should not fail encoding to xdr"); + Hash(Sha256::digest(preimage_xdr).into()) } pub mod rpc { From 7b74532807ac3f25f4b68e20c57d0b8bb607d0f6 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:01:10 +1000 Subject: [PATCH 56/66] fix missing issuer --- cmd/soroban-cli/src/commands/snapshot/create.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 476ffb122..23fd40f7c 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -333,6 +333,17 @@ impl Cmd { println!("ℹ️ Adding asset {name} to search"); let asset = parse_asset(&name) .map_err(|_| Error::ParseAssetName(name))?; + if let Some(issuer) = match &asset { + Asset::Native => None, + Asset::CreditAlphanum4(a4) => { + Some(a4.issuer.clone()) + } + Asset::CreditAlphanum12(a12) => { + Some(a12.issuer.clone()) + } + } { + next.account_ids.insert(issuer); + } next.assets.insert(asset); } } From 0e7245df168671348c9703c8ded3b02dcaf47efb Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:02:59 +1000 Subject: [PATCH 57/66] add println --- cmd/soroban-cli/src/commands/snapshot/create.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 23fd40f7c..c5e41f18a 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -342,6 +342,9 @@ impl Cmd { Some(a12.issuer.clone()) } } { + println!( + "ℹ️ Adding asset issuer {issuer} to search" + ); next.account_ids.insert(issuer); } next.assets.insert(asset); From 5cea3aca6e919a756d9dc997283d678758683a0c Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:05:53 +1000 Subject: [PATCH 58/66] fix getting name --- .../src/commands/snapshot/create.rs | 15 +++++----- cmd/soroban-cli/src/utils.rs | 29 +++++++++++++++++-- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index c5e41f18a..6685aeae6 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -28,7 +28,10 @@ use tokio::fs::OpenOptions; use crate::{ commands::{config::data, HEADING_RPC}, config::{self, locator, network::passphrase}, - utils::{contract_id_hash_from_asset, parsing::parse_asset}, + utils::{ + contract_id_hash_from_asset, get_name_from_stellar_asset_contract_storage, + parsing::parse_asset, + }, }; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] @@ -323,13 +326,9 @@ impl Cmd { executable: ContractExecutable::StellarAsset, storage: Some(storage), }) => { - if let Some(ScMapEntry { - val: ScVal::String(ScString(name)), - .. - }) = storage.iter().find(|ScMapEntry { key, .. }| { - key == &ScVal::Symbol("name".try_into().unwrap()) - }) { - let name = name.to_string(); + if let Some(name) = + get_name_from_stellar_asset_contract_storage(storage) + { println!("ℹ️ Adding asset {name} to search"); let asset = parse_asset(&name) .map_err(|_| Error::ParseAssetName(name))?; diff --git a/cmd/soroban-cli/src/utils.rs b/cmd/soroban-cli/src/utils.rs index 23f9184b6..9a91a9e80 100644 --- a/cmd/soroban-cli/src/utils.rs +++ b/cmd/soroban-cli/src/utils.rs @@ -4,9 +4,9 @@ use stellar_strkey::ed25519::PrivateKey; use soroban_env_host::xdr::{ Asset, ContractIdPreimage, DecoratedSignature, Error as XdrError, Hash, HashIdPreimage, - HashIdPreimageContractId, Limits, Signature, SignatureHint, Transaction, TransactionEnvelope, - TransactionSignaturePayload, TransactionSignaturePayloadTaggedTransaction, - TransactionV1Envelope, WriteXdr, + HashIdPreimageContractId, Limits, ScMap, Signature, SignatureHint, Transaction, + TransactionEnvelope, TransactionSignaturePayload, TransactionSignaturePayloadTaggedTransaction, + TransactionV1Envelope, WriteXdr,ScVal,ScMapEntry }; pub use soroban_spec_tools::contract as contract_spec; @@ -128,6 +128,29 @@ pub fn contract_id_hash_from_asset(asset: &Asset, network_passphrase: &str) -> H Hash(Sha256::digest(preimage_xdr).into()) } +pub fn get_name_from_stellar_asset_contract_storage(storage: &ScMap) -> Option { + if let Some(ScMapEntry { + val: ScVal::Map(Some(map)), + .. + }) = storage + .iter() + .find(|ScMapEntry { key, .. }| key == &ScVal::Symbol("METADATA".try_into().unwrap())) + { + if let Some(ScMapEntry { + val: ScVal::String(name), + .. + }) = map + .iter() + .find(|ScMapEntry { key, .. }| key == &ScVal::Symbol("METADATA".try_into().unwrap())) + { + Some(name.to_string()) + } + else { + None + } + } else { None } +} + pub mod rpc { use soroban_env_host::xdr; use soroban_rpc::{Client, Error}; From 74efd9886d480979a9da61a5e66689c4c791e76e Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:07:14 +1000 Subject: [PATCH 59/66] fix --- cmd/soroban-cli/src/commands/snapshot/create.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 6685aeae6..4228b0f0c 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -21,7 +21,7 @@ use stellar_xdr::curr::{ LedgerKeyClaimableBalance, LedgerKeyConfigSetting, LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, ScAddress, ScContractInstance, - ScMapEntry, ScString, ScVal, TrustLineAsset, + ScVal, TrustLineAsset, }; use tokio::fs::OpenOptions; From d415290ae981129fa9e5e17518410504cdf8e726 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:26:17 +1000 Subject: [PATCH 60/66] remove deep asset support --- .../src/commands/snapshot/create.rs | 54 +++---------------- cmd/soroban-cli/src/utils.rs | 15 +++--- 2 files changed, 14 insertions(+), 55 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 4228b0f0c..51be4c1cb 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -21,17 +21,14 @@ use stellar_xdr::curr::{ LedgerKeyClaimableBalance, LedgerKeyConfigSetting, LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, ScAddress, ScContractInstance, - ScVal, TrustLineAsset, + ScVal, }; use tokio::fs::OpenOptions; use crate::{ commands::{config::data, HEADING_RPC}, config::{self, locator, network::passphrase}, - utils::{ - contract_id_hash_from_asset, get_name_from_stellar_asset_contract_storage, - parsing::parse_asset, - }, + utils::{get_name_from_stellar_asset_contract_storage, parsing::parse_asset}, }; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum)] @@ -62,9 +59,6 @@ pub struct Cmd { /// WASM hashes to include in the snapshot. #[arg(long = "wasm-hash", help_heading = "Filter Options")] wasm_hashes: Vec, - /// WASM hashes to include in the snapshot. - #[arg(long = "asset", help_heading = "Filter Options", value_parser=parse_asset)] - assets: Vec, /// Format of the out file. #[arg(long)] output: Output, @@ -188,40 +182,25 @@ impl Cmd { account_ids: HashSet, contract_ids: HashSet, wasm_hashes: HashSet, - assets: HashSet, } impl SearchInputs { pub fn is_empty(&self) -> bool { self.account_ids.is_empty() && self.contract_ids.is_empty() && self.wasm_hashes.is_empty() - && self.assets.is_empty() } } // Search the buckets using the user inputs as the starting inputs. - let (mut account_ids, mut contract_ids): (HashSet, HashSet) = + let (account_ids, contract_ids): (HashSet, HashSet) = self.address.iter().cloned().partition_map(|a| match a { ScAddress::Account(account_id) => Either::Left(account_id), ScAddress::Contract(_) => Either::Right(a), }); - // Include accounts of issuers of any asset requested. - account_ids.extend(self.assets.iter().filter_map(|a| match a { - Asset::Native => None, - Asset::CreditAlphanum4(a4) => Some(a4.issuer.clone()), - Asset::CreditAlphanum12(a12) => Some(a12.issuer.clone()), - })); - // Include contracts of any asset requested. - contract_ids.extend( - self.assets - .iter() - .map(|a| ScAddress::Contract(contract_id_hash_from_asset(a, network_passphrase))), - ); let mut current = SearchInputs { account_ids, contract_ids, wasm_hashes: self.wasm_hashes.iter().cloned().collect(), - assets: self.assets.iter().cloned().collect(), }; let mut next = SearchInputs::default(); loop { @@ -229,11 +208,10 @@ impl Cmd { break; } println!( - "ℹ️ Searching for {} accounts, {} contracts, {} wasms, {} assets", + "ℹ️ Searching for {} accounts, {} contracts, {} wasms", current.account_ids.len(), current.contract_ids.len(), current.wasm_hashes.len(), - current.assets.len(), ); for (i, bucket) in buckets.iter().enumerate() { // Defined where the bucket will be read from, either from cache on @@ -272,26 +250,8 @@ impl Cmd { continue; } let keep = match &key { - LedgerKey::Account(k) => { - current.account_ids.contains(&k.account_id) - || current.assets.contains(&Asset::Native) - } - LedgerKey::Trustline(LedgerKeyTrustLine { - account_id, - asset: TrustLineAsset::CreditAlphanum4(a4), - }) => { - current.account_ids.contains(account_id) - || current.assets.contains(&Asset::CreditAlphanum4(a4.clone())) - } - LedgerKey::Trustline(LedgerKeyTrustLine { - account_id, - asset: TrustLineAsset::CreditAlphanum12(a12), - }) => { - current.account_ids.contains(account_id) - || current - .assets - .contains(&Asset::CreditAlphanum12(a12.clone())) - } + LedgerKey::Account(k) => current.account_ids.contains(&k.account_id), + LedgerKey::Trustline(k) => current.account_ids.contains(&k.account_id), LedgerKey::ContractData(k) => current.contract_ids.contains(&k.contract), LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash), _ => false, @@ -329,7 +289,6 @@ impl Cmd { if let Some(name) = get_name_from_stellar_asset_contract_storage(storage) { - println!("ℹ️ Adding asset {name} to search"); let asset = parse_asset(&name) .map_err(|_| Error::ParseAssetName(name))?; if let Some(issuer) = match &asset { @@ -346,7 +305,6 @@ impl Cmd { ); next.account_ids.insert(issuer); } - next.assets.insert(asset); } } _ => {} diff --git a/cmd/soroban-cli/src/utils.rs b/cmd/soroban-cli/src/utils.rs index 9a91a9e80..24b76b24f 100644 --- a/cmd/soroban-cli/src/utils.rs +++ b/cmd/soroban-cli/src/utils.rs @@ -4,9 +4,9 @@ use stellar_strkey::ed25519::PrivateKey; use soroban_env_host::xdr::{ Asset, ContractIdPreimage, DecoratedSignature, Error as XdrError, Hash, HashIdPreimage, - HashIdPreimageContractId, Limits, ScMap, Signature, SignatureHint, Transaction, - TransactionEnvelope, TransactionSignaturePayload, TransactionSignaturePayloadTaggedTransaction, - TransactionV1Envelope, WriteXdr,ScVal,ScMapEntry + HashIdPreimageContractId, Limits, ScMap, ScMapEntry, ScVal, Signature, SignatureHint, + Transaction, TransactionEnvelope, TransactionSignaturePayload, + TransactionSignaturePayloadTaggedTransaction, TransactionV1Envelope, WriteXdr, }; pub use soroban_spec_tools::contract as contract_spec; @@ -141,14 +141,15 @@ pub fn get_name_from_stellar_asset_contract_storage(storage: &ScMap) -> Option Date: Fri, 2 Aug 2024 16:33:35 +1000 Subject: [PATCH 61/66] help --- cmd/soroban-cli/src/commands/snapshot/create.rs | 10 ++++++++++ cmd/soroban-cli/src/commands/snapshot/mod.rs | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 51be4c1cb..89780e0b9 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -46,6 +46,16 @@ fn default_out_path() -> PathBuf { PathBuf::new().join("snapshot.json") } +/// Create a ledger snapshot using a history archive. +/// +/// Filters (address, wasm-hash) specify what ledger entries to include. +/// +/// Account addresses include the account, and trust lines. +/// +/// Contract addresses include the related wasm, contract data. +/// +/// If a contract is a Stellar asset contract, it includes the asset issuer's +/// account and trust lines. #[derive(Parser, Debug, Clone)] #[group(skip)] #[command(arg_required_else_help = true)] diff --git a/cmd/soroban-cli/src/commands/snapshot/mod.rs b/cmd/soroban-cli/src/commands/snapshot/mod.rs index 97ce414ae..344b320d2 100644 --- a/cmd/soroban-cli/src/commands/snapshot/mod.rs +++ b/cmd/soroban-cli/src/commands/snapshot/mod.rs @@ -2,9 +2,9 @@ use clap::Parser; pub mod create; +/// Create and operate on ledger snapshots. #[derive(Debug, Parser)] pub enum Cmd { - /// Create a snapshot using the archive Create(create::Cmd), } From fac13932a7bbe6c45f812304b672e4df25e098ba Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:34:45 +1000 Subject: [PATCH 62/66] help --- cmd/soroban-cli/src/commands/snapshot/create.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 89780e0b9..12b0d8ca5 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -55,7 +55,7 @@ fn default_out_path() -> PathBuf { /// Contract addresses include the related wasm, contract data. /// /// If a contract is a Stellar asset contract, it includes the asset issuer's -/// account and trust lines. +/// account and trust lines, but does not include all the trust lines of other accounts holding the asset. To inclu #[derive(Parser, Debug, Clone)] #[group(skip)] #[command(arg_required_else_help = true)] From 8c8a8070f26e7f1b93489ac98e7b15c1d3adc589 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:34:52 +1000 Subject: [PATCH 63/66] help --- cmd/soroban-cli/src/commands/snapshot/create.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 12b0d8ca5..790efaae5 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -55,7 +55,9 @@ fn default_out_path() -> PathBuf { /// Contract addresses include the related wasm, contract data. /// /// If a contract is a Stellar asset contract, it includes the asset issuer's -/// account and trust lines, but does not include all the trust lines of other accounts holding the asset. To inclu +/// account and trust lines, but does not include all the trust lines of other +/// accounts holding the asset. To include them specify the addresses of +/// relevant accounts. #[derive(Parser, Debug, Clone)] #[group(skip)] #[command(arg_required_else_help = true)] From 9ee88b82e410e7f6265bfea69174a11e4d4dde63 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 22:30:59 +1000 Subject: [PATCH 64/66] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9d02f93df..7e307b16c 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ endif install_rust: install install: - cargo install --force --locked --path ./cmd/stellar-cli #--debug + cargo install --force --locked --path ./cmd/stellar-cli --debug cargo install --force --locked --path ./cmd/crates/soroban-test/tests/fixtures/hello --root ./target --debug --quiet # regenerate the example lib in `cmd/crates/soroban-spec-typsecript/fixtures/ts` From d0680c0eaeb815a31bb87f2b5dc46cf76b23095e Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 23:42:05 +1000 Subject: [PATCH 65/66] fix --- cmd/soroban-cli/src/commands/snapshot/create.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 790efaae5..ea17785ae 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -382,7 +382,7 @@ impl Cmd { async fn get_history(archive_url: &Uri, ledger: Option) -> Result { let archive_url = archive_url.to_string(); - let archive_url = archive_url.strip_suffix("/").unwrap_or(&archive_url); + let archive_url = archive_url.strip_suffix('/').unwrap_or(&archive_url); let history_url = if let Some(ledger) = ledger { let ledger_hex = format!("{ledger:08x}"); let ledger_hex_0 = &ledger_hex[0..=1]; From 25a67b9de9a2c53dd11a5b0e71bbbd646890077f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 2 Aug 2024 23:51:25 +1000 Subject: [PATCH 66/66] help --- FULL_HELP_DOCS.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index 46342161b..b3782ec02 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -1007,13 +1007,21 @@ Download a snapshot of a ledger from an archive ###### **Subcommands:** -* `create` — Create a snapshot using the archive +* `create` — Create a ledger snapshot using a history archive ## `stellar snapshot create` -Create a snapshot using the archive +Create a ledger snapshot using a history archive. + +Filters (address, wasm-hash) specify what ledger entries to include. + +Account addresses include the account, and trust lines. + +Contract addresses include the related wasm, contract data. + +If a contract is a Stellar asset contract, it includes the asset issuer's account and trust lines, but does not include all the trust lines of other accounts holding the asset. To include them specify the addresses of relevant accounts. **Usage:** `stellar snapshot create [OPTIONS] --output `