From 937e728b509626a27b00ec01652a7ef29f135426 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Tue, 14 Jan 2025 21:30:57 +0100 Subject: [PATCH 1/7] electrum: handle re-org transactions --- src/indexers/electrum.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/indexers/electrum.rs b/src/indexers/electrum.rs index b08b3ee..01b9056 100644 --- a/src/indexers/electrum.rs +++ b/src/indexers/electrum.rs @@ -20,6 +20,7 @@ // limitations under the License. use std::collections::BTreeMap; +use std::mem; use std::num::NonZeroU32; use std::str::FromStr; @@ -78,6 +79,13 @@ impl Indexer for Client { ) -> MayError> { let mut errors = Vec::::new(); + // First, we scan all addresses. + // Addresses may be re-used, so known transactions doesn't help here. + // We collect these transactions, which contain the most recent information, into a new + // cache. We remove old transaction, since its data are now updated (for instance, if a + // transaction was re-orged, it may have a different height). + + let mut old_cache = mem::replace(&mut cache.tx, BTreeMap::new()); let mut address_index = BTreeMap::new(); for keychain in descriptor.keychains() { let mut empty_count = 0usize; @@ -104,6 +112,7 @@ impl Indexer for Client { empty_count = 0; + // TODO: Separate as `WalletTx::from_electrum_history` method. let mut process_history_entry = |hr: GetHistoryRes| -> Result { let txid = hr.tx_hash; @@ -202,6 +211,7 @@ impl Indexer for Client { for hr in hres { match process_history_entry(hr) { Ok(tx) => { + old_cache.remove(&tx.txid); cache.tx.insert(tx.txid, tx); } Err(e) => errors.push(e), @@ -213,6 +223,12 @@ impl Indexer for Client { } } + // The remaining transactions are unmined ones. + for (txid, mut tx) in old_cache { + tx.status = TxStatus::Unknown; + cache.tx.insert(txid, tx); + } + // TODO: Update headers & tip for (script, (wallet_addr, txids)) in &mut address_index { From 248eed3075a72a89a359825d613100df91351b63 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Tue, 14 Jan 2025 21:35:08 +0100 Subject: [PATCH 2/7] esplora: handle re-org transactions --- src/indexers/esplora.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/indexers/esplora.rs b/src/indexers/esplora.rs index a4a7349..9d14986 100644 --- a/src/indexers/esplora.rs +++ b/src/indexers/esplora.rs @@ -21,6 +21,7 @@ // limitations under the License. use std::collections::BTreeMap; +use std::mem; use std::num::NonZeroU32; use std::ops::{Deref, DerefMut}; @@ -208,6 +209,13 @@ impl Indexer for Client { ) -> MayError> { let mut errors = vec![]; + // First, we scan all addresses. + // Addresses may be re-used, so known transactions doesn't help here. + // We collect these transactions, which contain the most recent information, into a new + // cache. We remove old transaction, since its data are now updated (for instance, if a + // transaction was re-orged, it may have a different height). + + let mut old_cache = mem::replace(&mut cache.tx, BTreeMap::new()); let mut address_index = BTreeMap::new(); for keychain in descriptor.keychains() { let mut empty_count = 0usize; @@ -232,7 +240,10 @@ impl Indexer for Client { } Ok(txes) => { empty_count = 0; - txids = txes.iter().map(|tx| tx.txid).collect(); + txids.extend(txes.iter().map(|tx| tx.txid)); + for txid in &txids { + old_cache.remove(txid); + } cache .tx .extend(txes.into_iter().map(WalletTx::from).map(|tx| (tx.txid, tx))); @@ -244,6 +255,12 @@ impl Indexer for Client { } } + // The remaining transactions are unmined ones. + for (txid, mut tx) in old_cache { + tx.status = TxStatus::Unknown; + cache.tx.insert(txid, tx); + } + // TODO: Update headers & tip for (script, (wallet_addr, txids)) in &mut address_index { From 2bd637d7fc3e6c439b3d263cc9c04b9108519eb4 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Tue, 14 Jan 2025 21:45:30 +0100 Subject: [PATCH 3/7] chore: fix clippy lints --- src/indexers/electrum.rs | 2 +- src/indexers/esplora.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/indexers/electrum.rs b/src/indexers/electrum.rs index 01b9056..ccf2bf8 100644 --- a/src/indexers/electrum.rs +++ b/src/indexers/electrum.rs @@ -85,7 +85,7 @@ impl Indexer for Client { // cache. We remove old transaction, since its data are now updated (for instance, if a // transaction was re-orged, it may have a different height). - let mut old_cache = mem::replace(&mut cache.tx, BTreeMap::new()); + let mut old_cache = mem::take(&mut cache.tx); let mut address_index = BTreeMap::new(); for keychain in descriptor.keychains() { let mut empty_count = 0usize; diff --git a/src/indexers/esplora.rs b/src/indexers/esplora.rs index 9d14986..3c8e56d 100644 --- a/src/indexers/esplora.rs +++ b/src/indexers/esplora.rs @@ -215,7 +215,7 @@ impl Indexer for Client { // cache. We remove old transaction, since its data are now updated (for instance, if a // transaction was re-orged, it may have a different height). - let mut old_cache = mem::replace(&mut cache.tx, BTreeMap::new()); + let mut old_cache = mem::take(&mut cache.tx); let mut address_index = BTreeMap::new(); for keychain in descriptor.keychains() { let mut empty_count = 0usize; From aeea5e95534d41ef03a1ac19ecbe7848204f8a08 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Wed, 15 Jan 2025 13:20:36 +0100 Subject: [PATCH 4/7] wallet: add cache prune methods --- src/wallet.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wallet.rs b/src/wallet.rs index 214e180..880e14d 100644 --- a/src/wallet.rs +++ b/src/wallet.rs @@ -36,7 +36,7 @@ use psbt::{PsbtConstructor, Utxo}; use crate::{ BlockInfo, CoinRow, Indexer, Layer2, Layer2Cache, Layer2Data, Layer2Descriptor, Layer2Empty, - MayError, MiningInfo, NoLayer2, Party, TxRow, WalletAddr, WalletTx, WalletUtxo, + MayError, MiningInfo, NoLayer2, Party, TxRow, TxStatus, WalletAddr, WalletTx, WalletUtxo, }; #[derive(Copy, Clone, Eq, PartialEq, Debug, Display, Error)] @@ -337,6 +337,9 @@ impl WalletCache { res } + /// Prunes transaction cache by removing all transactions with `TxStatus::Unknown` + pub fn prune(&mut self) { self.tx.retain(|_, tx| tx.status != TxStatus::Unknown) } + pub fn addresses_on(&self, keychain: Keychain) -> &BTreeSet { self.addr.get(&keychain).unwrap_or_else(|| { panic!("keychain #{keychain} is not supported by the wallet descriptor") @@ -547,6 +550,9 @@ impl, L2: Layer2> Wallet { self.cache.update::(&self.descr, indexer).map(|_| ()) } + /// Prunes transaction cache by removing all transactions with `TxStatus::Unknown` + pub fn prune(&mut self) { self.cache.prune() } + pub fn to_deriver(&self) -> D where D: Clone, From da8c0d7b0b33f9dbe64d76997d3fa0631da2097b Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Wed, 15 Jan 2025 22:34:07 +0100 Subject: [PATCH 5/7] indexers: add prune argument to update, use default create impl --- src/cli/args.rs | 11 ++++++++--- src/indexers/any.rs | 39 ++++----------------------------------- src/indexers/electrum.rs | 17 ++++++----------- src/indexers/esplora.rs | 17 ++++++----------- src/indexers/mod.rs | 10 +++++++++- src/wallet.rs | 7 ++++--- 6 files changed, 37 insertions(+), 64 deletions(-) diff --git a/src/cli/args.rs b/src/cli/args.rs index 5f68710..f4ccf8b 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -41,7 +41,7 @@ use crate::{AnyIndexer, Wallet}; #[derive(Clone, Eq, PartialEq, Debug)] #[command(author, version, about)] pub struct Args { - /// Set verbosity level. + /// Set verbosity level /// /// Can be used multiple times to increase verbosity. #[clap(short, long, global = true, action = clap::ArgAction::Count)] @@ -53,10 +53,14 @@ pub struct Args Args { wallet: self.wallet.clone(), resolver: self.resolver.clone(), sync: self.sync, + prune: self.prune, general: self.general.clone(), command: cmd.clone(), } @@ -154,7 +159,7 @@ impl Args { if sync { let indexer = self.indexer()?; eprint!("Syncing"); - if let Some(errors) = wallet.update(&indexer).into_err() { + if let Some(errors) = wallet.update(&indexer, self.prune).into_err() { eprintln!(" partial, some requests has failed:"); for err in errors { eprintln!("- {err}"); diff --git a/src/indexers/any.rs b/src/indexers/any.rs index 950def4..5af527a 100644 --- a/src/indexers/any.rs +++ b/src/indexers/any.rs @@ -72,47 +72,16 @@ pub enum AnyIndexerError { impl Indexer for AnyIndexer { type Error = AnyIndexerError; - fn create, L2: Layer2>( - &self, - descr: &WalletDescr, - ) -> MayError, Vec> { - match self { - #[cfg(feature = "electrum")] - AnyIndexer::Electrum(inner) => { - let result = inner.create::(descr); - MayError { - ok: result.ok, - err: result.err.map(|v| v.into_iter().map(|e| e.into()).collect()), - } - } - #[cfg(feature = "esplora")] - AnyIndexer::Esplora(inner) => { - let result = inner.create::(descr); - MayError { - ok: result.ok, - err: result.err.map(|v| v.into_iter().map(|e| e.into()).collect()), - } - } - #[cfg(feature = "mempool")] - AnyIndexer::Mempool(inner) => { - let result = inner.create::(descr); - MayError { - ok: result.ok, - err: result.err.map(|v| v.into_iter().map(|e| e.into()).collect()), - } - } - } - } - fn update, L2: Layer2>( &self, descr: &WalletDescr, cache: &mut WalletCache, + prune: bool, ) -> MayError> { match self { #[cfg(feature = "electrum")] AnyIndexer::Electrum(inner) => { - let result = inner.update::(descr, cache); + let result = inner.update::(descr, cache, prune); MayError { ok: result.ok, err: result.err.map(|v| v.into_iter().map(|e| e.into()).collect()), @@ -120,7 +89,7 @@ impl Indexer for AnyIndexer { } #[cfg(feature = "esplora")] AnyIndexer::Esplora(inner) => { - let result = inner.update::(descr, cache); + let result = inner.update::(descr, cache, prune); MayError { ok: result.ok, err: result.err.map(|v| v.into_iter().map(|e| e.into()).collect()), @@ -128,7 +97,7 @@ impl Indexer for AnyIndexer { } #[cfg(feature = "mempool")] AnyIndexer::Mempool(inner) => { - let result = inner.update::(descr, cache); + let result = inner.update::(descr, cache, prune); MayError { ok: result.ok, err: result.err.map(|v| v.into_iter().map(|e| e.into()).collect()), diff --git a/src/indexers/electrum.rs b/src/indexers/electrum.rs index ccf2bf8..d78dbef 100644 --- a/src/indexers/electrum.rs +++ b/src/indexers/electrum.rs @@ -64,18 +64,11 @@ pub enum ElectrumError { impl Indexer for Client { type Error = ElectrumError; - fn create, L2: Layer2>( - &self, - descriptor: &WalletDescr, - ) -> MayError, Vec> { - let mut cache = WalletCache::new_nonsync(); - self.update::(descriptor, &mut cache).map(|_| cache) - } - fn update, L2: Layer2>( &self, descriptor: &WalletDescr, cache: &mut WalletCache, + prune: bool, ) -> MayError> { let mut errors = Vec::::new(); @@ -224,9 +217,11 @@ impl Indexer for Client { } // The remaining transactions are unmined ones. - for (txid, mut tx) in old_cache { - tx.status = TxStatus::Unknown; - cache.tx.insert(txid, tx); + if !prune { + for (txid, mut tx) in old_cache { + tx.status = TxStatus::Unknown; + cache.tx.insert(txid, tx); + } } // TODO: Update headers & tip diff --git a/src/indexers/esplora.rs b/src/indexers/esplora.rs index 3c8e56d..830de2d 100644 --- a/src/indexers/esplora.rs +++ b/src/indexers/esplora.rs @@ -194,18 +194,11 @@ fn get_scripthash_txs_all( impl Indexer for Client { type Error = Error; - fn create, L2: Layer2>( - &self, - descriptor: &WalletDescr, - ) -> MayError, Vec> { - let mut cache = WalletCache::new_nonsync(); - self.update::(descriptor, &mut cache).map(|_| cache) - } - fn update, L2: Layer2>( &self, descriptor: &WalletDescr, cache: &mut WalletCache, + prune: bool, ) -> MayError> { let mut errors = vec![]; @@ -256,9 +249,11 @@ impl Indexer for Client { } // The remaining transactions are unmined ones. - for (txid, mut tx) in old_cache { - tx.status = TxStatus::Unknown; - cache.tx.insert(txid, tx); + if !prune { + for (txid, mut tx) in old_cache { + tx.status = TxStatus::Unknown; + cache.tx.insert(txid, tx); + } } // TODO: Update headers & tip diff --git a/src/indexers/mod.rs b/src/indexers/mod.rs index a120ec6..9677c39 100644 --- a/src/indexers/mod.rs +++ b/src/indexers/mod.rs @@ -45,12 +45,20 @@ pub trait Indexer { fn create, L2: Layer2>( &self, descr: &WalletDescr, - ) -> MayError, Vec>; + ) -> MayError, Vec> { + let mut cache = WalletCache::new_nonsync(); + self.update::(descr, &mut cache, true).map(|_| cache) + } + /// Update the wallet transaction cache and balances + /// + /// If `prune` argument is set, removes all transactions which are not present in blockchain or + /// mempool (not known to the indexer, i.e. has `TxStatus::Unknown`). fn update, L2: Layer2>( &self, descr: &WalletDescr, cache: &mut WalletCache, + prune: bool, ) -> MayError>; fn publish(&self, tx: &Tx) -> Result<(), Self::Error>; diff --git a/src/wallet.rs b/src/wallet.rs index 880e14d..22832a4 100644 --- a/src/wallet.rs +++ b/src/wallet.rs @@ -331,8 +331,9 @@ impl WalletCache { &mut self, descriptor: &WalletDescr, indexer: &I, + prune: bool, ) -> MayError> { - let res = indexer.update::(descriptor, self); + let res = indexer.update::(descriptor, self, prune); self.mark_dirty(); res } @@ -546,8 +547,8 @@ impl, L2: Layer2> Wallet { res } - pub fn update(&mut self, indexer: &I) -> MayError<(), Vec> { - self.cache.update::(&self.descr, indexer).map(|_| ()) + pub fn update(&mut self, indexer: &I, prune: bool) -> MayError<(), Vec> { + self.cache.update::(&self.descr, indexer, prune).map(|_| ()) } /// Prunes transaction cache by removing all transactions with `TxStatus::Unknown` From 320eedd727db37932a485b62da9f3838cb2b9662 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Wed, 15 Jan 2025 22:37:38 +0100 Subject: [PATCH 6/7] indexers: rename publish into broadcast --- src/cli/command.rs | 4 ++-- src/indexers/any.rs | 8 ++++---- src/indexers/electrum.rs | 2 +- src/indexers/esplora.rs | 2 +- src/indexers/mod.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cli/command.rs b/src/cli/command.rs index bf9b5f2..9512661 100644 --- a/src/cli/command.rs +++ b/src/cli/command.rs @@ -323,7 +323,7 @@ impl Exec for Args { if *publish { let indexer = self.indexer()?; eprint!("Publishing transaction via {} ... ", indexer.name()); - indexer.publish(&tx)?; + indexer.broadcast(&tx)?; eprintln!("success"); } } @@ -343,7 +343,7 @@ impl Exec for Args { if *publish { let indexer = self.indexer()?; eprint!("Publishing transaction via {} ... ", indexer.name()); - indexer.publish(&tx)?; + indexer.broadcast(&tx)?; eprintln!("success"); } } diff --git a/src/indexers/any.rs b/src/indexers/any.rs index 5af527a..daa0888 100644 --- a/src/indexers/any.rs +++ b/src/indexers/any.rs @@ -106,14 +106,14 @@ impl Indexer for AnyIndexer { } } - fn publish(&self, tx: &Tx) -> Result<(), Self::Error> { + fn broadcast(&self, tx: &Tx) -> Result<(), Self::Error> { match self { #[cfg(feature = "electrum")] - AnyIndexer::Electrum(inner) => inner.publish(tx).map_err(|e| e.into()), + AnyIndexer::Electrum(inner) => inner.broadcast(tx).map_err(|e| e.into()), #[cfg(feature = "esplora")] - AnyIndexer::Esplora(inner) => inner.publish(tx).map_err(|e| e.into()), + AnyIndexer::Esplora(inner) => inner.broadcast(tx).map_err(|e| e.into()), #[cfg(feature = "mempool")] - AnyIndexer::Mempool(inner) => inner.publish(tx).map_err(|e| e.into()), + AnyIndexer::Mempool(inner) => inner.broadcast(tx).map_err(|e| e.into()), } } } diff --git a/src/indexers/electrum.rs b/src/indexers/electrum.rs index d78dbef..1b3cec0 100644 --- a/src/indexers/electrum.rs +++ b/src/indexers/electrum.rs @@ -300,7 +300,7 @@ impl Indexer for Client { } } - fn publish(&self, tx: &Tx) -> Result<(), Self::Error> { + fn broadcast(&self, tx: &Tx) -> Result<(), Self::Error> { self.transaction_broadcast(tx)?; Ok(()) } diff --git a/src/indexers/esplora.rs b/src/indexers/esplora.rs index 830de2d..a45177d 100644 --- a/src/indexers/esplora.rs +++ b/src/indexers/esplora.rs @@ -332,5 +332,5 @@ impl Indexer for Client { } } - fn publish(&self, tx: &Tx) -> Result<(), Self::Error> { self.inner.broadcast(tx) } + fn broadcast(&self, tx: &Tx) -> Result<(), Self::Error> { self.inner.broadcast(tx) } } diff --git a/src/indexers/mod.rs b/src/indexers/mod.rs index 9677c39..866f37c 100644 --- a/src/indexers/mod.rs +++ b/src/indexers/mod.rs @@ -61,5 +61,5 @@ pub trait Indexer { prune: bool, ) -> MayError>; - fn publish(&self, tx: &Tx) -> Result<(), Self::Error>; + fn broadcast(&self, tx: &Tx) -> Result<(), Self::Error>; } From affaa2020f60d787de4e5b5d489b1eef03c864ea Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Wed, 15 Jan 2025 22:40:44 +0100 Subject: [PATCH 7/7] indexers: move pruning to the end of calls --- src/indexers/electrum.rs | 16 ++++++++-------- src/indexers/esplora.rs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/indexers/electrum.rs b/src/indexers/electrum.rs index 1b3cec0..95fcbe3 100644 --- a/src/indexers/electrum.rs +++ b/src/indexers/electrum.rs @@ -216,14 +216,6 @@ impl Indexer for Client { } } - // The remaining transactions are unmined ones. - if !prune { - for (txid, mut tx) in old_cache { - tx.status = TxStatus::Unknown; - cache.tx.insert(txid, tx); - } - } - // TODO: Update headers & tip for (script, (wallet_addr, txids)) in &mut address_index { @@ -293,6 +285,14 @@ impl Indexer for Client { .insert(wallet_addr.expect_transmute()); } + // The remaining transactions are unmined ones. + if !prune { + for (txid, mut tx) in old_cache { + tx.status = TxStatus::Unknown; + cache.tx.insert(txid, tx); + } + } + if errors.is_empty() { MayError::ok(0) } else { diff --git a/src/indexers/esplora.rs b/src/indexers/esplora.rs index a45177d..c9495f6 100644 --- a/src/indexers/esplora.rs +++ b/src/indexers/esplora.rs @@ -248,14 +248,6 @@ impl Indexer for Client { } } - // The remaining transactions are unmined ones. - if !prune { - for (txid, mut tx) in old_cache { - tx.status = TxStatus::Unknown; - cache.tx.insert(txid, tx); - } - } - // TODO: Update headers & tip for (script, (wallet_addr, txids)) in &mut address_index { @@ -325,6 +317,14 @@ impl Indexer for Client { .insert(wallet_addr.expect_transmute()); } + // The remaining transactions are unmined ones. + if !prune { + for (txid, mut tx) in old_cache { + tx.status = TxStatus::Unknown; + cache.tx.insert(txid, tx); + } + } + if errors.is_empty() { MayError::ok(0) } else {