Indexers Speed-ups: Implement LRU Caching and Incremental Updates

Cargo.toml

@@ -66,6 +66,7 @@ rand = { version = "0.8.5", optional = true }
 rpassword = { version = "7.3.1", optional = true }
 aes-gcm = { version = "0.10.3", optional = true }
 bip39 = { version = "2.0.0", optional = true }
+lru = "0.12.4"
 
 serde_crate = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }
@@ -87,7 +88,7 @@ hot = ["bp-std/signers", "bip39", "rand", "aes-gcm", "rpassword"]
 cli = ["base64", "env_logger", "clap", "shellexpand", "fs", "serde", "electrum", "esplora", "mempool", "log", "colored"]
 log = ["env_logger"]
 electrum = ["bp-electrum", "serde", "serde_json"]
-esplora = ["bp-esplora"]
-mempool = ["esplora"]
+esplora = ["bp-esplora", "serde", "serde_json"]
+mempool = ["esplora", "serde", "serde_json"]
 fs = ["serde"]
 serde = ["serde_crate", "serde_yaml", "toml", "bp-std/serde"]

src/cli/args.rs

@@ -23,6 +23,7 @@
 use std::fmt::Debug;
 use std::path::PathBuf;
 use std::process::exit;
+use std::sync::OnceLock;
 
 use bpstd::XpubDerivable;
 use clap::Subcommand;
@@ -32,6 +33,7 @@
 use crate::cli::{
     Config, DescrStdOpts, DescriptorOpts, ExecError, GeneralOpts, ResolverOpt, WalletOpts,
 };
+use crate::indexers::electrum::ElectrumClient;
 use crate::indexers::esplora;
 use crate::{AnyIndexer, MayError, Wallet};
 
@@ -95,14 +97,36 @@
     }
 
     pub fn indexer(&self) -> Result<AnyIndexer, ExecError> {
+        use crate::indexers::IndexerCache;
+
+        // Define a static variable to store IndexerCache,
+        // Ensuring that all cached data is centralized
+        // Initialize only once
+        static INDEXER_CACHE: OnceLock<IndexerCache> = OnceLock::new();
+
+        let indexer_cache = INDEXER_CACHE
+            .get_or_init(|| {
+                IndexerCache::new(
+                    self.general
+                        .indexer_cache_factor
+                        .try_into()
+                        .expect("Error: indexer cache size is invalid"),
+                )
+            })
+            .clone();
+
         let network = self.general.network.to_string();
         Ok(match (&self.resolver.esplora, &self.resolver.electrum, &self.resolver.mempool) {
-            (None, Some(url), None) => AnyIndexer::Electrum(Box::new(electrum::Client::new(url)?)),
+            (None, Some(url), None) => {
+                AnyIndexer::Electrum(Box::new(ElectrumClient::new(url, indexer_cache)?))
+            }
             (Some(url), None, None) => AnyIndexer::Esplora(Box::new(esplora::Client::new_esplora(
                 &url.replace("{network}", &network),
+                indexer_cache,
             )?)),
             (None, None, Some(url)) => AnyIndexer::Mempool(Box::new(esplora::Client::new_mempool(
                 &url.replace("{network}", &network),
+                indexer_cache,
             )?)),
             _ => {
                 eprintln!(

src/cli/opts.rs

@@ -158,6 +158,12 @@
     /// Network to use.
     #[arg(short, long, global = true, default_value = "testnet3", env = "LNPBP_NETWORK")]
     pub network: Network,
+
+    /// Indexer cache factor.
+    /// `indexer_cache_factor`: A non-zero size value that determines the capacity of the LRU
+    /// caches for indexer.
+    #[arg(long, global = true, default_value = "1000")]
+    pub indexer_cache_factor: usize,
 }
 
 impl GeneralOpts {

src/data.rs

@@ -265,7 +265,7 @@
             Party::Subsidy => None,
             Party::Counterparty(addr) => Some(addr.script_pubkey()),
             Party::Unknown(script) => Some(script.clone()),
-            Party::Wallet(_) => None,
+            Party::Wallet(derive) => Some(derive.addr.script_pubkey()),
         }
     }
 }
@@ -419,3 +419,15 @@
         }
     }
 }
+
+impl WalletAddr<Sats> {
+    pub fn expect_transmute(self) -> WalletAddr<i64> {
+        WalletAddr {
+            terminal: self.terminal,
+            addr: self.addr,
+            used: self.used,
+            volume: self.volume,
+            balance: self.balance.sats_i64(),
+        }
+    }
+}

src/indexers/any.rs

@@ -22,6 +22,8 @@
 use bpstd::Tx;
 use descriptors::Descriptor;
 
+#[cfg(feature = "electrum")]
+use super::electrum::ElectrumClient;
 use crate::{Indexer, Layer2, MayError, WalletCache, WalletDescr};
 
 /// Type that contains any of the client types implementing the Indexer trait
@@ -31,7 +33,7 @@ pub enum AnyIndexer {
     #[cfg(feature = "electrum")]
     #[from]
     /// Electrum indexer
-    Electrum(Box<electrum::client::Client>),
+    Electrum(Box<ElectrumClient>),
     #[cfg(feature = "esplora")]
     #[from]
     /// Esplora indexer

src/indexers/cache.rs

@@ -0,0 +1,82 @@
+// Modern, minimalistic & standard-compliant cold wallet library.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Written in 2024 by
+//     Nicola Busanello <[email protected]>
+//
+// Copyright (C) 2024 LNP/BP Standards Association. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(feature = "electrum")]
+use std::collections::HashMap;
+use std::num::NonZeroUsize;
+use std::sync::{Arc, Mutex};
+
+#[cfg(feature = "electrum")]
+use bpstd::Txid;
+use bpstd::{BlockHash, DerivedAddr, Tx};
+#[cfg(feature = "electrum")]
+use electrum::GetHistoryRes;
+use lru::LruCache;
+
+#[derive(Clone, Eq, PartialEq, Hash, Debug)]
+pub struct TxDetail {
+    pub(crate) inner: Tx,
+    pub(crate) blockhash: Option<BlockHash>,
+    pub(crate) blocktime: Option<u64>,
+}
+
+#[derive(Debug, Clone)]
+pub struct IndexerCache {
+    // The cache is designed as Arc to ensure that
+    // all Indexers globally use the same cache
+    // (to maintain data consistency).
+    //
+    // The Mutex is used to ensure that the cache is thread-safe
+    // Make sure to get the updated cache from immutable references
+    // In the create/update processing logic of &Indexer
+    // addr_transactions: for esplora
+    #[cfg(feature = "esplora")]
+    pub(crate) addr_transactions: Arc<Mutex<LruCache<DerivedAddr, Vec<esplora::Tx>>>>,
+    // script_history: for electrum
+    #[cfg(feature = "electrum")]
+    pub(crate) script_history: Arc<Mutex<LruCache<DerivedAddr, HashMap<Txid, GetHistoryRes>>>>,
+    // tx_details: for electrum
+    #[cfg(feature = "electrum")]
+    pub(crate) tx_details: Arc<Mutex<LruCache<Txid, TxDetail>>>,
+}
+
+impl IndexerCache {
+    /// Creates a new `IndexerCache` with the specified cache factor.
+    ///
+    /// # Parameters
+    /// - `cache_factor`: A non-zero size value that determines the capacity of the LRU caches. This
+    ///   factor is used to initialize the `addr_transactions` and `script_history` caches. The
+    ///   `tx_details` cache is initialized with a capacity that is 20 times the size of the
+    ///   `script_history` cache.
+    pub fn new(cache_factor: NonZeroUsize) -> Self {
+        Self {
+            #[cfg(feature = "esplora")]
+            addr_transactions: Arc::new(Mutex::new(LruCache::new(cache_factor))),
+            #[cfg(feature = "electrum")]
+            script_history: Arc::new(Mutex::new(LruCache::new(cache_factor))),
+            // size of tx_details is 20 times the size of script_history for electrum
+            #[cfg(feature = "electrum")]
+            tx_details: Arc::new(Mutex::new(LruCache::new(
+                cache_factor.saturating_mul(NonZeroUsize::new(20).expect("20 is not zero")),
+            ))),
+        }
+    }
+}