New utxo snapshot format (#81)

* Separate out utxo_store.rs

* Introduce SnapshotProcessor

* Rework subcoin-utxo-snapshot

* write_utxo_snapshot_in_memory()

* New snapshot format compatible with Bitcoin Core

* Refactor snapshot_processor

* Refactor subcoin-utxo-snapshot

* Refactor dump_txout_set

* Nits

* Nits

* Fix test

* Remove local tests

* Nits

* fmt

* Fix clippy

* .

* Verify snapshot integrity in snapcake

* Rename snapshot_manager

* fmt

Cargo.toml

@@ -32,6 +32,7 @@ default-members = ["crates/subcoin-node"]
 array-bytes = "6.2.2"
 async-channel = "1.8.0"
 async-trait = "0.1"
+bincode = "1.3.3"
 bitcoin = { git = "https://github.com/liuchengxu/rust-bitcoin", branch = "0.32.x-subcoin", default-features = false }
 bitcoinconsensus = "0.105.0+25.1"
 bitcoin-explorer = { git = "https://github.com/liuchengxu/Rusty-Bitcoin-Explorer", branch = "main", default-features = false }
@@ -54,6 +55,7 @@ parking_lot = "0.12"
 scale-info = { version = "2.6.0", default-features = false }
 serde = "1"
 serde_json = "1"
+sha2 = "0.9.9"
 tempfile = "3.10.1"
 thiserror = "1.0"
 tokio = "1.41.1"

crates/subcoin-node/src/commands/blockchain/dump_txout_set.rs

@@ -1,10 +1,10 @@
 use super::MergedParams;
 use crate::commands::blockchain::{fetch_utxo_set_at, ClientParams};
 use crate::utils::Yield;
-use bitcoin::consensus::Encodable;
 use std::fs::File;
 use std::io::{Stdout, Write};
 use std::path::PathBuf;
+use std::sync::atomic::AtomicUsize;
 use std::sync::Arc;
 use subcoin_primitives::runtime::Coin;
 use subcoin_service::FullClient;
@@ -18,15 +18,13 @@ pub struct DumpTxOutSet {
     #[clap(long)]
     height: Option<u32>,
 
-    /// Export the dumped txout set to a CSV file.
+    /// Path to export the UTXO set in CSV format.
     #[clap(short, long, value_name = "PATH")]
     csv: Option<PathBuf>,
 
-    /// Path to the binary dump file for the UTXO set.
+    /// Path to export the UTXO set as a binary file.
     ///
     /// The binary dump is compatible with the format used by the Bitcoin Core client.
-    /// You can export the UTXO set from Subcoin and import it into the Bitcoin
-    /// Core client.
     ///
     /// If neither `--csv` nor `--binary` options are provided, the UTXO set will be
     /// printed to stdout in CSV format.
@@ -56,42 +54,57 @@ impl DumpTxOutSetCmd {
 
         let (block_number, bitcoin_block_hash, utxo_iter) = fetch_utxo_set_at(&client, height)?;
 
-        let mut output_file = if let Some(path) = csv {
-            println!(
-                "Dumping UTXO set at #{block_number},{bitcoin_block_hash} to {}",
-                path.display()
-            );
-            UtxoSetOutput::Csv(std::fs::File::create(path)?)
-        } else if let Some(path) = binary {
+        if let Some(path) = binary {
             let utxo_set_size = fetch_utxo_set_at(&client, height)?.2.count() as u64;
 
             println!(
-                "Dumping UTXO set at #{block_number},{bitcoin_block_hash} to {}",
+                "Exporting UTXO set ({utxo_set_size}) at #{block_number},{bitcoin_block_hash} to binary file: {}",
                 path.display()
             );
-            println!("UTXO set size: {utxo_set_size}");
 
-            let mut file = std::fs::File::create(&path)?;
+            let file = std::fs::File::create(&path)?;
 
-            let mut data = Vec::new();
-            bitcoin_block_hash
-                .consensus_encode(&mut data)
-                .map_err(|err| sc_cli::Error::Application(Box::new(err)))?;
-            utxo_set_size
-                .consensus_encode(&mut data)
-                .map_err(|err| sc_cli::Error::Application(Box::new(err)))?;
+            let mut snapshot_generator =
+                UtxoSnapshotGenerator::new(path, file, bitcoin::Network::Bitcoin);
 
-            let _ = file.write(data.as_slice())?;
+            snapshot_generator.generate_snapshot_in_mem(
+                bitcoin_block_hash,
+                utxo_set_size,
+                utxo_iter.map(Into::into),
+            )?;
 
-            UtxoSetOutput::Snapshot(UtxoSnapshotGenerator::new(path, file))
+            return Ok(());
+        }
+
+        let mut output_file = if let Some(path) = csv {
+            println!(
+                "Exporting UTXO set at #{block_number},{bitcoin_block_hash} to CSV file: {}",
+                path.display()
+            );
+            UtxoSetOutput::Csv(std::fs::File::create(path)?)
         } else {
-            println!("Dumping UTXO set at #{block_number},{bitcoin_block_hash}");
+            println!("Exporting UTXO set at #{block_number},{bitcoin_block_hash} to stdout:");
             UtxoSetOutput::Stdout(std::io::stdout())
         };
 
-        for (txid, vout, coin) in utxo_iter {
-            output_file.write(txid, vout, coin)?;
+        let processed = Arc::new(AtomicUsize::new(0));
+
+        let grouped_utxos =
+            subcoin_utxo_snapshot::group_utxos_by_txid(utxo_iter.into_iter().map(Into::into));
 
+        let total_txids = grouped_utxos.len();
+
+        crate::utils::show_progress_in_background(
+            processed.clone(),
+            total_txids as u64,
+            format!("Dumping UTXO set at block #{block_number}..."),
+        );
+
+        for (txid, coins) in grouped_utxos {
+            for subcoin_utxo_snapshot::OutputEntry { vout, coin } in coins {
+                output_file.write(txid, vout, coin)?;
+            }
+            processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
             // Yield here allows to make the process interruptible by ctrl_c.
             Yield::new().await;
         }
@@ -101,17 +114,13 @@ impl DumpTxOutSetCmd {
 }
 
 enum UtxoSetOutput {
-    Snapshot(UtxoSnapshotGenerator),
     Csv(File),
     Stdout(Stdout),
 }
 
 impl UtxoSetOutput {
     fn write(&mut self, txid: bitcoin::Txid, vout: u32, coin: Coin) -> std::io::Result<()> {
         match self {
-            Self::Snapshot(snapshot_generator) => {
-                snapshot_generator.write_utxo_entry(txid, vout, coin)?;
-            }
             Self::Csv(ref mut file) => {
                 let Coin {
                     is_coinbase,

crates/subcoin-node/src/commands/blockchain/get_txout_set_info.rs

@@ -1,7 +1,6 @@
 use super::MergedParams;
 use crate::commands::blockchain::{fetch_utxo_set_at, ClientParams};
 use crate::utils::Yield;
-use indicatif::{ProgressBar, ProgressStyle};
 use sc_client_api::HeaderBackend;
 use sp_api::ProvideRuntimeApi;
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -108,8 +107,11 @@ async fn gettxoutsetinfo(
     let mut muhash = subcoin_crypto::muhash::MuHash3072::new();
 
     if progress_bar {
-        let loaded = txouts.clone();
-        std::thread::spawn(move || show_progress(loaded, total_coins, block_number));
+        crate::utils::show_progress_in_background(
+            txouts.clone(),
+            total_coins,
+            format!("Loading UTXO set at block #{block_number}..."),
+        );
     }
 
     for (txid, vout, coin) in utxo_iter {
@@ -155,32 +157,6 @@ async fn gettxoutsetinfo(
     Ok(tx_out_set_info)
 }
 
-fn show_progress(loaded: Arc<AtomicUsize>, total_coins: u64, block_number: u32) {
-    let pb = ProgressBar::new(total_coins);
-
-    pb.set_message(format!("Loading UTXO set at block #{block_number}..."));
-
-    pb.set_style(
-        ProgressStyle::default_bar()
-            .template("{msg} [{bar:40}] {percent:.2}% ({pos}/{len}, {eta})")
-            .unwrap()
-            .progress_chars("##-"),
-    );
-
-    loop {
-        let new = loaded.load(Ordering::Relaxed) as u64;
-
-        if new == total_coins {
-            pb.finish_and_clear();
-            return;
-        }
-
-        pb.set_position(new);
-
-        std::thread::sleep(Duration::from_millis(200));
-    }
-}
-
 // Custom serializer for total_amount to display 8 decimal places
 fn serialize_as_btc<S>(amount: &u64, serializer: S) -> Result<S::Ok, S::Error>
 where

crates/subcoin-node/src/utils.rs

@@ -1,3 +1,8 @@
+use indicatif::{ProgressBar, ProgressStyle};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
 /// A future that will always `yield` on the first call of `poll` but schedules the
 /// current task for re-execution.
 ///
@@ -29,3 +34,33 @@ impl futures::Future for Yield {
         }
     }
 }
+
+pub(crate) fn show_progress_in_background(processed: Arc<AtomicUsize>, total: u64, msg: String) {
+    std::thread::spawn(move || show_progress(processed, total, msg));
+}
+
+fn show_progress(processed: Arc<AtomicUsize>, total: u64, msg: String) {
+    let pb = ProgressBar::new(total);
+
+    pb.set_message(msg);
+
+    pb.set_style(
+        ProgressStyle::default_bar()
+            .template("{msg} [{bar:40}] {percent:.2}% ({pos}/{len}, {eta})")
+            .unwrap()
+            .progress_chars("##-"),
+    );
+
+    loop {
+        let new = processed.load(Ordering::Relaxed) as u64;
+
+        if new == total {
+            pb.finish_and_clear();
+            return;
+        }
+
+        pb.set_position(new);
+
+        std::thread::sleep(Duration::from_millis(200));
+    }
+}

crates/subcoin-service/Cargo.toml

@@ -28,8 +28,7 @@ sc-network = { workspace = true }
 sc-network-sync = { workspace = true }
 sc-rpc = { workspace = true }
 sc-rpc-api = { workspace = true }
-# `test-helpers` for the exposed Client type.
-sc-service = { workspace = true, features = ["test-helpers"], default-features = false }
+sc-service = { workspace = true, default-features = false }
 sc-storage-monitor = { workspace = true }
 sc-sysinfo = { workspace = true }
 sc-telemetry = { workspace = true }

crates/subcoin-snapcake/Cargo.toml

@@ -11,6 +11,7 @@ name = "snapcake"
 path = "src/main.rs"
 
 [dependencies]
+bincode = { workspace = true }
 bitcoin = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 codec = { workspace = true }
@@ -30,6 +31,7 @@ sc-network-sync = { workspace = true }
 sc-service = { workspace = true }
 sc-transaction-pool = { workspace = true }
 serde = { workspace = true }
+sha2 = { workspace = true }
 sp-api = { workspace = true }
 sp-blockchain = { workspace = true }
 sp-consensus = { workspace = true }
@@ -43,3 +45,4 @@ subcoin-service = { workspace = true }
 subcoin-utxo-snapshot = { workspace = true }
 tracing = { workspace = true }
 tokio = { workspace = true }
+rocksdb = "0.21.0"

crates/subcoin-snapcake/src/main.rs

@@ -35,6 +35,7 @@
 
 mod cli;
 mod params;
+mod snapshot_manager;
 mod state_sync_wrapper;
 mod syncing_strategy;