feat: utreexo accumulator interface

.gitignore

@@ -17,4 +17,6 @@ Cargo.lock
 **/dist
 **/.DS_Store
 **/.idea
-**/.vscode
+**/.vscode
+
+.env

README.md

@@ -111,6 +111,12 @@ This will run the test-suite:
 scarb test
 ```
 
+Re-generate test data:
+
+```base
+scarb run get_blocks
+```
+
 ## References
 
 * [STWO](https://github.com/starkware-libs/stwo)

scripts/data/block_filter.jq

@@ -5,10 +5,11 @@ def txin_coinbase:
         previous_output: OutPoint {
             txid: 0x\(0)_u256.into(),
             vout: 0xffffffff_u32,
-            txo_index: 0, 
-            amount: 0 
+            data: Default::default(),
+            block_height: Default::default(),
+            block_time: Default::default(),
         },
-        witness: LITERAL_AT_QUOTES
+        witness: array![].span()
     }"
 ;
 
@@ -19,10 +20,11 @@ def txin_regular:
         previous_output: OutPoint {
             txid: 0x\(.txid)_u256.into(),
             vout: \(.vout),
-            txo_index: 0, // TODO: implement
-            amount: 0 // TODO: implement
+            data: Default::default(),
+            block_height: Default::default(),
+            block_time: Default::default(),
         },
-        witness: LITERAL_AT_QUOTES
+        witness: array![].span()
     }"
 ;
 
@@ -38,6 +40,7 @@ def txout:
     "TxOut {
         value: \((.value*100000000) | round)_u64,
         pk_script: @from_hex(\"\(.scriptPubKey.hex)\"),
+        cached: false,
     }"
 ;
 
@@ -56,15 +59,18 @@ def block:
         header : Header {	
             version: \(.version)_u32,
             time: \(.time)_u32,
-            bits: 0, // TODO
+            bits: 0x\(.bits)_u32,
             nonce: \(.nonce)_u32
         },
 		txs: array![\(.tx | map(tx) | join(",\n"))].span()
    }"
 ;
 
 def fixture:
-"use raito::state::{Block, Header, Transaction, OutPoint, TxIn, TxOut};
+"
+// THIS CODE IS GENERATED BY SCRIPT, DO NOT EDIT IT MANUALLY
+
+use raito::state::{Block, Header, Transaction, OutPoint, TxIn, TxOut};
 use raito::test_utils::from_hex;
 
 pub fn block_\(.height)() -> Block {
@@ -74,4 +80,3 @@ pub fn block_\(.height)() -> Block {
 ;
 
 .result | fixture
-

scripts/data/get_block.sh

@@ -2,7 +2,12 @@
 set -e;
 set -o pipefail;
 
-HEIGHT=$(curl -s --user $USERPWD -s -d '{"jsonrpc": "1.0", "id":"curltest", "method": "getblockheader", "params": ["'${1}'"] }' -H 'content-type: text/plain;' $BITCOIN_RPC | jq -r '.result.height')
+if [ -f .env ]
+then
+  export $(cat .env | xargs)
+fi
+
+HEIGHT=$(curl -s --user $USERPWD -d '{"jsonrpc": "1.0", "id":"curltest", "method": "getblockheader", "params": ["'${1}'"] }' -H 'content-type: text/plain;' $BITCOIN_RPC | jq -r '.result.height')
 
 curl \
  -s \

src/lib.cairo

@@ -5,5 +5,6 @@ pub mod codec;
 
 mod main;
 mod merkle_tree;
+mod utreexo;
 
 pub mod test_utils;

src/state.cairo

@@ -5,7 +5,7 @@
 //! and to avoid repetitive computations.
 
 use raito::utils::Hash;
-use raito::test_utils::from_hex;
+pub use super::utreexo::UtreexoState;
 
 /// Represents the state of the blockchain.
 #[derive(Drop, Copy)]
@@ -38,7 +38,7 @@ impl ChainStateDefault of Default<ChainState> {
             epoch_start_time: 0,
             prev_timestamps: [
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-            ].span(), utreexo_state: UtreexoState { roots: [].span() },
+            ].span(), utreexo_state: Default::default(),
         }
     }
 }
@@ -84,81 +84,104 @@ pub struct Transaction {
     pub inputs: Span<TxIn>,
     /// The outputs of the transaction.
     pub outputs: Span<TxOut>,
-    /// The lock time of the transaction.
+    /// Block height / time after which this transaction can be mined.
+    /// Locktime feature is enabled if at least one input has sequence <= 0xfffffffe.
     pub lock_time: u32,
 }
 
-/// Represents an input of a transaction.
+/// Input of a transaction.
 /// https://learnmeabitcoin.com/technical/transaction/input/
-///
-/// NOTE that `txid` and `vout` fields can be resolved via Utreexo set using the TXO index.
 #[derive(Drop, Copy)]
 pub struct TxIn {
     /// The signature script which satisfies the conditions placed in the txo pubkey script
     /// or coinbase script that contains block height (since 227,836) and miner nonce (optional).
     pub script: @ByteArray,
-    /// The sequence number of the input.
+    /// This field enables absolute or relative locktime feature, basically how much time or how
+    /// many blocks must pass (since genesis or since the referenced output was mined) before this
+    /// transaction can be mined.
     pub sequence: u32,
-    /// The reference to the previous output that is being used as an input.
+    /// The reference to the previous output that is being spent by this input.
     pub previous_output: OutPoint,
     /// The witness data for transactions.
-    pub witness: @ByteArray
+    /// A list of items (of different size) pushed onto stack before sig script execution.
+    pub witness: Span<ByteArray>,
 }
 
-/// Represents a reference to a transaction output.
+/// A reference to a transaction output.
+///
+/// NOTE that `data` and `block_height` meta fields are not serialized with the rest of
+/// the transaction and hence are not constrained with the transaction hash.
+///
+/// There are four possible cases:
+///   1. Coinbase input that does not spend any outputs (zero txid)
+///   2. Input that spends an output created within the same block (cached)
+///   3. Input that spends a coinbase output
+///   4. Input that spends an output from a past block
+///
+/// For (1) we don't need to add extra constraints, because meta fields are not used.
+/// For (2) we need to check that the referenced output is indeed cached:
+///     * Calculate cache key by hashing (txid, vout, data)
+///     * Check if the key is present in the cache
+///     * Remove item from the cache
+/// For (3) we need to check that the referenced output is in the utreexo accumulator:
+///     * Calculate utreexo leaf hash from (txid, vout, data, block_height)
+///     * Verify inclusion proof (either individual or batched) against the roots
+///     * Delete the leaf from the accumulator
+/// For (4) we need to additionally check if the coinbase output is older than 100 blocks
+///
+/// IMPORTANT:
+///     * Utreexo proofs can be verified at any point of block validation because accumulator
+///       is not changing until the end of the block;
+///     * Cache lookups MUST be done in a sequential order, i.e. transactions are validated
+///       one by one, first inputs then outputs. Output validation might put something to the
+///       cache while input validation might remove an item, thus it's important to maintain
+///       the order.
 #[derive(Drop, Copy)]
 pub struct OutPoint {
     /// The hash of the referenced transaction.
     pub txid: Hash,
     /// The index of the specific output in the transaction.
     pub vout: u32,
-    /// The index of output in the utreexo set (meta field).
-    pub txo_index: u64,
-    /// Amount calculated with the txid and vout.
-    pub amount: u64
+    /// Referenced output data (meta field).
+    /// Must be set to default for coinbase inputs.
+    pub data: TxOut,
+    /// The height of the block that contains this output (meta field).
+    /// Used to validate coinbase tx spending (not sooner than 100 blocks) and relative timelocks
+    /// (it has been more than X block since the transaction containing this output was mined).
+    /// Can be set to default for non-coinbase outputs & if locktime feature (height relative) is
+    /// disabled.
+    pub block_height: u32,
+    /// The time of the block that contains this output (meta field).
+    /// Used to validate relative timelocks (it has been more than X seconds since the transaction
+    /// containing this output was mined).
+    /// Can be set to default if locktime feature (time relative) is disabled.
+    pub block_time: u32,
 }
 
-/// Represents an output of a transaction.
+/// Output of a transaction.
 /// https://learnmeabitcoin.com/technical/transaction/output/
+///
+/// Upon processing (validating) an output one of three actions must be taken:
+///     - Add output with some extra info (see [OutPoint]) to the Utreexo accumulator
+///     - Add output to the cache in case it is going to be spent in the same block
+///     - Do nothing in case of a provably unspendable output
+///
+/// Read more: https://en.bitcoin.it/wiki/Script#Provably_Unspendable/Prunable_Outputs
 #[derive(Drop, Copy)]
 pub struct TxOut {
     /// The value of the output in satoshis.
+    /// Can be in range [0, 21_000_000] BTC (including both ends).
     pub value: u64,
     /// The spending script (aka locking code) for this output.
     pub pk_script: @ByteArray,
+    /// Meta flag indicating that this output will be spent within the current block(s).
+    /// This output won't be added to the utreexo accumulator.
+    /// Note that coinbase outputs cannot be spent sooner than 100 blocks after inclusion.
+    pub cached: bool,
 }
 
-/// Accumulator representation of the state aka "Compact State Node".
-#[derive(Drop, Copy)]
-pub struct UtreexoState {
-    /// Roots of Merkle tree forest.
-    pub roots: Span<felt252>,
-}
-
-/// Utreexo set is used to retrieve TXOs spent by particular inputs.
-#[derive(Drop, Copy)]
-pub struct UtreexoSet {
-    /// A list of extended transaction outputs spent in a particular block(s).
-    pub outputs: Span<UtreexoOutput>,
-}
-
-/// TXO extended with info about parent transaction and the position within it.
-/// The hash of this structure is a leaf node in the Utreexo Merkle tree forest.
-#[derive(Drop, Copy)]
-pub struct UtreexoOutput {
-    /// The TXID this output belongs to.
-    pub txid: Hash,
-    /// The index of this output.
-    pub vout: u32,
-    /// Output data.
-    pub output: TxOut,
-}
-
-/// Inclusion proof for multiple leaves.
-#[derive(Drop, Copy)]
-pub struct UtreexoBatchProof {
-    /// Indices of tree leaves, one for each output in the utreexo set.
-    pub targets: Span<u64>,
-    /// All the nodes required to calculate the root.
-    pub proof: Span<felt252>,
+impl TxOutDefault of Default<TxOut> {
+    fn default() -> TxOut {
+        TxOut { value: 0, pk_script: @"", cached: false, }
+    }
 }

src/utreexo.cairo

@@ -0,0 +1,101 @@
+//! Utreexo is an accumulator for the Bitcoin unspent transaction set.
+//!
+//! It allows to verify that a certain transaction output exists
+//! and still unspent at a particular block while maintaining only
+//! a very compact state.
+//!
+//! It is also useful for transaction validation (our case) since it
+//! allows to "fetch" the output spent by a particular input in the
+//! validated transaction. This is typically required to calculate
+//! transaction fee and also to check that script execution succeeds.
+//!
+//! The expected workflow is the following:
+//!   - For coinbase and inputs spending TXOs created in the same block
+//!     utreexo accumulator is not updated (local cache is used instead);
+//!   - For all other inputs we provide respective TXOs (extended) plus
+//!     plus inclusion proof (can be individual, batched, or hybrid);
+//!   - The client has to verify the inclusion proof and then remove all
+//!     the TXOs from the utreexo set, that way updating the state;
+//!   - For every output that is not spent in the same block we add the
+//!     extended (additionally contains txid and output index aka vout) output
+//!     to the accumulator (i.e. update the utreexo state).
+//!
+//! Note that utreexo data and proofs are provided via program input so
+//! it is not part of prover/verifier arguments. Utreexo state (which
+//! is part of the chain state) is what allows us to constrain
+//! these inputs and ensure integrity.
+//!
+//! Read more about utreexo: https://eprint.iacr.org/2019/611.pdf
+
+use super::state::OutPoint;
+
+/// Accumulator representation of the state aka "Compact State Node".
+/// Part of the chain state.
+#[derive(Drop, Copy)]
+pub struct UtreexoState {
+    /// Roots of the Merkle tree forest.
+    /// Index is the root height, None means a gap.
+    pub roots: Span<Option<felt252>>,
+    /// Total number of leaves (in the bottom-most row).
+    /// Required to calculate the number of nodes in a particular row.
+    /// Can be reconstructed from the roots, but cached for convenience.
+    pub num_leaves: u64,
+}
+
+/// Accumulator interface
+pub trait UtreexoAccumulator {
+    /// Adds single output to the accumulator.
+    /// The order *is important*: adding A,B and B,A would result in different states.
+    ///
+    /// Note that this call also pushes old UTXOs "to the left", to a larger subtree.
+    /// This mechanism ensures that short-lived outputs have small inclusion proofs.
+    fn add(ref self: UtreexoState, output: OutPoint);
+
+    /// Verifies inclusion proof for a single output.
+    fn verify(
+        self: @UtreexoState, output: @OutPoint, proof: @UtreexoProof
+    ) -> Result<(), UtreexoError>;
+
+    /// Removes single output from the accumlator (order is important).
+    ///
+    /// Note that once verified, the output itself is not required for deletion,
+    /// the leaf index plus inclusion proof is enough.
+    fn delete(ref self: UtreexoState, proof: @UtreexoProof);
+
+    /// Verifies batch proof for multiple outputs (e.g. all outputs in a block).
+    fn verify_batch(
+        self: @UtreexoState, outputs: Span<OutPoint>, proof: @UtreexoBatchProof
+    ) -> Result<(), UtreexoError>;
+
+    /// Removes multiple outputs from the accumulator.
+    fn delete_batch(ref self: UtreexoState, proof: @UtreexoBatchProof);
+}
+
+#[derive(Drop, Copy, PartialEq)]
+pub enum UtreexoError {}
+
+/// Utreexo inclusion proof for a single transaction output.
+#[derive(Drop, Copy)]
+pub struct UtreexoProof {
+    /// Index of the leaf in the forest, but also an encoded binary path,
+    /// specifying which sibling node is left and which is right.
+    pub leaf_index: u64,
+    /// List of sibling nodes required to calculate the root.
+    pub proof: Span<felt252>,
+}
+
+/// Utreexo inclusion proof for multiple outputs.
+/// Compatible with https://github.com/utreexo/utreexo
+#[derive(Drop, Copy)]
+pub struct UtreexoBatchProof {
+    /// Indices of leaves to be deleted (ordered starting from 0, left to right).
+    pub targets: Span<u64>,
+    /// List of sibling nodes required to calculate the root.
+    pub proof: Span<felt252>,
+}
+
+pub impl UtreexoStateDefault of Default<UtreexoState> {
+    fn default() -> UtreexoState {
+        UtreexoState { roots: array![].span(), num_leaves: 0, }
+    }
+}