Merge branch 'main' into clean_codebase

keep-starknet-strange · Nov 5, 2024 · 2ff8bb8 · 2ff8bb8
2 parents ec7135f + bf4b89d
commit 2ff8bb8
Show file tree

Hide file tree

Showing 21 changed files with 651 additions and 494 deletions.
diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,7 @@ __pycache__
 
 .client_cache/
 .utxo_data/
+.utreexo_data
 .timestamps_data/
 .arguments*.json
 

diff --git a/README.md b/README.md
@@ -96,18 +96,21 @@ Tasks:
 
 ### Milestone 5 - Full consensus validation
 
-Validate full block execution, including the Bitcoin scripts checks and Utreexo.
+Validate full block execution over large number of blocks, including the Bitcoin scripts checks and Utreexo proofs.
 
-### Milestone 6 - Optimizations
+* [x] consensus logic
+* [ ] consensus logic + utreexo proofs  
+* [ ] consensus logic + utreexo proofs + scripts
 
-* [ ] identify Cairo code botlenecks
-* [ ] consider using garaga msm to batch signature verifications
-
-### Milestone 7 - Proving the validation
+### Milestone 6 - Proving
 
-Recursively verify STARK proofs of chain state updates. Still largely tbd.
+Recursively verify STARK proofs of chain state updates. Still largely tbd. From initial observations it is clear that a series of optimizations will be necessary.
 
-* [ ] use garaga to implement signature verifications
+* [ ] sha256 optimization
+* [ ] don't use ByteArray when serializing data
+* [ ] blocklevel recursion
+* [ ] consider using garaga msm to batch signature verifications
+* [ ] identify other Cairo code botlenecks
 
 # Contact
 
@@ -139,6 +142,7 @@ pip install -r scripts/data/requirements.txt
 ## References
 
 * [Data processing notes](./docs/data.md)
+* [Utreexo implementation notes](./docs/utreexo.md)
 * [ZeroSync](https://github.com/ZeroSync/ZeroSync)
 * [Shinigami Script](https://github.com/keep-starknet-strange/shinigami)
 * [STWO](https://github.com/starkware-libs/stwo)

diff --git a/docs/data.md b/docs/data.md
@@ -16,4 +16,36 @@ Input data is processed in multiple steps:
 5. Script [generate_data](../scripts/data/generate_data.py) generates data that can be consumed by the `validate_and_apply` function.
 
 ## UtxoSet
-tbd
+tbd
+
+## Utreexo data
+
+In order to generate Utreexo states and batch proofs for every block in the Bitcoin history we need to use a special [Bridge node](https://github.com/Davidson-Souza/bridge).
+
+### Install
+
+```sh
+cargo install --git https://github.com/Davidson-Souza/bridge.git --no-default-features --features shinigami
+```
+
+### Configure
+
+You need to configure connection to the Bitcoin RPC via environment variables:
+
+```sh
+export BITCOIN_CORE_RPC_URL=http://localhost:8332
+export BITCOIN_CORE_RPC_USER=username
+export BITCOIN_CORE_RPC_PASSWORD=password
+```
+
+### Run
+
+Run via `screen`, `nohup`, or set up a systemd service:
+
+```sh
+~/.cargo/bin/bridge
+```
+
+You can access per-block data at `~/.bridge/blocks/<bucket>/<block_height>.json`:
+- Bucket size is 10k blocks;
+- The data directory can be changed by setting the `DATA_DIR` environment variable.
diff --git a/packages/client/src/test.cairo b/packages/client/src/test.cairo
@@ -3,8 +3,9 @@ use core::testing::get_available_gas;
 use consensus::types::block::Block;
 use consensus::types::chain_state::{ChainState, BlockValidatorImpl};
 use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait};
-use utreexo::vanilla::proof::UtreexoProof;
-use utreexo::vanilla::state::{UtreexoState, UtreexoStateTrait};
+use utreexo::stump::accumulator::StumpUtreexoAccumulator;
+use utreexo::stump::state::UtreexoStumpState;
+use utreexo::stump::proof::UtreexoBatchProof;
 
 /// Integration testing program arguments.
 #[derive(Drop)]
@@ -17,30 +18,36 @@ struct Args {
     expected_chain_state: ChainState,
     /// Optional Utreexo arguments.
     utreexo_args: Option<UtreexoArgs>,
+    /// If this flag is set, locking scripts will be executed
+    execute_script: bool,
 }
 
 /// Utreexo arguments necessary for constraining the UTXO set.
 #[derive(Drop, Serde)]
 struct UtreexoArgs {
     /// Current (initial) accumulator state.
-    state: UtreexoState,
-    /// Inclusion proofs for TXOs spent during program run.
-    proofs: Array<UtreexoProof>,
+    state: UtreexoStumpState,
+    /// Batch inclusion proof for TXOs spent during the current block.
+    /// Note that it doesn't support flow with multiple blocks applied
+    /// in a single program run.
+    proof: UtreexoBatchProof,
     /// Expected accumulator state at the end of the execution.
-    expected_state: UtreexoState,
+    expected_state: UtreexoStumpState,
 }
 
 /// Integration testing program entrypoint.
 ///
 /// Receives arguments in a serialized format (Cairo serde).
 /// Panics in case of a validation error or chain state mismatch.
 /// Prints result to the stdout.
-pub(crate) fn main(mut arguments: Span<felt252>, execute_script: bool) {
+fn main(arguments: Array<felt252>) -> Array<felt252> {
     println!("Running integration test... ");
     let mut gas_before = get_available_gas();
+    let mut args = arguments.span();
 
-    let Args { mut chain_state, blocks, expected_chain_state, utreexo_args } = Serde::deserialize(
-        ref arguments
+    let Args { mut chain_state, blocks, expected_chain_state, utreexo_args, execute_script } =
+        Serde::deserialize(
+        ref args
     )
         .expect('Failed to deserialize');
 
@@ -71,18 +78,17 @@ pub(crate) fn main(mut arguments: Span<felt252>, execute_script: bool) {
         panic!();
     }
 
-    if let Option::Some(UtreexoArgs { mut state, proofs, expected_state }) = utreexo_args {
-        match state
-            .validate_and_apply(
-                utxo_set.leaves_to_add.span(), utxo_set.leaves_to_delete.span(), proofs.span(),
-            ) {
+    if let Option::Some(UtreexoArgs { mut state, proof, expected_state }) = utreexo_args {
+        match state.verify_and_delete(@proof, utxo_set.leaves_to_delete.span()) {
             Result::Ok(new_state) => { state = new_state; },
             Result::Err(err) => {
                 println!("FAIL: gas_spent={} error='{:?}'", gas_before - get_available_gas(), err);
                 panic!();
             }
         }
 
+        state = state.add(utxo_set.leaves_to_add.span());
+
         if state != expected_state {
             println!(
                 "FAIL: gas_spent={} error='expected utreexo state {:?}, actual {:?}'",
@@ -95,6 +101,7 @@ pub(crate) fn main(mut arguments: Span<felt252>, execute_script: bool) {
     }
 
     println!("OK: gas_spent={}", gas_before - get_available_gas());
+    array![]
 }
 
 /// Workaround for handling missing `utreexo_args` field.
@@ -109,11 +116,14 @@ impl ArgsSerde of Serde<Args> {
         let blocks: Array<Block> = Serde::deserialize(ref serialized).expect('blocks');
         let expected_chain_state: ChainState = Serde::deserialize(ref serialized)
             .expect('expected_chain_state');
-        let utreexo_args: Option<UtreexoArgs> = if serialized.len() > 0 {
+        let utreexo_args: Option<UtreexoArgs> = if serialized.len() > 1 {
             Option::Some(Serde::deserialize(ref serialized).expect('utreexo_args'))
         } else {
             Option::None
         };
-        Option::Some(Args { chain_state, blocks, expected_chain_state, utreexo_args, })
+        let execute_script: bool = Serde::deserialize(ref serialized).expect('execute_script');
+        Option::Some(
+            Args { chain_state, blocks, expected_chain_state, utreexo_args, execute_script, }
+        )
     }
 }
diff --git a/packages/client/tests/data/utreexo_0.json b/packages/client/tests/data/utreexo_0.json
@@ -0,0 +1,90 @@
+{
+  "chain_state": {
+    "block_height": 0,
+    "total_work": "4295032833",
+    "best_block_hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",
+    "current_target": "26959535291011309493156476344723991336010898738574164086137773096960",
+    "epoch_start_time": 1231006505,
+    "prev_timestamps": [
+      1231006505
+    ]
+  },
+  "blocks": [
+    {
+      "header": {
+        "version": 1,
+        "time": 1231469665,
+        "bits": 486604799,
+        "nonce": 2573394689
+      },
+      "data": {
+        "variant_id": 1,
+        "transactions": [
+          {
+            "version": 1,
+            "is_segwit": false,
+            "inputs": [
+              {
+                "script": "0x04ffff001d0104",
+                "sequence": 4294967295,
+                "previous_output": {
+                  "txid": "0000000000000000000000000000000000000000000000000000000000000000",
+                  "vout": 4294967295,
+                  "data": {
+                    "value": 0,
+                    "pk_script": "0x",
+                    "cached": false
+                  },
+                  "block_height": 0,
+                  "median_time_past": 0,
+                  "is_coinbase": false
+                },
+                "witness": [
+                  "0x0000000000000000000000000000000000000000000000000000000000000000"
+                ]
+              }
+            ],
+            "outputs": [
+              {
+                "value": 5000000000,
+                "pk_script": "0x410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac",
+                "cached": false
+              }
+            ],
+            "lock_time": 0
+          }
+        ]
+      }
+    }
+  ],
+  "expected": {
+    "block_height": 1,
+    "total_work": "8590065666",
+    "best_block_hash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048",
+    "current_target": "26959535291011309493156476344723991336010898738574164086137773096960",
+    "epoch_start_time": 1231006505,
+    "prev_timestamps": [
+      1231006505,
+      1231469665
+    ]
+  },
+  "utreexo": {
+    "state": {
+      "roots": [],
+      "num_leaves": 0
+    },
+    "proof": {
+      "proof": [],
+      "targets": []
+    },
+    "expected_state": {
+      "roots": [
+        {
+          "variant_id": 0,
+          "value": 49459078824306138476779209834441505868925737545954320330266544605873965565
+        }
+      ],
+      "num_leaves": 1
+    }
+  }
+}
diff --git a/packages/client/tests/data/utreexo_1.json b/packages/client/tests/data/utreexo_1.json
@@ -0,0 +1,97 @@
+{
+  "chain_state": {
+    "block_height": 1,
+    "total_work": "8590065666",
+    "best_block_hash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048",
+    "current_target": "26959535291011309493156476344723991336010898738574164086137773096960",
+    "epoch_start_time": 1231006505,
+    "prev_timestamps": [
+      1231006505,
+      1231469665
+    ]
+  },
+  "blocks": [
+    {
+      "header": {
+        "version": 1,
+        "time": 1231469744,
+        "bits": 486604799,
+        "nonce": 1639830024
+      },
+      "data": {
+        "variant_id": 1,
+        "transactions": [
+          {
+            "version": 1,
+            "is_segwit": false,
+            "inputs": [
+              {
+                "script": "0x04ffff001d010b",
+                "sequence": 4294967295,
+                "previous_output": {
+                  "txid": "0000000000000000000000000000000000000000000000000000000000000000",
+                  "vout": 4294967295,
+                  "data": {
+                    "value": 0,
+                    "pk_script": "0x",
+                    "cached": false
+                  },
+                  "block_height": 0,
+                  "median_time_past": 0,
+                  "is_coinbase": false
+                },
+                "witness": [
+                  "0x0000000000000000000000000000000000000000000000000000000000000000"
+                ]
+              }
+            ],
+            "outputs": [
+              {
+                "value": 5000000000,
+                "pk_script": "0x41047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac",
+                "cached": false
+              }
+            ],
+            "lock_time": 0
+          }
+        ]
+      }
+    }
+  ],
+  "expected": {
+    "block_height": 2,
+    "total_work": "12885098499",
+    "best_block_hash": "000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd",
+    "current_target": "26959535291011309493156476344723991336010898738574164086137773096960",
+    "epoch_start_time": 1231006505,
+    "prev_timestamps": [
+      1231006505,
+      1231469665,
+      1231469744
+    ]
+  },
+  "utreexo": {
+    "state": {
+      "roots": [
+        {
+          "variant_id": 0,
+          "value": 49459078824306138476779209834441505868925737545954320330266544605873965565
+        }
+      ],
+      "num_leaves": 1
+    },
+    "proof": {
+      "proof": [],
+      "targets": []
+    },
+    "expected_state": {
+      "roots": [
+        {
+          "variant_id": 0,
+          "value": 2915651996892698521196383484133475266682410400159810761618404984775762993564
+        }
+      ],
+      "num_leaves": 2
+    }
+  }
+}