From 3822c05515497be8446ffcdbf275efaa08d138bd Mon Sep 17 00:00:00 2001 From: Michael Zaikin Date: Mon, 5 Aug 2024 18:45:48 +0100 Subject: [PATCH 01/12] Extend Bitcoin types with extra fields for validation --- src/state.cairo | 54 +++++++++++++++++++++++++++++++------------- src/validation.cairo | 2 ++ 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/state.cairo b/src/state.cairo index eb9c1950..8928a518 100644 --- a/src/state.cairo +++ b/src/state.cairo @@ -1,3 +1,9 @@ +//! Bitcoin data type objects extended with validation context. +//! +//! The data is expected to be prepared in advance and passed as program arguments. +//! The extended set of fields allows to recursively validate entities in a stateless manner, +//! and to avoid repetitive computations. + /// Represents the state of the blockchain. #[derive(Drop, Copy)] pub struct ChainState { @@ -13,23 +19,21 @@ pub struct ChainState { pub epoch_start_time: u32, /// Previous timestamps. pub prev_timestamps: Span, - // TODO: utreexo_roots? + // Utreexo roots (for checking [TxIn] inclusion proofs) + pub utreexo_roots: Span } /// Represents a block in the blockchain. -/// #[derive(Drop, Copy)] pub struct Block { /// block header pub header: Header, - // TODO: how to handle coinbase transactions? - /// Transactions pub txs: Span, } /// Block header -/// https://developer.bitcoin.org/reference/block_chain.html#block-headers +/// https://learnmeabitcoin.com/technical/block/ #[derive(Drop, Copy)] pub struct Header { /// The version of the block. @@ -46,22 +50,34 @@ pub struct Header { pub nonce: u32, } -/// Transaction -/// https://developer.bitcoin.org/reference/transactions.html#raw-transaction-format +/// Extended transaction. +/// https://learnmeabitcoin.com/technical/transaction/ +/// +/// Contains additional "meta" fields required for validation. #[derive(Drop, Copy)] pub struct Transaction { /// The version of the transaction. pub version: i32, + /// Flag which indicates the presence of witness data. + /// Segwit marker and flag do not contribute to TXID (transaction hash), + /// but do contribute to wTXID. + pub is_segwit: bool, /// The inputs of the transaction. pub inputs: Span, /// The outputs of the transaction. pub outputs: Span, + /// The list of witnesses, one for each input. + /// Each witness is a list of elements that are to be pushed onto stack. + /// Witnesses do not contribute to TXID but do contribute to wTXID. + pub witnesses: Span>, /// The lock time of the transaction. pub lock_time: u32, + /// Transaction fee which is diff between total input and output amounts (meta field) + pub fee: i64, } /// Output of a transaction. -/// https://developer.bitcoin.org/reference/transactions.html#txout-a-transaction-output +/// https://learnmeabitcoin.com/technical/transaction/output/ #[derive(Drop, Copy)] pub struct TxOut { /// The value of the output. @@ -70,17 +86,23 @@ pub struct TxOut { pub pk_script: @ByteArray, } -/// Input of a transaction. -/// https://developer.bitcoin.org/reference/transactions.html#txin-a-transaction-input-non-coinbase +/// Extended input of a transaction. +/// https://learnmeabitcoin.com/technical/transaction/input/ +/// +/// Contains additional "meta" fields required for validation. #[derive(Drop, Copy)] pub struct TxIn { - /// The transaction ID of the input. + /// The previous TXID this input spends. pub txid: u256, - /// The index of the input. - pub index: u32, - /// The script of the input. + /// The previous transaction output index this input spends. + pub vout: u32, + /// The signature script which satisfies the conditions placed in the txo pubkey script + /// or coinbase script that contains block height (since 227,836) and miner nonce (optional). pub script: @ByteArray, - /// The sequence of the input. + /// The sequence number of the input (). pub sequence: u32, + /// The previous transaction output this input spends (meta field) + pub txo: @TxOut, + /// Utreexo inclusion proof of the spent output (meta field) + pub txo_proof: Span, } - diff --git a/src/validation.cairo b/src/validation.cairo index 68bf7eb8..427f387d 100644 --- a/src/validation.cairo +++ b/src/validation.cairo @@ -85,6 +85,7 @@ mod tests { current_target: 1, epoch_start_time: 1, prev_timestamps: array![1, 2, 3, 4, 5].span(), + utreexo_roots: array![].span(), }; let mut block = Block { header: Header { @@ -116,6 +117,7 @@ mod tests { current_target: 1, epoch_start_time: 1, prev_timestamps: array![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].span(), + utreexo_roots: array![].span(), }; let mut block = Block { header: Header { From 49c707da33e415b81f87443be722bd132617f5fc Mon Sep 17 00:00:00 2001 From: Michael Zaikin Date: Tue, 6 Aug 2024 13:54:14 +0100 Subject: [PATCH 02/12] Introduce utreexo state, set, and abstract proofs --- src/state.cairo | 69 ++++++++++++++++++++++++++++++-------------- src/validation.cairo | 8 ++--- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/src/state.cairo b/src/state.cairo index 8928a518..2f5e3653 100644 --- a/src/state.cairo +++ b/src/state.cairo @@ -19,8 +19,43 @@ pub struct ChainState { pub epoch_start_time: u32, /// Previous timestamps. pub prev_timestamps: Span, - // Utreexo roots (for checking [TxIn] inclusion proofs) - pub utreexo_roots: Span + /// Utreexo state + pub utreexo_state: UtreexoState, +} + +/// Accumulator representation of the state aka "Compact State Node" +#[derive(Drop, Copy)] +pub struct UtreexoState { + /// Roots of Merkle tree forest + pub roots: Span, +} + +/// Utreexo set is used to retrieve TXOs spent by particular inputs +#[derive(Drop, Copy)] +pub struct UtreexoSet { + /// A list of extended transaction outputs spent in a particular block(s). + pub outputs: Span, +} + +/// TXO extended with info about parent transaction and the position within it. +/// The hash of this structure is a leaf node in the Utreexo Merkle tree forest. +#[derive(Drop, Copy)] +pub struct UtreexoOutput { + /// The TXID this output belongs to. + pub txid: u256, + /// The index of this output. + pub vout: u32, + /// Output data + pub output: TxOut, +} + +/// Inclusion proof for multiple leaves +#[derive(Drop, Copy)] +pub struct UtreexoBatchProof { + /// Indices of tree leaves, one for each output in the utreexo set + pub targets: Span, + /// All the nodes required to calculate the root + pub proof: Span, } /// Represents a block in the blockchain. @@ -50,15 +85,15 @@ pub struct Header { pub nonce: u32, } -/// Extended transaction. +/// Transaction /// https://learnmeabitcoin.com/technical/transaction/ -/// -/// Contains additional "meta" fields required for validation. #[derive(Drop, Copy)] pub struct Transaction { /// The version of the transaction. - pub version: i32, + pub version: u32, /// Flag which indicates the presence of witness data. + /// It combines `marker` and `flag` fields for now but in the future + /// we might need to separate them if transaction structure changes. /// Segwit marker and flag do not contribute to TXID (transaction hash), /// but do contribute to wTXID. pub is_segwit: bool, @@ -72,37 +107,29 @@ pub struct Transaction { pub witnesses: Span>, /// The lock time of the transaction. pub lock_time: u32, - /// Transaction fee which is diff between total input and output amounts (meta field) - pub fee: i64, } /// Output of a transaction. /// https://learnmeabitcoin.com/technical/transaction/output/ #[derive(Drop, Copy)] pub struct TxOut { - /// The value of the output. + /// The value of the output in satoshis. pub value: i64, - /// The public key script of the output. + /// The spending script (aka locking code) for this output. pub pk_script: @ByteArray, } -/// Extended input of a transaction. +/// Input of a transaction. /// https://learnmeabitcoin.com/technical/transaction/input/ /// -/// Contains additional "meta" fields required for validation. +/// NOTE that `txid` and `vout` fields can be resolved via Utreexo set using the TXO index. #[derive(Drop, Copy)] pub struct TxIn { - /// The previous TXID this input spends. - pub txid: u256, - /// The previous transaction output index this input spends. - pub vout: u32, /// The signature script which satisfies the conditions placed in the txo pubkey script /// or coinbase script that contains block height (since 227,836) and miner nonce (optional). pub script: @ByteArray, - /// The sequence number of the input (). + /// The sequence number of the input. pub sequence: u32, - /// The previous transaction output this input spends (meta field) - pub txo: @TxOut, - /// Utreexo inclusion proof of the spent output (meta field) - pub txo_proof: Span, + /// The index of output in the utreexo set (meta field). + pub txo_index: u64, } diff --git a/src/validation.cairo b/src/validation.cairo index 427f387d..2b65e832 100644 --- a/src/validation.cairo +++ b/src/validation.cairo @@ -1,4 +1,4 @@ -use super::state::{Block, ChainState}; +use super::state::{Block, ChainState, UtreexoState}; #[generate_trait] impl BlockValidatorImpl of BlockValidator { @@ -73,7 +73,7 @@ fn validate_merkle_root(self: @ChainState, block: @Block) -> Result<(), ByteArra #[cfg(test)] mod tests { use super::{validate_target, validate_timestamp}; - use super::{Block, ChainState}; + use super::{Block, ChainState, UtreexoState}; use super::super::state::{Header, Transaction, TxIn, TxOut}; #[test] @@ -85,7 +85,7 @@ mod tests { current_target: 1, epoch_start_time: 1, prev_timestamps: array![1, 2, 3, 4, 5].span(), - utreexo_roots: array![].span(), + utreexo_state: UtreexoState { roots: array![].span() }, }; let mut block = Block { header: Header { @@ -117,7 +117,7 @@ mod tests { current_target: 1, epoch_start_time: 1, prev_timestamps: array![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].span(), - utreexo_roots: array![].span(), + utreexo_state: UtreexoState { roots: array![].span() }, }; let mut block = Block { header: Header { From c3cc53fe0715e633228f2682a20d2c55ca4462e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Tue, 6 Aug 2024 22:20:30 +0200 Subject: [PATCH 03/12] :busts_in_silhouette: Add @m-kus as a contributor --- .all-contributorsrc | 9 +++++++++ README.md | 1 + 2 files changed, 10 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 7dcb8445..89d255a6 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -63,6 +63,15 @@ "contributions": [ "code" ] + }, + { + "login": "m-kus", + "name": "Michael Zaikin", + "avatar_url": "https://avatars.githubusercontent.com/u/44951260?v=4", + "profile": "https://nodeguardians.io/character/m-kus", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 1050ab93..2df41baa 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Tristan
Tristan
💻 Jean-Michel
Jean-Michel
💻 lomasson
lomasson
💻 + Michael Zaikin
Michael Zaikin
💻 From d598d209f00a3f7aafd197861d0c7802c328f4c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:02:06 +0200 Subject: [PATCH 04/12] updated roadmap --- .github/CODEOWNERS | 2 +- README.md | 28 +++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 65ea647d..dc1830d5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @b-j-roberts @m-kus @maciejka +* @m-kus @maciejka diff --git a/README.md b/README.md index 2df41baa..874d5053 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,29 @@ Although this is a highly experimental project without immediate plans for deplo ## Roadmap -* [ ] verify block header (block hash, previous block hash, Merkle root, proof-of-work, median time, and difficulty adjustment) -* [ ] verify transactions -* [ ] integrate with Shinigami and verify scripts -* [ ] verify previous chain proofs -* [ ] add utreexo accumulator to the chain state +### Milestone 1 - Block Verification + * header verification + * [ ] block hash + * [ ] previous block hash + * [ ] tx merkle root, + * [ ] proof-of-work, + * [ ] median time, + * [ ] difficulty adjustment + * transaction verification + * [ ] verify inputs + * [ ] verify outputs + * [ ] verify coinbase tx + * [ ] use utreexo to verify tx inputs + * verify scripts(integrate with Shinigami) + * tbd + * integration testing + * [ ] test on individual historical blocks +### Milestone 2 - Real Data + * [ ] feed it with real data + * [ ] produce and verify proofs of verrification of individual blocks +### Milestone 3 - Recursive Verification + * verify chain proofs with cairo verifier, tbd + ## Name reference From 2f11c3cef1a2e3027d29d47b191468ecb2e8e833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:07:42 +0200 Subject: [PATCH 05/12] fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 874d5053..27d6538f 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Although this is a highly experimental project without immediate plans for deplo * [ ] test on individual historical blocks ### Milestone 2 - Real Data * [ ] feed it with real data - * [ ] produce and verify proofs of verrification of individual blocks + * [ ] produce and verify proofs of verification of individual blocks ### Milestone 3 - Recursive Verification * verify chain proofs with cairo verifier, tbd From 5a258d8cdde38d700811adde1f961b01049fbcc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:08:37 +0200 Subject: [PATCH 06/12] fix 2 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 27d6538f..32a222f3 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,8 @@ Although this is a highly experimental project without immediate plans for deplo * [ ] verify outputs * [ ] verify coinbase tx * [ ] use utreexo to verify tx inputs - * verify scripts(integrate with Shinigami) - * tbd + * verify scripts + * integrate with Shinigami, tbd * integration testing * [ ] test on individual historical blocks ### Milestone 2 - Real Data From d188296b3d8b636844b16499ef26cda3376e532c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:10:11 +0200 Subject: [PATCH 07/12] fix 3 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32a222f3..4f9a36fb 100644 --- a/README.md +++ b/README.md @@ -63,12 +63,12 @@ Although this is a highly experimental project without immediate plans for deplo * [ ] verify coinbase tx * [ ] use utreexo to verify tx inputs * verify scripts - * integrate with Shinigami, tbd + * integration with Shinigami, tbd * integration testing * [ ] test on individual historical blocks ### Milestone 2 - Real Data * [ ] feed it with real data - * [ ] produce and verify proofs of verification of individual blocks + * [ ] test that you can produce and verify proofs of individual blocks ### Milestone 3 - Recursive Verification * verify chain proofs with cairo verifier, tbd From bfa4521533d4cc205dd0f8821031185dc8028fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:26:11 +0200 Subject: [PATCH 08/12] fix 4 --- README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4f9a36fb..2a66f8c2 100644 --- a/README.md +++ b/README.md @@ -53,17 +53,20 @@ Although this is a highly experimental project without immediate plans for deplo * header verification * [ ] block hash * [ ] previous block hash - * [ ] tx merkle root, - * [ ] proof-of-work, - * [ ] median time, + * [ ] proof-of-work + * [ ] median time * [ ] difficulty adjustment * transaction verification - * [ ] verify inputs - * [ ] verify outputs - * [ ] verify coinbase tx + * [ ] tx hash + * [ ] tx merkle root + * [ ] verify transaction fee + * utreexo + * [ ] fetch utreexo from some kind of bridge node, tbd * [ ] use utreexo to verify tx inputs * verify scripts * integration with Shinigami, tbd + * block verification + * [ ] verify coinbase tx * integration testing * [ ] test on individual historical blocks ### Milestone 2 - Real Data From 56e6ac4949db0fc7da50e75105ce375bea45ae9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:30:09 +0200 Subject: [PATCH 09/12] fix markdown --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a66f8c2..1f698af2 100644 --- a/README.md +++ b/README.md @@ -50,31 +50,40 @@ Although this is a highly experimental project without immediate plans for deplo ## Roadmap ### Milestone 1 - Block Verification + * header verification * [ ] block hash * [ ] previous block hash * [ ] proof-of-work * [ ] median time * [ ] difficulty adjustment + * transaction verification * [ ] tx hash * [ ] tx merkle root * [ ] verify transaction fee + * utreexo * [ ] fetch utreexo from some kind of bridge node, tbd * [ ] use utreexo to verify tx inputs + * verify scripts * integration with Shinigami, tbd + * block verification * [ ] verify coinbase tx + * integration testing * [ ] test on individual historical blocks + ### Milestone 2 - Real Data + * [ ] feed it with real data * [ ] test that you can produce and verify proofs of individual blocks + ### Milestone 3 - Recursive Verification - * verify chain proofs with cairo verifier, tbd + * verify chain proofs with cairo verifier, tbd ## Name reference From 9fafacd6840aba5d0e208349c82e2d1555380d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:31:53 +0200 Subject: [PATCH 10/12] fix markdown 2 --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 1f698af2..57dcfd31 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Although this is a highly experimental project without immediate plans for deplo ### Milestone 1 - Block Verification * header verification + * [ ] block hash * [ ] previous block hash * [ ] proof-of-work @@ -59,21 +60,26 @@ Although this is a highly experimental project without immediate plans for deplo * [ ] difficulty adjustment * transaction verification + * [ ] tx hash * [ ] tx merkle root * [ ] verify transaction fee * utreexo + * [ ] fetch utreexo from some kind of bridge node, tbd * [ ] use utreexo to verify tx inputs * verify scripts + * integration with Shinigami, tbd * block verification + * [ ] verify coinbase tx * integration testing + * [ ] test on individual historical blocks ### Milestone 2 - Real Data From 3bf60fd2947309c3a76dd840627687953c988b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:35:37 +0200 Subject: [PATCH 11/12] fix markdown 3 --- README.md | 55 ++++++++++++++++++++++--------------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 57dcfd31..005469b4 100644 --- a/README.md +++ b/README.md @@ -51,45 +51,34 @@ Although this is a highly experimental project without immediate plans for deplo ### Milestone 1 - Block Verification - * header verification - - * [ ] block hash - * [ ] previous block hash - * [ ] proof-of-work - * [ ] median time - * [ ] difficulty adjustment - - * transaction verification - - * [ ] tx hash - * [ ] tx merkle root - * [ ] verify transaction fee - - * utreexo - - * [ ] fetch utreexo from some kind of bridge node, tbd - * [ ] use utreexo to verify tx inputs - - * verify scripts - - * integration with Shinigami, tbd - - * block verification - - * [ ] verify coinbase tx - - * integration testing - - * [ ] test on individual historical blocks +* header verification + * [ ] block hash + * [ ] previous block hash + * [ ] proof-of-work + * [ ] median time + * [ ] difficulty adjustment +* transaction verification + * [ ] tx hash + * [ ] tx merkle root + * [ ] verify transaction fee +* utreexo + * [ ] fetch utreexo from some kind of bridge node, tbd + * [ ] use utreexo to verify tx inputs +* verify scripts + * integration with Shinigami, tbd +* block verification + * [ ] verify coinbase tx +* integration testing + * [ ] test on individual historical blocks ### Milestone 2 - Real Data - * [ ] feed it with real data - * [ ] test that you can produce and verify proofs of individual blocks +* [ ] feed it with real data +* [ ] test that you can produce and verify proofs of individual blocks ### Milestone 3 - Recursive Verification - * verify chain proofs with cairo verifier, tbd +* verify chain proofs with cairo verifier, tbd ## Name reference From 71fb420ed6faa12e1ab8752f2daf62e148c03f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kami=C5=84ski?= Date: Wed, 7 Aug 2024 14:37:12 +0200 Subject: [PATCH 12/12] fix markdown 4 --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 005469b4..acbb1900 100644 --- a/README.md +++ b/README.md @@ -52,24 +52,24 @@ Although this is a highly experimental project without immediate plans for deplo ### Milestone 1 - Block Verification * header verification - * [ ] block hash - * [ ] previous block hash - * [ ] proof-of-work - * [ ] median time - * [ ] difficulty adjustment + * [ ] block hash + * [ ] previous block hash + * [ ] proof-of-work + * [ ] median time + * [ ] difficulty adjustment * transaction verification - * [ ] tx hash - * [ ] tx merkle root - * [ ] verify transaction fee + * [ ] tx hash + * [ ] tx merkle root + * [ ] verify transaction fee * utreexo - * [ ] fetch utreexo from some kind of bridge node, tbd - * [ ] use utreexo to verify tx inputs + * [ ] fetch utreexo from some kind of bridge node, tbd + * [ ] use utreexo to verify tx inputs * verify scripts - * integration with Shinigami, tbd + * integration with Shinigami, tbd * block verification - * [ ] verify coinbase tx + * [ ] verify coinbase tx * integration testing - * [ ] test on individual historical blocks + * [ ] test on individual historical blocks ### Milestone 2 - Real Data