diff --git a/.gitignore b/.gitignore index e18ee446..d9e114b6 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ __pycache__ .client_cache/ .utxo_data/ +.utreexo_data .timestamps_data/ .arguments*.json diff --git a/docs/data.md b/docs/data.md index ff53a929..b588d8e5 100644 --- a/docs/data.md +++ b/docs/data.md @@ -16,4 +16,36 @@ Input data is processed in multiple steps: 5. Script [generate_data](../scripts/data/generate_data.py) generates data that can be consumed by the `validate_and_apply` function. ## UtxoSet -tbd \ No newline at end of file +tbd + +## Utreexo data + +In order to generate Utreexo states and batch proofs for every block in the Bitcoin history we need to use a special [Bridge node](https://github.com/Davidson-Souza/bridge). + +### Install + +```sh +cargo install --git https://github.com/Davidson-Souza/bridge.git --no-default-features --features shinigami +``` + +### Configure + +You need to configure connection to the Bitcoin RPC via environment variables: + +```sh +export BITCOIN_CORE_RPC_URL=http://localhost:8332 +export BITCOIN_CORE_RPC_USER=username +export BITCOIN_CORE_RPC_PASSWORD=password +``` + +### Run + +Run via `screen`, `nohup`, or set up a systemd service: + +```sh +~/.cargo/bin/bridge +``` + +You can access per-block data at `~/.bridge/blocks//.json`: +- Bucket size is 10k blocks; +- The data directory can be changed by setting the `DATA_DIR` environment variable. diff --git a/packages/client/src/test.cairo b/packages/client/src/test.cairo index b45dc8a2..cdaa7b98 100644 --- a/packages/client/src/test.cairo +++ b/packages/client/src/test.cairo @@ -1,8 +1,9 @@ use consensus::types::block::Block; use consensus::types::chain_state::{ChainState, BlockValidatorImpl}; use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait}; -use utreexo::vanilla::state::{UtreexoState, UtreexoStateTrait}; -use utreexo::vanilla::proof::UtreexoProof; +use utreexo::stump::accumulator::StumpUtreexoAccumulator; +use utreexo::stump::state::UtreexoStumpState; +use utreexo::stump::proof::UtreexoBatchProof; use core::testing::get_available_gas; use core::serde::Serde; @@ -17,17 +18,21 @@ struct Args { expected_chain_state: ChainState, /// Optional Utreexo arguments utreexo_args: Option, + /// If this flag is set, locking scripts will be executed + execute_script: bool, } /// Utreexo arguments necessary for constraining the UTXO set #[derive(Drop, Serde)] struct UtreexoArgs { /// Current (initial) accumulator state - state: UtreexoState, - /// Inclusion proofs for TXOs spent during program run - proofs: Array, + state: UtreexoStumpState, + /// Batch inclusion proof for TXOs spent during the current block. + /// Note that it doesn't support flow with multiple blocks applied + /// in a single program run. + proof: UtreexoBatchProof, /// Expected accumulator state at the end of the execution - expected_state: UtreexoState, + expected_state: UtreexoStumpState, } /// Integration testing program entrypoint. @@ -35,12 +40,14 @@ struct UtreexoArgs { /// Receives arguments in a serialized format (Cairo serde). /// Panics in case of a validation error or chain state mismatch. /// Prints result to the stdout. -pub(crate) fn main(mut arguments: Span, execute_script: bool) { +fn main(arguments: Array) -> Array { println!("Running integration test... "); let mut gas_before = get_available_gas(); + let mut args = arguments.span(); - let Args { mut chain_state, blocks, expected_chain_state, utreexo_args } = Serde::deserialize( - ref arguments + let Args { mut chain_state, blocks, expected_chain_state, utreexo_args, execute_script } = + Serde::deserialize( + ref args ) .expect('Failed to deserialize'); @@ -71,11 +78,8 @@ pub(crate) fn main(mut arguments: Span, execute_script: bool) { panic!(); } - if let Option::Some(UtreexoArgs { mut state, proofs, expected_state }) = utreexo_args { - match state - .validate_and_apply( - utxo_set.leaves_to_add.span(), utxo_set.leaves_to_delete.span(), proofs.span(), - ) { + if let Option::Some(UtreexoArgs { mut state, proof, expected_state }) = utreexo_args { + match state.verify_and_delete(@proof, utxo_set.leaves_to_delete.span()) { Result::Ok(new_state) => { state = new_state; }, Result::Err(err) => { println!("FAIL: gas_spent={} error='{:?}'", gas_before - get_available_gas(), err); @@ -83,6 +87,8 @@ pub(crate) fn main(mut arguments: Span, execute_script: bool) { } } + state = state.add(utxo_set.leaves_to_add.span()); + if state != expected_state { println!( "FAIL: gas_spent={} error='expected utreexo state {:?}, actual {:?}'", @@ -95,6 +101,7 @@ pub(crate) fn main(mut arguments: Span, execute_script: bool) { } println!("OK: gas_spent={}", gas_before - get_available_gas()); + array![] } /// Workaround for handling missing `utreexo_args` field. @@ -109,11 +116,14 @@ impl ArgsSerde of Serde { let blocks: Array = Serde::deserialize(ref serialized).expect('blocks'); let expected_chain_state: ChainState = Serde::deserialize(ref serialized) .expect('expected_chain_state'); - let utreexo_args: Option = if serialized.len() > 0 { + let utreexo_args: Option = if serialized.len() > 1 { Option::Some(Serde::deserialize(ref serialized).expect('utreexo_args')) } else { Option::None }; - Option::Some(Args { chain_state, blocks, expected_chain_state, utreexo_args, }) + let execute_script: bool = Serde::deserialize(ref serialized).expect('execute_script'); + Option::Some( + Args { chain_state, blocks, expected_chain_state, utreexo_args, execute_script, } + ) } } diff --git a/packages/client/tests/data/utreexo_0.json b/packages/client/tests/data/utreexo_0.json new file mode 100644 index 00000000..3949d462 --- /dev/null +++ b/packages/client/tests/data/utreexo_0.json @@ -0,0 +1,90 @@ +{ + "chain_state": { + "block_height": 0, + "total_work": "4295032833", + "best_block_hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505 + ] + }, + "blocks": [ + { + "header": { + "version": 1, + "time": 1231469665, + "bits": 486604799, + "nonce": 2573394689 + }, + "data": { + "variant_id": 1, + "transactions": [ + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x04ffff001d0104", + "sequence": 4294967295, + "previous_output": { + "txid": "0000000000000000000000000000000000000000000000000000000000000000", + "vout": 4294967295, + "data": { + "value": 0, + "pk_script": "0x", + "cached": false + }, + "block_height": 0, + "median_time_past": 0, + "is_coinbase": false + }, + "witness": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "outputs": [ + { + "value": 5000000000, + "pk_script": "0x410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac", + "cached": false + } + ], + "lock_time": 0 + } + ] + } + } + ], + "expected": { + "block_height": 1, + "total_work": "8590065666", + "best_block_hash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505, + 1231469665 + ] + }, + "utreexo": { + "state": { + "roots": [], + "num_leaves": 0 + }, + "proof": { + "proof": [], + "targets": [] + }, + "expected_state": { + "roots": [ + { + "variant_id": 0, + "value": 49459078824306138476779209834441505868925737545954320330266544605873965565 + } + ], + "num_leaves": 1 + } + } +} \ No newline at end of file diff --git a/packages/client/tests/data/utreexo_1.json b/packages/client/tests/data/utreexo_1.json new file mode 100644 index 00000000..395a5d62 --- /dev/null +++ b/packages/client/tests/data/utreexo_1.json @@ -0,0 +1,97 @@ +{ + "chain_state": { + "block_height": 1, + "total_work": "8590065666", + "best_block_hash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505, + 1231469665 + ] + }, + "blocks": [ + { + "header": { + "version": 1, + "time": 1231469744, + "bits": 486604799, + "nonce": 1639830024 + }, + "data": { + "variant_id": 1, + "transactions": [ + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x04ffff001d010b", + "sequence": 4294967295, + "previous_output": { + "txid": "0000000000000000000000000000000000000000000000000000000000000000", + "vout": 4294967295, + "data": { + "value": 0, + "pk_script": "0x", + "cached": false + }, + "block_height": 0, + "median_time_past": 0, + "is_coinbase": false + }, + "witness": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "outputs": [ + { + "value": 5000000000, + "pk_script": "0x41047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac", + "cached": false + } + ], + "lock_time": 0 + } + ] + } + } + ], + "expected": { + "block_height": 2, + "total_work": "12885098499", + "best_block_hash": "000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505, + 1231469665, + 1231469744 + ] + }, + "utreexo": { + "state": { + "roots": [ + { + "variant_id": 0, + "value": 49459078824306138476779209834441505868925737545954320330266544605873965565 + } + ], + "num_leaves": 1 + }, + "proof": { + "proof": [], + "targets": [] + }, + "expected_state": { + "roots": [ + { + "variant_id": 0, + "value": 2915651996892698521196383484133475266682410400159810761618404984775762993564 + } + ], + "num_leaves": 2 + } + } +} \ No newline at end of file diff --git a/packages/client/tests/data/utreexo_169.json b/packages/client/tests/data/utreexo_169.json index 76b4be3e..83ec15f0 100644 --- a/packages/client/tests/data/utreexo_169.json +++ b/packages/client/tests/data/utreexo_169.json @@ -128,68 +128,57 @@ "roots": [ { "variant_id": 0, - "value": 1075667171590848154982556435125102517390553068689710620455153660349876510186 + "value": 1502558161404968896970257403264055788192189695540729001360864566276543485500 }, - null, - null, { "variant_id": 0, - "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171 + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 }, - null, { "variant_id": 0, - "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 + "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171 }, - null, { "variant_id": 0, - "value": 2789527937826812553983355805324006356113540169680511206602120580468925994144 - }, - null + "value": 1075667171590848154982556435125102517390553068689710620455153660349876510186 + } + ], + "num_leaves": 169 + }, + "proof": { + "proof": [ + 1341831566043387106325844845909158799402866001662974927108069625603583971480, + 1443675700228503369543683218943466499245617322985505154945738943716743694265, + 519646392377825945466547611274092870030802470736981675895864465165188803020, + 1080650929466224224603091215856096240942830750577279058777853241919006292201, + 3429608730564493038725327726731886582242605618763607208860625722555112702870, + 1464346753344216951108436013919475968846135803696833681088250234646995926893, + 2059532798546650597701371553400476891060690283065963625811202716166665983015 + ], + "targets": [ + 8 ] }, - "proofs": [ - { - "proof": [ - 1341831566043387106325844845909158799402866001662974927108069625603583971480, - 1443675700228503369543683218943466499245617322985505154945738943716743694265, - 519646392377825945466547611274092870030802470736981675895864465165188803020, - 3219090166904938598688939525052593187314786152670445479780817128254606574061, - 3429608730564493038725327726731886582242605618763607208860625722555112702870, - 1464346753344216951108436013919475968846135803696833681088250234646995926893, - 2059532798546650597701371553400476891060690283065963625811202716166665983015 - ], - "leaf_index": 8 - } - ], - "expected": { + "expected_state": { "roots": [ { "variant_id": 0, - "value": 2798110709967837858515372760817606172577565229842771835911030873647893889774 + "value": 472495115707702931286975202981613081341496930913817199425502977245962405497 }, { "variant_id": 0, - "value": 143588888139305297948079116321466646636133238970252601366055285460885727445 + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 }, - null, { "variant_id": 0, "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171 }, - null, - { - "variant_id": 0, - "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 - }, - null, { "variant_id": 0, - "value": 72427602467111712670190708647558062003041829148690045408562661193902370953 - }, - null - ] + "value": 940126151801873300266416875114055738009783862579557360123104711005733824770 + } + ], + "num_leaves": 172 } } } \ No newline at end of file diff --git a/packages/client/tests/data/utreexo_182.json b/packages/client/tests/data/utreexo_182.json new file mode 100644 index 00000000..929c425d --- /dev/null +++ b/packages/client/tests/data/utreexo_182.json @@ -0,0 +1,195 @@ +{ + "chain_state": { + "block_height": 182, + "total_work": "785991008439", + "best_block_hash": "0000000054487811fc4ff7a95be738aa5ad9320c394c482b27c0da28b227ad5d", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231731853, + 1231732245, + 1231732861, + 1231733876, + 1231735142, + 1231736557, + 1231737131, + 1231738005, + 1231739452, + 1231740133, + 1231740736 + ] + }, + "blocks": [ + { + "header": { + "version": 1, + "time": 1231742062, + "bits": 486604799, + "nonce": 3235118355 + }, + "data": { + "variant_id": 1, + "transactions": [ + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x04ffff001d012a", + "sequence": 4294967295, + "previous_output": { + "txid": "0000000000000000000000000000000000000000000000000000000000000000", + "vout": 4294967295, + "data": { + "value": 0, + "pk_script": "0x", + "cached": false + }, + "block_height": 0, + "median_time_past": 0, + "is_coinbase": false + }, + "witness": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "outputs": [ + { + "value": 5000000000, + "pk_script": "0x41041915d670aa55621ed1c438477c5da654344e10eecea90ede3a048103fc3cf4beccda70ddf71ce55a5083b7bade4319d159b44374234590e9296cbe08c67774b2ac", + "cached": false + } + ], + "lock_time": 0 + }, + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x483045022052ffc1929a2d8bd365c6a2a4e3421711b4b1e1b8781698ca9075807b4227abcb0221009984107ddb9e3813782b095d0d84361ed4c76e5edaf6561d252ae162c2341cfb01", + "sequence": 4294967295, + "previous_output": { + "txid": "591e91f809d716912ca1d4a9295e70c3e78bab077683f79350f101da64588073", + "vout": 1, + "data": { + "value": 2900000000, + "pk_script": "0x410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac", + "cached": false + }, + "block_height": 182, + "median_time_past": 1231735142, + "is_coinbase": false + }, + "witness": [] + } + ], + "outputs": [ + { + "value": 100000000, + "pk_script": "0x4104baa9d36653155627c740b3409a734d4eaf5dcca9fb4f736622ee18efcf0aec2b758b2ec40db18fbae708f691edb2d4a2a3775eb413d16e2e3c0f8d4c69119fd1ac", + "cached": false + }, + { + "value": 2800000000, + "pk_script": "0x410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac", + "cached": false + } + ], + "lock_time": 0 + } + ] + } + } + ], + "expected": { + "block_height": 183, + "total_work": "790286041272", + "best_block_hash": "00000000f46e513f038baf6f2d9a95b2a28d8a6c985bcf24b9e07f0f63a29888", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231732245, + 1231732861, + 1231733876, + 1231735142, + 1231736557, + 1231737131, + 1231738005, + 1231739452, + 1231740133, + 1231740736, + 1231742062 + ] + }, + "utreexo": { + "state": { + "roots": [ + { + "variant_id": 0, + "value": 472495115707702931286975202981613081341496930913817199425502977245962405497 + }, + { + "variant_id": 0, + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 + }, + { + "variant_id": 0, + "value": 3379475774617422517749239484786779535800164983837536926832995269432589051493 + }, + { + "variant_id": 0, + "value": 2960972179536349754495570588856684293379166070401208198804481828943354090177 + }, + { + "variant_id": 0, + "value": 1027525398361326601103535357192346699264220866701033959615605604739328879107 + } + ], + "num_leaves": 188 + }, + "proof": { + "proof": [ + 471794939040413771741937605933609163704938509795149935005523699179740318544, + 963498998615873495259761020963144237838429346272910499751572193538456625178 + ], + "targets": [ + 187 + ] + }, + "expected_state": { + "roots": [ + { + "variant_id": 0, + "value": 472495115707702931286975202981613081341496930913817199425502977245962405497 + }, + { + "variant_id": 0, + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 + }, + { + "variant_id": 0, + "value": 3379475774617422517749239484786779535800164983837536926832995269432589051493 + }, + { + "variant_id": 0, + "value": 2960972179536349754495570588856684293379166070401208198804481828943354090177 + }, + { + "variant_id": 0, + "value": 1393936218426697313222000203973633691677533429852178919818058911865240713624 + }, + { + "variant_id": 0, + "value": 388190300499697056611762555012906818486802305728968255357693461417536026290 + }, + { + "variant_id": 0, + "value": 493770350110029404732941515600367633414267409352068154843236614730043385883 + } + ], + "num_leaves": 191 + } + } +} \ No newline at end of file diff --git a/packages/consensus/src/types/transaction.cairo b/packages/consensus/src/types/transaction.cairo index 5631d9b3..96ad8662 100644 --- a/packages/consensus/src/types/transaction.cairo +++ b/packages/consensus/src/types/transaction.cairo @@ -274,7 +274,7 @@ mod tests { } #[test] - pub fn test_outpoint_poseidon_hash() { + pub fn test_outpoint_poseidon_hash_cb9() { let mut coinbase_9_utxo = OutPoint { txid: hex_to_hash_rev( "0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9" @@ -316,4 +316,48 @@ mod tests { 761592244424273723796345514960638980240531938129162865626185984897576522513, hash ); } + + #[test] + pub fn test_outpoint_poseidon_hash_cb1() { + let mut coinbase_9_utxo = OutPoint { + txid: hex_to_hash_rev( + "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098" + ), + vout: 0, + data: TxOut { + value: 5000000000, + pk_script: @from_hex( + "410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac" + ), + cached: false, + }, + block_height: 1, + median_time_past: 1231006505, + is_coinbase: true, + }; + + let mut state: HashState = Default::default(); + state = state.update_with(coinbase_9_utxo); + + let expected: Array = array![ + 18931831195212887181660290436187791739, + 137019159177035157628276746705882390680, + 0, + 5000000000, + 2, + 114876729272917404712191936498804624660105992100397383656070609774475449467, + 406791163401893627439198994794895943141891052128672824792182596804809637667, + 286829113004, + 5, + 1, + 1231006505, + 1 + ]; + assert_eq!(expected, state.value); + + let hash = coinbase_9_utxo.hash(); + assert_eq!( + 49459078824306138476779209834441505868925737545954320330266544605873965565, hash + ); + } } diff --git a/packages/consensus/src/validation/timestamp.cairo b/packages/consensus/src/validation/timestamp.cairo index 91d220a3..f77bfeb5 100644 --- a/packages/consensus/src/validation/timestamp.cairo +++ b/packages/consensus/src/validation/timestamp.cairo @@ -37,7 +37,7 @@ pub fn compute_median_time_past(prev_timestamps: Span) -> u32 { }; }; - *sorted_prev_timestamps.at(sorted_prev_timestamps.len() - 6) + *sorted_prev_timestamps.at(sorted_prev_timestamps.len() / 2) } /// Check that the block time is greater than the Median Time Past (MTP). @@ -54,7 +54,9 @@ pub fn validate_timestamp(median_time_past: u32, block_time: u32) -> Result<(), /// Update the list of the recent timestamps, removing the oldest and appending the most recent one. pub fn next_prev_timestamps(prev_timestamps: Span, block_time: u32) -> Span { let mut timestamps: Array = prev_timestamps.into(); - timestamps.pop_front().unwrap(); // remove the oldest timestamp (not necessarily the min) + if timestamps.len() == 11 { + timestamps.pop_front().unwrap(); // remove the oldest timestamp (not necessarily the min) + } timestamps.append(block_time); // append the most recent timestamp (not necessarily the max) timestamps.span() } @@ -124,4 +126,13 @@ mod tests { let result = validate_timestamp(mtp, block_time); assert!(result.is_err(), "MTP is greater than block's timestamp"); } + + #[test] + fn test_few_prev_timestamps() { + assert_eq!(1, compute_median_time_past(array![1].span())); + assert_eq!(2, compute_median_time_past(array![1, 2].span())); + assert_eq!(2, compute_median_time_past(array![1, 2, 3].span())); + assert_eq!(3, compute_median_time_past(array![1, 2, 3, 4].span())); + assert_eq!(3, compute_median_time_past(array![1, 2, 3, 4, 5].span())); + } } diff --git a/packages/utreexo/src/test.cairo b/packages/utreexo/src/test.cairo index 25d589bf..24266f28 100644 --- a/packages/utreexo/src/test.cairo +++ b/packages/utreexo/src/test.cairo @@ -10,12 +10,14 @@ struct Args { leaves_to_del: Array, leaves_to_add: Array, expected_state: UtreexoStumpState, + _unused: felt252, } -fn main(mut arguments: Span, _flags: felt252) { +fn main(args: Array) -> Array { let mut gas_before = get_available_gas(); + let mut arguments = args.span(); - let Args { mut state, proof, leaves_to_del, leaves_to_add, expected_state } = + let Args { mut state, proof, leaves_to_del, leaves_to_add, expected_state, _unused: _ } = Serde::deserialize( ref arguments ) @@ -50,4 +52,5 @@ fn main(mut arguments: Span, _flags: felt252) { } println!("OK: gas_spent={}", gas_before - get_available_gas()); + array![] } diff --git a/scripts/data/format_args.py b/scripts/data/format_args.py index 05ace152..c6c5f7ab 100755 --- a/scripts/data/format_args.py +++ b/scripts/data/format_args.py @@ -65,7 +65,7 @@ def serialize(obj): raise NotImplementedError(obj) -def flatten_tuples(src): +def flatten_tuples(src) -> list: """Recursively flattens tuples. Example: (0, (1, 2), [(3, 4, [5, 6])]) -> [0, 1, 2, [3, 4, [5, 6]]] @@ -117,11 +117,11 @@ def format_args(input_file, execute_script, cairo1_run): """ args = json.loads(Path(input_file).read_text()) res = flatten_tuples(serialize(args)) - flag = 1 if execute_script else 0 + res.append(1 if execute_script else 0) if cairo1_run: - return f"{format_cairo1_run(res)} {flag}" + return format_cairo1_run(res) else: - return [res, flag] + return [res] if __name__ == "__main__": diff --git a/scripts/data/generate_data.py b/scripts/data/generate_data.py index 1335966e..43f6667e 100755 --- a/scripts/data/generate_data.py +++ b/scripts/data/generate_data.py @@ -11,7 +11,7 @@ import requests from generate_timestamp_data import get_timestamp_data -from generate_utreexo_data import UtreexoData +from generate_utreexo_data import get_utreexo_data from generate_utxo_data import get_utxo_set logger = logging.getLogger(__name__) @@ -84,7 +84,7 @@ def fetch_chain_state(block_height: int): prev_timestamps = [int(head["time"])] for _ in range(10): if prev_header["height"] == 0: - prev_timestamps.insert(0, 0) + break else: prev_header = request_rpc( "getblockheader", [prev_header["previousblockhash"]] @@ -323,6 +323,7 @@ def generate_data( "utreexo" — only last block from the batch is included, but it is extended with Utreexo state/proofs :param initial_height: The block height of the initial chain state (0 means the state after genesis) :param num_blocks: The number of blocks to apply on top of it (has to be at least 1) + :param fast: Use data exported from BigQuery rather than Bitcoin node :return: tuple (arguments, expected output) """ @@ -335,12 +336,10 @@ def generate_data( if fast else fetch_chain_state(initial_height) ) - prev_chain_state = None initial_chain_state = chain_state next_block_hash = chain_state["nextblockhash"] blocks = [] - utreexo_data = {} for i in range(num_blocks): logger.debug(f"Fetching block {initial_height + i + 1} {i + 1}/{num_blocks}...") @@ -380,7 +379,6 @@ def generate_data( raise NotImplementedError(mode) blocks.append(block) - prev_chain_state = chain_state chain_state = next_chain_state(chain_state, block) next_block_hash = block["nextblockhash"] @@ -396,13 +394,8 @@ def generate_data( } if mode == "utreexo": - utreexo_data = UtreexoData() - result["utreexo"] = utreexo_data.apply_blocks( - blocks, int(chain_state["mediantime"]) - ) - result["blocks"] = [result["blocks"][-1]] - if num_blocks > 1: - result["chain_state"] = format_chain_state(prev_chain_state) + assert len(blocks) == 1, "Cannot handle more than one block in Utreexo mode" + result["utreexo"] = get_utreexo_data(blocks[0]["height"]) return result diff --git a/scripts/data/generate_utreexo_data.py b/scripts/data/generate_utreexo_data.py index 307752e5..2f923f6e 100644 --- a/scripts/data/generate_utreexo_data.py +++ b/scripts/data/generate_utreexo_data.py @@ -1,227 +1,59 @@ #!/usr/bin/env python3 -import typing as t -from collections import OrderedDict -from utreexo import Utreexo -from poseidon_py.poseidon_hash import poseidon_hash_many +import os +import json +BASE_DIR = f"{os.path.dirname(os.path.realpath(__file__))}/.utreexo_data" +BUCKET_SIZE = 10000 -class TxOut: - def __init__(self, value, pk_script, cached): - self.value = value - self.pk_script = pk_script - self.cached = cached - - def _calculate_sub_data(self) -> t.List[int]: - sub_data = [] - pending_word = 0 - pending_word_len = 0 - - # Convert `pk_script` to bytes - if self.pk_script.startswith("0x") or self.pk_script.startswith("0X"): - hex_str = self.pk_script[2:] - byte_data = bytes.fromhex(hex_str) - else: - byte_data = self.pk_script.encode("utf-8") - - for byte in byte_data: - if not (0 <= byte <= 255): - raise ValueError("Not between 0 and 255.") - - if pending_word_len == 0: - pending_word = byte - pending_word_len = 1 - continue - - new_pending = pending_word * 0x100 + byte - if pending_word_len < 31 - 1: - pending_word = new_pending - pending_word_len += 1 - continue - - # Convert new_pending to 31 bytes (big endian) - new_pending_bytes = new_pending.to_bytes(31, byteorder="big") - word_int = int.from_bytes(new_pending_bytes, "big") - sub_data.append(word_int) - - pending_word = 0 - pending_word_len = 0 - - sub_data.append(pending_word) - sub_data.append(pending_word_len) - - return sub_data - - def serialize(self) -> OrderedDict: - res = OrderedDict() - - sub_data = self._calculate_sub_data() - - res["value"] = self.value - # length of the array containing full words - res["sub_data_len"] = len(sub_data) - 2 - - for idx, word in enumerate(sub_data): - res["sub_data_{}".format(idx)] = word - - return res - - def __repr__(self): - return f"TxOut(\n\ - value={self.value}\n\ - pk_script={self.pk_script}\n\ - cached={self.cached})" - - -class OutPoint: - def __init__( - self, txid, vout, data: TxOut, block_height, median_time_past, is_coinbase - ): - self.txid = txid - self.vout = vout - self.data = data # Instance de TxOut - self.block_height = block_height - self.median_time_past = median_time_past - self.is_coinbase = is_coinbase - - def hash(self): - tab = [] - - # txid (2x u128 in little endian high/low) - txid_bytes = bytes.fromhex(self.txid) - tab.append(int.from_bytes(txid_bytes[:16], "big")) - tab.append(int.from_bytes(txid_bytes[16:], "big")) - - # vout - tab.append(self.vout) - - # prev output - for _, e in self.data.serialize().items(): - tab.append(e) - - tab.append(self.block_height) - tab.append(self.median_time_past) - - tab.append(int(self.is_coinbase)) - hash = poseidon_hash_many(tab) - return hash - - def __repr__(self): - return f"OutPoint(\n\ - txid={self.txid}\n\ - vout={self.vout}\n\ - tx_out={self.data}\n\ - block_height={self.block_height}\n\ - median_time_past={self.median_time_past}\n\ - is_coinbase={self.is_coinbase}\n\ - hash={self.hash()})" +def get_utreexo_data(block_height): + bucket_number = block_height // BUCKET_SIZE + utreexo_data = {} - -class UtreexoData: - def __init__(self) -> None: - self.utreexo = Utreexo() - - def snapshot_state(self) -> dict: - return { - "roots": list(map(format_root_node, self.utreexo.root_nodes)), + if block_height > 1: + with open(f"{BASE_DIR}/{bucket_number}/{block_height - 1}.json") as f: + data = json.loads(f.read()) + utreexo_data["state"] = convert_state(data["utreexo_state"]) + else: + utreexo_data["state"] = { + "roots": [], + "num_leaves": 0, } - def apply_blocks(self, blocks: list, prev_mtp: int) -> dict: - state = {} - proofs = [] - for block_idx, block in enumerate(blocks): - if block_idx == len(blocks) - 1: - state = self.snapshot_state() - - # First we remove all STXOs - for i, (txid, tx) in enumerate(block["data"].items()): - # Skip coinbase tx input - if i != 0: - inc_proofs = self.handle_txin(tx["inputs"]) - # Storing proofs for last block only - if block_idx == len(blocks) - 1: - proofs.extend(inc_proofs) - - # Then we add all new UTXOs - for i, (txid, tx) in enumerate(block["data"].items()): - self.handle_txout( - tx["outputs"], - block["height"], - prev_mtp, - txid, - i == 0, - ) - prev_mtp = block["mediantime"] - - return {"state": state, "proofs": proofs, "expected": self.snapshot_state()} - - def handle_txin(self, inputs: list) -> list: - proofs = [] - for input in inputs: - outpoint = input["previous_output"] - - # Skip if output is cached - if outpoint["data"]["cached"]: - continue - - outpoint = OutPoint( - txid=outpoint["txid"], - vout=outpoint["vout"], - data=TxOut( - value=outpoint["data"]["value"], - pk_script=outpoint["data"]["pk_script"], - cached=outpoint["data"]["cached"], - ), - block_height=outpoint["block_height"], - median_time_past=outpoint["median_time_past"], - is_coinbase=outpoint["is_coinbase"], - ) - - # Remove OutPoint from accumulator and get proof - proof, leaf_index = self.utreexo.delete(outpoint.hash()) - proofs.append( - {"proof": list(map(format_node, proof)), "leaf_index": leaf_index} - ) - - return proofs - - def handle_txout( - self, - outputs: list, - block_height: int, - median_time_past: int, - txid: str, - is_coinbase: bool, - ): - for i, output in enumerate(outputs): - # Skip if output is cached - if output["cached"]: - continue - - new_outpoint = OutPoint( - txid=txid, - vout=i, - data=TxOut( - value=output["value"], - pk_script=output["pk_script"], - cached=output["cached"], - ), - block_height=block_height, - median_time_past=median_time_past, - is_coinbase=is_coinbase, - ) - - # Add OutPoint to accumulator - self.utreexo.add(new_outpoint.hash()) - - -def format_root_node(node) -> str: - if node: - return {"variant_id": 0, "value": format_node(node)} - else: + with open(f"{BASE_DIR}/{bucket_number}/{block_height}.json") as f: + data = json.loads(f.read()) + utreexo_data["proof"] = convert_proof(data["inclusion_proof"]) + utreexo_data["expected_state"] = convert_state(data["utreexo_state"]) + + return utreexo_data + + +def convert_proof(proof): + return { + "proof": list(map(convert_felt, proof["hashes"])), + "targets": proof["targets"], + } + + +def convert_state(state): + return { + "roots": list(map(convert_root, state["roots"])), + "num_leaves": state["num_leaves"], + } + + +def convert_root(root: str) -> int: + if root is None: return None + else: + return {"variant_id": 0, "value": convert_felt(root)} + + +def convert_felt(felt: str) -> int: + return int.from_bytes(bytes.fromhex(felt[2:]), "big") -def format_node(node) -> int: - return int.from_bytes(bytes.fromhex(node.val[2:]), "big") +if __name__ == "__main__": + # TODO: download ~/.bridge/blocks folder bucket by bucket from a remote server + raise NotImplementedError diff --git a/scripts/data/regenerate_tests.sh b/scripts/data/regenerate_tests.sh index 53ff811b..a5ba7fe4 100755 --- a/scripts/data/regenerate_tests.sh +++ b/scripts/data/regenerate_tests.sh @@ -66,7 +66,10 @@ full_test_cases=( ) utreexo_test_cases=( + 0 # To test leaf serialization compliance + 1 169 # Block containing first P2P tx to Hal Finney (170) + 182 ) mkdir $data_dir || true @@ -94,5 +97,5 @@ done for test_case in "${utreexo_test_cases[@]}"; do echo -e "\nGenerating test data: utreexo mode, chain state @ $test_case, single block" - generate_test "utreexo" 0 $(($test_case+1)) + generate_test "utreexo" $test_case done diff --git a/scripts/data/utreexo.py b/scripts/data/utreexo.py deleted file mode 100644 index 19865098..00000000 --- a/scripts/data/utreexo.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 ZeroSync -# -# SPDX-License-Identifier: MIT -from poseidon_py.poseidon_hash import poseidon_hash_many - - -class Node: - def __init__(self, key, left=None, right=None): - self.val = key - self.left = left - self.right = right - self.parent = None - - def __repr__(self) -> str: - return str(self.val) - - -class Utreexo: - def __init__(self): - self.root_nodes = [] - self.leaf_nodes = {} - - def print_state(self): - print("--- Utreexo State ---") - print("Root Nodes:") - for idx, node in enumerate(self.root_nodes): - if node is not None: - print(f" Height {idx}: {node.val}") - print("\nLeaf Nodes:") - for leaf_val, node in self.leaf_nodes.items(): - print(f" Leaf {leaf_val}: {node.val}") - print("---------------------\n") - - def print_tree(self): - print("--- Utreexo Tree ---") - - def _print_subtree(node, prefix=""): - if node is not None: - print(f"{prefix}{node.val}") - if node.left is not None or node.right is not None: - # Display branches of the left subtree - _print_subtree(node.left, prefix + "├── ") - # Display branches of the right subtree - _print_subtree(node.right, prefix + "└── ") - - for idx, root in enumerate(self.root_nodes): - if root is not None: - print(f"Root at height {idx}:") - _print_subtree(root) - print("---------------------\n") - - def parent_node(self, root1, root2): - val1 = int(root1.val, 16) - val2 = int(root2.val, 16) - root = poseidon_hash_many([val1, val2]) - root_hex = f"0x{root:064x}" - root_node = Node(root_hex, root1, root2) - root1.parent = root_node - root2.parent = root_node - return root_node - - def add(self, leaf): - if leaf in self.leaf_nodes: - raise Exception("Leaf exists already") - - n = Node(f"0x{leaf:064x}") - self.leaf_nodes[leaf] = n - h = 0 - - while len(self.root_nodes) <= h: - self.root_nodes.append(None) - - r = self.root_nodes[h] - while r is not None: - n = self.parent_node(r, n) - self.root_nodes[h] = None - h += 1 - r = self.root_nodes[h] - self.root_nodes[h] = n - - if h == len(self.root_nodes) - 1: - self.root_nodes.append(None) - - return self.root_nodes - - def delete(self, leaf): - if leaf not in self.leaf_nodes: - raise Exception(f"Leaf {leaf} does not exist in the Utreexo accumulator") - - leaf_node = self.leaf_nodes[leaf] - del self.leaf_nodes[leaf] - - proof, leaf_index = self.inclusion_proof(leaf_node) - - n = None - h = 0 - while h < len(proof): - p = proof[h] - if n is not None: - n = self.parent_node(p, n) - elif self.root_nodes[h] is None: - p.parent = None - self.root_nodes[h] = p - else: - n = self.parent_node(p, self.root_nodes[h]) - self.root_nodes[h] = None - h += 1 - - if n is not None: - n.parent = None - - self.root_nodes[h] = n - - return proof, leaf_index - - def inclusion_proof(self, node): - if node.parent is None: - return [], 0 - - parent = node.parent - path, leaf_index = self.inclusion_proof(parent) - - if node == parent.left: - path.insert(0, parent.right) - leaf_index *= 2 - else: - path.insert(0, parent.left) - leaf_index = leaf_index * 2 + 1 - - return path, leaf_index - - def reset(self): - self.root_nodes = [] - self.leaf_nodes = {} - - def print_roots(self): - roots = [node.val if node is not None else "" for node in self.root_nodes] - print("Roots:", roots) - - -if __name__ == "__main__": - utreexo = Utreexo() - - # Add some elements - utreexo.add(0x111111111111111111111111) - utreexo.add(0x222222222222222222222222) - utreexo.add(0x333333333333333333333333) - utreexo.add(0x444444444444444444444444) - utreexo.add(0x555555555555555555555555) - utreexo.add(0x666666666666666666666666) - utreexo.add(0x777777777777777777777777) - utreexo.add(0x888888888888888888888888) - utreexo.add(0x999999999999999999999999) - utreexo.add(0xAAAAAAAAAAAAAAAAAAAAAAAA) - utreexo.print_roots() - - # Reset the Utreexo - utreexo.reset() - utreexo.print_roots()