Use serde to deserialize program arguments

README.md

@@ -158,10 +158,11 @@ scarb run integration_tests tests/data/light_481823.json
 Re-generate integration test data:
 
 ```base
-scarb run regenerate_tests
+scarb run regenerate_tests --force
 ```
 
-* Files will be created in [tests/data/](https://github.com/keep-starknet-strange/raito/blob/main/tests/data)
+* Without `--force` flag only non-existent files will be created
+* Files are located in [tests/data/](https://github.com/keep-starknet-strange/raito/blob/main/tests/data)
 * If you want to add a new test case, edit [scripts/data/regenerate_tests.sh](https://github.com/keep-starknet-strange/raito/blob/main/scripts/data/regenerate_tests.sh)
 
 ## Build dependencies

scripts/data/format_args.py

@@ -13,7 +13,7 @@ def serialize(obj):
         dec string (0-9) -> (int, int) -> u256 = { lo: felt252, hi: felt252 }
         hex string (0-F), 64 len -> (int, int, int, int, int, int, int, int) -> Hash !reversed!
         hex string 0x prefixed -> ([int, ...], int, int) -> ByteArray
-        list -> []
+        list -> tuple(len(list), *list)
         dict -> tuple(dict.values)
     """
     if isinstance(obj, bool):
@@ -23,7 +23,11 @@ def serialize(obj):
         assert(obj >= 0 and obj < 2 ** 252)
         return obj
     elif isinstance(obj, str):
-        if obj.isdigit():
+        if obj == "0" * 64:
+            # special case - zero hash
+            return (0, 0, 0, 0, 0, 0, 0, 0)
+        elif obj.isdigit():
+            # TODO: there might still be collisions with hashes
             # Try to cast to int and then to low/high parts
             num = int(obj)
             assert(num >= 0 and num < 2 ** 256)
@@ -39,14 +43,15 @@ def serialize(obj):
             main = [int.from_bytes(src[i:i+31], 'big') for i in range(0, main_len, 31)]
             # TODO: check if this is how byte31 is implemented
             rem = int.from_bytes(src[main_len:].rjust(31, b'\x00'), 'big')
-            return (main, rem, rem_len)
+            return tuple([len(main)] + main + [rem, rem_len])
         else:
             # Reversed hex string into 4-byte words then into BE u32
             assert(len(obj) == 64)
             rev = list(reversed(bytes.fromhex(obj)))
             return tuple(int.from_bytes(rev[i:i+4], 'big') for i in range(0, 32, 4))
     elif isinstance(obj, list):
-        return list(map(serialize, obj))
+        arr = list(map(serialize, obj))
+        return tuple([len(arr)] + arr)
     elif isinstance(obj, dict):
         return tuple(map(serialize, obj.values()))
     elif isinstance(obj, tuple):
@@ -105,7 +110,7 @@ def format_args():
         raise TypeError("Expected single argument")
     args = json.loads(Path(sys.argv[1]).read_text())
     res = flatten_tuples(serialize(args))
-    print(res)
+    print([res])
 
 
 if __name__ == '__main__':

scripts/data/generate_data.py

@@ -130,34 +130,35 @@ def fetch_block(block_hash: str):
 
 
 def resolve_transaction(transaction: dict):
-    """
+    """Resolves transaction inputs and formats the content according to the Cairo type.
     """
     return {
         "version": transaction['version'],
         # Skip the first 4 bytes (version) and take the next 4 bytes (marker + flag)
         "is_segwit": transaction["hex"][8:12] == "0001",
-        "lock_time": transaction['locktime'],
         "inputs": [resolve_input(input) for input in transaction['vin']],
         "outputs": [format_output(output) for output in transaction['vout']],
+        "lock_time": transaction['locktime'],
     }
 
 
 def resolve_input(input: dict):
-    """
+    """Resolves referenced UTXO and formats the transaction inputs according to the Cairo type.
     """
     if input.get('coinbase'):
         return format_coinbase_input(input)
     else:
         return {
             "script": f'0x{input["scriptSig"]["hex"]}',
             "sequence": input['sequence'],
-            "witness": [f'0x{item}' for item in input.get('txinwitness', [])],
             "previous_output": resolve_outpoint(input),
+            "witness": [f'0x{item}' for item in input.get('txinwitness', [])],
         }
 
 
 def resolve_outpoint(input: dict):
-    """
+    """Fetches transaction and block header for the referenced output,
+    formats resulting outpoint according to the Cairo type.
     """
     tx = request_rpc("getrawtransaction", [input['txid'], True])
     block = request_rpc("getblockheader", [tx['blockhash']])
@@ -172,12 +173,11 @@ def resolve_outpoint(input: dict):
 
 
 def format_coinbase_input(input: dict):
-    """
+    """Formats coinbase input according to the Cairo type.
     """
     return {
         "script": f'0x{input["coinbase"]}',
         "sequence": input["sequence"],
-        "witness": [],
         "previous_output": {
             "txid": "0" * 64,
             "vout": 0xffffffff,
@@ -190,17 +190,17 @@ def format_coinbase_input(input: dict):
             "block_time": 0,
             "is_coinbase": False,
         },
+        "witness": []
     }
 
 
 def format_output(output: dict):
-    """
+    """Formats transaction output according to the Cairo type.
     """
     return {
         "value": int(output["value"] * 100000000),
         "pk_script": f'0x{output["scriptPubKey"]["hex"]}',
         "cached": False,
-        # TODO: create a hash set of all inputs/outputs in the block and propagate down
     }
 
 

scripts/data/integration_tests.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 set -e;
-set -o pipefail;
 
 GREEN='\033[0;32m'
 RED='\033[1;31m'
@@ -13,11 +12,10 @@ num_ignored=0
 failures=()
 test_files="tests/data"/*
 
-# TODO: fix bugs
 ignored_files=(
     "tests/data/light_481823.json"
     "tests/data/light_709631.json"
-    "tests/data/full_169.json"
+    "tests/data/full_757738.json"
 )
 ignored="${ignored_files[@]}"
 
@@ -41,11 +39,16 @@ for test_file in $test_files; do
             if [[ "$output" == *"OK"* ]]; then
                 echo -e "${GREEN} ok ${RESET}(gas usage est.: $gas_spent)"
                 num_ok=$((num_ok + 1))
-            else
+            elif [[ "$output" == *"FAIL"* ]]; then
                 echo -e "${RED} fail ${RESET}(gas usage est.: $gas_spent)"
                 num_fail=$((num_fail + 1))
                 error=$(echo $output | grep -o "error='[^']*'" | sed "s/error=//")
                 failures+="\te2e:$test_file — Panicked with $error\n"
+            else
+                echo -e "${RED} fail ${RESET}(gas usage est.: 0)"
+                num_fail=$((num_fail + 1))
+                error=$(echo "$output" | sed '1d')
+                failures+="\te2e:$test_file — $error\n"
             fi
         fi
     fi

scripts/data/regenerate_tests.sh

@@ -8,7 +8,13 @@ then
   export $(cat .env | xargs)
 fi
 
-test_cases=(
+force=0
+
+if [[ "$1" == "--force" ]]; then
+  force=1
+fi
+
+light_test_cases=(
     169     # Block containing first P2P tx to Hal Finney (170)
     24834   # Block containing first off ramp tx from Martti Malmi (24835)
     57042   # Block containing pizza tx (57043)
@@ -25,13 +31,29 @@ test_cases=(
     839999  # Fourth halving block (840000)
 )
 
+full_test_cases=(
+    169     # Block containing first P2P tx to Hal Finney (170)
+    757738  # Block with witness (757739)
+)
+
 mkdir tests/data || true
 
-# Loop through the test cases and generate data
-for test_case in "${test_cases[@]}"; do
+# Generate test file if it does not exist yet or if "force" flag is set
+generate_test() {
+  local mode=$1
+  local height=$2
+  test_file="tests/data/${mode}_${test_case}.json"
+  if [[ ! -f "$test_file" || $force -eq 1 ]]; then
+    python scripts/data/generate_data.py $mode $height 1 true $test_file
+  fi
+}
+
+for test_case in "${light_test_cases[@]}"; do
     echo "Generating test data: light mode, chain state @ $test_case, single block"
-    python scripts/data/generate_data.py "light" $test_case 1 true tests/data/light_${test_case}.json
+    generate_test "light" $test_case
 done
 
-# TODO: generate more full blocks
-python scripts/data/generate_data.py "full" $test_case 1 true tests/data/full_169.json
+for test_case in "${full_test_cases[@]}"; do
+    echo "Generating test data: full mode, chain state @ $test_case, single block"
+    generate_test "full" $test_case
+done

src/main.cairo

@@ -1,13 +1,25 @@
 use crate::types::block::Block;
 use crate::types::chain_state::{ChainState, BlockValidator};
 
+/// Raito program arguments.
+#[derive(Serde)]
+struct Args {
+    /// Current (initial) chain state
+    chain_state: ChainState,
+    /// Batch of blocks that have to be applied to the current chain state
+    blocks: Array<Block>,
+}
+
 /// Raito program entrypoint.
 ///
 /// Receives current chain state and pending blocks,
 /// then validates and applies them one by one.
 /// Returns new chain state in case of succes, otherwise raises an error.
-fn main(mut chain_state: ChainState, mut blocks: Array<Block>) -> ChainState {
-    while let Option::Some(block) = blocks.pop_front() {
+fn main(mut arguments: Span<felt252>) -> ChainState {
+    let Args { mut chain_state, blocks, } = Serde::deserialize(ref arguments)
+        .expect('Failed to deserialize');
+
+    for block in blocks {
         chain_state = chain_state.validate_and_apply(block).expect('Validation failed');
     };
     chain_state

src/test.cairo

@@ -2,15 +2,29 @@ use crate::types::block::Block;
 use crate::types::chain_state::{ChainState, BlockValidator};
 use core::testing::get_available_gas;
 
+/// Integration testing program arguments.
+#[derive(Serde)]
+struct Args {
+    /// Current (initial) chain state
+    chain_state: ChainState,
+    /// Batch of blocks that have to be applied to the current chain state
+    blocks: Array<Block>,
+    /// Expected chain state (that we want to compare the result with)
+    expected_chain_state: ChainState,
+}
+
 /// Integration testing program entrypoint.
 ///
-/// Receives current chain state, pending blocks, and expected result.
-/// Validates and applies blocks one by one, exits on first failure.
-fn test(
-    mut chain_state: ChainState, mut blocks: Array<Block>, mut expected_chain_state: ChainState
-) {
+/// Receives arguments in a serialized format (Cairo serde).
+/// Panics in case of a validation error or chain state mismatch.
+/// Prints result to the stdout.
+fn test(mut arguments: Span<felt252>) {
+    let Args { mut chain_state, blocks, expected_chain_state } = Serde::deserialize(ref arguments)
+        .expect('Failed to deserialize');
+
     let mut gas_before = get_available_gas();
-    while let Option::Some(block) = blocks.pop_front() {
+
+    for block in blocks {
         let height = chain_state.block_height + 1;
         match chain_state.validate_and_apply(block) {
             Result::Ok(new_chain_state) => {
@@ -28,6 +42,7 @@ fn test(
             }
         }
     };
+
     if chain_state != expected_chain_state {
         println!(
             "FAIL: block={} error='expected state {:?}, actual {:?}'",

src/types/block.cairo

@@ -8,7 +8,7 @@ use crate::utils::numeric::u32_byte_reverse;
 use super::transaction::Transaction;
 
 /// Represents a block in the blockchain.
-#[derive(Drop, Copy, Debug, PartialEq, Default)]
+#[derive(Drop, Copy, Debug, PartialEq, Default, Serde)]
 pub struct Block {
     /// Block header.
     pub header: Header,
@@ -17,7 +17,7 @@ pub struct Block {
 }
 
 /// Represents block contents.
-#[derive(Drop, Copy, Debug, PartialEq)]
+#[derive(Drop, Copy, Debug, PartialEq, Serde)]
 pub enum TransactionData {
     /// Merkle root of all transactions in the block.
     /// This variant is used for header-only validation mode (light client).
@@ -37,7 +37,7 @@ pub enum TransactionData {
 /// In order to do the calculation we just need data about the block that is strictly necessary,
 /// but not the data we can calculate like merkle root or data that we already have
 /// like previous_block_hash (in the previous chain state).
-#[derive(Drop, Copy, Debug, PartialEq, Default)]
+#[derive(Drop, Copy, Debug, PartialEq, Default, Serde)]
 pub struct Header {
     /// The version of the block.
     pub version: u32,

src/types/chain_state.cairo

@@ -13,7 +13,7 @@ use crate::validation::{
 use super::block::{BlockHash, Block, TransactionData};
 
 /// Represents the state of the blockchain.
-#[derive(Drop, Copy, Debug, PartialEq)]
+#[derive(Drop, Copy, Debug, PartialEq, Serde)]
 pub struct ChainState {
     /// Height of the current block.
     pub block_height: u32,

src/types/transaction.cairo

@@ -8,7 +8,7 @@ use crate::codec::{Encode, TransactionCodec};
 
 /// Represents a transaction.
 /// https://learnmeabitcoin.com/technical/transaction/
-#[derive(Drop, Copy, Debug, PartialEq)]
+#[derive(Drop, Copy, Debug, PartialEq, Serde)]
 pub struct Transaction {
     /// The version of the transaction.
     pub version: u32,
@@ -29,7 +29,7 @@ pub struct Transaction {
 
 /// Input of a transaction.
 /// https://learnmeabitcoin.com/technical/transaction/input/
-#[derive(Drop, Copy, Debug, PartialEq)]
+#[derive(Drop, Copy, Debug, PartialEq, Serde)]
 pub struct TxIn {
     /// The signature script which satisfies the conditions placed in the txo pubkey script
     /// or coinbase script that contains block height (since 227,836) and miner nonce (optional).
@@ -77,7 +77,7 @@ pub struct TxIn {
 ///       one by one, first inputs then outputs. Output validation might put something to the
 ///       cache while input validation might remove an item, thus it's important to maintain
 ///       the order.
-#[derive(Drop, Copy, Debug, PartialEq)]
+#[derive(Drop, Copy, Debug, PartialEq, Serde)]
 pub struct OutPoint {
     /// The hash of the referenced transaction.
     pub txid: Hash,
@@ -112,7 +112,7 @@ pub struct OutPoint {
 ///     - Do nothing in case of a provably unspendable output
 ///
 /// Read more: https://en.bitcoin.it/wiki/Script#Provably_Unspendable/Prunable_Outputs
-#[derive(Drop, Copy, Debug, PartialEq)]
+#[derive(Drop, Copy, Debug, PartialEq, Serde)]
 pub struct TxOut {
     /// The value of the output in satoshis.
     /// Can be in range [0, 21_000_000] BTC (including both ends).
@@ -125,6 +125,19 @@ pub struct TxOut {
     pub cached: bool,
 }
 
+impl ByteArraySnapSerde of Serde<@ByteArray> {
+    fn serialize(self: @@ByteArray, ref output: Array<felt252>) {
+        (*self).serialize(ref output);
+    }
+
+    fn deserialize(ref serialized: Span<felt252>) -> Option<@ByteArray> {
+        match Serde::deserialize(ref serialized) {
+            Option::Some(res) => Option::Some(@res),
+            Option::None => Option::None,
+        }
+    }
+}
+
 impl TxOutDefault of Default<TxOut> {
     fn default() -> TxOut {
         TxOut { value: 0, pk_script: @"", cached: false, }