Light client pipeline

.gitignore

@@ -23,3 +23,5 @@ Cargo.lock
 .venv
 .python-version
 __pycache__
+
+.light_client

README.md

@@ -61,8 +61,8 @@ Tasks:
 * [x] block time validation/computation
 * [x] block difficulty adjustment
 * [x] script for fetching arbitrary block data
-* [ ] script for preparing program arguments
-* [ ] script for running the program e2e for multiple block
+* [x] script for preparing program arguments
+* [x] script for running the program e2e for multiple blocks
 
 ### Milestone 2 - Partial transaction validation
 
@@ -71,7 +71,7 @@ Extend light client with partial transaction validation, but without UTXO checks
 Tasks:
 
 * [ ] reassess validation check list (analyze Bitcoin core codebase)
-* [ ] generate & run integration tests e2e instead of Cairo codegen
+* [x] generate & run integration tests e2e instead of Cairo codegen
 * [x] transaction ID calculation
 * [x] transaction root computation
 * [x] validate transaction fee
@@ -135,29 +135,42 @@ This will compile all the components:
 
 ```bash
 scarb build
-
 ```
 
-This will run the test-suite:
+This will run unit and integration tests:
 
 ```bash
 scarb test
 ```
 
-Re-generate test data:
+For integration tests ony:
+
+```bash
+scarb run integration_tests
+```
+
+Run for specific test file(s):
+
+```bash
+scarb run integration_tests tests/data/light_481823.json
+```
+
+Re-generate integration test data:
 
 ```base
-scarb run get_blocks
-scarb run get_block <BlockHash>
-scarb run get_block_py <BlockHash>
+scarb run regenerate_tests
 ```
 
-* File will be created in [tests/blocks/](https://github.com/keep-starknet-strange/raito/blob/main/tests/blocks)block_\<block_height\>.cairo
-* If you want to modify the blockHash for get_blocks change [scripts/data/get_blocks.sh](https://github.com/keep-starknet-strange/raito/blob/main/scripts/data/get_blocks.sh#L11)
+* Files will be created in [tests/data/](https://github.com/keep-starknet-strange/raito/blob/main/tests/data)
+* If you want to add a new test case, edit [scripts/data/regenerate_tests.sh](https://github.com/keep-starknet-strange/raito/blob/main/scripts/data/regenerate_tests.sh)
 
 ## Build dependencies
 
-* ```get_blocks``` and ```get_block_py``` need [that Python dependencies](https://github.com/keep-starknet-strange/raito/tree/main/scripts/data/requirements.txt)
+Install necessary packages required by Python scripts:
+
+```bash
+pip install -r scripts/data/requirements.txt
+```
 
 ## References
 

Scarb.toml

@@ -4,9 +4,10 @@ version = "0.1.0"
 edition = "2024_07"
 
 [scripts]
-get_block= "python3 ./scripts/data/get_block.py"
-get_blocks= "./scripts/data/get_blocks.sh"
-light_client= "./scripts/data/light_client.sh"
+regenerate_tests= "./scripts/data/regenerate_tests.sh"
+integration_tests = "scarb build && ./scripts/data/integration_tests.sh"
+light_client= "scarb build && ./scripts/data/light_client.sh"
+test = "scarb cairo-test && scarb run integration_tests"
 
 [dependencies]
 

scripts/data/format_args.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+import sys
+import json
+from pathlib import Path
+
+
+def serialize(obj):
+    """Serializes Cairo data in JSON format to a Python object with reduced types.
+    Supports the following conversions:
+        integer -> int  # felt252
+        dec string (0-9) -> (int, int) -> u256 = { lo: felt252, hi: felt252 }
+        hex string (0-F), 64 len -> (int, int, int, int, int, int, int, int) -> Hash !reversed!
+        hex string 0x prefixed -> ([int, ...], int, int) -> ByteArray
+        list -> []
+        dict -> tuple(dict.values)
+    """
+    if isinstance(obj, int):
+        # This covers u8, u16, u32, u64, u128, felt252
+        assert(obj >= 0 and obj < 2 ** 252)
+        return obj
+    elif isinstance(obj, str):
+        if obj.isdigit():
+            # Try to cast to int and then to low/high parts
+            num = int(obj)
+            assert(num >= 0 and num < 2 ** 256)
+            lo = num % 2 ** 128
+            hi = num // 2 ** 128
+            return (lo, hi)
+        elif obj.startswith('0x'):
+            # Split into 31-byte chunks and save the remainder
+            src = bytes.fromhex(obj[2:])
+            num_chunks = len(src) // 31
+            main_len = num_chunks * 31
+            rem_len = len(src) - main_len
+            main = [int.from_bytes(src[i:i+31], 'big') for i in range(0, main_len, 31)]
+            # TODO: check if this is how byte31 is implemented
+            rem = int.from_bytes(src[main_len:].rjust(31, b'\x00'), 'big')
+            return (main, rem, rem_len)
+        else:
+            # Reversed hex string into 4-byte words then into BE u32
+            assert(len(obj) == 64)
+            rev = list(reversed(bytes.fromhex(obj)))
+            return tuple(int.from_bytes(rev[i:i+4], 'big') for i in range(0, 32, 4))
+    elif isinstance(obj, list):
+        return list(map(serialize, obj))
+    elif isinstance(obj, dict):
+        return tuple(map(serialize, obj.values()))
+    elif isinstance(obj, tuple):
+        return obj
+    else:
+        raise NotImplementedError(obj)
+
+
+def flatten_tuples(src):
+    """Recursively flattens tuples.
+    Example: (0, (1, 2), [(3, 4, [5, 6])]) -> [0, 1, 2, [3, 4, [5, 6]]]
+
+    :param src: an object that can be int|list|tuple or their nested combination.
+    :return: an object that can only contain integers and lists, top-level tuple converts to a list.
+    """
+    res = []
+    def append_obj(obj, to):
+        if isinstance(obj, int):
+            to.append(obj)
+        elif isinstance(obj, list):
+            inner = []
+            for item in obj:
+                append_obj(item, inner)
+            to.append(inner)
+        elif isinstance(obj, tuple):
+            for item in obj:
+                append_obj(item, to)
+        else:
+            raise NotImplementedError(obj)
+    append_obj(src, res)
+    return res
+
+
+def format_cairo1_run(args: list) -> str:
+    """Formats arguments for usage with cairo1-run.
+    Example: [0, 1, [2, 3, 4]] -> "0 1 [2 3 4]"
+
+    :param args: Python object containing already processed arguments.
+    :return: Removes outer array brackets [] and commas, returns string.
+    """
+    def format_item(item, root=False):
+        if isinstance(item, list):
+            arr = " ".join(map(format_item, item))
+            return arr if root else f'[{arr}]'
+        else:
+            return str(item)
+    return format_item(args, root=True)
+
+
+def format_args():
+    """Reads arguments from JSON file and prints formatted result.
+    Expects a single CLI argument containing file path.
+    Output is compatible with the Scarb runner arguments format.
+    """
+    if (len(sys.argv) != 2):
+        raise TypeError("Expected single argument")
+    args = json.loads(Path(sys.argv[1]).read_text())
+    res = flatten_tuples(serialize(args))
+    print(res)
+
+
+if __name__ == '__main__':
+    format_args()