Set up memory + trace output comparison for contracts run in starknet-devnet tests

.github/workflows/compare_memory_devnet.yml

@@ -0,0 +1,36 @@
+name: compare_devnet_memory
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ '*' ]
+
+env:
+    CARGO_TERM_COLOR: always
+jobs:
+  compare_memory_devnet:
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Install Rust 1.61.0
+      uses: actions-rs/toolchain@v1
+      with:
+          toolchain: 1.61.0
+          override: true
+          components: rustfmt, clippy
+    - uses: actions/checkout@v3
+    - name: Python3 Build
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Checkout starknet-devnet
+      uses: actions/checkout@v3
+      with:
+        repository: Shard-Labs/starknet-devnet
+        path: starknet-devnet
+    - name: Install test dependencies
+      run: pip install ecdsa fastecdsa sympy cairo-lang==0.9.1 maturin
+    - name: Run devnet tests & compare memory outputs
+      run: make compare_memory_devnet_ci

Makefile

@@ -1,4 +1,4 @@
-.PHONY: deps deps-macos deps-default-version build run check test clippy clean run-python-test full-test run-comparer-tracer compare_trace_memory compare_trace compare_memory
+.PHONY: deps deps-macos deps-default-version build run check test clippy clean run-python-test full-test run-comparer-tracer compare_trace_memory compare_trace compare_memory compare_memory_devnet compare_memory_devnet_ci
 
 TEST_DIR=cairo_programs
 TEST_FILES:=$(wildcard $(TEST_DIR)/*.cairo)
@@ -78,6 +78,8 @@ clean:
 	rm -f $(BAD_TEST_DIR)/*.memory
 	rm -f $(BAD_TEST_DIR)/*.trace
 	rm -rf cairo-rs-py-env
+	rm -rf starknet-devnet
+	rm -rf scripts/memory_comparator/cairo*
 
 run-python-test: $(COMPILED_TESTS) $(COMPILED_BAD_TESTS)
 	PYENV_VERSION=pypy3.7-7.3.9 . cairo-rs-py-env/bin/activate && \
@@ -104,4 +106,50 @@ compare_trace: $(CAIRO_RS_TRACE) $(CAIRO_TRACE)
 compare_memory: $(CAIRO_RS_MEM) $(CAIRO_MEM)
 	cd tests; ./compare_vm_state.sh memory
 
-
+compare_memory_devnet:
+# Set up the virtual envs
+	scripts/memory_comparator/build_envs.sh
+# Clone the starknet-devnet from github
+	git clone [email protected]:Shard-Labs/starknet-devnet.git
+# Set up the starknet-devnet in each env
+# cairo-rs-py
+	. scripts/memory_comparator/cairo-rs-py/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# cairo-lang
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# Create the folder where we will store the memory outputs
+	cd starknet-devnet; mkdir memory_files; mkdir trace_files
+# Compile test files
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	cd starknet-devnet; scripts/compile_contracts.sh
+# Patch both envs
+	patch --directory scripts/memory_comparator/cairo-rs-py/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-rs-py.patch
+	patch --directory scripts/memory_comparator/cairo-lang/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-lang.patch
+# Run each test one by one in each env and run the memory comparator
+	. ./scripts/memory_comparator/run_tests_compare_memory.sh
+
+compare_memory_devnet_ci:
+# Set up the virtual envs
+	scripts/memory_comparator/build_envs.sh
+# Set up the starknet-devnet in each env
+# cairo-rs-py
+	. scripts/memory_comparator/cairo-rs-py/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# cairo-lang
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# Create the folder where we will store the memory outputs
+	cd starknet-devnet; mkdir memory_files; mkdir trace_files
+# Compile test files
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	cd starknet-devnet; scripts/compile_contracts.sh
+# Patch both envs
+	patch --directory scripts/memory_comparator/cairo-rs-py/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-rs-py.patch
+	patch --directory scripts/memory_comparator/cairo-lang/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-lang.patch
+# Run each test one by one in each env and run the memory comparator
+	. ./scripts/memory_comparator/run_tests_compare_memory.sh

scripts/memory_comparator/build_envs.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+# This is not reaaaaally a robust way to find it, but you need to be actively
+# trying to break it for this to fail :)
+SCRIPT_DIR="scripts/memory_comparator"
+
+python3.9 -m venv --upgrade-deps ${SCRIPT_DIR}/cairo-lang ${SCRIPT_DIR}/cairo-rs-py
+${SCRIPT_DIR}/cairo-lang/bin/pip install cairo-lang==0.10.1
+${SCRIPT_DIR}/cairo-rs-py/bin/pip install maturin==0.14.1 cairo-lang==0.10.1
+${SCRIPT_DIR}/cairo-rs-py/bin/maturin build --manifest-path Cargo.toml --release --strip --interpreter 3.9 --no-default-features --features extension
+${SCRIPT_DIR}/cairo-rs-py/bin/pip install target/wheels/cairo_rs_py-*.whl
+
+${SCRIPT_DIR}/cairo-rs-py/bin/cairo-run --version
+${SCRIPT_DIR}/cairo-rs-py/bin/starknet --version

scripts/memory_comparator/memory_comparator.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import sys
+
+def main():
+    filename1 = sys.argv[1]
+    filename2 = sys.argv[2]
+    cairo_mem = {}
+    cairo_rs_mem = {}
+    name = filename2.split("/")[-1]
+    with open(filename1, 'rb') as f:
+        cairo_raw = f.read()
+        assert len(cairo_raw) % 40 == 0, f'{filename1}: malformed memory file from Cairo VM'
+        chunks = len(cairo_raw) // 40
+        for i in range(0, chunks):
+            chunk = cairo_raw[i*40:(i+1)*40]
+            k, v = int.from_bytes(chunk[:8], 'little'), int.from_bytes(chunk[8:], 'little')
+            assert k not in cairo_mem, f'{filename1}: address {k} has two values'
+            cairo_mem[k] = v
+        assert len(cairo_mem) * 40 == len(cairo_raw), f'{filename1}: {len(cairo_mem) * 40} != {len(cairo_raw)}'
+
+    with open(filename2, 'rb') as f:
+        cairo_rs_raw = f.read()
+        assert len(cairo_rs_raw) % 40 == 0, f'{filename2}: malformed memory file from cairo-rs'
+        chunks = len(cairo_rs_raw) // 40
+        for i in range(0, chunks):
+            chunk = cairo_rs_raw[i*40:(i+1)*40]
+            k, v = int.from_bytes(chunk[:8], 'little'), int.from_bytes(chunk[8:], 'little')
+            assert k not in cairo_rs_mem, f'{filename2}: address {k} has two values'
+            cairo_rs_mem[k] = v
+        assert len(cairo_rs_mem) * 40 == len(cairo_rs_raw), f'{filename2}: {len(cairo_rs_mem) * 40} != {len(cairo_rs_raw)}'
+
+    assert len(cairo_mem) == len(cairo_rs_mem), f'{filename2}: len(cairo_mem)={len(cairo_mem)} len(cairo_mem)={len(cairo_rs_mem)}'
+    if cairo_mem != cairo_rs_mem:
+        print(f'Mismatch between {filename1} (Cairo) and {filename2} (cairo_rs)')
+        print('keys in Cairo but not cairo-rs:')
+        for k in cairo_mem:
+            if k in cairo_rs_mem:
+                continue
+            print(f'{k}:{v}')
+        print('keys in cairo_rs but not Cairo:')
+        for k in cairo_rs_mem:
+            if k in cairo_mem:
+                continue
+            print(f'{k}:{v}')
+        print('mismatched values (Cairo <-> cairo_rs)):')
+        for k in cairo_rs_mem:
+            if k not in cairo_mem:
+                continue
+            if cairo_rs_mem[k] == cairo_mem[k]:
+                continue
+            print(f'{k}:({cairo_mem[k]} <-> {cairo_rs_mem[k]})')
+        exit(1)
+
+
+if __name__ == '__main__':
+    main()

scripts/memory_comparator/output-memory-cairo-lang.patch

@@ -0,0 +1,124 @@
+diff --git a/src/starkware/starknet/business_logic/execution/execute_entry_point.py b/src/starkware/starknet/business_logic/execution/execute_entry_point.py
+index 09dfd89..3ae7c9a 100644
+--- a/src/starkware/starknet/business_logic/execution/execute_entry_point.py
++++ b/src/starkware/starknet/business_logic/execution/execute_entry_point.py
+@@ -9,6 +9,8 @@ from starkware.cairo.lang.vm.relocatable import RelocatableValue
+ from starkware.cairo.lang.vm.security import SecurityError
+ from starkware.cairo.lang.vm.utils import ResourcesError
+ from starkware.cairo.lang.vm.vm_exceptions import HintException, VmException, VmExceptionBase
++from starkware.cairo.lang.vm.cairo_run import write_binary_memory, write_binary_trace
++import math
+ from starkware.python.utils import to_bytes
+ from starkware.starknet.business_logic.execution.execute_entry_point_base import (
+     ExecuteEntryPointBase,
+@@ -45,8 +47,9 @@ from starkware.starkware_utils.error_handling import (
+
+ logger = logging.getLogger(__name__)
+
+-FAULTY_CLASS_HASH = to_bytes(0x1A7820094FEAF82D53F53F214B81292D717E7BB9A92BB2488092CD306F3993F)
++call_number = 0
+
++FAULTY_CLASS_HASH = to_bytes(0x1A7820094FEAF82D53F53F214B81292D717E7BB9A92BB2488092CD306F3993F)
+
+ class ExecuteEntryPoint(ExecuteEntryPointBase):
+     """
+@@ -174,6 +177,7 @@ class ExecuteEntryPoint(ExecuteEntryPointBase):
+         Returns the corresponding CairoFunctionRunner and BusinessLogicSysCallHandler in order to
+         retrieve the execution information.
+         """
++        global call_number
+         # Prepare input for Cairo function runner.
+         class_hash = self._get_code_class_hash(state=state)
+
+@@ -284,6 +288,13 @@ class ExecuteEntryPoint(ExecuteEntryPointBase):
+         assert isinstance(args_ptr, RelocatableValue)  # Downcast.
+         runner.mark_as_accessed(address=args_ptr, size=len(entry_points_args))
+
++        memory_file = open("memory_files/execute_entry_point_{}.memory".format(call_number), "wb")
++        trace_file = open("trace_files/execute_entry_point_{}.trace".format(call_number), "wb")
++        field_bytes = math.ceil(contract_class.program.prime.bit_length() / 8)
++        runner.relocate()
++        write_binary_memory(memory_file, runner.relocated_memory, field_bytes)
++        write_binary_trace(trace_file, runner.relocated_trace)
++        call_number += 1
+         return runner, syscall_handler
+
+     def _get_selected_entry_point(
+diff --git a/src/starkware/starknet/cli/starknet_cli.py b/src/starkware/starknet/cli/starknet_cli.py
+index ddca7c6..fbd4e89 100755
+--- a/src/starkware/starknet/cli/starknet_cli.py
++++ b/src/starkware/starknet/cli/starknet_cli.py
+@@ -60,6 +60,8 @@ from starkware.starknet.wallets.account import DEFAULT_ACCOUNT_DIR, Account
+ from starkware.starknet.wallets.starknet_context import StarknetContext
+ from starkware.starkware_utils.error_handling import StarkErrorCode
+
++custom_salt = 0
++
+ NETWORKS = {
+     "alpha-goerli": "alpha4.starknet.io",
+     "alpha-goerli2": "alpha4-2.starknet.io",
+@@ -839,12 +841,14 @@ def parse_hex_arg(arg: str, arg_name: str) -> int:
+
+
+ def get_salt(salt: Optional[str]) -> int:
++    global custom_salt
++    custom_salt +=1
+     """
+     Validates the given salt and returns it as an integer.
+     If salt is None, returns a random salt.
+     """
+     if salt is None:
+-        return fields.ContractAddressSalt.get_random_value()
++        return custom_salt
+
+     return parse_hex_arg(arg=salt, arg_name="salt")
+
+diff --git a/src/starkware/starknet/core/os/class_hash.py b/src/starkware/starknet/core/os/class_hash.py
+index 132fb21..46bd820 100644
+--- a/src/starkware/starknet/core/os/class_hash.py
++++ b/src/starkware/starknet/core/os/class_hash.py
+@@ -3,9 +3,11 @@ import dataclasses
+ import itertools
+ import json
+ import os
++import math
+ from contextvars import ContextVar
+ from functools import lru_cache
+ from typing import Callable, List, Optional
++from starkware.cairo.lang.vm.cairo_run import write_binary_memory, write_binary_trace
+
+ import cachetools
+
+@@ -26,6 +28,8 @@ from starkware.starknet.services.api.contract_class import ContractClass, EntryP
+
+ CAIRO_FILE = os.path.join(os.path.dirname(__file__), "contracts.cairo")
+
++call_number = 0
++
+ class_hash_cache_ctx_var: ContextVar[Optional[cachetools.LRUCache]] = ContextVar(
+     "class_hash_cache", default=None
+ )
+@@ -73,6 +77,7 @@ def compute_class_hash(
+ def compute_class_hash_inner(
+     contract_class: ContractClass, hash_func: Callable[[int, int], int]
+ ) -> int:
++    global call_number
+     program = load_program()
+     contract_class_struct = get_contract_class_struct(
+         identifiers=program.identifiers, contract_class=contract_class
+@@ -93,6 +98,15 @@ def compute_class_hash_inner(
+         verify_secure=False,
+     )
+     _, class_hash = runner.get_return_values(2)
++
++    memory_file = open("memory_files/class_hash_{}.memory".format(call_number), "wb")
++    trace_file = open("trace_files/class_hash_{}.trace".format(call_number), "wb")
++    field_bytes = math.ceil(program.prime.bit_length() / 8)
++    runner.relocate()
++    write_binary_memory(memory_file, runner.relocated_memory, field_bytes)
++    write_binary_trace(trace_file, runner.relocated_trace)
++    call_number += 1
++
+     return class_hash
+
+