[REPLACED] Set up memory output comparison for contracts run in starknet-devnet tests

.github/workflows/compare_memory_devnet.yml

@@ -0,0 +1,36 @@
+name: compare_devnet_memory
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ '*' ]
+
+env:
+    CARGO_TERM_COLOR: always
+jobs:
+  compare_memory_devnet:
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Install Rust 1.61.0
+      uses: actions-rs/toolchain@v1
+      with:
+          toolchain: 1.61.0
+          override: true
+          components: rustfmt, clippy
+    - uses: actions/checkout@v3
+    - name: Python3 Build
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Checkout starknet-devnet
+      uses: actions/checkout@v3
+      with:
+        repository: Shard-Labs/starknet-devnet
+        path: starknet-devnet
+    - name: Install test dependencies
+      run: pip install ecdsa fastecdsa sympy cairo-lang==0.9.1 maturin
+    - name: Run devnet tests & compare memory outputs
+      run: make compare_memory_devnet_ci

Makefile

@@ -1,4 +1,4 @@
-.PHONY: deps deps-macos deps-default-version build run check test clippy clean run-python-test full-test run-comparer-tracer compare_trace_memory compare_trace compare_memory
+.PHONY: deps deps-macos deps-default-version build run check test clippy clean run-python-test full-test run-comparer-tracer compare_trace_memory compare_trace compare_memory compare_memory_devnet compare_memory_devnet_ci
 
 TEST_DIR=cairo_programs
 TEST_FILES:=$(wildcard $(TEST_DIR)/*.cairo)
@@ -78,6 +78,8 @@ clean:
 	rm -f $(BAD_TEST_DIR)/*.memory
 	rm -f $(BAD_TEST_DIR)/*.trace
 	rm -rf cairo-rs-py-env
+	rm -rf starknet-devnet
+	rm -rf scripts/memory_comparator/cairo*
 
 run-python-test: $(COMPILED_TESTS) $(COMPILED_BAD_TESTS)
 	PYENV_VERSION=pypy3.7-7.3.9 . cairo-rs-py-env/bin/activate && \
@@ -104,4 +106,50 @@ compare_trace: $(CAIRO_RS_TRACE) $(CAIRO_TRACE)
 compare_memory: $(CAIRO_RS_MEM) $(CAIRO_MEM)
 	cd tests; ./compare_vm_state.sh memory
 
-
+compare_memory_devnet:
+# Set up the virtual envs
+	scripts/memory_comparator/build_envs.sh
+# Clone the starknet-devnet from github
+	git clone [email protected]:Shard-Labs/starknet-devnet.git
+# Set up the starknet-devnet in each env
+# cairo-rs-py
+	. scripts/memory_comparator/cairo-rs-py/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# cairo-lang
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# Create the folder where we will store the memory outputs
+	cd starknet-devnet; mkdir memory_files
+# Compile test files
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	cd starknet-devnet; scripts/compile_contracts.sh
+# Patch both envs
+	patch --directory scripts/memory_comparator/cairo-rs-py/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-rs-py.patch
+	patch --directory scripts/memory_comparator/cairo-lang/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-lang.patch
+# Run each test one by one in each env and run the memory comparator
+	. ./scripts/memory_comparator/run_tests_compare_memory.sh
+
+compare_memory_devnet_ci:
+# Set up the virtual envs
+	scripts/memory_comparator/build_envs.sh
+# Set up the starknet-devnet in each env
+# cairo-rs-py
+	. scripts/memory_comparator/cairo-rs-py/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# cairo-lang
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	pip install starknet-devnet && \
+	cd starknet-devnet; scripts/install_dev_tools.sh
+# Create the folder where we will store the memory outputs
+	cd starknet-devnet; mkdir memory_files
+# Compile test files
+	. scripts/memory_comparator/cairo-lang/bin/activate && \
+	cd starknet-devnet; scripts/compile_contracts.sh
+# Patch both envs
+	patch --directory scripts/memory_comparator/cairo-rs-py/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-rs-py.patch
+	patch --directory scripts/memory_comparator/cairo-lang/lib/python3.9/site-packages/ --strip 2 < scripts/memory_comparator/output-memory-cairo-lang.patch
+# Run each test one by one in each env and run the memory comparator
+	. ./scripts/memory_comparator/run_tests_compare_memory.sh

scripts/memory_comparator/build_envs.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+# This is not reaaaaally a robust way to find it, but you need to be actively
+# trying to break it for this to fail :)
+SCRIPT_DIR="scripts/memory_comparator"
+
+python3.9 -m venv --upgrade-deps ${SCRIPT_DIR}/cairo-lang ${SCRIPT_DIR}/cairo-rs-py
+${SCRIPT_DIR}/cairo-lang/bin/pip install cairo-lang==0.10.1
+${SCRIPT_DIR}/cairo-rs-py/bin/pip install maturin==0.14.1 cairo-lang==0.10.1
+${SCRIPT_DIR}/cairo-rs-py/bin/maturin build --manifest-path Cargo.toml --release --strip --interpreter 3.9 --no-default-features --features extension
+${SCRIPT_DIR}/cairo-rs-py/bin/pip install target/wheels/cairo_rs_py-*.whl
+
+${SCRIPT_DIR}/cairo-rs-py/bin/cairo-run --version
+${SCRIPT_DIR}/cairo-rs-py/bin/starknet --version

scripts/memory_comparator/memory_comparator.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import sys
+
+def main():
+    filename1 = sys.argv[1]
+    filename2 = sys.argv[2]
+    cairo_mem = {}
+    cairo_rs_mem = {}
+    name = filename2.split("/")[-1]
+    with open(filename1, 'rb') as f:
+        cairo_raw = f.read()
+        assert len(cairo_raw) % 40 == 0, f'{filename1}: malformed memory file from Cairo VM'
+        chunks = len(cairo_raw) // 40
+        for i in range(0, chunks):
+            chunk = cairo_raw[i*40:(i+1)*40]
+            k, v = int.from_bytes(chunk[:8], 'little'), int.from_bytes(chunk[8:], 'little')
+            assert k not in cairo_mem, f'{filename1}: address {k} has two values'
+            cairo_mem[k] = v
+        assert len(cairo_mem) * 40 == len(cairo_raw), f'{filename1}: {len(cairo_mem) * 40} != {len(cairo_raw)}'
+
+    with open(filename2, 'rb') as f:
+        cairo_rs_raw = f.read()
+        assert len(cairo_rs_raw) % 40 == 0, f'{filename2}: malformed memory file from cairo-rs'
+        chunks = len(cairo_rs_raw) // 40
+        for i in range(0, chunks):
+            chunk = cairo_rs_raw[i*40:(i+1)*40]
+            k, v = int.from_bytes(chunk[:8], 'little'), int.from_bytes(chunk[8:], 'little')
+            assert k not in cairo_rs_mem, f'{filename2}: address {k} has two values'
+            cairo_rs_mem[k] = v
+        assert len(cairo_rs_mem) * 40 == len(cairo_rs_raw), f'{filename2}: {len(cairo_rs_mem) * 40} != {len(cairo_rs_raw)}'
+
+    assert len(cairo_mem) == len(cairo_rs_mem), f'{filename2}: len(cairo_mem)={len(cairo_mem)} len(cairo_mem)={len(cairo_rs_mem)}'
+    if cairo_mem != cairo_rs_mem:
+        print(f'Mismatch between {filename1} (Cairo) and {filename2} (cairo_rs)')
+        print('keys in Cairo but not cairo-rs:')
+        for k in cairo_mem:
+            if k in cairo_rs_mem:
+                continue
+            print(f'{k}:{v}')
+        print('keys in cairo_rs but not Cairo:')
+        for k in cairo_rs_mem:
+            if k in cairo_mem:
+                continue
+            print(f'{k}:{v}')
+        print('mismatched values (Cairo <-> cairo_rs)):')
+        for k in cairo_rs_mem:
+            if k not in cairo_mem:
+                continue
+            if cairo_rs_mem[k] == cairo_mem[k]:
+                continue
+            print(f'{k}:({cairo_mem[k]} <-> {cairo_rs_mem[k]})')
+        exit(1)
+
+
+if __name__ == '__main__':
+    main()

scripts/memory_comparator/output-memory-cairo-lang.patch

@@ -0,0 +1,51 @@
+diff --git a/src/starkware/starknet/business_logic/execution/execute_entry_point.py b/src/starkware/starknet/business_logic/execution/execute_entry_point.py
+index 62d4290..f4a0d8b 100644
+--- a/src/starkware/starknet/business_logic/execution/execute_entry_point.py
++++ b/src/starkware/starknet/business_logic/execution/execute_entry_point.py
+@@ -9,6 +9,8 @@ from starkware.cairo.lang.vm.relocatable import RelocatableValue
+ from starkware.cairo.lang.vm.security import SecurityError
+ from starkware.cairo.lang.vm.utils import ResourcesError
+ from starkware.cairo.lang.vm.vm_exceptions import HintException, VmException, VmExceptionBase
++from starkware.cairo.lang.vm.cairo_run import write_binary_memory
++import math
+ from starkware.starknet.business_logic.execution.execute_entry_point_base import (
+     ExecuteEntryPointBase,
+ )
+@@ -274,6 +276,10 @@ class ExecuteEntryPoint(ExecuteEntryPointBase):
+         assert isinstance(args_ptr, RelocatableValue)  # Downcast.
+         runner.mark_as_accessed(address=args_ptr, size=len(entry_points_args))
+
++        f = open("memory_files/execute_entry_point.memory", "wb")
++        field_bytes = math.ceil(contract_class.program.prime.bit_length() / 8)
++        runner.relocate()
++        write_binary_memory(f, runner.relocated_memory, field_bytes)
+         return runner, syscall_handler
+
+     def _get_selected_entry_point(
+diff --git a/src/starkware/starknet/core/os/class_hash.py b/src/starkware/starknet/core/os/class_hash.py
+index 132fb21..321c63a 100644
+--- a/src/starkware/starknet/core/os/class_hash.py
++++ b/src/starkware/starknet/core/os/class_hash.py
+@@ -3,9 +3,11 @@ import dataclasses
+ import itertools
+ import json
+ import os
++import math
+ from contextvars import ContextVar
+ from functools import lru_cache
+ from typing import Callable, List, Optional
++from starkware.cairo.lang.vm.cairo_run import write_binary_memory
+
+ import cachetools
+
+@@ -92,6 +94,10 @@ def compute_class_hash_inner(
+         use_full_name=True,
+         verify_secure=False,
+     )
++    f = open("memory_files/class_hash.memory", "wb")
++    field_bytes = math.ceil(program.prime.bit_length() / 8)
++    runner.relocate()
++    write_binary_memory(f, runner.relocated_memory, field_bytes)
+     _, class_hash = runner.get_return_values(2)
+     return class_hash
+