dev: implement compile-time filtering for tests (#997)

* dev: implement compile-time filtering

* fmt

.gitignore

@@ -7,3 +7,4 @@ node_modules/
 cache/
 
 scripts/libcairo_native_runtime.a
+scripts/__pycache__

Makefile

@@ -1,2 +1,19 @@
-install: 
+install:
 	bash scripts/install_hook.sh
+
+test-unit:
+	@PACKAGE="$(word 2,$(MAKECMDGOALS))" && \
+	FILTER="$(word 3,$(MAKECMDGOALS))" && \
+	if [ -z "$$PACKAGE" ] && [ -z "$$FILTER" ]; then \
+		scarb test; \
+	elif [ -n "$$PACKAGE" ] && [ -z "$$FILTER" ]; then \
+		scarb test -p $$PACKAGE; \
+	elif [ -n "$$PACKAGE" ] && [ -n "$$FILTER" ]; then \
+		uv run scripts/run_filtered_tests.py $$PACKAGE $$FILTER; \
+	else \
+		echo "Usage: make test-unit [PACKAGE] [FILTER]"; \
+		exit 1; \
+	fi
+
+%:
+	@:

crates/evm/src/backend/starknet_backend.cairo

@@ -479,37 +479,39 @@ mod tests {
         assert_called(starknet_address, selector!("set_code_hash"));
         assert_called(starknet_address, selector!("set_nonce"));
     }
-
-    #[test]
-    #[ignore]
-    //TODO(starknet-foundry): it's impossible to deploy an un-declared class, nor is it possible to
-    //mock_deploy.
-    fn test_exec_sstore_finalized() { // // Given
-    // setup_test_environment();
-    // let mut vm = VMBuilderTrait::new_with_presets().build();
-    // let evm_address = vm.message().target.evm;
-    // let starknet_address = compute_starknet_address(
-    //     test_address(), evm_address, uninitialized_account()
-    // );
-    // let account = Account {
-    //     address: Address { evm: evm_address, starknet: starknet_address },
-    //     code: [].span(),
-    //     nonce: 1,
-    //     balance: 0,
-    //     selfdestruct: false,
-    //     is_created: false,
-    // };
-    // let key: u256 = 0x100000000000000000000000000000001;
-    // let value: u256 = 0xABDE1E11A5;
-    // vm.stack.push(value).expect('push failed');
-    // vm.stack.push(key).expect('push failed');
-
-    // // When
-
-    // vm.exec_sstore().expect('exec_sstore failed');
-    // starknet_backend::commit(ref vm.env.state).expect('commit storage failed');
-
-    // // Then
-    // assert(fetch_original_storage(@account, key) == value, 'wrong committed value')
-    }
 }
+// #[test]
+// #[ignore]
+//TODO(starknet-foundry): it's impossible to deploy an un-declared class, nor is it possible to
+//mock_deploy.
+// fn test_exec_sstore_finalized() { // // Given
+// setup_test_environment();
+// let mut vm = VMBuilderTrait::new_with_presets().build();
+// let evm_address = vm.message().target.evm;
+// let starknet_address = compute_starknet_address(
+//     test_address(), evm_address, uninitialized_account()
+// );
+// let account = Account {
+//     address: Address { evm: evm_address, starknet: starknet_address },
+//     code: [].span(),
+//     nonce: 1,
+//     balance: 0,
+//     selfdestruct: false,
+//     is_created: false,
+// };
+// let key: u256 = 0x100000000000000000000000000000001;
+// let value: u256 = 0xABDE1E11A5;
+// vm.stack.push(value).expect('push failed');
+// vm.stack.push(key).expect('push failed');
+
+// // When
+
+// vm.exec_sstore().expect('exec_sstore failed');
+// starknet_backend::commit(ref vm.env.state).expect('commit storage failed');
+
+// // Then
+// assert(fetch_original_storage(@account, key) == value, 'wrong committed value')
+// }
+// }
+
+

scripts/filter_tests.py

@@ -0,0 +1,50 @@
+import os
+import re
+import sys
+
+
+def filter_tests(directory, filter_string):
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".cairo"):
+                file_path = os.path.join(root, file)
+                filter_file(file_path, filter_string)
+
+    print(f"Filtered tests for {filter_string}")
+
+
+def filter_file(file_path, filter_string):
+    with open(file_path, "r") as f:
+        content = f.read()
+
+    # Regular expression to match test functions, including nested braces
+    test_pattern = re.compile(
+        r"#\[test\]\s*(?:#\[available_gas\([^\)]+\)\]\s*)?fn\s+(\w+)\s*\([^)]*\)\s*(\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})",
+        re.DOTALL,
+    )
+
+    def replace_func(match):
+        full_match = match.group(0)
+        func_name = match.group(1)
+        if filter_string.lower() in func_name.lower():
+            return full_match
+        else:
+            return ""
+
+    new_content = test_pattern.sub(replace_func, content)
+
+    if new_content != content:
+        with open(file_path, "w") as f:
+            f.write(new_content)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python filter_tests.py <filter_string>")
+        sys.exit(1)
+
+    filter_string = sys.argv[1]
+    crates_dir = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "crates"
+    )
+    filter_tests(crates_dir, filter_string)

scripts/run_filtered_tests.py

@@ -0,0 +1,73 @@
+import os
+import pty
+import select
+import shutil
+import subprocess
+import sys
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+from filter_tests import filter_tests
+
+PROJECT_FILES = ["Scarb.toml", "Scarb.lock", ".tool-versions"]
+
+
+@contextmanager
+def temporary_project_copy(src_dir):
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+        src_path = Path(src_dir)
+
+        for file in PROJECT_FILES:
+            if (src_file := src_path / file).exists():
+                shutil.copy2(src_file, temp_path / file)
+
+        if (src_crates := src_path / "crates").exists():
+            shutil.copytree(src_crates, temp_path / "crates", symlinks=True)
+
+        yield temp_path
+
+
+def stream_output(fd):
+    while True:
+        try:
+            r, _, _ = select.select([fd], [], [], 0.1)
+            if r:
+                data = os.read(fd, 1024)
+                if not data:
+                    break
+                sys.stdout.buffer.write(data)
+                sys.stdout.buffer.flush()
+        except OSError:
+            break
+
+
+def run_scarb_command(command, cwd):
+    master, slave = pty.openpty()
+    with subprocess.Popen(
+        command, shell=True, stdout=slave, stderr=slave, close_fds=True, cwd=cwd
+    ) as process:
+        os.close(slave)
+        stream_output(master)
+        return_code = process.wait()
+
+    if return_code != 0:
+        print(f"Error: Scarb command failed with return code {return_code}")
+        sys.exit(return_code)
+
+
+def run_filtered_tests(package, filter_name):
+    project_root = Path(__file__).parent.parent
+
+    with temporary_project_copy(project_root) as temp_project_dir:
+        filter_tests(temp_project_dir / "crates", filter_name)
+        run_scarb_command(f"scarb test -p {package} {filter_name}", temp_project_dir)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python run_filtered_tests.py <package> <filter_name>")
+        sys.exit(1)
+
+    run_filtered_tests(sys.argv[1], sys.argv[2])