feature(invariant) - persist and replay failure (#7899)

* feature(invariant) - persist and replay failure

* Fix unit test

* Changes after review:
- replace test cache rm macro with closure
- use commons for load / persist failure sequence

* Changes after review: display proper message if replayed sequence reverts before checking invariant

* Changes after review: simplify check sequence logic

crates/config/src/invariant.rs

@@ -8,9 +8,10 @@ use crate::{
     },
 };
 use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
 
 /// Contains for invariant testing
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct InvariantConfig {
     /// The number of runs that must execute for each invariant test group.
     pub runs: u32,
@@ -31,6 +32,8 @@ pub struct InvariantConfig {
     pub max_assume_rejects: u32,
     /// Number of runs to execute and include in the gas report.
     pub gas_report_samples: u32,
+    /// Path where invariant failures are recorded and replayed.
+    pub failure_persist_dir: Option<PathBuf>,
 }
 
 impl Default for InvariantConfig {
@@ -44,10 +47,36 @@ impl Default for InvariantConfig {
             shrink_run_limit: 2usize.pow(18_u32),
             max_assume_rejects: 65536,
             gas_report_samples: 256,
+            failure_persist_dir: None,
         }
     }
 }
 
+impl InvariantConfig {
+    /// Creates invariant configuration to write failures in `{PROJECT_ROOT}/cache/fuzz` dir.
+    pub fn new(cache_dir: PathBuf) -> Self {
+        InvariantConfig {
+            runs: 256,
+            depth: 15,
+            fail_on_revert: false,
+            call_override: false,
+            dictionary: FuzzDictionaryConfig { dictionary_weight: 80, ..Default::default() },
+            shrink_run_limit: 2usize.pow(18_u32),
+            max_assume_rejects: 65536,
+            gas_report_samples: 256,
+            failure_persist_dir: Some(cache_dir),
+        }
+    }
+
+    /// Returns path to failure dir of given invariant test contract.
+    pub fn failure_dir(self, contract_name: &str) -> PathBuf {
+        self.failure_persist_dir
+            .unwrap()
+            .join("failures")
+            .join(contract_name.split(':').last().unwrap())
+    }
+}
+
 impl InlineConfigParser for InvariantConfig {
     fn config_key() -> String {
         INLINE_CONFIG_INVARIANT_KEY.into()
@@ -60,8 +89,7 @@ impl InlineConfigParser for InvariantConfig {
             return Ok(None)
         }
 
-        // self is Copy. We clone it with dereference.
-        let mut conf_clone = *self;
+        let mut conf_clone = self.clone();
 
         for pair in overrides {
             let key = pair.0;
@@ -71,6 +99,9 @@ impl InlineConfigParser for InvariantConfig {
                 "depth" => conf_clone.depth = parse_config_u32(key, value)?,
                 "fail-on-revert" => conf_clone.fail_on_revert = parse_config_bool(key, value)?,
                 "call-override" => conf_clone.call_override = parse_config_bool(key, value)?,
+                "failure-persist-dir" => {
+                    conf_clone.failure_persist_dir = Some(PathBuf::from(value))
+                }
                 _ => Err(InlineConfigParserError::InvalidConfigProperty(key.to_string()))?,
             }
         }

crates/config/src/lib.rs

@@ -21,7 +21,7 @@ use foundry_compilers::{
     },
     cache::SOLIDITY_FILES_CACHE_FILENAME,
     compilers::{solc::SolcVersionManager, CompilerVersionManager},
-    error::{SolcError, SolcIoError},
+    error::SolcError,
     remappings::{RelativeRemapping, Remapping},
     CompilerConfig, ConfigurableArtifacts, EvmVersion, Project, ProjectPathsConfig, Solc,
     SolcConfig,
@@ -793,13 +793,17 @@ impl Config {
     pub fn cleanup(&self, project: &Project) -> Result<(), SolcError> {
         project.cleanup()?;
 
-        // Remove fuzz cache directory.
-        if let Some(fuzz_cache) = &self.fuzz.failure_persist_dir {
-            let path = project.root().join(fuzz_cache);
-            if path.exists() {
-                std::fs::remove_dir_all(&path).map_err(|e| SolcIoError::new(e, path))?;
+        // Remove fuzz and invariant cache directories.
+        let remove_test_dir = |test_dir: &Option<PathBuf>| {
+            if let Some(test_dir) = test_dir {
+                let path = project.root().join(test_dir);
+                if path.exists() {
+                    let _ = fs::remove_dir_all(&path);
+                }
             }
-        }
+        };
+        remove_test_dir(&self.fuzz.failure_persist_dir);
+        remove_test_dir(&self.invariant.failure_persist_dir);
 
         Ok(())
     }
@@ -1958,7 +1962,7 @@ impl Default for Config {
             path_pattern: None,
             path_pattern_inverse: None,
             fuzz: FuzzConfig::new("cache/fuzz".into()),
-            invariant: Default::default(),
+            invariant: InvariantConfig::new("cache/invariant".into()),
             always_use_create_2_factory: false,
             ffi: false,
             prompt_timeout: 120,
@@ -4461,7 +4465,12 @@ mod tests {
             let loaded = Config::load().sanitized();
             assert_eq!(
                 loaded.invariant,
-                InvariantConfig { runs: 512, depth: 10, ..Default::default() }
+                InvariantConfig {
+                    runs: 512,
+                    depth: 10,
+                    failure_persist_dir: Some(PathBuf::from("cache/invariant")),
+                    ..Default::default()
+                }
             );
 
             Ok(())

crates/evm/evm/src/executors/fuzz/mod.rs

@@ -175,15 +175,10 @@ impl FuzzedExecutor {
                 } else {
                     vec![]
                 };
-                result.counterexample = Some(CounterExample::Single(BaseCounterExample {
-                    sender: None,
-                    addr: None,
-                    signature: None,
-                    contract_name: None,
-                    traces: call.traces,
-                    calldata,
-                    args,
-                }));
+
+                result.counterexample = Some(CounterExample::Single(
+                    BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
+                ));
             }
             _ => {}
         }

crates/evm/evm/src/executors/invariant/mod.rs

@@ -40,6 +40,7 @@ mod result;
 pub use result::InvariantFuzzTestResult;
 
 mod shrink;
+pub use shrink::check_sequence;
 
 sol! {
     interface IInvariantTest {

crates/evm/evm/src/executors/invariant/replay.rs

@@ -8,7 +8,7 @@ use foundry_evm_core::constants::CALLER;
 use foundry_evm_coverage::HitMaps;
 use foundry_evm_fuzz::{
     invariant::{BasicTxDetails, InvariantContract},
-    BaseCounterExample, CounterExample,
+    BaseCounterExample,
 };
 use foundry_evm_traces::{load_contracts, TraceKind, Traces};
 use parking_lot::RwLock;
@@ -28,7 +28,7 @@ pub fn replay_run(
     traces: &mut Traces,
     coverage: &mut Option<HitMaps>,
     inputs: Vec<BasicTxDetails>,
-) -> Result<Option<CounterExample>> {
+) -> Result<Vec<BaseCounterExample>> {
     // We want traces for a failed case.
     executor.set_tracing(true);
 
@@ -60,31 +60,34 @@ pub fn replay_run(
         ));
 
         // Create counter example to be used in failed case.
-        counterexample_sequence.push(BaseCounterExample::create(
+        counterexample_sequence.push(BaseCounterExample::from_invariant_call(
             tx.sender,
             tx.call_details.target,
             &tx.call_details.calldata,
             &ided_contracts,
             call_result.traces,
         ));
-
-        // Replay invariant to collect logs and traces.
-        let error_call_result = executor.call_raw(
-            CALLER,
-            invariant_contract.address,
-            invariant_contract
-                .invariant_function
-                .abi_encode_input(&[])
-                .expect("invariant should have no inputs")
-                .into(),
-            U256::ZERO,
-        )?;
-        traces.push((TraceKind::Execution, error_call_result.traces.clone().unwrap()));
-        logs.extend(error_call_result.logs);
     }
 
-    Ok((!counterexample_sequence.is_empty())
-        .then_some(CounterExample::Sequence(counterexample_sequence)))
+    // Replay invariant to collect logs and traces.
+    // We do this only once at the end of the replayed sequence.
+    // Checking after each call doesn't add valuable info for passing scenario
+    // (invariant call result is always success) nor for failed scenarios
+    // (invariant call result is always success until the last call that breaks it).
+    let invariant_result = executor.call_raw(
+        CALLER,
+        invariant_contract.address,
+        invariant_contract
+            .invariant_function
+            .abi_encode_input(&[])
+            .expect("invariant should have no inputs")
+            .into(),
+        U256::ZERO,
+    )?;
+    traces.push((TraceKind::Execution, invariant_result.traces.clone().unwrap()));
+    logs.extend(invariant_result.logs);
+
+    Ok(counterexample_sequence)
 }
 
 /// Replays the error case, shrinks the failing sequence and collects all necessary traces.
@@ -98,15 +101,16 @@ pub fn replay_error(
     logs: &mut Vec<Log>,
     traces: &mut Traces,
     coverage: &mut Option<HitMaps>,
-) -> Result<Option<CounterExample>> {
+) -> Result<Vec<BaseCounterExample>> {
     match failed_case.test_error {
         // Don't use at the moment.
-        TestError::Abort(_) => Ok(None),
+        TestError::Abort(_) => Ok(vec![]),
         TestError::Fail(_, ref calls) => {
             // Shrink sequence of failed calls.
             let calls = shrink_sequence(failed_case, calls, &executor)?;
 
             set_up_inner_replay(&mut executor, &failed_case.inner_sequence);
+
             // Replay calls to get the counterexample and to collect logs, traces and coverage.
             replay_run(
                 invariant_contract,

crates/evm/evm/src/executors/invariant/shrink.rs

@@ -1,5 +1,5 @@
 use crate::executors::{invariant::error::FailedInvariantCaseData, Executor};
-use alloy_primitives::U256;
+use alloy_primitives::{Address, Bytes, U256};
 use foundry_evm_core::constants::CALLER;
 use foundry_evm_fuzz::invariant::BasicTxDetails;
 use proptest::bits::{BitSetLike, VarBitSet};
@@ -97,28 +97,39 @@ pub(crate) fn shrink_sequence(
     let mut shrinker = CallSequenceShrinker::new(calls.len());
     for _ in 0..failed_case.shrink_run_limit {
         // Check candidate sequence result.
-        match check_sequence(failed_case, executor.clone(), calls, shrinker.current().collect()) {
+        match check_sequence(
+            executor.clone(),
+            calls,
+            shrinker.current().collect(),
+            failed_case.addr,
+            failed_case.func.clone(),
+            failed_case.fail_on_revert,
+        ) {
             // If candidate sequence still fails then shrink more if possible.
-            Ok(false) if !shrinker.simplify() => break,
+            Ok((false, _)) if !shrinker.simplify() => break,
             // If candidate sequence pass then restore last removed call and shrink other
             // calls if possible.
-            Ok(true) if !shrinker.complicate() => break,
+            Ok((true, _)) if !shrinker.complicate() => break,
             _ => {}
         }
     }
 
     Ok(shrinker.current().map(|idx| &calls[idx]).cloned().collect())
 }
 
-/// Checks if the shrinked sequence fails test, if it does then we can try simplifying more.
-fn check_sequence(
-    failed_case: &FailedInvariantCaseData,
+/// Checks if the given call sequence breaks the invariant.
+/// Used in shrinking phase for checking candidate sequences and in replay failures phase to test
+/// persisted failures.
+/// Returns the result of invariant check and if sequence was entirely applied.
+pub fn check_sequence(
     mut executor: Executor,
     calls: &[BasicTxDetails],
     sequence: Vec<usize>,
-) -> eyre::Result<bool> {
-    let mut sequence_failed = false;
-    // Apply the shrinked candidate sequence.
+    test_address: Address,
+    test_function: Bytes,
+    fail_on_revert: bool,
+) -> eyre::Result<(bool, bool)> {
+    // Apply the call sequence.
     for call_index in sequence {
         let tx = &calls[call_index];
         let call_result = executor.call_raw_committing(
@@ -127,30 +138,22 @@ fn check_sequence(
             tx.call_details.calldata.clone(),
             U256::ZERO,
         )?;
-        if call_result.reverted && failed_case.fail_on_revert {
+        if call_result.reverted && fail_on_revert {
             // Candidate sequence fails test.
             // We don't have to apply remaining calls to check sequence.
-            sequence_failed = true;
-            break;
+            return Ok((false, false));
         }
     }
-    // Return without checking the invariant if we already have failing sequence.
-    if sequence_failed {
-        return Ok(false);
-    };
 
-    // Check the invariant for candidate sequence.
-    // If sequence fails then we can continue with shrinking - the removed call does not affect
-    // failure.
-    //
-    // If sequence doesn't fail then we have to restore last removed call and continue with next
-    // call - removed call is a required step for reproducing the failure.
-    let mut call_result =
-        executor.call_raw(CALLER, failed_case.addr, failed_case.func.clone(), U256::ZERO)?;
-    Ok(executor.is_raw_call_success(
-        failed_case.addr,
-        Cow::Owned(call_result.state_changeset.take().unwrap()),
-        &call_result,
-        false,
+    // Check the invariant for call sequence.
+    let mut call_result = executor.call_raw(CALLER, test_address, test_function, U256::ZERO)?;
+    Ok((
+        executor.is_raw_call_success(
+            test_address,
+            Cow::Owned(call_result.state_changeset.take().unwrap()),
+            &call_result,
+            false,
+        ),
+        true,
     ))
 }