feat(forge): Remove proptest runner from fuzzed tests (#11061)

* Remove proptest from fuzzed tests

* Persist and replay fuzz failure

* Nits

* add fuzz coverage metrics config, cleanup

* rm unec RefCell, refactor

* Add inline fuzz coverage metrics

* Same corpus manager for fuzz and invariant tests

* Fix fmt

* Unify configs

* Use corpus manager / abi mutation for fuzz tests - Move hit count in corpus - add mutate_abi and evict_oldes_corpus fns TODO: load persisted corpus from file on fuzz test start

* Replay corpus for fuzz tests as well

* Add corpus replay failures for fuzz tests too

* cleanup fuzz corpus on forge clean, display errors

* more abi mutations

* rand bit flip mutation + add support for more types

* clippy

* More mutations - interesting byte and word

* More mutations, traces

* Review changes

* add mutators traits

* Simplify

* Update crates/evm/fuzz/src/strategies/mod.rs

Co-authored-by: DaniPopes <[email protected]>

* Avoid clone when mutating array and tuples

* Update crates/evm/evm/src/executors/corpus.rs

Co-authored-by: DaniPopes <[email protected]>

* Update crates/evm/evm/src/executors/corpus.rs

Co-authored-by: DaniPopes <[email protected]>

* Update crates/evm/evm/src/executors/corpus.rs

Co-authored-by: DaniPopes <[email protected]>

* Fix typo

* use wrapping add / sub for inc/dec mutation

* Validate inc/dec, better word mutation, tests

* Remove unused validation, early return on validation

* mutators to operate on &mut [u8], instrument, cleanup

* Mutate fixedbytes, more address mutations

* Unify mutators

* Add word mutation unit test

* Address nits

* Update crates/evm/fuzz/src/strategies/param.rs

Co-authored-by: DaniPopes <[email protected]>

* Update crates/evm/fuzz/src/strategies/param.rs

Co-authored-by: DaniPopes <[email protected]>

* Rename mutate_random_array/tuple_value

---------

Co-authored-by: DaniPopes <[email protected]>

Author: grandizzy

crates/config/src/fuzz.rs

@@ -24,10 +24,11 @@ pub struct FuzzConfig {
     pub dictionary: FuzzDictionaryConfig,
     /// Number of runs to execute and include in the gas report.
     pub gas_report_samples: u32,
+    /// The fuzz corpus configuration.
+    #[serde(flatten)]
+    pub corpus: FuzzCorpusConfig,
     /// Path where fuzz failures are recorded and replayed.
     pub failure_persist_dir: Option<PathBuf>,
-    /// Name of the file to record fuzz failures, defaults to `failures`.
-    pub failure_persist_file: Option<String>,
     /// show `console.log` in fuzz test, defaults to `false`
     pub show_logs: bool,
     /// Optional timeout (in seconds) for each property test
@@ -43,8 +44,8 @@ impl Default for FuzzConfig {
             seed: None,
             dictionary: FuzzDictionaryConfig::default(),
             gas_report_samples: 256,
+            corpus: FuzzCorpusConfig::default(),
             failure_persist_dir: None,
-            failure_persist_file: None,
             show_logs: false,
             timeout: None,
         }
@@ -54,11 +55,7 @@ impl Default for FuzzConfig {
 impl FuzzConfig {
     /// Creates fuzz configuration to write failures in `{PROJECT_ROOT}/cache/fuzz` dir.
     pub fn new(cache_dir: PathBuf) -> Self {
-        Self {
-            failure_persist_dir: Some(cache_dir),
-            failure_persist_file: Some("failures".to_string()),
-            ..Default::default()
-        }
+        Self { failure_persist_dir: Some(cache_dir), ..Default::default() }
     }
 }
 
@@ -97,3 +94,49 @@ impl Default for FuzzDictionaryConfig {
         }
     }
 }
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FuzzCorpusConfig {
+    // Path to corpus directory, enabled coverage guided fuzzing mode.
+    // If not set then sequences producing new coverage are not persisted and mutated.
+    pub corpus_dir: Option<PathBuf>,
+    // Whether corpus to use gzip file compression and decompression.
+    pub corpus_gzip: bool,
+    // Number of mutations until entry marked as eligible to be flushed from in-memory corpus.
+    // Mutations will be performed at least `corpus_min_mutations` times.
+    pub corpus_min_mutations: usize,
+    // Number of corpus that won't be evicted from memory.
+    pub corpus_min_size: usize,
+    /// Whether to collect and display edge coverage metrics.
+    pub show_edge_coverage: bool,
+}
+
+impl FuzzCorpusConfig {
+    pub fn with_test_name(&mut self, test_name: &String) {
+        if let Some(corpus_dir) = &self.corpus_dir {
+            self.corpus_dir = Some(corpus_dir.join(test_name));
+        }
+    }
+
+    /// Whether edge coverage should be collected and displayed.
+    pub fn collect_edge_coverage(&self) -> bool {
+        self.corpus_dir.is_some() || self.show_edge_coverage
+    }
+
+    /// Whether coverage guided fuzzing is enabled.
+    pub fn is_coverage_guided(&self) -> bool {
+        self.corpus_dir.is_some()
+    }
+}
+
+impl Default for FuzzCorpusConfig {
+    fn default() -> Self {
+        Self {
+            corpus_dir: None,
+            corpus_gzip: true,
+            corpus_min_mutations: 5,
+            corpus_min_size: 0,
+            show_edge_coverage: false,
+        }
+    }
+}

crates/config/src/invariant.rs

@@ -1,6 +1,6 @@
 //! Configuration for invariant testing
 
-use crate::fuzz::FuzzDictionaryConfig;
+use crate::fuzz::{FuzzCorpusConfig, FuzzDictionaryConfig};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
@@ -26,15 +26,9 @@ pub struct InvariantConfig {
     pub max_assume_rejects: u32,
     /// Number of runs to execute and include in the gas report.
     pub gas_report_samples: u32,
-    /// Path where invariant corpus is stored, enables coverage guided fuzzing and edge coverage
-    /// metrics.
-    pub corpus_dir: Option<PathBuf>,
-    /// Whether corpus to use gzip file compression and decompression.
-    pub corpus_gzip: bool,
-    // Number of corpus mutations until marked as eligible to be flushed from memory.
-    pub corpus_min_mutations: usize,
-    // Number of corpus that won't be evicted from memory.
-    pub corpus_min_size: usize,
+    /// The fuzz corpus configuration.
+    #[serde(flatten)]
+    pub corpus: FuzzCorpusConfig,
     /// Path where invariant failures are recorded and replayed.
     pub failure_persist_dir: Option<PathBuf>,
     /// Whether to collect and display fuzzed selectors metrics.
@@ -43,8 +37,6 @@ pub struct InvariantConfig {
     pub timeout: Option<u32>,
     /// Display counterexample as solidity calls.
     pub show_solidity: bool,
-    /// Whether to collect and display edge coverage metrics.
-    pub show_edge_coverage: bool,
 }
 
 impl Default for InvariantConfig {
@@ -58,15 +50,11 @@ impl Default for InvariantConfig {
             shrink_run_limit: 5000,
             max_assume_rejects: 65536,
             gas_report_samples: 256,
-            corpus_dir: None,
-            corpus_gzip: true,
-            corpus_min_mutations: 5,
-            corpus_min_size: 0,
+            corpus: FuzzCorpusConfig::default(),
             failure_persist_dir: None,
             show_metrics: true,
             timeout: None,
             show_solidity: false,
-            show_edge_coverage: false,
         }
     }
 }
@@ -83,15 +71,11 @@ impl InvariantConfig {
             shrink_run_limit: 5000,
             max_assume_rejects: 65536,
             gas_report_samples: 256,
-            corpus_dir: None,
-            corpus_gzip: true,
-            corpus_min_mutations: 5,
-            corpus_min_size: 0,
+            corpus: FuzzCorpusConfig::default(),
             failure_persist_dir: Some(cache_dir),
             show_metrics: true,
             timeout: None,
             show_solidity: false,
-            show_edge_coverage: false,
         }
     }
 }

crates/config/src/lib.rs

@@ -107,7 +107,7 @@ pub use providers::Remappings;
 use providers::*;
 
 mod fuzz;
-pub use fuzz::{FuzzConfig, FuzzDictionaryConfig};
+pub use fuzz::{FuzzConfig, FuzzCorpusConfig, FuzzDictionaryConfig};
 
 mod invariant;
 pub use invariant::InvariantConfig;
@@ -1095,7 +1095,8 @@ impl Config {
             }
         };
         remove_test_dir(&self.fuzz.failure_persist_dir);
-        remove_test_dir(&self.invariant.corpus_dir);
+        remove_test_dir(&self.fuzz.corpus.corpus_dir);
+        remove_test_dir(&self.invariant.corpus.corpus_dir);
         remove_test_dir(&self.invariant.failure_persist_dir);
 
         Ok(())
@@ -4614,7 +4615,6 @@ mod tests {
                     runs: 512,
                     depth: 10,
                     failure_persist_dir: Some(PathBuf::from("cache/invariant")),
-                    corpus_dir: None,
                     ..Default::default()
                 }
             );

crates/evm/core/src/constants.rs

@@ -37,9 +37,6 @@ pub const MAGIC_ASSUME: &[u8] = b"FOUNDRY::ASSUME";
 /// Magic return value returned by the `skip` cheatcode. Optionally appended with a reason.
 pub const MAGIC_SKIP: &[u8] = b"FOUNDRY::SKIP";
 
-/// Test timeout return value.
-pub const TEST_TIMEOUT: &str = "FOUNDRY::TEST_TIMEOUT";
-
 /// The address that deploys the default CREATE2 deployer contract.
 pub const DEFAULT_CREATE2_DEPLOYER_DEPLOYER: Address =
     address!("0x3fAB184622Dc19b6109349B94811493BF2a45362");