From 262bdf2cecedfb34522dc75ec3452ea4416fc729 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 4 Aug 2022 16:27:15 +0100 Subject: [PATCH] feat(forge): Invariant Testing v2 (#1572) * init * invariant testing kinda working * updates * fmt * wip * wip * wip * check if there is a fuzzer for invariants * less clones * add support for targetContracts on invariant tests * move load_contracts * add TestOptions and invariant_depth as param * pass TestOptions on fuzz tests * fuzz senders as well * light cleanup * make counterexample list concise * show reverts on invariants test reports * add excludeContracts() * refactor address fetching * move invariant to fuzz module * fuzz calldata from state changes * move block into assert_invariances * add union between selected senders and random * fix sender on get_addresses * wip * add targetSelectors * add fail_on_revert for invariant tests * dont stop on the first invariant failure on each case * create a new strategy tree if a new contract is created * only collect contract addresses from NewlyCreated * display contract and sig on displaying counter example * add documentation * generate the sequence lazily instead * wip * refactor invariants into multi file module * refactor get_addresses to get_list * add test cases * add reentrancy_strat * set reentrancy target as an union with random * merge master * make call_override a flag * add inspector_config() and inspector_config_mut() * always collect data, even without override set * docs * more docs * more docs * remove unnecessary changeset clone & docs * refactor +prepare_fuzzing * more explanations and better var names * replace TestKindGas for a more generic TestKindReport * add docs to strategies * smol fixes * format failure sequence * pass TestOptions instead of fuzzer to multicontractrunner * small fixes * make counterexample an enum * add InvariantFailures * turn add_function into get_function * improve error report on assert_invariants * simplify refs * only override_call_strat needs to be sboxed, revert others * fix invariant test regression * fix: set_replay after setting the last_sequence * fix: test_contract address comparison on call gen * check invariants before calling anything * improve doc on invariant_call_override * remove unused error map from testrunner * reset executor instead of db * add type alias InvariantPreparation * move InvariantExecutor into the same file * add return status * small refactor * const instead of static * merge fixes: backend + testoptions * use iterator for functions * FuzzRunIdentifiedContracts now uses Mutex * from_utf8_lossy instead of unsafe unchecked * use Mutex for runner of RandomCallGenerator * move RandomCallGenerator to its own module * write to fmt * small refactor: error.replay * remove newlines * add remaining is_invariant_test Co-authored-by: Brock --- Cargo.lock | 1 + cli/src/cmd/forge/coverage.rs | 20 +- cli/src/cmd/forge/snapshot.rs | 34 +- cli/src/cmd/forge/test/mod.rs | 93 ++-- cli/src/opts/multi_wallet.rs | 2 +- cli/tests/it/config.rs | 4 + common/src/traits.rs | 14 + config/src/lib.rs | 17 + evm/src/executor/builder.rs | 19 +- evm/src/executor/inspector/fuzzer.rs | 118 +++++ evm/src/executor/inspector/mod.rs | 7 + evm/src/executor/inspector/stack.rs | 6 +- evm/src/executor/mod.rs | 10 + evm/src/fuzz/invariant/call_override.rs | 102 +++++ evm/src/fuzz/invariant/executor.rs | 421 ++++++++++++++++++ evm/src/fuzz/invariant/mod.rs | 271 +++++++++++ evm/src/fuzz/mod.rs | 86 +++- evm/src/fuzz/strategies/invariants.rs | 169 +++++++ evm/src/fuzz/strategies/mod.rs | 6 +- evm/src/fuzz/strategies/param.rs | 28 +- evm/src/fuzz/strategies/state.rs | 53 ++- evm/src/lib.rs | 5 + evm/src/trace/mod.rs | 38 +- forge/Cargo.toml | 1 + forge/src/lib.rs | 27 ++ forge/src/multi_runner.rs | 123 +++-- forge/src/result.rs | 49 +- forge/src/runner.rs | 194 +++++++- testdata/foundry.toml | 2 + .../invariant/InvariantInnerContract.t.sol | 49 ++ .../fuzz/invariant/InvariantReentrancy.t.sol | 44 ++ testdata/fuzz/invariant/InvariantTest1.t.sol | 34 ++ .../invariant/target/ExcludeContracts.t.sol | 31 ++ .../invariant/target/TargetContracts.t.sol | 32 ++ .../invariant/target/TargetSelectors.t.sol | 41 ++ .../fuzz/invariant/target/TargetSenders.t.sol | 31 ++ 36 files changed, 2012 insertions(+), 170 deletions(-) create mode 100644 evm/src/executor/inspector/fuzzer.rs create mode 100644 evm/src/fuzz/invariant/call_override.rs create mode 100644 evm/src/fuzz/invariant/executor.rs create mode 100644 evm/src/fuzz/invariant/mod.rs create mode 100644 evm/src/fuzz/strategies/invariants.rs create mode 100644 testdata/fuzz/invariant/InvariantInnerContract.t.sol create mode 100644 testdata/fuzz/invariant/InvariantReentrancy.t.sol create mode 100644 testdata/fuzz/invariant/InvariantTest1.t.sol create mode 100644 testdata/fuzz/invariant/target/ExcludeContracts.t.sol create mode 100644 testdata/fuzz/invariant/target/TargetContracts.t.sol create mode 100644 testdata/fuzz/invariant/target/TargetSelectors.t.sol create mode 100644 testdata/fuzz/invariant/target/TargetSenders.t.sol diff --git a/Cargo.lock b/Cargo.lock index 2623443a10ea..b79ed7e2ad74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1897,6 +1897,7 @@ dependencies = [ "glob", "hex", "once_cell", + "parking_lot 0.12.0", "proptest", "rayon", "regex", diff --git a/cli/src/cmd/forge/coverage.rs b/cli/src/cmd/forge/coverage.rs index f01fad17dce9..64848d0ac8cd 100644 --- a/cli/src/cmd/forge/coverage.rs +++ b/cli/src/cmd/forge/coverage.rs @@ -26,7 +26,7 @@ use forge::{ executor::{inspector::CheatsConfig, opts::EvmOpts}, result::SuiteResult, trace::identifier::LocalTraceIdentifier, - MultiContractRunnerBuilder, + MultiContractRunnerBuilder, TestOptions, }; use foundry_common::{evm::EvmArgs, fs}; use foundry_config::{figment::Figment, Config}; @@ -251,16 +251,13 @@ impl CoverageArgs { config: Config, evm_opts: EvmOpts, ) -> eyre::Result<()> { - // Setup the fuzzer - // TODO: Add CLI Options to modify the persistence - let cfg = proptest::test_runner::Config { - failure_persistence: None, - cases: config.fuzz_runs, - max_local_rejects: config.fuzz_max_local_rejects, - max_global_rejects: config.fuzz_max_global_rejects, + let test_options = TestOptions { + fuzz_runs: config.fuzz_runs, + fuzz_max_local_rejects: config.fuzz_max_local_rejects, + fuzz_max_global_rejects: config.fuzz_max_global_rejects, ..Default::default() }; - let fuzzer = proptest::test_runner::TestRunner::new(cfg); + let root = project.paths.root; let env = evm_opts.evm_env_blocking(); @@ -268,12 +265,12 @@ impl CoverageArgs { // Build the contract runner let evm_spec = utils::evm_spec(&config.evm_version); let mut runner = MultiContractRunnerBuilder::default() - .fuzzer(fuzzer) .initial_balance(evm_opts.initial_balance) .evm_spec(evm_spec) .sender(evm_opts.sender) .with_fork(evm_opts.get_fork(&config, env.clone())) .with_cheats_config(CheatsConfig::new(&config, &evm_opts)) + .with_test_options(test_options) .set_coverage(true) .build(root.clone(), output, env, evm_opts)?; @@ -283,7 +280,8 @@ impl CoverageArgs { let local_identifier = LocalTraceIdentifier::new(&runner.known_contracts); // TODO: Coverage for fuzz tests - let handle = thread::spawn(move || runner.test(&self.filter, Some(tx)).unwrap()); + let handle = + thread::spawn(move || runner.test(&self.filter, Some(tx), Default::default()).unwrap()); for mut result in rx.into_iter().flat_map(|(_, suite)| suite.test_results.into_values()) { if let Some(hit_map) = result.coverage.take() { for (_, trace) in &mut result.traces { diff --git a/cli/src/cmd/forge/snapshot.rs b/cli/src/cmd/forge/snapshot.rs index b25e5b0ca01a..2332e1be0163 100644 --- a/cli/src/cmd/forge/snapshot.rs +++ b/cli/src/cmd/forge/snapshot.rs @@ -10,7 +10,7 @@ use crate::cmd::{ use clap::{Parser, ValueHint}; use ethers::types::U256; use eyre::Context; -use forge::result::TestKindGas; +use forge::result::TestKindReport; use once_cell::sync::Lazy; use regex::Regex; use std::{ @@ -210,7 +210,7 @@ impl SnapshotConfig { pub struct SnapshotEntry { pub contract_name: String, pub signature: String, - pub gas_used: TestKindGas, + pub gas_used: TestKindReport, } impl FromStr for SnapshotEntry { @@ -226,7 +226,9 @@ impl FromStr for SnapshotEntry { Some(SnapshotEntry { contract_name: file.as_str().to_string(), signature: sig.as_str().to_string(), - gas_used: TestKindGas::Standard(gas.as_str().parse().unwrap()), + gas_used: TestKindReport::Standard { + gas: gas.as_str().parse().unwrap(), + }, }) } else { cap.name("runs") @@ -237,10 +239,10 @@ impl FromStr for SnapshotEntry { .map(|(runs, avg, med)| SnapshotEntry { contract_name: file.as_str().to_string(), signature: sig.as_str().to_string(), - gas_used: TestKindGas::Fuzz { + gas_used: TestKindReport::Fuzz { runs: runs.as_str().parse().unwrap(), - median: med.as_str().parse().unwrap(), - mean: avg.as_str().parse().unwrap(), + median_gas: med.as_str().parse().unwrap(), + mean_gas: avg.as_str().parse().unwrap(), }, }) } @@ -274,13 +276,7 @@ fn write_to_snapshot_file( ) -> eyre::Result<()> { let mut out = String::new(); for test in tests { - writeln!( - out, - "{}:{} {}", - test.contract_name(), - test.signature, - test.result.kind.gas_used() - )?; + writeln!(out, "{}:{} {}", test.contract_name(), test.signature, test.result.kind.report())?; } Ok(fs::write(path, out)?) } @@ -289,8 +285,8 @@ fn write_to_snapshot_file( #[derive(Debug, Clone, Eq, PartialEq)] pub struct SnapshotDiff { pub signature: String, - pub source_gas_used: TestKindGas, - pub target_gas_used: TestKindGas, + pub source_gas_used: TestKindReport, + pub target_gas_used: TestKindReport, } impl SnapshotDiff { @@ -321,7 +317,7 @@ fn check(tests: Vec, snaps: Vec) -> bool { if let Some(target_gas) = snaps.get(&(test.contract_name().to_string(), test.signature.clone())).cloned() { - let source_gas = test.result.kind.gas_used(); + let source_gas = test.result.kind.report(); if source_gas.gas() != target_gas.gas() { eprintln!( "Diff in \"{}::{}\": consumed \"{}\" gas, expected \"{}\" gas ", @@ -363,7 +359,7 @@ fn diff(tests: Vec, snaps: Vec) -> eyre::Result<()> { })?; diffs.push(SnapshotDiff { - source_gas_used: test.result.kind.gas_used(), + source_gas_used: test.result.kind.report(), signature: test.signature, target_gas_used, }); @@ -429,7 +425,7 @@ mod tests { SnapshotEntry { contract_name: "Test".to_string(), signature: "deposit()".to_string(), - gas_used: TestKindGas::Standard(7222) + gas_used: TestKindReport::Standard { gas: 7222 } } ); } @@ -443,7 +439,7 @@ mod tests { SnapshotEntry { contract_name: "Test".to_string(), signature: "deposit()".to_string(), - gas_used: TestKindGas::Fuzz { runs: 256, median: 200, mean: 100 } + gas_used: TestKindReport::Fuzz { runs: 256, median_gas: 200, mean_gas: 100 } } ); } diff --git a/cli/src/cmd/forge/test/mod.rs b/cli/src/cmd/forge/test/mod.rs index dd16982d300b..86ef0d923dd0 100644 --- a/cli/src/cmd/forge/test/mod.rs +++ b/cli/src/cmd/forge/test/mod.rs @@ -8,6 +8,7 @@ use crate::{ compile::ProjectCompiler, suggestions, utils, }; +use cast::fuzz::CounterExample; use clap::{AppSettings, Parser}; use ethers::{solc::utils::RuntimeOrHandle, types::U256}; use forge::{ @@ -19,11 +20,10 @@ use forge::{ identifier::{EtherscanIdentifier, LocalTraceIdentifier}, CallTraceDecoderBuilder, TraceKind, }, - MultiContractRunner, MultiContractRunnerBuilder, + MultiContractRunner, MultiContractRunnerBuilder, TestOptions, }; use foundry_common::evm::EvmArgs; use foundry_config::{figment, figment::Figment, Config}; -use proptest::test_runner::{RngAlgorithm, TestRng}; use regex::Regex; use std::{collections::BTreeMap, path::PathBuf, sync::mpsc::channel, thread, time::Duration}; use tracing::trace; @@ -181,7 +181,7 @@ pub struct Test { impl Test { pub fn gas_used(&self) -> u64 { - self.result.kind.gas_used().gas() + self.result.kind.report().gas() } /// Returns the contract name of the artifact id @@ -280,47 +280,47 @@ fn short_test_result(name: &str, result: &TestResult) { let status = if result.success { Paint::green("[PASS]".to_string()) } else { - let txt = match (&result.reason, &result.counterexample) { - (Some(ref reason), Some(ref counterexample)) => { - format!("[FAIL. Reason: {reason}. Counterexample: {counterexample}]") - } - (None, Some(ref counterexample)) => { - format!("[FAIL. Counterexample: {counterexample}]") - } - (Some(ref reason), None) => { - format!("[FAIL. Reason: {reason}]") - } - (None, None) => "[FAIL]".to_string(), - }; + let reason = result + .reason + .as_ref() + .map(|reason| format!("Reason: {reason}")) + .unwrap_or_else(|| "Reason: Undefined.".to_string()); + + let counterexample = result + .counterexample + .as_ref() + .map(|example| match example { + CounterExample::Single(eg) => format!(" Counterexample: {eg}]"), + CounterExample::Sequence(sequence) => { + let mut inner_txt = String::new(); + + for checkpoint in sequence { + inner_txt += format!("\t\t{checkpoint}\n").as_str(); + } + format!("]\n\t[Sequence]\n{inner_txt}\n") + } + }) + .unwrap_or_else(|| "]".to_string()); - Paint::red(txt) + Paint::red(format!("[FAIL. {reason}{counterexample}")) }; - println!("{} {} {}", status, name, result.kind.gas_used()); + println!("{} {} {}", status, name, result.kind.report()); } pub fn custom_run(args: TestArgs) -> eyre::Result { // Merge all configs let (config, mut evm_opts) = args.config_and_evm_opts()?; - // Setup the fuzzer - // TODO: Add CLI Options to modify the persistence - let cfg = proptest::test_runner::Config { - failure_persistence: None, - cases: config.fuzz_runs, - max_local_rejects: config.fuzz_max_local_rejects, - max_global_rejects: config.fuzz_max_global_rejects, - ..Default::default() - }; - - let fuzzer = if let Some(ref fuzz_seed) = config.fuzz_seed { - let mut bytes: [u8; 32] = [0; 32]; - fuzz_seed.to_big_endian(&mut bytes); - trace!(target: "forge::test", "executing test command"); - let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &bytes); - proptest::test_runner::TestRunner::new_with_rng(cfg, rng) - } else { - proptest::test_runner::TestRunner::new(cfg) + let test_options = TestOptions { + fuzz_runs: config.fuzz_runs, + fuzz_max_local_rejects: config.fuzz_max_local_rejects, + fuzz_max_global_rejects: config.fuzz_max_global_rejects, + fuzz_seed: config.fuzz_seed, + invariant_runs: config.invariant_runs, + invariant_depth: config.invariant_depth, + invariant_fail_on_revert: config.invariant_fail_on_revert, + invariant_call_override: config.invariant_call_override, }; let mut filter = args.filter(&config); @@ -350,20 +350,21 @@ pub fn custom_run(args: TestArgs) -> eyre::Result { let evm_spec = utils::evm_spec(&config.evm_version); let mut runner = MultiContractRunnerBuilder::default() - .fuzzer(fuzzer) .initial_balance(evm_opts.initial_balance) .evm_spec(evm_spec) .sender(evm_opts.sender) .with_fork(evm_opts.get_fork(&config, env.clone())) .with_cheats_config(CheatsConfig::new(&config, &evm_opts)) + .with_test_options(test_options) .build(project.paths.root, output, env, evm_opts)?; if args.debug.is_some() { filter.test_pattern = args.debug; + match runner.count_filtered_tests(&filter) { 1 => { // Run the test - let results = runner.test(&filter, None)?; + let results = runner.test(&filter, None, test_options)?; // Get the result of the single test let (id, sig, test_kind, counterexample) = results.iter().map(|(id, SuiteResult{ test_results, .. })| { @@ -375,7 +376,7 @@ pub fn custom_run(args: TestArgs) -> eyre::Result { // Build debugger args if this is a fuzz test let sig = match test_kind { TestKind::Fuzz(cases) => { - if let Some(counterexample) = counterexample { + if let Some(CounterExample::Single(counterexample)) = counterexample { counterexample.calldata.to_string() } else { cases.cases().first().expect("no fuzz cases run").calldata.to_string() @@ -407,7 +408,16 @@ pub fn custom_run(args: TestArgs) -> eyre::Result { } else if args.list { list(runner, filter, args.json) } else { - test(config, runner, verbosity, filter, args.json, args.allow_failure, args.gas_report) + test( + config, + runner, + verbosity, + filter, + args.json, + args.allow_failure, + test_options, + args.gas_report, + ) } } @@ -438,6 +448,7 @@ fn test( filter: Filter, json: bool, allow_failure: bool, + test_options: TestOptions, gas_reporting: bool, ) -> eyre::Result { trace!(target: "forge::test", "running all tests"); @@ -462,7 +473,7 @@ fn test( } if json { - let results = runner.test(&filter, None)?; + let results = runner.test(&filter, None, test_options)?; println!("{}", serde_json::to_string(&results)?); Ok(TestOutcome::new(results, allow_failure)) } else { @@ -483,7 +494,7 @@ fn test( let (tx, rx) = channel::<(String, SuiteResult)>(); // Run tests - let handle = thread::spawn(move || runner.test(&filter, Some(tx)).unwrap()); + let handle = thread::spawn(move || runner.test(&filter, Some(tx), test_options).unwrap()); let mut results: BTreeMap = BTreeMap::new(); let mut gas_report = GasReport::new(config.gas_reports); diff --git a/cli/src/opts/multi_wallet.rs b/cli/src/opts/multi_wallet.rs index c9081d7b6643..be4fbb1b4e71 100644 --- a/cli/src/opts/multi_wallet.rs +++ b/cli/src/opts/multi_wallet.rs @@ -219,7 +219,7 @@ impl MultiWallet { } ); - let mut error_msg = "".to_string(); + let mut error_msg = String::new(); // This is an actual used address if addresses.contains(&Config::DEFAULT_SENDER) { diff --git a/cli/tests/it/config.rs b/cli/tests/it/config.rs index e129569447c0..dbb2efcf9734 100644 --- a/cli/tests/it/config.rs +++ b/cli/tests/it/config.rs @@ -60,6 +60,10 @@ forgetest!(can_extract_config_values, |prj: TestProject, mut cmd: TestCommand| { fuzz_max_local_rejects: 2000, fuzz_max_global_rejects: 100203, fuzz_seed: Some(1000.into()), + invariant_runs: 256, + invariant_depth: 15, + invariant_fail_on_revert: false, + invariant_call_override: false, ffi: true, sender: "00a329c0648769A73afAc7F9381D08FB43dBEA72".parse().unwrap(), tx_origin: "00a329c0648769A73afAc7F9F81E08FB43dBEA72".parse().unwrap(), diff --git a/common/src/traits.rs b/common/src/traits.rs index 97d7a88f6718..1669635a5f4b 100644 --- a/common/src/traits.rs +++ b/common/src/traits.rs @@ -14,6 +14,8 @@ pub trait TestFilter: Send + Sync { /// Extension trait for `Function` pub trait TestFunctionExt { + /// Whether this function should be executed as invariant test + fn is_invariant_test(&self) -> bool; /// Whether this function should be executed as fuzz test fn is_fuzz_test(&self) -> bool; /// Whether this function is a test @@ -25,6 +27,10 @@ pub trait TestFunctionExt { } impl TestFunctionExt for Function { + fn is_invariant_test(&self) -> bool { + self.name.is_invariant_test() + } + fn is_fuzz_test(&self) -> bool { // test functions that have inputs are considered fuzz tests as those inputs will be fuzzed !self.inputs.is_empty() @@ -44,6 +50,10 @@ impl TestFunctionExt for Function { } impl<'a> TestFunctionExt for &'a str { + fn is_invariant_test(&self) -> bool { + self.starts_with("invariant") + } + fn is_fuzz_test(&self) -> bool { unimplemented!("no naming convention for fuzz tests.") } @@ -62,6 +72,10 @@ impl<'a> TestFunctionExt for &'a str { } impl TestFunctionExt for String { + fn is_invariant_test(&self) -> bool { + self.as_str().is_invariant_test() + } + fn is_fuzz_test(&self) -> bool { self.as_str().is_fuzz_test() } diff --git a/config/src/lib.rs b/config/src/lib.rs index 456d7287e2bb..583d3d94958f 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -187,6 +187,15 @@ pub struct Config { pub path_pattern_inverse: Option, /// The number of test cases that must execute for each property test pub fuzz_runs: u32, + /// The number of runs that must execute for each invariant test group. + pub invariant_runs: u32, + /// The number of calls executed to attempt to break invariants in one run. + pub invariant_depth: u32, + /// Fails the invariant fuzzing if a revert occurs + pub invariant_fail_on_revert: bool, + /// Allows overriding an unsafe external call when running invariant tests. eg. reetrancy + /// checks + pub invariant_call_override: bool, /// Whether to allow ffi cheatcodes in test pub ffi: bool, /// The address which will be executing all tests @@ -1466,6 +1475,10 @@ impl Default for Config { fuzz_max_local_rejects: 1024, fuzz_max_global_rejects: 65536, fuzz_seed: None, + invariant_runs: 256, + invariant_depth: 15, + invariant_fail_on_revert: false, + invariant_call_override: false, ffi: false, sender: Config::DEFAULT_SENDER, tx_origin: Config::DEFAULT_SENDER, @@ -2857,6 +2870,10 @@ mod tests { fuzz_max_local_rejects = 1024 fuzz_runs = 256 fuzz_seed = '0x3e8' + invariant_runs = 256 + invariant_depth = 15 + invariant_fail_on_revert = false + invariant_call_override = false gas_limit = 9223372036854775807 gas_price = 0 gas_reports = ['*'] diff --git a/evm/src/executor/builder.rs b/evm/src/executor/builder.rs index a2ec987cbb4a..7c6e04f6b4fd 100644 --- a/evm/src/executor/builder.rs +++ b/evm/src/executor/builder.rs @@ -1,10 +1,12 @@ use super::{ - inspector::{Cheatcodes, InspectorStackConfig}, + inspector::{Cheatcodes, Fuzzer, InspectorStackConfig}, Executor, }; -use crate::executor::{backend::Backend, inspector::CheatsConfig}; +use crate::{ + executor::{backend::Backend, inspector::CheatsConfig}, + fuzz::{invariant::RandomCallGenerator, strategies::EvmFuzzState}, +}; use ethers::types::U256; - use revm::{Env, SpecId}; #[derive(Default, Debug)] @@ -48,6 +50,17 @@ impl ExecutorBuilder { self } + /// Enables the fuzzer for data collection and maybe call overriding + #[must_use] + pub fn with_fuzzer( + mut self, + call_generator: Option, + fuzz_state: EvmFuzzState, + ) -> Self { + self.inspector_config.fuzzer = Some(Fuzzer { call_generator, fuzz_state, collect: false }); + self + } + /// Sets the EVM spec to use #[must_use] pub fn with_spec(mut self, spec: SpecId) -> Self { diff --git a/evm/src/executor/inspector/fuzzer.rs b/evm/src/executor/inspector/fuzzer.rs new file mode 100644 index 000000000000..5b254c246c99 --- /dev/null +++ b/evm/src/executor/inspector/fuzzer.rs @@ -0,0 +1,118 @@ +use crate::{ + fuzz::{invariant::RandomCallGenerator, strategies::EvmFuzzState}, + utils, +}; +use bytes::Bytes; +use revm::{db::Database, CallInputs, CallScheme, EVMData, Gas, Inspector, Interpreter, Return}; + +/// An inspector that can fuzz and collect data for that effect. +#[derive(Clone, Debug)] +pub struct Fuzzer { + /// Given a strategy, it generates a random call. + pub call_generator: Option, + /// If set, it collects `stack` and `memory` values for fuzzing purposes. + pub collect: bool, + /// If `collect` is set, we store the collected values in this fuzz dictionary. + pub fuzz_state: EvmFuzzState, +} + +impl Inspector for Fuzzer +where + DB: Database, +{ + fn call( + &mut self, + data: &mut EVMData<'_, DB>, + call: &mut CallInputs, + _: bool, + ) -> (Return, Gas, Bytes) { + // We don't want to override the very first call made to the test contract. + if self.call_generator.is_some() && data.env.tx.caller != call.context.caller { + self.override_call(call); + } + + // We only collect `stack` and `memory` data before and after calls. + // this will be turned off on the next `step` + self.collect = true; + + (Return::Continue, Gas::new(call.gas_limit), Bytes::new()) + } + + fn step( + &mut self, + interpreter: &mut Interpreter, + _: &mut EVMData<'_, DB>, + _is_static: bool, + ) -> Return { + // We only collect `stack` and `memory` data before and after calls. + if self.collect { + self.collect_data(interpreter); + self.collect = false; + } + Return::Continue + } + + fn call_end( + &mut self, + _: &mut EVMData<'_, DB>, + _: &CallInputs, + remaining_gas: Gas, + status: Return, + retdata: Bytes, + _: bool, + ) -> (Return, Gas, Bytes) { + if let Some(ref mut call_generator) = self.call_generator { + call_generator.used = false; + } + + // We only collect `stack` and `memory` data before and after calls. + // this will be turned off on the next `step` + self.collect = true; + + (status, remaining_gas, retdata) + } +} + +impl Fuzzer { + /// Collects `stack` and `memory` values into the fuzz dictionary. + fn collect_data(&mut self, interpreter: &mut Interpreter) { + let mut state = self.fuzz_state.write(); + + for slot in interpreter.stack().data() { + state.insert(utils::u256_to_h256_be(*slot).into()); + } + + for index in 0..interpreter.memory.len() / 32 { + let mut slot = [0u8; 32]; + slot.clone_from_slice(interpreter.memory.get_slice(index * 32, 32)); + + state.insert(slot); + } + } + + /// Overrides an external call and tries to call any method of msg.sender. + fn override_call(&mut self, call: &mut CallInputs) { + if let Some(ref mut call_generator) = self.call_generator { + // We only override external calls which are not coming from the test contract. + if call.context.caller != call_generator.test_address && + call.context.scheme == CallScheme::Call && + !call_generator.used + { + // There's only a 30% chance that an override happens. + if let Some((sender, (contract, input))) = + call_generator.next(call.context.caller, call.contract) + { + call.input = input.0; + call.context.caller = sender; + call.contract = contract; + + // TODO: in what scenarios can the following be problematic + call.context.code_address = contract; + call.context.address = contract; + + call_generator.used = true; + } + } + } + } +} diff --git a/evm/src/executor/inspector/mod.rs b/evm/src/executor/inspector/mod.rs index 113ef502f9e7..d8fd767090b3 100644 --- a/evm/src/executor/inspector/mod.rs +++ b/evm/src/executor/inspector/mod.rs @@ -23,6 +23,9 @@ use ethers::types::U256; use revm::BlockEnv; +mod fuzzer; +pub use fuzzer::Fuzzer; + #[derive(Default, Clone, Debug)] pub struct InspectorStackConfig { /// The cheatcode inspector and its state, if cheatcodes are enabled. @@ -42,6 +45,8 @@ pub struct InspectorStackConfig { pub tracing: bool, /// Whether or not the debugger is enabled pub debugger: bool, + /// The fuzzer inspector and its state, if it exists. + pub fuzzer: Option, /// Whether or not coverage info should be collected pub coverage: bool, } @@ -63,6 +68,8 @@ impl InspectorStackConfig { if self.debugger { stack.debugger = Some(Debugger::default()); } + stack.fuzzer = self.fuzzer.clone(); + if self.coverage { stack.coverage = Some(CoverageCollector::default()); } diff --git a/evm/src/executor/inspector/stack.rs b/evm/src/executor/inspector/stack.rs index 684a12bc20a9..e1578c9b78ce 100644 --- a/evm/src/executor/inspector/stack.rs +++ b/evm/src/executor/inspector/stack.rs @@ -1,4 +1,4 @@ -use super::{Cheatcodes, Debugger, LogCollector, Tracer}; +use super::{Cheatcodes, Debugger, Fuzzer, LogCollector, Tracer}; use crate::{ coverage::HitMaps, debug::DebugArena, @@ -41,6 +41,7 @@ pub struct InspectorStack { pub logs: Option, pub cheatcodes: Option, pub debugger: Option, + pub fuzzer: Option, pub coverage: Option, } @@ -102,6 +103,7 @@ where call_inspectors!( inspector, [ + &mut self.fuzzer, &mut self.debugger, &mut self.tracer, &mut self.coverage, @@ -165,6 +167,7 @@ where call_inspectors!( inspector, [ + &mut self.fuzzer, &mut self.debugger, &mut self.tracer, &mut self.coverage, @@ -196,6 +199,7 @@ where call_inspectors!( inspector, [ + &mut self.fuzzer, &mut self.debugger, &mut self.tracer, &mut self.coverage, diff --git a/evm/src/executor/mod.rs b/evm/src/executor/mod.rs index cd0b9e576fd0..aae6bc9a738a 100644 --- a/evm/src/executor/mod.rs +++ b/evm/src/executor/mod.rs @@ -106,6 +106,16 @@ impl Executor { &self.backend } + /// Returns an immutable reference to the InspectorStackConfig + pub fn inspector_config(&self) -> &InspectorStackConfig { + &self.inspector_config + } + + /// Returns a mutable reference to the InspectorStackConfig + pub fn inspector_config_mut(&mut self) -> &mut InspectorStackConfig { + &mut self.inspector_config + } + /// Creates the default CREATE2 Contract Deployer for local tests and scripts. pub fn deploy_create2_deployer(&mut self) -> eyre::Result<()> { let create2_deployer_account = self.backend_mut().basic(DEFAULT_CREATE2_DEPLOYER); diff --git a/evm/src/fuzz/invariant/call_override.rs b/evm/src/fuzz/invariant/call_override.rs new file mode 100644 index 000000000000..f06fe6717c82 --- /dev/null +++ b/evm/src/fuzz/invariant/call_override.rs @@ -0,0 +1,102 @@ +use super::BasicTxDetails; +use crate::executor::Executor; +use ethers::types::{Address, Bytes}; +use parking_lot::{Mutex, RwLock}; +use proptest::{ + option::weighted, + strategy::{SBoxedStrategy, Strategy, ValueTree}, + test_runner::TestRunner, +}; +use std::sync::Arc; + +/// Given a TestRunner and a strategy, it generates calls. Used inside the Fuzzer inspector to +/// override external calls to test for potential reentrancy vulnerabilities.. +#[derive(Debug, Clone)] +pub struct RandomCallGenerator { + /// Address of the test contract. + pub test_address: Address, + /// Runner that will generate the call from the strategy. + pub runner: Arc>, + /// Strategy to be used to generate calls from `target_reference`. + pub strategy: SBoxedStrategy>, + /// Reference to which contract we want a fuzzed calldata from. + pub target_reference: Arc>, + /// Flag to know if a call has been overriden. Don't allow nesting for now. + pub used: bool, + /// If set to `true`, consumes the next call from `last_sequence`, otherwise queries it from + /// the strategy. + pub replay: bool, + /// Saves the sequence of generated calls that can be replayed later on. + pub last_sequence: Arc>>>, +} + +impl RandomCallGenerator { + pub fn new( + test_address: Address, + runner: TestRunner, + strategy: SBoxedStrategy<(Address, Bytes)>, + target_reference: Arc>, + ) -> Self { + let strategy = weighted(0.9, strategy).sboxed(); + + RandomCallGenerator { + test_address, + runner: Arc::new(Mutex::new(runner)), + strategy, + target_reference, + last_sequence: Arc::new(RwLock::new(vec![])), + replay: false, + used: false, + } + } + + /// All `self.next()` calls will now pop `self.last_sequence`. Used to replay an invariant + /// failure. + pub fn set_replay(&mut self, status: bool) { + self.replay = status; + if status { + // So it can later be popped. + self.last_sequence.write().reverse(); + } + } + + /// Gets the next call. Random if replay is not set. Otherwise, it pops from `last_sequence`. + pub fn next( + &mut self, + original_caller: Address, + original_target: Address, + ) -> Option { + if self.replay { + self.last_sequence.write().pop().expect( + "to have same size as the number of (unsafe) external calls of the sequence.", + ) + } else { + // TODO: Do we want it to be 80% chance only too ? + let new_caller = original_target; + + // Set which contract we mostly (80% chance) want to generate calldata from. + *self.target_reference.write() = original_caller; + + // `original_caller` has a 80% chance of being the `new_target`. + let choice = self + .strategy + .new_tree(&mut self.runner.lock()) + .unwrap() + .current() + .map(|(new_target, calldata)| (new_caller, (new_target, calldata))); + + self.last_sequence.write().push(choice.clone()); + choice + } + } +} + +/// Sets up the calls generated by the internal fuzzer, if they exist. +pub fn set_up_inner_replay(executor: &mut Executor, inner_sequence: &[Option]) { + if let Some(ref mut fuzzer) = executor.inspector_config_mut().fuzzer { + if let Some(ref mut call_generator) = fuzzer.call_generator { + call_generator.last_sequence = Arc::new(RwLock::new(inner_sequence.to_owned())); + call_generator.set_replay(true); + } + } +} diff --git a/evm/src/fuzz/invariant/executor.rs b/evm/src/fuzz/invariant/executor.rs new file mode 100644 index 000000000000..34172886f25c --- /dev/null +++ b/evm/src/fuzz/invariant/executor.rs @@ -0,0 +1,421 @@ +use super::{ + assert_invariants, BasicTxDetails, FuzzRunIdentifiedContracts, InvariantContract, + InvariantFuzzError, InvariantFuzzTestResult, InvariantTestOptions, RandomCallGenerator, + TargetedContracts, +}; +use crate::{ + executor::{ + inspector::Fuzzer, Executor, RawCallResult, CHEATCODE_ADDRESS, HARDHAT_CONSOLE_ADDRESS, + }, + fuzz::{ + strategies::{ + build_initial_state, collect_created_contracts, collect_state_from_call, + invariant_strat, override_call_strat, EvmFuzzState, + }, + FuzzCase, FuzzedCases, + }, + CALLER, +}; +use ethers::{ + abi::{Abi, Address, Detokenize, FixedBytes, Function, Tokenizable, TokenizableItem}, + prelude::{ArtifactId, U256}, +}; +use eyre::ContextCompat; +use parking_lot::{Mutex, RwLock}; +use proptest::{ + strategy::{BoxedStrategy, Strategy, ValueTree}, + test_runner::{TestCaseError, TestRunner}, +}; +use revm::DatabaseCommit; +use std::{cell::RefCell, collections::BTreeMap, sync::Arc}; +use tracing::warn; + +/// Alias for (Dictionary for fuzzing, initial contracts to fuzz and an InvariantStrategy). +type InvariantPreparation = + (EvmFuzzState, FuzzRunIdentifiedContracts, BoxedStrategy>); + +/// Wrapper around any [`Executor`] implementor which provides fuzzing support using [`proptest`](https://docs.rs/proptest/1.0.0/proptest/). +/// +/// After instantiation, calling `fuzz` will proceed to hammer the deployed smart contracts with +/// inputs, until it finds a counterexample sequence. The provided [`TestRunner`] contains all the +/// configuration which can be overridden via [environment variables](https://docs.rs/proptest/1.0.0/proptest/test_runner/struct.Config.html) +pub struct InvariantExecutor<'a> { + pub executor: &'a mut Executor, + /// Proptest runner. + runner: TestRunner, + /// Contracts deployed with `setUp()` + setup_contracts: &'a BTreeMap, + /// Contracts that are part of the project but have not been deployed yet. We need the bytecode + /// to identify them from the stateset changes. + project_contracts: &'a BTreeMap)>, +} + +impl<'a> InvariantExecutor<'a> { + /// Instantiates a fuzzed executor EVM given a testrunner + pub fn new( + executor: &'a mut Executor, + runner: TestRunner, + setup_contracts: &'a BTreeMap, + project_contracts: &'a BTreeMap)>, + ) -> Self { + Self { executor, runner, setup_contracts, project_contracts } + } + + /// Fuzzes any deployed contract and checks any broken invariant at `invariant_address` + /// Returns a list of all the consumed gas and calldata of every invariant fuzz case + pub fn invariant_fuzz( + &mut self, + invariant_contract: InvariantContract, + test_options: InvariantTestOptions, + ) -> eyre::Result> { + let (fuzz_state, targeted_contracts, strat) = + self.prepare_fuzzing(&invariant_contract, test_options)?; + + // Stores the consumed gas and calldata of every successful fuzz call. + let fuzz_cases: RefCell> = RefCell::new(Default::default()); + + // Stores data related to reverts or failed assertions of the test. + let failures = + RefCell::new(InvariantFailures::new(&invariant_contract.invariant_functions)); + + let blank_executor = RefCell::new(&mut *self.executor); + + // Make sure invariants are sound even before starting to fuzz + if assert_invariants( + &invariant_contract, + &blank_executor.borrow(), + &[], + &mut failures.borrow_mut(), + ) + .is_err() + { + fuzz_cases.borrow_mut().push(FuzzedCases::new(vec![])); + } + + if failures.borrow().broken_invariants_count < invariant_contract.invariant_functions.len() + { + // The strategy only comes with the first `input`. We fill the rest of the `inputs` + // until the desired `depth` so we can use the evolving fuzz dictionary + // during the run. We need another proptest runner to query for random + // values. + let branch_runner = RefCell::new(self.runner.clone()); + let _ = self.runner.run(&strat, |mut inputs| { + // Scenarios where we want to fail as soon as possible. + { + if test_options.fail_on_revert && failures.borrow().reverts == 1 { + return Err(TestCaseError::fail("Revert occurred.")) + } + + if failures.borrow().broken_invariants_count == + invariant_contract.invariant_functions.len() + { + return Err(TestCaseError::fail("All invariants have been broken.")) + } + } + + // Before each run, we must reset the backend state. + let mut executor = blank_executor.borrow().clone(); + + // Used for stat reports (eg. gas usage). + let mut fuzz_runs = vec![]; + + // Created contracts during a run. + let mut created_contracts = vec![]; + + 'fuzz_run: for _ in 0..test_options.depth { + let (sender, (address, calldata)) = + inputs.last().expect("to have the next randomly generated input."); + + // Executes the call from the randomly generated sequence. + let call_result = executor + .call_raw(*sender, *address, calldata.0.clone(), U256::zero()) + .expect("could not make raw evm call"); + + // Collect data for fuzzing from the state changeset. + let state_changeset = + call_result.state_changeset.to_owned().expect("to have a state changeset."); + + collect_state_from_call( + &call_result.logs, + &state_changeset, + fuzz_state.clone(), + ); + collect_created_contracts( + &state_changeset, + self.project_contracts, + self.setup_contracts, + targeted_contracts.clone(), + &mut created_contracts, + ); + + // Commit changes to the database. + executor.backend_mut().commit(state_changeset); + + fuzz_runs.push(FuzzCase { + calldata: calldata.clone(), + gas: call_result.gas, + stipend: call_result.stipend, + }); + + if !can_continue( + &invariant_contract, + call_result, + &executor, + &inputs, + &mut failures.borrow_mut(), + test_options, + ) { + break 'fuzz_run + } + + // Generates the next call from the run using the recently updated + // dictionary. + inputs.extend( + strat + .new_tree(&mut branch_runner.borrow_mut()) + .map_err(|_| TestCaseError::Fail("Could not generate case".into()))? + .current(), + ); + } + + // We clear all the targeted contracts created during this run. + if !created_contracts.is_empty() { + let mut writable_targeted = targeted_contracts.lock(); + for addr in created_contracts.iter() { + writable_targeted.remove(addr); + } + } + + fuzz_cases.borrow_mut().push(FuzzedCases::new(fuzz_runs)); + + Ok(()) + }); + } + + let (reverts, invariants) = failures.into_inner().into_inner(); + + Ok(Some(InvariantFuzzTestResult { invariants, cases: fuzz_cases.into_inner(), reverts })) + } + + /// Prepares certain structures to execute the invariant tests: + /// * Fuzz dictionary + /// * Targeted contracts + /// * Invariant Strategy + fn prepare_fuzzing( + &mut self, + invariant_contract: &InvariantContract, + test_options: InvariantTestOptions, + ) -> eyre::Result { + // Finds out the chosen deployed contracts and/or senders. + let (targeted_senders, targeted_contracts) = + self.select_contracts_and_senders(invariant_contract.address, invariant_contract.abi)?; + + if targeted_contracts.is_empty() { + eyre::bail!("No contracts to fuzz."); + } + + // Stores fuzz state for use with [fuzz_calldata_from_state]. + let fuzz_state: EvmFuzzState = build_initial_state(self.executor.backend().mem_db()); + + // During execution, any newly created contract is added here and used through the rest of + // the fuzz run. + let targeted_contracts: FuzzRunIdentifiedContracts = + Arc::new(Mutex::new(targeted_contracts)); + + // Creates the invariant strategy. + let strat = + invariant_strat(fuzz_state.clone(), targeted_senders, targeted_contracts.clone()) + .no_shrink() + .boxed(); + + // Allows `override_call_strat` to use the address given by the Fuzzer inspector during + // EVM execution. + let mut call_generator = None; + if test_options.call_override { + let target_contract_ref = Arc::new(RwLock::new(Address::zero())); + + call_generator = Some(RandomCallGenerator::new( + invariant_contract.address, + self.runner.clone(), + override_call_strat( + fuzz_state.clone(), + targeted_contracts.clone(), + target_contract_ref.clone(), + ), + target_contract_ref, + )); + } + + self.executor.inspector_config_mut().fuzzer = + Some(Fuzzer { call_generator, fuzz_state: fuzz_state.clone(), collect: true }); + + // Tracing should be off when running all runs. It will be turned on later for the failure + // cases. + self.executor.set_tracing(false); + + Ok((fuzz_state, targeted_contracts, strat)) + } + + /// Selects senders and contracts based on the contract methods `targetSenders() -> address[]`, + /// `targetContracts() -> address[]` and `excludeContracts() -> address[]`. + pub fn select_contracts_and_senders( + &self, + invariant_address: Address, + abi: &Abi, + ) -> eyre::Result<(Vec
, TargetedContracts)> { + let [senders, selected, excluded] = + ["targetSenders", "targetContracts", "excludeContracts"] + .map(|method| self.get_list::
(invariant_address, abi, method)); + + let mut contracts: TargetedContracts = self + .setup_contracts + .clone() + .into_iter() + .filter(|(addr, _)| { + *addr != invariant_address && + *addr != CHEATCODE_ADDRESS && + *addr != HARDHAT_CONSOLE_ADDRESS && + (selected.is_empty() || selected.contains(addr)) && + (excluded.is_empty() || !excluded.contains(addr)) + }) + .map(|(addr, (name, abi))| (addr, (name, abi, vec![]))) + .collect(); + + self.select_selectors(invariant_address, abi, &mut contracts)?; + + Ok((senders, contracts)) + } + + /// Selects the functions to fuzz based on the contract method `targetSelectors() -> (address, + /// bytes4[])[]`. + pub fn select_selectors( + &self, + address: Address, + abi: &Abi, + targeted_contracts: &mut TargetedContracts, + ) -> eyre::Result<()> { + let selectors = + self.get_list::<(Address, Vec)>(address, abi, "targetSelectors"); + + fn get_function(name: &str, selector: FixedBytes, abi: &Abi) -> eyre::Result { + abi.functions() + .into_iter() + .find(|func| func.short_signature().as_slice() == selector.as_slice()) + .cloned() + .wrap_err(format!("{name} does not have the selector {:?}", selector)) + } + + for (address, bytes4_array) in selectors.into_iter() { + if let Some((name, abi, address_selectors)) = targeted_contracts.get_mut(&address) { + // The contract is already part of our filter, and all we do is specify that we're + // only looking at specific functions coming from `bytes4_array`. + for selector in bytes4_array { + address_selectors.push(get_function(name, selector, abi)?); + } + } else { + let (name, abi) = self.setup_contracts.get(&address).wrap_err(format!( + "[targetSelectors] address does not have an associated contract: {}", + address + ))?; + + let functions = bytes4_array + .into_iter() + .map(|selector| get_function(name, selector, abi)) + .collect::, _>>()?; + + targeted_contracts.insert(address, (name.to_string(), abi.clone(), functions)); + } + } + Ok(()) + } + + /// Gets list of `T` by calling the contract `method_name` function. + fn get_list(&self, address: Address, abi: &Abi, method_name: &str) -> Vec + where + T: Tokenizable + Detokenize + TokenizableItem, + { + if let Some(func) = abi.functions().into_iter().find(|func| func.name == method_name) { + if let Ok(call_result) = self.executor.call::, _, _>( + CALLER, + address, + func.clone(), + (), + U256::zero(), + Some(abi), + ) { + return call_result.result + } else { + warn!( + "The function {} was found but there was an error querying its data.", + method_name + ); + } + }; + + Vec::new() + } +} + +/// Verifies that the invariant run execution can continue. +fn can_continue( + invariant_contract: &InvariantContract, + call_result: RawCallResult, + executor: &Executor, + calldata: &[BasicTxDetails], + failures: &mut InvariantFailures, + test_options: InvariantTestOptions, +) -> bool { + if !call_result.reverted { + if assert_invariants(invariant_contract, executor, calldata, failures).is_err() { + return false + } + } else { + failures.reverts += 1; + + // The user might want to stop all execution if a revert happens to + // better bound their testing space. + if test_options.fail_on_revert { + let error = + InvariantFuzzError::new(invariant_contract, None, calldata, call_result, &[]); + + failures.revert_reason = Some(error.revert_reason.clone()); + + // Hacky to provide the full error to the user. + for invariant in invariant_contract.invariant_functions.iter() { + failures.failed_invariants.insert(invariant.name.clone(), Some(error.clone())); + } + + return false + } + } + true +} + +#[derive(Clone)] +/// Stores information about failures and reverts of the invariant tests. +pub struct InvariantFailures { + /// The latest revert reason of a run. + pub revert_reason: Option, + /// Total number of reverts. + pub reverts: usize, + /// How many different invariants have been broken. + pub broken_invariants_count: usize, + /// Maps a broken invariant to its specific error. + pub failed_invariants: BTreeMap>, +} + +impl InvariantFailures { + fn new(invariants: &[&Function]) -> Self { + InvariantFailures { + reverts: 0, + broken_invariants_count: 0, + failed_invariants: invariants.iter().map(|f| (f.name.to_string(), None)).collect(), + revert_reason: None, + } + } + + /// Moves `reverts` and `failed_invariants` out of the struct. + fn into_inner(self) -> (usize, BTreeMap>) { + (self.reverts, self.failed_invariants) + } +} diff --git a/evm/src/fuzz/invariant/mod.rs b/evm/src/fuzz/invariant/mod.rs new file mode 100644 index 000000000000..cbc566e119e4 --- /dev/null +++ b/evm/src/fuzz/invariant/mod.rs @@ -0,0 +1,271 @@ +//! Fuzzing support abstracted over the [`Evm`](crate::Evm) used +use crate::{ + fuzz::*, + trace::{load_contracts, TraceKind}, + CALLER, +}; +mod call_override; +pub use call_override::{set_up_inner_replay, RandomCallGenerator}; +mod executor; +use crate::{ + decode::decode_revert, + executor::{Executor, RawCallResult}, +}; +use ethers::{ + abi::{Abi, Function}, + solc::ArtifactId, + types::{Address, Bytes, U256}, +}; +pub use executor::{InvariantExecutor, InvariantFailures}; +use parking_lot::Mutex; +pub use proptest::test_runner::Config as FuzzConfig; +use proptest::test_runner::TestError; +use std::{collections::BTreeMap, sync::Arc}; + +pub type TargetedContracts = BTreeMap)>; +pub type FuzzRunIdentifiedContracts = Arc>; + +/// (Sender, (TargetContract, Calldata)) +pub type BasicTxDetails = (Address, (Address, Bytes)); + +/// Test contract which is testing its invariants. +#[derive(Debug, Clone)] +pub struct InvariantContract<'a> { + /// Address of the test contract. + pub address: Address, + /// Invariant functions present in the test contract. + pub invariant_functions: Vec<&'a Function>, + /// Abi of the test contract. + pub abi: &'a Abi, +} + +/// Metadata on how to run invariant tests +#[derive(Debug, Clone, Copy, Default)] +pub struct InvariantTestOptions { + /// The number of calls executed to attempt to break invariants in one run. + pub depth: u32, + /// Fails the invariant fuzzing if a revert occurs + pub fail_on_revert: bool, + /// Allows overriding an unsafe external call when running invariant tests. eg. reetrancy + /// checks + pub call_override: bool, +} + +/// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the +/// external `invariant_failures.failed_invariant` map and returns a generic error. +pub fn assert_invariants( + invariant_contract: &InvariantContract, + executor: &Executor, + calldata: &[BasicTxDetails], + invariant_failures: &mut InvariantFailures, +) -> eyre::Result<()> { + let mut found_case = false; + let mut inner_sequence = vec![]; + + if let Some(ref fuzzer) = executor.inspector_config().fuzzer { + if let Some(ref call_generator) = fuzzer.call_generator { + inner_sequence.extend(call_generator.last_sequence.read().iter().cloned()); + } + } + + for func in &invariant_contract.invariant_functions { + let mut call_result = executor + .call_raw( + CALLER, + invariant_contract.address, + func.encode_input(&[]).expect("invariant should have no inputs").into(), + U256::zero(), + ) + .expect("EVM error"); + + let err = if call_result.reverted { + Some(*func) + } else { + // This will panic and get caught by the executor + if !executor.is_success( + invariant_contract.address, + call_result.reverted, + call_result.state_changeset.take().expect("we should have a state changeset"), + false, + ) { + Some(*func) + } else { + None + } + }; + + if let Some(broken_invariant) = err { + let invariant_error = invariant_failures + .failed_invariants + .get(&broken_invariant.name) + .expect("to have been initialized."); + + // We only care about invariants which we haven't broken yet. + if invariant_error.is_none() { + invariant_failures.failed_invariants.insert( + broken_invariant.name.clone(), + Some(InvariantFuzzError::new( + invariant_contract, + Some(broken_invariant), + calldata, + call_result, + &inner_sequence, + )), + ); + found_case = true; + } + } + } + + if found_case { + let before = invariant_failures.broken_invariants_count; + + invariant_failures.broken_invariants_count = invariant_failures + .failed_invariants + .iter() + .filter(|(_function, error)| error.is_some()) + .count(); + + eyre::bail!( + "{} new invariants have been broken.", + invariant_failures.broken_invariants_count - before + ); + } + Ok(()) +} + +/// The outcome of an invariant fuzz test +#[derive(Debug)] +pub struct InvariantFuzzTestResult { + pub invariants: BTreeMap>, + /// Every successful fuzz test case + pub cases: Vec, + /// Number of reverted fuzz calls + pub reverts: usize, +} + +#[derive(Debug, Clone)] +pub struct InvariantFuzzError { + /// The proptest error occurred as a result of a test case. + pub test_error: TestError>, + /// The return reason of the offending call. + pub return_reason: Reason, + /// The revert string of the offending call. + pub revert_reason: String, + /// Address of the invariant asserter. + pub addr: Address, + /// Function data for invariant check. + pub func: Option, + /// Inner fuzzing Sequence coming from overriding calls. + pub inner_sequence: Vec>, +} + +impl InvariantFuzzError { + fn new( + invariant_contract: &InvariantContract, + error_func: Option<&Function>, + calldata: &[BasicTxDetails], + call_result: RawCallResult, + inner_sequence: &[Option], + ) -> Self { + let mut func = None; + let origin: String; + + if let Some(f) = error_func { + func = Some(f.short_signature().into()); + origin = f.name.clone(); + } else { + origin = "Revert".to_string(); + } + + InvariantFuzzError { + test_error: proptest::test_runner::TestError::Fail( + format!( + "{}, reason: '{}'", + origin, + match decode_revert( + call_result.result.as_ref(), + Some(invariant_contract.abi), + Some(call_result.status) + ) { + Ok(e) => e, + Err(e) => e.to_string(), + } + ) + .into(), + calldata.to_vec(), + ), + return_reason: "".into(), + revert_reason: decode_revert( + call_result.result.as_ref(), + Some(invariant_contract.abi), + Some(call_result.status), + ) + .unwrap_or_default(), + addr: invariant_contract.address, + func, + inner_sequence: inner_sequence.to_vec(), + } + } + + /// Replays the error case and collects all necessary traces. + pub fn replay( + &self, + mut executor: Executor, + known_contracts: Option<&BTreeMap)>>, + mut ided_contracts: BTreeMap, + logs: &mut Vec, + traces: &mut Vec<(TraceKind, CallTraceArena)>, + ) -> Option { + let mut counterexample_sequence = vec![]; + let calls = match self.test_error { + // Don't use at the moment. + TestError::Abort(_) => return None, + TestError::Fail(_, ref calls) => calls, + }; + + // We want traces for a failed case. + executor.set_tracing(true); + + set_up_inner_replay(&mut executor, &self.inner_sequence); + + // Replay each call from the sequence until we break the invariant. + for (sender, (addr, bytes)) in calls.iter() { + let call_result = executor + .call_raw_committing(*sender, *addr, bytes.0.clone(), 0.into()) + .expect("bad call to evm"); + + logs.extend(call_result.logs); + traces.push((TraceKind::Execution, call_result.traces.clone().unwrap())); + + // Identify newly generated contracts, if they exist. + ided_contracts.extend(load_contracts( + vec![(TraceKind::Execution, call_result.traces.unwrap())], + known_contracts, + )); + + counterexample_sequence.push(BaseCounterExample::create( + *sender, + *addr, + bytes, + &ided_contracts, + )); + + // Checks the invariant. + if let Some(func) = &self.func { + let error_call_result = executor + .call_raw(CALLER, self.addr, func.0.clone(), 0.into()) + .expect("bad call to evm"); + + if error_call_result.reverted { + logs.extend(error_call_result.logs); + traces.push((TraceKind::Execution, error_call_result.traces.unwrap())); + break + } + } + } + + (!counterexample_sequence.is_empty()) + .then_some(CounterExample::Sequence(counterexample_sequence)) + } +} diff --git a/evm/src/fuzz/mod.rs b/evm/src/fuzz/mod.rs index e6c08d6f5d76..660b6a54544e 100644 --- a/evm/src/fuzz/mod.rs +++ b/evm/src/fuzz/mod.rs @@ -16,8 +16,8 @@ use strategies::{ build_initial_state, collect_state_from_call, fuzz_calldata, fuzz_calldata_from_state, EvmFuzzState, }; - -mod strategies; +pub mod invariant; +pub mod strategies; /// Magic return code for the `assume` cheatcode pub const ASSUME_MAGIC_RETURN_CODE: &[u8] = b"FOUNDRY::ASSUME"; @@ -146,7 +146,15 @@ impl<'a> FuzzedExecutor<'a> { let args = func .decode_input(&calldata.as_ref()[4..]) .expect("could not decode fuzzer inputs"); - result.counterexample = Some(CounterExample { calldata, args }); + + result.counterexample = Some(CounterExample::Single(BaseCounterExample { + sender: None, + addr: None, + signature: None, + contract_name: None, + calldata, + args, + })); } _ => (), } @@ -156,17 +164,81 @@ impl<'a> FuzzedExecutor<'a> { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CounterExample { - pub calldata: Bytes, +pub enum CounterExample { + /// Call used as a counter example for fuzz tests. + Single(BaseCounterExample), + /// Sequence of calls used as a counter example for invariant tests. + Sequence(Vec), +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BaseCounterExample { + /// Address which makes the call + pub sender: Option
, + /// Address to which to call to + pub addr: Option
, + /// The data to provide + pub calldata: Bytes, + /// Function signature if it exists + pub signature: Option, + /// Contract name if it exists + pub contract_name: Option, + // Token does not implement Serde (lol), so we just serialize the calldata #[serde(skip)] pub args: Vec, } -impl fmt::Display for CounterExample { +impl BaseCounterExample { + pub fn create( + sender: Address, + addr: Address, + bytes: &Bytes, + contracts: &BTreeMap, + ) -> Self { + let (name, abi) = &contracts.get(&addr).expect("Couldnt call unknown contract"); + + let func = abi + .functions() + .find(|f| f.short_signature() == bytes.0.as_ref()[0..4]) + .expect("Couldnt find function"); + + // skip the function selector when decoding + let args = func.decode_input(&bytes.0.as_ref()[4..]).expect("Unable to decode input"); + + BaseCounterExample { + sender: Some(sender), + addr: Some(addr), + calldata: bytes.clone(), + signature: Some(func.signature()), + contract_name: Some(name.clone()), + args, + } + } +} + +impl fmt::Display for BaseCounterExample { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let args = foundry_utils::format_tokens(&self.args).collect::>().join(", "); - write!(f, "calldata=0x{}, args=[{}]", hex::encode(&self.calldata), args) + + if let Some(sender) = self.sender { + write!(f, "sender={:?} addr=", sender)? + } + + if let Some(name) = &self.contract_name { + write!(f, "[{}]", name)? + } + + if let Some(addr) = &self.addr { + write!(f, "{:?} ", addr)? + } + + if let Some(sig) = &self.signature { + write!(f, "calldata={}", &sig)? + } else { + write!(f, "calldata=0x{}", hex::encode(&self.calldata))? + } + + write!(f, ", args=[{}]", args) } } diff --git a/evm/src/fuzz/strategies/invariants.rs b/evm/src/fuzz/strategies/invariants.rs new file mode 100644 index 000000000000..b638cf392043 --- /dev/null +++ b/evm/src/fuzz/strategies/invariants.rs @@ -0,0 +1,169 @@ +use crate::fuzz::{ + fuzz_calldata, fuzz_calldata_from_state, + invariant::{BasicTxDetails, FuzzRunIdentifiedContracts}, + strategies::fuzz_param, + EvmFuzzState, +}; +use ethers::{ + abi::{Abi, Function, ParamType}, + types::{Address, Bytes}, +}; +use parking_lot::RwLock; +use proptest::prelude::*; +pub use proptest::test_runner::Config as FuzzConfig; +use std::sync::Arc; + +/// Given a target address, we generate random calldata. +pub fn override_call_strat( + fuzz_state: EvmFuzzState, + contracts: FuzzRunIdentifiedContracts, + target: Arc>, +) -> SBoxedStrategy<(Address, Bytes)> { + let contracts_ref = contracts.clone(); + + let random_contract = any::() + .prop_map(move |selector| *selector.select(contracts_ref.lock().keys())); + let target = any::().prop_map(move |_| *target.read()); + + proptest::strategy::Union::new_weighted(vec![ + (80, target.sboxed()), + (20, random_contract.sboxed()), + ]) + .prop_flat_map(move |target_address| { + let fuzz_state = fuzz_state.clone(); + let (_, abi, functions) = contracts.lock().get(&target_address).unwrap().clone(); + + let func = select_random_function(abi, functions); + func.prop_flat_map(move |func| { + fuzz_contract_with_calldata(fuzz_state.clone(), target_address, func) + }) + }) + .sboxed() +} + +/// Creates the invariant strategy. +/// +/// Given the known and future contracts, it generates the next call by fuzzing the `caller`, +/// `calldata` and `target`. The generated data is evaluated lazily for every single call to fully +/// leverage the evolving fuzz dictionary. +/// +/// The fuzzed parameters can be filtered through different methods implemented in the test +/// contract: +/// +/// `targetContracts()`, `targetSenders()`, `excludeContracts()`, `targetSelectors()` +pub fn invariant_strat( + fuzz_state: EvmFuzzState, + senders: Vec
, + contracts: FuzzRunIdentifiedContracts, +) -> BoxedStrategy> { + // We only want to seed the first value, since we want to generate the rest as we mutate the + // state + vec![generate_call(fuzz_state, senders, contracts); 1].boxed() +} + +/// Strategy to generate a transaction where the `sender`, `target` and `calldata` are all generated +/// through specific strategies. +fn generate_call( + fuzz_state: EvmFuzzState, + senders: Vec
, + contracts: FuzzRunIdentifiedContracts, +) -> BoxedStrategy { + let random_contract = select_random_contract(contracts); + random_contract + .prop_flat_map(move |(contract, abi, functions)| { + let func = select_random_function(abi, functions); + let senders = senders.clone(); + let fuzz_state = fuzz_state.clone(); + func.prop_flat_map(move |func| { + let sender = select_random_sender(senders.clone()); + (sender, fuzz_contract_with_calldata(fuzz_state.clone(), contract, func)) + }) + }) + .boxed() +} + +/// Strategy to select a sender address: +/// * If `senders` is empty, then it's a completely random address. +/// * If `senders` is not empty, then there's an 80% chance that one from the list is selected. The +/// remaining 20% will be random. +fn select_random_sender(senders: Vec
) -> impl Strategy { + let fuzz_strategy = + fuzz_param(&ParamType::Address).prop_map(move |addr| addr.into_address().unwrap()).boxed(); + + if !senders.is_empty() { + let selector = + any::().prop_map(move |selector| *selector.select(&*senders)); + proptest::strategy::Union::new_weighted(vec![(80, selector.boxed()), (20, fuzz_strategy)]) + .boxed() + } else { + fuzz_strategy + } +} + +/// Strategy to randomly select a contract from the `contracts` list. +fn select_random_contract( + contracts: FuzzRunIdentifiedContracts, +) -> impl Strategy)> { + let selectors = any::(); + + selectors.prop_map(move |selector| { + let contracts = contracts.lock(); + let (addr, (_, abi, functions)) = selector.select(contracts.iter()); + (*addr, abi.clone(), functions.clone()) + }) +} + +/// Strategy to select a random mutable function from the abi. +/// +/// If `targeted_functions` is not empty, select one from it. Otherwise, take any +/// of the available abi functions. +fn select_random_function( + abi: Abi, + targeted_functions: Vec, +) -> impl Strategy { + let selectors = any::(); + let possible_funcs: Vec = abi + .functions() + .filter(|func| { + !matches!( + func.state_mutability, + ethers::abi::StateMutability::Pure | ethers::abi::StateMutability::View + ) + }) + .cloned() + .collect(); + + let total_random = selectors.prop_map(move |selector| { + let func = selector.select(&possible_funcs); + func.clone() + }); + + if !targeted_functions.is_empty() { + let selector = any::() + .prop_map(move |selector| selector.select(targeted_functions.clone())); + + selector.boxed() + } else { + total_random.boxed() + } +} + +/// Given a function, it returns a proptest strategy which generates valid abi-encoded calldata +/// for that function's input types. +pub fn fuzz_contract_with_calldata( + fuzz_state: EvmFuzzState, + contract: Address, + func: Function, +) -> impl Strategy { + // // We need to compose all the strategies generated for each parameter in all + // // possible combinations + let strats = proptest::strategy::Union::new_weighted(vec![ + (60, fuzz_calldata(func.clone())), + (40, fuzz_calldata_from_state(func, fuzz_state)), + ]); + + strats.prop_map(move |calldata| { + tracing::trace!(input = ?calldata); + (contract, calldata) + }) +} diff --git a/evm/src/fuzz/strategies/mod.rs b/evm/src/fuzz/strategies/mod.rs index b5004d2ccb0e..8add232c8953 100644 --- a/evm/src/fuzz/strategies/mod.rs +++ b/evm/src/fuzz/strategies/mod.rs @@ -11,5 +11,9 @@ pub use calldata::fuzz_calldata; mod state; pub use state::{ - build_initial_state, collect_state_from_call, fuzz_calldata_from_state, EvmFuzzState, + build_initial_state, collect_created_contracts, collect_state_from_call, + fuzz_calldata_from_state, EvmFuzzState, }; + +mod invariants; +pub use invariants::*; diff --git a/evm/src/fuzz/strategies/param.rs b/evm/src/fuzz/strategies/param.rs index eae373261a45..e1e044156402 100644 --- a/evm/src/fuzz/strategies/param.rs +++ b/evm/src/fuzz/strategies/param.rs @@ -55,15 +55,16 @@ pub fn fuzz_param(param: &ParamType) -> impl Strategy { /// fuzz state. /// /// Works with ABI Encoder v2 tuples. -pub fn fuzz_param_from_state(param: &ParamType, state: EvmFuzzState) -> BoxedStrategy { +pub fn fuzz_param_from_state(param: &ParamType, arc_state: EvmFuzzState) -> BoxedStrategy { // These are to comply with lifetime requirements - let state_len = state.borrow().len(); - let s = state.clone(); + let state = arc_state.read(); + let state_len = state.len(); // Select a value from the state + let st = arc_state.clone(); let value = any::() .prop_map(move |index| index.index(state_len)) - .prop_map(move |index| *s.borrow().iter().nth(index).unwrap()); + .prop_map(move |index| *st.read().iter().nth(index).unwrap()); // Convert the value based on the parameter type match param { @@ -98,28 +99,27 @@ pub fn fuzz_param_from_state(param: &ParamType, state: EvmFuzzState) -> BoxedStr }, ParamType::Bool => value.prop_map(move |value| Token::Bool(value[31] == 1)).boxed(), ParamType::String => value - .prop_map(move |value| { - Token::String(unsafe { std::str::from_utf8_unchecked(&value[..]).to_string() }) - }) + .prop_map(move |value| Token::String(String::from_utf8_lossy(&value[..]).to_string())) .boxed(), - ParamType::Array(param) => { - proptest::collection::vec(fuzz_param_from_state(param, state), 0..MAX_ARRAY_LEN) - .prop_map(Token::Array) - .boxed() - } + ParamType::Array(param) => proptest::collection::vec( + fuzz_param_from_state(param, arc_state.clone()), + 0..MAX_ARRAY_LEN, + ) + .prop_map(Token::Array) + .boxed(), ParamType::FixedBytes(size) => { let size = *size; value.prop_map(move |value| Token::FixedBytes(value[32 - size..].to_vec())).boxed() } ParamType::FixedArray(param, size) => { let fixed_size = *size; - proptest::collection::vec(fuzz_param_from_state(param, state), fixed_size) + proptest::collection::vec(fuzz_param_from_state(param, arc_state.clone()), fixed_size) .prop_map(Token::FixedArray) .boxed() } ParamType::Tuple(params) => params .iter() - .map(|p| fuzz_param_from_state(p, state.clone())) + .map(|p| fuzz_param_from_state(p, arc_state.clone())) .collect::>() .prop_map(Token::Tuple) .boxed(), diff --git a/evm/src/fuzz/strategies/state.rs b/evm/src/fuzz/strategies/state.rs index 1f6d65b62d55..58af8b8fa30c 100644 --- a/evm/src/fuzz/strategies/state.rs +++ b/evm/src/fuzz/strategies/state.rs @@ -1,21 +1,28 @@ use super::fuzz_param_from_state; -use crate::{executor::StateChangeset, utils}; +use crate::{executor::StateChangeset, fuzz::invariant::FuzzRunIdentifiedContracts, utils}; use bytes::Bytes; use ethers::{ - abi::Function, + abi::{Abi, Function}, + prelude::ArtifactId, types::{Address, Log, H256, U256}, }; +use foundry_utils::diff_score; +use parking_lot::RwLock; use proptest::prelude::{BoxedStrategy, Strategy}; use revm::{ db::{CacheDB, DatabaseRef}, - opcode, spec_opcode_gas, SpecId, + opcode, spec_opcode_gas, Filth, SpecId, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + io::Write, + sync::Arc, }; -use std::{cell::RefCell, collections::BTreeSet, io::Write, rc::Rc}; /// A set of arbitrary 32 byte data from the VM used to generate values for the strategy. /// /// Wrapped in a shareable container. -pub type EvmFuzzState = Rc>>; +pub type EvmFuzzState = Arc>>; /// Given a function and some state, it returns a strategy which generated valid calldata for the /// given function's input types, based on state taken from the EVM. @@ -71,7 +78,7 @@ pub fn build_initial_state(db: &CacheDB) -> EvmFuzzState { state.insert(H256::from(Address::random()).into()); } - Rc::new(RefCell::new(state)) + Arc::new(RwLock::new(state)) } /// Collects state changes from a [StateChangeset] and logs into an [EvmFuzzState]. @@ -80,7 +87,7 @@ pub fn collect_state_from_call( state_changeset: &StateChangeset, state: EvmFuzzState, ) { - let state = &mut *state.borrow_mut(); + let mut state = state.write(); for (address, account) in state_changeset { // Insert basic account information @@ -157,3 +164,35 @@ fn collect_push_bytes(code: Bytes) -> Vec<[u8; 32]> { bytes } + +/// Collects all created contracts from a StateChangeset which haven't been discovered yet. Stores +/// them at `targeted_contracts` and `created_contracts`. +pub fn collect_created_contracts( + state_changeset: &StateChangeset, + project_contracts: &BTreeMap)>, + setup_contracts: &BTreeMap, + targeted_contracts: FuzzRunIdentifiedContracts, + created_contracts: &mut Vec
, +) -> bool { + let mut writable_targeted = targeted_contracts.lock(); + let before = created_contracts.len(); + + for (address, account) in state_changeset { + if !setup_contracts.contains_key(address) { + if let (Filth::NewlyCreated, Some(code)) = (&account.filth, &account.info.code) { + if !code.is_empty() { + if let Some((artifact, (abi, _))) = project_contracts + .iter() + .find(|(_, (_, known_code))| diff_score(known_code, code.bytes()) < 0.1) + { + created_contracts.push(*address); + writable_targeted + .insert(*address, (artifact.name.clone(), abi.clone(), vec![])); + } + } + } + } + } + + created_contracts.len() > before +} diff --git a/evm/src/lib.rs b/evm/src/lib.rs index a3b32f8b9126..69b13ca991b1 100644 --- a/evm/src/lib.rs +++ b/evm/src/lib.rs @@ -41,6 +41,11 @@ pub const CALLER: Address = H160([ 0x30, 0x9D, 0x1F, 0x38, ]); +/// Stores the default test contract address: 0xb4c79daB8f259C7Aee6E5b2Aa729821864227e84 +pub const TEST_CONTRACT_ADDRESS: Address = H160([ + 180, 199, 157, 171, 143, 37, 156, 122, 238, 110, 91, 42, 167, 41, 130, 24, 100, 34, 126, 132, +]); + #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum CallKind { Call, diff --git a/evm/src/trace/mod.rs b/evm/src/trace/mod.rs index 3cd00b0845fb..2c92eb5481f1 100644 --- a/evm/src/trace/mod.rs +++ b/evm/src/trace/mod.rs @@ -9,16 +9,17 @@ mod utils; pub use decoder::{CallTraceDecoder, CallTraceDecoderBuilder}; -use crate::{abi::CHEATCODE_ADDRESS, CallKind}; +use crate::{abi::CHEATCODE_ADDRESS, trace::identifier::LocalTraceIdentifier, CallKind}; use ethers::{ - abi::{Address, RawLog}, + abi::{Abi, Address, RawLog}, + prelude::ArtifactId, types::U256, }; use node::CallTraceNode; use revm::{CallContext, Return}; use serde::{Deserialize, Serialize}; use std::{ - collections::HashSet, + collections::{BTreeMap, HashSet}, fmt::{self, Write}, }; use yansi::{Color, Paint}; @@ -436,3 +437,34 @@ fn trace_color(trace: &CallTrace) -> Color { Color::Red } } + +/// Given a list of traces and artifacts, it returns a map connecting address to abi +pub fn load_contracts( + traces: Vec<(TraceKind, CallTraceArena)>, + known_contracts: Option<&BTreeMap)>>, +) -> BTreeMap { + if let Some(contracts) = known_contracts { + let local_identifier = LocalTraceIdentifier::new(contracts); + let mut decoder = CallTraceDecoderBuilder::new().build(); + for (_, trace) in &traces { + decoder.identify(trace, &local_identifier); + } + + decoder + .contracts + .iter() + .map(|(addr, name)| { + let (_, (abi, _)) = contracts + .iter() + .find(|(artifact, _)| { + artifact.name == + *name.split(':').last().expect("invalid contract").to_string() + }) + .expect("no contract"); + (*addr, (name.clone(), abi.clone())) + }) + .collect() + } else { + BTreeMap::new() + } +} diff --git a/forge/Cargo.toml b/forge/Cargo.toml index a94b6f2e96a7..b17823a4cbb2 100644 --- a/forge/Cargo.toml +++ b/forge/Cargo.toml @@ -27,6 +27,7 @@ rayon = "1.5" rlp = "0.5.1" once_cell = "1.13" comfy-table = "6.0.0" +parking_lot = "0.12" [dev-dependencies] ethers = { git = "https://github.com/gakonst/ethers-rs", default-features = false, features = ["solc-full", "solc-tests"] } diff --git a/forge/src/lib.rs b/forge/src/lib.rs index dfdae9465445..a258ee1be9d1 100644 --- a/forge/src/lib.rs +++ b/forge/src/lib.rs @@ -6,6 +6,7 @@ pub mod coverage; /// The Forge test runner mod runner; +use ethers::types::U256; pub use runner::ContractRunner; /// Forge test runners for multiple contracts @@ -22,3 +23,29 @@ mod test_helpers; /// The Forge EVM backend pub use foundry_evm::*; + +/// Metadata on how to run fuzz/invariant tests +#[derive(Debug, Clone, Copy, Default)] +pub struct TestOptions { + /// The number of test cases that must execute for each fuzz test + pub fuzz_runs: u32, + /// The maximum number of global test case rejections allowed + /// by proptest, to be encountered during usage of `vm.assume` + /// cheatcode. + pub fuzz_max_local_rejects: u32, + /// The maximum number of local test case rejections allowed + /// by proptest, to be encountered during usage of `vm.assume` + /// cheatcode. + pub fuzz_max_global_rejects: u32, + /// Optional seed for the fuzzing RNG algorithm + pub fuzz_seed: Option, + /// The number of runs that must execute for each invariant test group. + pub invariant_runs: u32, + /// The number of calls executed to attempt to break invariants in one run. + pub invariant_depth: u32, + /// Fails the invariant fuzzing if a revert occurs + pub invariant_fail_on_revert: bool, + /// Allows overriding an unsafe external call when running invariant tests. eg. reetrancy + /// checks + pub invariant_call_override: bool, +} diff --git a/forge/src/multi_runner.rs b/forge/src/multi_runner.rs index 326865a2336b..709076b7a16a 100644 --- a/forge/src/multi_runner.rs +++ b/forge/src/multi_runner.rs @@ -1,4 +1,4 @@ -use crate::{result::SuiteResult, ContractRunner, TestFilter}; +use crate::{result::SuiteResult, ContractRunner, TestFilter, TestOptions}; use ethers::{ abi::Abi, prelude::{artifacts::CompactContractBytecode, ArtifactId, ArtifactOutput}, @@ -15,7 +15,6 @@ use foundry_evm::{ revm, }; use foundry_utils::PostLinkInput; -use proptest::test_runner::TestRunner; use rayon::prelude::*; use std::{collections::BTreeMap, path::Path, sync::mpsc::Sender}; @@ -37,8 +36,6 @@ pub struct MultiContractRunner { pub evm_spec: SpecId, /// All known errors, used for decoding reverts pub errors: Option, - /// The fuzzer which will be used to run parametric tests (w/ non-0 solidity args) - fuzzer: Option, /// The address which will be used as the `from` field in all EVM calls sender: Option
, /// A map of contract names to absolute source file paths @@ -49,6 +46,8 @@ pub struct MultiContractRunner { pub cheats_config: CheatsConfig, /// Whether to collect coverage info pub coverage: bool, + /// Settings related to fuzz and/or invariant tests + pub test_options: TestOptions, } impl MultiContractRunner { @@ -119,6 +118,7 @@ impl MultiContractRunner { &mut self, filter: &impl TestFilter, stream_result: Option>, + test_options: TestOptions, ) -> Result> { let db = Backend::spawn(self.fork.take()); @@ -153,7 +153,7 @@ impl MultiContractRunner { executor, deploy_code.clone(), libs, - filter + (filter, test_options), )?; tracing::trace!(contract= ?identifier, "executed all tests in contract"); @@ -187,7 +187,7 @@ impl MultiContractRunner { executor: Executor, deploy_code: Bytes, libs: &[Bytes], - filter: &impl TestFilter, + (filter, test_options): (&impl TestFilter, TestOptions), ) -> Result { let runner = ContractRunner::new( executor, @@ -198,15 +198,13 @@ impl MultiContractRunner { self.errors.as_ref(), libs, ); - runner.run_tests(filter, self.fuzzer.clone()) + runner.run_tests(filter, test_options, Some(&self.known_contracts)) } } /// Builder used for instantiating the multi-contract runner #[derive(Debug, Default)] pub struct MultiContractRunnerBuilder { - /// The fuzzer to be used for running fuzz tests - pub fuzzer: Option, /// The address which will be used to deploy the initial contracts and send all /// transactions pub sender: Option
, @@ -220,6 +218,8 @@ pub struct MultiContractRunnerBuilder { pub cheats_config: Option, /// Whether or not to collect coverage info pub coverage: bool, + /// Settings related to fuzz and/or invariant tests + pub test_options: Option, } impl MultiContractRunnerBuilder { @@ -280,7 +280,8 @@ impl MultiContractRunnerBuilder { let abi = contract.abi.expect("We should have an abi by now"); // if it's a test, add it to deployable contracts if abi.constructor.as_ref().map(|c| c.inputs.is_empty()).unwrap_or(true) && - abi.functions().any(|func| func.name.is_test()) + abi.functions() + .any(|func| func.name.is_test() || func.name.is_invariant_test()) { deployable_contracts.insert( id.clone(), @@ -312,12 +313,12 @@ impl MultiContractRunnerBuilder { env, evm_spec: self.evm_spec.unwrap_or(SpecId::LONDON), sender: self.sender, - fuzzer: self.fuzzer, errors: Some(execution_info.2), source_paths, fork: self.fork, cheats_config: self.cheats_config.unwrap_or_default(), coverage: self.coverage, + test_options: self.test_options.unwrap_or_default(), }) } @@ -333,12 +334,6 @@ impl MultiContractRunnerBuilder { self } - #[must_use] - pub fn fuzzer(mut self, fuzzer: TestRunner) -> Self { - self.fuzzer = Some(fuzzer); - self - } - #[must_use] pub fn evm_spec(mut self, spec: SpecId) -> Self { self.evm_spec = Some(spec); @@ -357,6 +352,12 @@ impl MultiContractRunnerBuilder { self } + #[must_use] + pub fn with_test_options(mut self, test_options: TestOptions) -> Self { + self.test_options = Some(test_options); + self + } + #[must_use] pub fn set_coverage(mut self, enable: bool) -> Self { self.coverage = enable; @@ -378,6 +379,17 @@ mod tests { use foundry_evm::trace::TraceKind; use std::env; + static TEST_OPTS: TestOptions = TestOptions { + fuzz_runs: 256, + fuzz_max_local_rejects: 1024, + fuzz_max_global_rejects: 65536, + fuzz_seed: None, + invariant_runs: 256, + invariant_depth: 15, + invariant_fail_on_revert: false, + invariant_call_override: false, + }; + /// Builds a base runner fn base_runner() -> MultiContractRunnerBuilder { MultiContractRunnerBuilder::default().sender(EVM_OPTS.sender) @@ -516,7 +528,7 @@ mod tests { #[test] fn test_core() { let mut runner = runner(); - let results = runner.test(&Filter::new(".*", ".*", ".*core"), None).unwrap(); + let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, TEST_OPTS).unwrap(); assert_multiple( &results, @@ -591,7 +603,7 @@ mod tests { #[test] fn test_logs() { let mut runner = runner(); - let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None).unwrap(); + let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, TEST_OPTS).unwrap(); assert_multiple( &results, @@ -1154,7 +1166,7 @@ mod tests { // test `setEnv` first, and confirm that it can correctly set environment variables, // so that we can use it in subsequent `env*` tests - runner.test(&Filter::new("testSetEnv", ".*", ".*"), None).unwrap(); + runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, TEST_OPTS).unwrap(); let env_var_key = "_foundryCheatcodeSetEnvTestKey"; let env_var_val = "_foundryCheatcodeSetEnvTestVal"; let res = env::var(env_var_key); @@ -1179,6 +1191,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR), ), None, + TEST_OPTS, ) .unwrap(); assert_eq!(suite_result.len(), 1); @@ -1202,6 +1215,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", &Filter::new(".*", ".*", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR)) .exclude_tests(".*Revert"), None, + TEST_OPTS, ) .unwrap(); assert!(!suite_result.is_empty()); @@ -1225,7 +1239,11 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", fn test_cheats_local() { let mut runner = runner(); let suite_result = runner - .test(&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), None) + .test( + &Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), + None, + TEST_OPTS, + ) .unwrap(); assert!(!suite_result.is_empty()); @@ -1246,10 +1264,9 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", #[test] fn test_fuzz() { let mut runner = runner(); - let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() }; - runner.fuzzer = Some(proptest::test_runner::TestRunner::new(cfg)); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None).unwrap(); + let suite_result = + runner.test(&Filter::new(".*", ".*", ".*fuzz/[^invariant]"), None, TEST_OPTS).unwrap(); assert!(!suite_result.is_empty()); @@ -1283,7 +1300,8 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", #[test] fn test_trace() { let mut runner = tracing_runner(); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None).unwrap(); + let suite_result = + runner.test(&Filter::new(".*", ".*", ".*trace"), None, TEST_OPTS).unwrap(); // TODO: This trace test is very basic - it is probably a good candidate for snapshot // testing. @@ -1321,7 +1339,8 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", fn test_fork() { let rpc_url = foundry_utils::rpc::next_http_archive_rpc_endpoint(); let mut runner = forked_runner(&rpc_url); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None).unwrap(); + let suite_result = + runner.test(&Filter::new(".*", ".*", ".*fork"), None, TEST_OPTS).unwrap(); for (_, SuiteResult { test_results, .. }) in suite_result { for (test_name, result) in test_results { @@ -1342,9 +1361,59 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", fn test_doesnt_run_abstract_contract() { let mut runner = runner(); let results = runner - .test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None) + .test( + &Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), + None, + TEST_OPTS, + ) .unwrap(); assert!(results.get("core/Abstract.t.sol:AbstractTestBase").is_none()); assert!(results.get("core/Abstract.t.sol:AbstractTest").is_some()); } + + #[test] + fn test_invariant() { + let mut runner = runner(); + + let mut opts = TEST_OPTS; + opts.invariant_call_override = true; + runner.test_options = opts; + + let results = + runner.test(&Filter::new(".*", ".*", ".*fuzz/invariant/"), None, opts).unwrap(); + + assert_multiple( + &results, + BTreeMap::from([ + ( + "fuzz/invariant/InvariantInnerContract.t.sol:InvariantInnerContract", + vec![("invariantHideJesus", false, Some("jesus betrayed.".into()), None, None)], + ), + ( + "fuzz/invariant/InvariantReentrancy.t.sol:InvariantReentrancy", + vec![("invariantNotStolen", false, Some("stolen.".into()), None, None)], + ), + ( + "fuzz/invariant/InvariantTest1.t.sol:InvariantTest", + vec![("invariant_neverFalse", false, Some("false.".into()), None, None)], + ), + ( + "fuzz/invariant/target/ExcludeContracts.t.sol:ExcludeContracts", + vec![("invariantTrueWorld", true, None, None, None)], + ), + ( + "fuzz/invariant/target/TargetContracts.t.sol:TargetContracts", + vec![("invariantTrueWorld", true, None, None, None)], + ), + ( + "fuzz/invariant/target/TargetSenders.t.sol:TargetSenders", + vec![("invariantTrueWorld", false, Some("false world.".into()), None, None)], + ), + ( + "fuzz/invariant/target/TargetSelectors.t.sol:TargetSelectors", + vec![("invariantTrueWorld", true, None, None, None)], + ), + ]), + ); + } } diff --git a/forge/src/result.rs b/forge/src/result.rs index d22aee936fea..d69d06c73b7c 100644 --- a/forge/src/result.rs +++ b/forge/src/result.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fmt, time::Duration}; /// Results and duration for a set of tests included in the same test contract -#[derive(Clone, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct SuiteResult { /// Total duration of the test run for this block of tests pub duration: Duration, @@ -51,7 +51,7 @@ pub struct TestResult { /// still be successful (i.e self.success == true) when it's expected to fail. pub reason: Option, - /// Minimal reproduction test case for failing fuzz tests + /// Minimal reproduction test case for failing test pub counterexample: Option, /// Any captured & parsed as strings logs along the test's execution which should @@ -79,33 +79,39 @@ impl TestResult { } } -/// Used gas by a test +/// Data report by a test. #[derive(Debug, Clone, Eq, PartialEq)] -pub enum TestKindGas { - Standard(u64), - Fuzz { runs: usize, mean: u64, median: u64 }, +pub enum TestKindReport { + Standard { gas: u64 }, + Fuzz { runs: usize, mean_gas: u64, median_gas: u64 }, + Invariant { runs: usize, calls: usize, reverts: usize }, } -impl fmt::Display for TestKindGas { +impl fmt::Display for TestKindReport { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - TestKindGas::Standard(gas) => { + TestKindReport::Standard { gas } => { write!(f, "(gas: {})", gas) } - TestKindGas::Fuzz { runs, mean, median } => { - write!(f, "(runs: {}, μ: {}, ~: {})", runs, mean, median) + TestKindReport::Fuzz { runs, mean_gas, median_gas } => { + write!(f, "(runs: {}, μ: {}, ~: {})", runs, mean_gas, median_gas) + } + TestKindReport::Invariant { runs, calls, reverts } => { + write!(f, "(runs: {}, calls: {}, reverts: {})", runs, calls, reverts) } } } } -impl TestKindGas { +impl TestKindReport { /// Returns the main gas value to compare against pub fn gas(&self) -> u64 { match self { - TestKindGas::Standard(gas) => *gas, + TestKindReport::Standard { gas } => *gas, // We use the median for comparisons - TestKindGas::Fuzz { median, .. } => *median, + TestKindReport::Fuzz { median_gas, .. } => *median_gas, + // We return 0 since it's not applicable + TestKindReport::Invariant { .. } => 0, } } } @@ -119,17 +125,24 @@ pub enum TestKind { Standard(u64), /// A solidity fuzz test, that stores all test cases Fuzz(FuzzedCases), + /// A solidity invariant test, that stores all test cases + Invariant(Vec, usize), } impl TestKind { /// The gas consumed by this test - pub fn gas_used(&self) -> TestKindGas { + pub fn report(&self) -> TestKindReport { match self { - TestKind::Standard(gas) => TestKindGas::Standard(*gas), - TestKind::Fuzz(fuzzed) => TestKindGas::Fuzz { + TestKind::Standard(gas) => TestKindReport::Standard { gas: *gas }, + TestKind::Fuzz(fuzzed) => TestKindReport::Fuzz { runs: fuzzed.cases().len(), - median: fuzzed.median_gas(false), - mean: fuzzed.mean_gas(false), + median_gas: fuzzed.median_gas(false), + mean_gas: fuzzed.mean_gas(false), + }, + TestKind::Invariant(fuzzed, reverts) => TestKindReport::Invariant { + runs: fuzzed.len(), + calls: fuzzed.iter().map(|sequence| sequence.cases().len()).sum(), + reverts: *reverts, }, } } diff --git a/forge/src/runner.rs b/forge/src/runner.rs index a677af68a846..feb7b50a78c9 100644 --- a/forge/src/runner.rs +++ b/forge/src/runner.rs @@ -1,20 +1,26 @@ use crate::{ result::{SuiteResult, TestKind, TestResult, TestSetup}, - TestFilter, + TestFilter, TestOptions, }; use ethers::{ abi::{Abi, Function}, + prelude::ArtifactId, types::{Address, Bytes, U256}, }; use eyre::Result; use foundry_common::TestFunctionExt; use foundry_evm::{ executor::{CallResult, DeployResult, EvmError, Executor}, - fuzz::FuzzedExecutor, - trace::TraceKind, + fuzz::{ + invariant::{ + InvariantContract, InvariantExecutor, InvariantFuzzTestResult, InvariantTestOptions, + }, + FuzzedExecutor, + }, + trace::{load_contracts, TraceKind}, CALLER, }; -use proptest::test_runner::TestRunner; +use proptest::test_runner::{RngAlgorithm, TestError, TestRng, TestRunner}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{collections::BTreeMap, time::Instant}; use tracing::{error, trace}; @@ -175,7 +181,8 @@ impl<'a> ContractRunner<'a> { pub fn run_tests( mut self, filter: &impl TestFilter, - fuzzer: Option, + test_options: TestOptions, + known_contracts: Option<&BTreeMap)>>, ) -> Result { tracing::info!("starting tests"); let start = Instant::now(); @@ -218,6 +225,15 @@ impl<'a> ContractRunner<'a> { )) } + let has_invariants = + self.contract.functions().into_iter().any(|func| func.name.is_invariant_test()); + + if has_invariants && needs_setup { + // invariant testing requires tracing to figure + // out what contracts were created + self.executor.set_tracing(true); + } + let setup = self.setup(needs_setup)?; if setup.setup_failed { // The setup failed, so we return a single test result for `setUp` @@ -250,20 +266,91 @@ impl<'a> ContractRunner<'a> { .map(|func| (func, func.is_test_fail())) .collect(); - let test_results = tests - .par_iter() - .filter_map(|(func, should_fail)| { - let result = if func.is_fuzz_test() { - fuzzer.as_ref().map(|fuzzer| { - self.run_fuzz_test(func, *should_fail, fuzzer.clone(), setup.clone()) + let mut test_results = BTreeMap::new(); + if !tests.is_empty() { + // TODO: Add Options to modify the persistence + let cfg = proptest::test_runner::Config { + failure_persistence: None, + cases: test_options.fuzz_runs, + max_local_rejects: test_options.fuzz_max_local_rejects, + max_global_rejects: test_options.fuzz_max_global_rejects, + ..Default::default() + }; + + let fuzzer = if let Some(ref fuzz_seed) = test_options.fuzz_seed { + let mut bytes: [u8; 32] = [0; 32]; + fuzz_seed.to_big_endian(&mut bytes); + trace!(target: "forge::test", "executing test command"); + let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &bytes); + proptest::test_runner::TestRunner::new_with_rng(cfg, rng) + } else { + proptest::test_runner::TestRunner::new(cfg) + }; + + test_results.extend( + tests + .par_iter() + .flat_map(|(func, should_fail)| { + let result = if func.is_fuzz_test() { + self.run_fuzz_test(func, *should_fail, fuzzer.clone(), setup.clone()) + } else { + self.clone().run_test(func, *should_fail, setup.clone()) + }; + + result.map(|result| Ok((func.signature(), result))) }) - } else { - Some(self.clone().run_test(func, *should_fail, setup.clone())) - }; + .collect::>>()?, + ); + } + + if has_invariants { + // TODO: Add Options to modify the persistence + let cfg = proptest::test_runner::Config { + failure_persistence: None, + cases: test_options.invariant_runs, + max_local_rejects: test_options.fuzz_max_local_rejects, + max_global_rejects: test_options.fuzz_max_global_rejects, + ..Default::default() + }; + + let fuzzer = if let Some(ref fuzz_seed) = test_options.fuzz_seed { + let mut bytes: [u8; 32] = [0; 32]; + fuzz_seed.to_big_endian(&mut bytes); + trace!(target: "forge::test", "executing invariant test command"); + let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &bytes); + proptest::test_runner::TestRunner::new_with_rng(cfg, rng) + } else { + proptest::test_runner::TestRunner::new(cfg) + }; - result.map(|result| Ok((func.signature(), result?))) - }) - .collect::>>()?; + let identified_contracts = load_contracts(setup.traces.clone(), known_contracts); + let functions: Vec<&Function> = self + .contract + .functions() + .into_iter() + .filter(|func| { + func.name.is_invariant_test() && filter.matches_test(func.signature()) + }) + .collect(); + + let results = self.run_invariant_test( + fuzzer, + setup, + test_options, + functions.clone(), + known_contracts, + identified_contracts, + )?; + + results.into_iter().zip(functions.iter()).for_each(|(result, function)| { + match result.kind { + TestKind::Invariant(ref _cases, _) => { + test_results.insert(function.name.clone(), result); + } + _ => unreachable!(), + } + }); + } let duration = start.elapsed(); if !test_results.is_empty() { @@ -275,6 +362,7 @@ impl<'a> ContractRunner<'a> { test_results.len() ); } + Ok(SuiteResult::new(duration, test_results, warnings)) } @@ -367,6 +455,78 @@ impl<'a> ContractRunner<'a> { }) } + #[tracing::instrument(name = "invariant-test", skip_all)] + pub fn run_invariant_test( + &mut self, + runner: TestRunner, + setup: TestSetup, + test_options: TestOptions, + functions: Vec<&Function>, + known_contracts: Option<&BTreeMap)>>, + identified_contracts: BTreeMap, + ) -> Result> { + let empty = BTreeMap::new(); + let project_contracts = known_contracts.unwrap_or(&empty); + let TestSetup { address, logs, traces, labeled_addresses, .. } = setup; + + let mut evm = InvariantExecutor::new( + &mut self.executor, + runner, + &identified_contracts, + project_contracts, + ); + + let invariant_contract = + InvariantContract { address, invariant_functions: functions, abi: self.contract }; + + if let Some(InvariantFuzzTestResult { invariants, cases, reverts }) = evm.invariant_fuzz( + invariant_contract, + InvariantTestOptions { + depth: test_options.invariant_depth, + fail_on_revert: test_options.invariant_fail_on_revert, + call_override: test_options.invariant_call_override, + }, + )? { + let results = invariants + .iter() + .map(|(_, test_error)| { + let mut counterexample = None; + let mut logs = logs.clone(); + let mut traces = traces.clone(); + + if let Some(ref error) = test_error { + if let TestError::Fail(_, _) = &error.test_error { + counterexample = error.replay( + self.executor.clone(), + known_contracts, + identified_contracts.clone(), + &mut logs, + &mut traces, + ); + } + } + + TestResult { + success: test_error.is_none(), + reason: test_error.as_ref().and_then(|err| { + (!err.revert_reason.is_empty()).then(|| err.revert_reason.clone()) + }), + counterexample, + logs, + kind: TestKind::Invariant(cases.clone(), reverts), + coverage: None, // todo? + traces, + labeled_addresses: labeled_addresses.clone(), + } + }) + .collect(); + + Ok(results) + } else { + Ok(vec![]) + } + } + #[tracing::instrument(name = "fuzz-test", skip_all, fields(name = %func.signature(), %should_fail))] pub fn run_fuzz_test( &self, diff --git a/testdata/foundry.toml b/testdata/foundry.toml index 12868a790f6e..7a7bfe7c6bcc 100644 --- a/testdata/foundry.toml +++ b/testdata/foundry.toml @@ -16,6 +16,8 @@ force = false fuzz_max_global_rejects = 65536 fuzz_max_local_rejects = 1024 fuzz_runs = 256 +invariant_fail_on_revert = false +invariant_call_override = false gas_limit = 9223372036854775807 gas_price = 0 gas_reports = ['*'] diff --git a/testdata/fuzz/invariant/InvariantInnerContract.t.sol b/testdata/fuzz/invariant/InvariantInnerContract.t.sol new file mode 100644 index 000000000000..3f844e98f6b0 --- /dev/null +++ b/testdata/fuzz/invariant/InvariantInnerContract.t.sol @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +/*////////////////////////////////////////////////////////////// + Here we test that the fuzz engine can include a contract created during the fuzz + in its fuzz dictionary and eventually break the invariant. + Specifically, can Judas, a created contract from Jesus, break Jesus contract + by revealing his identity. +/*////////////////////////////////////////////////////////////// + +contract Jesus { + address fren; + bool public identity_revealed; + + function create_fren() public { + fren = address(new Judas()); + } + + function kiss() public { + require(msg.sender == fren); + identity_revealed = true; + } +} + +contract Judas { + Jesus jesus; + + constructor() { + jesus = Jesus(msg.sender); + } + + function betray() public { + jesus.kiss(); + } +} + +contract InvariantInnerContract is DSTest { + Jesus jesus; + + function setUp() public { + jesus = new Jesus(); + } + + function invariantHideJesus() public { + require(jesus.identity_revealed() == false, "jesus betrayed."); + } +} \ No newline at end of file diff --git a/testdata/fuzz/invariant/InvariantReentrancy.t.sol b/testdata/fuzz/invariant/InvariantReentrancy.t.sol new file mode 100644 index 000000000000..d330e787f584 --- /dev/null +++ b/testdata/fuzz/invariant/InvariantReentrancy.t.sol @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract Malicious { + function world() public { + // Does not matter, since it will get overriden. + } +} + +contract Vulnerable { + bool public open_door = false; + bool public stolen = false; + Malicious mal; + + constructor(address _mal) { + mal = Malicious(_mal); + } + + function hello() public { + open_door = true; + mal.world(); + open_door = false; + } + + function backdoor() public { + require(open_door, ""); + stolen = true; + } +} + +contract InvariantReentrancy is DSTest { + Vulnerable vuln; + Malicious mal; + + function setUp() public { + mal = new Malicious(); + vuln = new Vulnerable(address(mal)); + } + function invariantNotStolen() public { + require(vuln.stolen() == false, "stolen."); + } +} diff --git a/testdata/fuzz/invariant/InvariantTest1.t.sol b/testdata/fuzz/invariant/InvariantTest1.t.sol new file mode 100644 index 000000000000..86c3c9cc801d --- /dev/null +++ b/testdata/fuzz/invariant/InvariantTest1.t.sol @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract InvariantBreaker { + + bool public flag0 = true; + bool public flag1 = true; + + function set0(int val) public returns (bool){ + if (val % 100 == 0) + flag0 = false; + return flag0; + } + + function set1(int val) public returns (bool){ + if (val % 10 == 0 && !flag0) + flag1 = false; + return flag1; + } +} + +contract InvariantTest is DSTest { + InvariantBreaker inv; + + function setUp() public { + inv = new InvariantBreaker(); + } + + function invariant_neverFalse() public { + require(inv.flag1(), "false."); + } +} \ No newline at end of file diff --git a/testdata/fuzz/invariant/target/ExcludeContracts.t.sol b/testdata/fuzz/invariant/target/ExcludeContracts.t.sol new file mode 100644 index 000000000000..c703bf54a89f --- /dev/null +++ b/testdata/fuzz/invariant/target/ExcludeContracts.t.sol @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract Hello { + bool public world = true; + + function change() public { + world = false; + } +} + +contract ExcludeContracts is DSTest { + Hello hello; + + function setUp() public { + hello = new Hello(); + new Hello(); + } + + function excludeContracts() public returns (address[] memory) { + address[] memory addrs = new address[](1); + addrs[0] = address(hello); + return addrs; + } + + function invariantTrueWorld() public { + require(hello.world() == true, "false world."); + } +} \ No newline at end of file diff --git a/testdata/fuzz/invariant/target/TargetContracts.t.sol b/testdata/fuzz/invariant/target/TargetContracts.t.sol new file mode 100644 index 000000000000..e3f7fb91e0a2 --- /dev/null +++ b/testdata/fuzz/invariant/target/TargetContracts.t.sol @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract Hello { + bool public world = true; + + function change() public { + world = false; + } +} + +contract TargetContracts is DSTest { + Hello hello1; + Hello hello2; + + function setUp() public { + hello1 = new Hello(); + hello2 = new Hello(); + } + + function targetContracts() public returns (address[] memory) { + address[] memory addrs = new address[](1); + addrs[0] = address(hello1); + return addrs; + } + + function invariantTrueWorld() public { + require(hello2.world() == true, "false world."); + } +} \ No newline at end of file diff --git a/testdata/fuzz/invariant/target/TargetSelectors.t.sol b/testdata/fuzz/invariant/target/TargetSelectors.t.sol new file mode 100644 index 000000000000..c8ca74e524c1 --- /dev/null +++ b/testdata/fuzz/invariant/target/TargetSelectors.t.sol @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +struct FuzzSelector { + address addr; + bytes4[] selectors; +} + +contract Hello { + bool public world = true; + + function change() public { + world = true; + } + + function real_change() public { + world = false; + } +} + +contract TargetSelectors is DSTest { + Hello hello; + + function setUp() public { + hello = new Hello(); + } + + function targetSelectors() public returns (FuzzSelector[] memory) { + FuzzSelector[] memory targets = new FuzzSelector[](1); + bytes4[] memory selectors = new bytes4[](1); + selectors[0] = Hello.change.selector; + targets[0] = FuzzSelector(address(hello), selectors); + return targets; + } + + function invariantTrueWorld() public { + require(hello.world() == true, "false world."); + } +} \ No newline at end of file diff --git a/testdata/fuzz/invariant/target/TargetSenders.t.sol b/testdata/fuzz/invariant/target/TargetSenders.t.sol new file mode 100644 index 000000000000..145d65c1cd1a --- /dev/null +++ b/testdata/fuzz/invariant/target/TargetSenders.t.sol @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract Hello { + bool public world = true; + + function change() public { + require(msg.sender == address(0xdeadbeef)); + world = false; + } +} + +contract TargetSenders is DSTest { + Hello hello; + + function setUp() public { + hello = new Hello(); + } + + function targetSenders() public returns (address[] memory) { + address[] memory addrs = new address[](1); + addrs[0] = address(0xdeadbeef); + return addrs; + } + + function invariantTrueWorld() public { + require(hello.world() == true, "false world."); + } +} \ No newline at end of file