Enable all testing modes by default, update property mode testing, improve UX, allow for contracts to have starting balances, and fix coverage panic (crytic#216)

* - merged all three testing modes
- created testing provider utils file to evaluate whether an abi.method is an optimization / property test
- updated fuzzer tests
- made default property test prefix "invariant_"

* enable all testing modes, update fuzzer tests

* add verification for config

* improve config-related errors

* update deploymentOrder to targetContracts

* update edge case where coverage is 0/0 lines

* add support for payable constructors

* linting

* fix bug

* fix panic and improve return data printing in execution trace

* encode bytes and byteX as hex strings

* fix console

* updates from PR review

* change config language

Author: anishnaik

cmd/fuzz_flags.go

@@ -21,8 +21,8 @@ func addFuzzFlags() error {
 	// Config file
 	fuzzCmd.Flags().String("config", "", "path to config file")
 
-	// Target
-	fuzzCmd.Flags().String("target", "", TargetFlagDescription)
+	// Compilation Target
+	fuzzCmd.Flags().String("compilation-target", "", TargetFlagDescription)
 
 	// Number of workers
 	fuzzCmd.Flags().Int("workers", 0,
@@ -40,14 +40,13 @@ func addFuzzFlags() error {
 	fuzzCmd.Flags().Int("seq-len", 0,
 		fmt.Sprintf("maximum transactions to run in sequence (unless a config file is provided, default is %d)", defaultConfig.Fuzzing.CallSequenceLength))
 
-	// Deployment order
-	fuzzCmd.Flags().StringSlice("deployment-order", []string{},
-		fmt.Sprintf("order in which to deploy target contracts (unless a config file is provided, default is %v)", defaultConfig.Fuzzing.DeploymentOrder))
+	// Target contracts
+	fuzzCmd.Flags().StringSlice("target-contracts", []string{},
+		fmt.Sprintf("target contracts for fuzz testing (unless a config file is provided, default is %v)", defaultConfig.Fuzzing.TargetContracts))
 
 	// Corpus directory
-	// TODO: Update description when we add "coverage reports" feature
 	fuzzCmd.Flags().String("corpus-dir", "",
-		fmt.Sprintf("directory path for corpus items (unless a config file is provided, default is %q)", defaultConfig.Fuzzing.CorpusDirectory))
+		fmt.Sprintf("directory path for corpus items and coverage reports (unless a config file is provided, default is %q)", defaultConfig.Fuzzing.CorpusDirectory))
 
 	// Senders
 	fuzzCmd.Flags().StringSlice("senders", []string{},
@@ -57,14 +56,6 @@ func addFuzzFlags() error {
 	fuzzCmd.Flags().String("deployer", "",
 		"account address used to deploy contracts")
 
-	// Assertion mode
-	fuzzCmd.Flags().Bool("assertion-mode", false,
-		fmt.Sprintf("enable assertion mode (unless a config file is provided, default is %t)", defaultConfig.Fuzzing.Testing.AssertionTesting.Enabled))
-
-	// Optimization mode
-	fuzzCmd.Flags().Bool("optimization-mode", false,
-		fmt.Sprintf("enable optimization mode (unless a config file is provided, default is %t)", defaultConfig.Fuzzing.Testing.OptimizationTesting.Enabled))
-
 	// Trace all
 	fuzzCmd.Flags().Bool("trace-all", false,
 		fmt.Sprintf("print the execution trace for every element in a shrunken call sequence instead of only the last element (unless a config file is provided, default is %t)", defaultConfig.Fuzzing.Testing.TraceAll))
@@ -79,10 +70,10 @@ func addFuzzFlags() error {
 func updateProjectConfigWithFuzzFlags(cmd *cobra.Command, projectConfig *config.ProjectConfig) error {
 	var err error
 
-	// If --target was used
-	if cmd.Flags().Changed("target") {
+	// If --compilation-target was used
+	if cmd.Flags().Changed("compilation-target") {
 		// Get the new target
-		newTarget, err := cmd.Flags().GetString("target")
+		newTarget, err := cmd.Flags().GetString("compilation-target")
 		if err != nil {
 			return err
 		}
@@ -125,9 +116,9 @@ func updateProjectConfigWithFuzzFlags(cmd *cobra.Command, projectConfig *config.
 		}
 	}
 
-	// Update deployment order
-	if cmd.Flags().Changed("deployment-order") {
-		projectConfig.Fuzzing.DeploymentOrder, err = cmd.Flags().GetStringSlice("deployment-order")
+	// Update target contracts
+	if cmd.Flags().Changed("target-contracts") {
+		projectConfig.Fuzzing.TargetContracts, err = cmd.Flags().GetStringSlice("target-contracts")
 		if err != nil {
 			return err
 		}
@@ -157,22 +148,6 @@ func updateProjectConfigWithFuzzFlags(cmd *cobra.Command, projectConfig *config.
 		}
 	}
 
-	// Update assertion mode enablement
-	if cmd.Flags().Changed("assertion-mode") {
-		projectConfig.Fuzzing.Testing.AssertionTesting.Enabled, err = cmd.Flags().GetBool("assertion-mode")
-		if err != nil {
-			return err
-		}
-	}
-
-	// Update optimization mode enablement
-	if cmd.Flags().Changed("optimization-mode") {
-		projectConfig.Fuzzing.Testing.OptimizationTesting.Enabled, err = cmd.Flags().GetBool("optimization-mode")
-		if err != nil {
-			return err
-		}
-	}
-
 	// Update trace all enablement
 	if cmd.Flags().Changed("trace-all") {
 		projectConfig.Fuzzing.Testing.TraceAll, err = cmd.Flags().GetBool("trace-all")

cmd/init_flags.go

@@ -10,18 +10,18 @@ func addInitFlags() error {
 	// Output path for configuration
 	initCmd.Flags().String("out", "", "output path for the new project configuration file")
 
-	// Target file / directory
-	initCmd.Flags().String("target", "", TargetFlagDescription)
+	// Target file / directory for compilation
+	initCmd.Flags().String("compilation-target", "", TargetFlagDescription)
 
 	return nil
 }
 
 // updateProjectConfigWithInitFlags will update the given projectConfig with any CLI arguments that were provided to the init command
 func updateProjectConfigWithInitFlags(cmd *cobra.Command, projectConfig *config.ProjectConfig) error {
-	// If --target was used
-	if cmd.Flags().Changed("target") {
+	// If --compilation-target was used
+	if cmd.Flags().Changed("compilation-target") {
 		// Get the new target
-		newTarget, err := cmd.Flags().GetString("target")
+		newTarget, err := cmd.Flags().GetString("compilation-target")
 		if err != nil {
 			return err
 		}

fuzzing/config/config.go

@@ -3,15 +3,21 @@ package config
 import (
 	"encoding/json"
 	"errors"
-	"os"
-
 	"github.com/crytic/medusa/chain/config"
-	"github.com/rs/zerolog"
-
 	"github.com/crytic/medusa/compilation"
+	"github.com/crytic/medusa/logging"
 	"github.com/crytic/medusa/utils"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/rs/zerolog"
+	"math/big"
+	"os"
 )
 
+// The following directives will be picked up by the `go generate` command to generate JSON marshaling code from
+// templates defined below. They should be preserved for re-use in case we change our structures.
+//go:generate go get github.com/fjl/gencodec
+//go:generate go run github.com/fjl/gencodec -type FuzzingConfig -field-override fuzzingConfigMarshaling -out gen_fuzzing_config.go
+
 type ProjectConfig struct {
 	// Fuzzing describes the configuration used in fuzzing campaigns.
 	Fuzzing FuzzingConfig `json:"fuzzing"`
@@ -50,10 +56,15 @@ type FuzzingConfig struct {
 	// CoverageEnabled describes whether to use coverage-guided fuzzing
 	CoverageEnabled bool `json:"coverageEnabled"`
 
-	// DeploymentOrder determines the order in which the contracts should be deployed
-	DeploymentOrder []string `json:"deploymentOrder"`
+	// TargetContracts are the target contracts for fuzz testing
+	TargetContracts []string `json:"targetContracts"`
+
+	// TargetContractsBalances holds the amount of wei that should be sent during deployment for one or more contracts in
+	// TargetContracts
+	TargetContractsBalances []*big.Int `json:"targetContractsBalances"`
 
-	// Constructor arguments for contracts deployment. It is available only in init mode
+	// ConstructorArgs holds the constructor arguments for TargetContracts deployments. It is available via the project
+	// configuration
 	ConstructorArgs map[string]map[string]any `json:"constructorArgs"`
 
 	// DeployerAddress describe the account address to be used to deploy contracts.
@@ -85,6 +96,13 @@ type FuzzingConfig struct {
 	TestChainConfig config.TestChainConfig `json:"chainConfig"`
 }
 
+// fuzzingConfigMarshaling is a structure that overrides field types during JSON marshaling. It allows FuzzingConfig to
+// have its custom marshaling methods auto-generated and will handle type conversions for serialization purposes.
+// For example, this enables serialization of big.Int but specifying a different field type to control serialization.
+type fuzzingConfigMarshaling struct {
+	TargetContractsBalances []*hexutil.Big
+}
+
 // TestingConfig describes the configuration options used for testing
 type TestingConfig struct {
 	// StopOnFailedTest describes whether the fuzzing.Fuzzer should stop after detecting the first failed test.
@@ -111,7 +129,7 @@ type TestingConfig struct {
 	AssertionTesting AssertionTestingConfig `json:"assertionTesting"`
 
 	// PropertyTesting describes the configuration used for property testing.
-	PropertyTesting PropertyTestConfig `json:"propertyTesting"`
+	PropertyTesting PropertyTestingConfig `json:"propertyTesting"`
 
 	// OptimizationTesting describes the configuration used for optimization testing.
 	OptimizationTesting OptimizationTestingConfig `json:"optimizationTesting"`
@@ -125,13 +143,12 @@ type AssertionTestingConfig struct {
 	// TestViewMethods dictates whether constant/pure/view methods should be tested.
 	TestViewMethods bool `json:"testViewMethods"`
 
-	// AssertionModes describes the various panic codes that can be enabled and be treated as a "failing case"
-	AssertionModes AssertionModesConfig `json:"assertionModes"`
+	// PanicCodeConfig describes the various panic codes that can be enabled and be treated as a "failing case"
+	PanicCodeConfig PanicCodeConfig `json:"panicCodeConfig"`
 }
 
-// AssertionModesConfig describes the configuration options for the various modes that can be enabled for assertion
-// testing
-type AssertionModesConfig struct {
+// PanicCodeConfig describes the various panic codes that can be enabled and be treated as a failing assertion test
+type PanicCodeConfig struct {
 	// FailOnCompilerInsertedPanic describes whether a generic compiler inserted panic should be treated as a failing case
 	FailOnCompilerInsertedPanic bool `json:"failOnCompilerInsertedPanic"`
 
@@ -163,8 +180,8 @@ type AssertionModesConfig struct {
 	FailOnCallUninitializedVariable bool `json:"failOnCallUninitializedVariable"`
 }
 
-// PropertyTestConfig describes the configuration options used for property testing
-type PropertyTestConfig struct {
+// PropertyTestingConfig describes the configuration options used for property testing
+type PropertyTestingConfig struct {
 	// Enabled describes whether testing is enabled.
 	Enabled bool `json:"enabled"`
 
@@ -255,27 +272,51 @@ func (p *ProjectConfig) WriteToFile(path string) error {
 // Validate validates that the ProjectConfig meets certain requirements.
 // Returns an error if one occurs.
 func (p *ProjectConfig) Validate() error {
+	// Create logger instance if global logger is available
+	logger := logging.NewLogger(zerolog.Disabled)
+	if logging.GlobalLogger != nil {
+		logger = logging.GlobalLogger.NewSubLogger("module", "fuzzer config")
+	}
+
 	// Verify the worker count is a positive number.
 	if p.Fuzzing.Workers <= 0 {
 		return errors.New("project configuration must specify a positive number for the worker count")
 	}
 
 	// Verify that the sequence length is a positive number
 	if p.Fuzzing.CallSequenceLength <= 0 {
-		return errors.New("project configuration must specify a positive number for the transaction sequence length")
+		return errors.New("project configuration must specify a positive number for the transaction sequence lengt")
 	}
 
 	// Verify the worker reset limit is a positive number
 	if p.Fuzzing.WorkerResetLimit <= 0 {
 		return errors.New("project configuration must specify a positive number for the worker reset limit")
 	}
 
+	// Verify timeout
+	if p.Fuzzing.Timeout < 0 {
+		return errors.New("project configuration must specify a positive number for the timeout")
+	}
+
 	// Verify gas limits are appropriate
 	if p.Fuzzing.BlockGasLimit < p.Fuzzing.TransactionGasLimit {
 		return errors.New("project configuration must specify a block gas limit which is not less than the transaction gas limit")
 	}
 	if p.Fuzzing.BlockGasLimit == 0 || p.Fuzzing.TransactionGasLimit == 0 {
-		return errors.New("project configuration must specify a block and transaction gas limit which is non-zero")
+		return errors.New("project configuration must specify a block and transaction gas limit which are non-zero")
+	}
+
+	// Log warning if max block delay is zero
+	if p.Fuzzing.MaxBlockNumberDelay == 0 {
+		logger.Warn("The maximum block number delay is set to zero. Please be aware that transactions will " +
+			"always be fit in the same block until the block gas limit is reached and that the block number will always " +
+			"increment by one.")
+	}
+
+	// Log warning if max timestamp delay is zero
+	if p.Fuzzing.MaxBlockTimestampDelay == 0 {
+		logger.Warn("The maximum timestamp delay is set to zero. Please be aware that block time jumps will " +
+			"always be exactly one.")
 	}
 
 	// Verify that senders are well-formed addresses
@@ -288,17 +329,10 @@ func (p *ProjectConfig) Validate() error {
 		return errors.New("project configuration must specify only a well-formed deployer address")
 	}
 
-	// Verify property testing fields.
-	if p.Fuzzing.Testing.PropertyTesting.Enabled {
-		// Test prefixes must be supplied if property testing is enabled.
-		if len(p.Fuzzing.Testing.PropertyTesting.TestPrefixes) == 0 {
-			return errors.New("project configuration must specify test name prefixes if property testing is enabled")
-		}
-	}
-
 	// Ensure that the log level is a valid one
-	if _, err := zerolog.ParseLevel(p.Logging.Level.String()); err != nil {
-		return err
+	level, err := zerolog.ParseLevel(p.Logging.Level.String())
+	if err != nil || level == zerolog.FatalLevel {
+		return errors.New("project config must specify a valid log level (trace, debug, info, warn, error, or panic)")
 	}
 
 	return nil

fuzzing/config/config_defaults.go

@@ -4,6 +4,7 @@ import (
 	testChainConfig "github.com/crytic/medusa/chain/config"
 	"github.com/crytic/medusa/compilation"
 	"github.com/rs/zerolog"
+	"math/big"
 )
 
 // GetDefaultProjectConfig obtains a default configuration for a project. It populates a default compilation config
@@ -32,15 +33,16 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 	// Create a project configuration
 	projectConfig := &ProjectConfig{
 		Fuzzing: FuzzingConfig{
-			Workers:            10,
-			WorkerResetLimit:   50,
-			Timeout:            0,
-			TestLimit:          0,
-			CallSequenceLength: 100,
-			DeploymentOrder:    []string{},
-			ConstructorArgs:    map[string]map[string]any{},
-			CorpusDirectory:    "",
-			CoverageEnabled:    true,
+			Workers:                 10,
+			WorkerResetLimit:        50,
+			Timeout:                 0,
+			TestLimit:               0,
+			CallSequenceLength:      100,
+			TargetContracts:         []string{},
+			TargetContractsBalances: []*big.Int{},
+			ConstructorArgs:         map[string]map[string]any{},
+			CorpusDirectory:         "",
+			CoverageEnabled:         true,
 			SenderAddresses: []string{
 				"0x10000",
 				"0x20000",
@@ -58,20 +60,20 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 				TestAllContracts:             false,
 				TraceAll:                     false,
 				AssertionTesting: AssertionTestingConfig{
-					Enabled:         false,
+					Enabled:         true,
 					TestViewMethods: false,
-					AssertionModes: AssertionModesConfig{
+					PanicCodeConfig: PanicCodeConfig{
 						FailOnAssertion: true,
 					},
 				},
-				PropertyTesting: PropertyTestConfig{
+				PropertyTesting: PropertyTestingConfig{
 					Enabled: true,
 					TestPrefixes: []string{
-						"fuzz_",
+						"property_",
 					},
 				},
 				OptimizationTesting: OptimizationTestingConfig{
-					Enabled: false,
+					Enabled: true,
 					TestPrefixes: []string{
 						"optimize_",
 					},