feat: nat; per-validator configuration.

op-nat/gate.go

@@ -12,24 +12,34 @@ var _ Validator = &Gate{}
 // A Gate is a collection of suites and/or tests.
 type Gate struct {
 	ID         string
-	Validators []Validator
+	Validators []Validator // Validators can be Suites or Tests
+	Params     map[string]interface{}
 }
 
 // Run runs all the tests in the gate.
 // Returns the overall result of the gate and an error if any of the tests failed.
-func (g Gate) Run(ctx context.Context, log log.Logger, cfg Config) (ValidatorResult, error) {
+// Gate-specific params are passed in as `_` because we haven't implemented them yet.
+func (g Gate) Run(ctx context.Context, log log.Logger, cfg Config, _ interface{}) (ValidatorResult, error) {
 	log.Info("", "type", g.Type(), "id", g.Name())
 	allPassed := true
 	results := []ValidatorResult{}
 	var allErrors error
 	for _, validator := range g.Validators {
-		res, err := validator.Run(ctx, log, cfg)
+		// We don't want Gates to have Gates
+		if validator == nil || validator.Type() == "Gate" {
+			continue
+		}
+		// Get params
+		params := g.Params[validator.Name()]
+
+		res, err := validator.Run(ctx, log, cfg, params)
 		if err != nil || !res.Passed {
 			allPassed = false
 			allErrors = errors.Join(allErrors, err)
 		}
 		results = append(results, res)
 	}
+	log.Info("", "type", g.Type(), "id", g.Name(), "passed", allPassed, "error", allErrors)
 	return ValidatorResult{
 		ID:         g.ID,
 		Type:       g.Type(),

op-nat/gate_test.go

@@ -0,0 +1,96 @@
+package nat
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGate(t *testing.T) {
+	t.Run("passes when all validators pass", func(t *testing.T) {
+		gate := &Gate{
+			Validators: []Validator{
+				&Test{
+					ID: "test1",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						return true, nil
+					},
+				},
+				&Test{
+					ID: "test2",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						return true, nil
+					},
+				},
+			},
+		}
+
+		result, err := gate.Run(context.Background(), log.New(), Config{}, nil)
+
+		require.NoError(t, err)
+		assert.True(t, result.Passed)
+	})
+
+	t.Run("fails if any validator fails", func(t *testing.T) {
+		gate := &Gate{
+			Validators: []Validator{
+				&Test{
+					ID: "test1",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						return true, nil
+					},
+				},
+				&Test{
+					ID: "test2",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						return false, nil
+					},
+				},
+			},
+		}
+
+		result, err := gate.Run(context.Background(), log.New(), Config{}, nil)
+
+		require.NoError(t, err)
+		assert.False(t, result.Passed)
+	})
+
+	t.Run("doesnt stop on validator failure", func(t *testing.T) {
+		executionOrder := []string{}
+
+		gate := &Gate{
+			Validators: []Validator{
+				&Test{
+					ID: "test1",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						executionOrder = append(executionOrder, "test1")
+						return true, nil
+					},
+				},
+				&Test{
+					ID: "test2",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						executionOrder = append(executionOrder, "test2")
+						return false, nil
+					},
+				},
+				&Test{
+					ID: "test3",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						executionOrder = append(executionOrder, "test3")
+						return true, nil
+					},
+				},
+			},
+		}
+
+		result, err := gate.Run(context.Background(), log.New(), Config{}, nil)
+
+		require.NoError(t, err)
+		assert.False(t, result.Passed)
+		assert.Equal(t, []string{"test1", "test2", "test3"}, executionOrder, "shouldnt stop on validator failure")
+	})
+}

op-nat/nat.go

@@ -19,6 +19,7 @@ type nat struct {
 	ctx     context.Context
 	log     log.Logger
 	config  *Config
+	params  map[string]interface{}
 	version string
 	results []ValidatorResult
 
@@ -36,6 +37,7 @@ func New(ctx context.Context, config *Config, log log.Logger, version string) (*
 	return &nat{
 		ctx:     ctx,
 		config:  config,
+		params:  map[string]interface{}{},
 		log:     log,
 		version: version,
 	}, nil
@@ -48,7 +50,11 @@ func (n *nat) Start(ctx context.Context) error {
 	n.running.Store(true)
 	for _, validator := range n.config.Validators {
 		n.log.Info("Running acceptance tests...")
-		result, err := validator.Run(ctx, n.log, *n.config)
+
+		// Get test-specific parameters if they exist
+		params := n.params[validator.Name()]
+
+		result, err := validator.Run(ctx, n.log, *n.config, params)
 		n.log.Info("Completed validator", "validator", validator.Name(), "type", validator.Type(), "passed", result.Passed, "error", err)
 		if err != nil {
 			n.log.Error("Error running validator", "validator", validator.Name(), "error", err)

op-nat/nat_test.go

@@ -0,0 +1,117 @@
+package nat
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNATParameterization(t *testing.T) {
+	// Create a test that records its received parameters
+	var receivedParams interface{}
+	testFn := func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+		receivedParams = params
+		return true, nil
+	}
+
+	test := &Test{
+		ID:            "test-with-params",
+		DefaultParams: map[string]string{"value": "default"},
+		Fn:            testFn,
+	}
+
+	// Create a basic config with our test
+	cfg := &Config{
+		Validators:          []Validator{test},
+		SenderSecretKey:     "0x0",
+		ReceiverPublicKeys:  []string{"0x0"},
+		ReceiverPrivateKeys: []string{"0x0"},
+	}
+	logger := log.New()
+
+	t.Run("uses default parameters when none provided", func(t *testing.T) {
+		nat, err := New(context.Background(), cfg, logger, "test")
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := nat.Stop(context.Background())
+			require.NoError(t, err)
+		})
+
+		err = nat.Start(context.Background())
+		require.NoError(t, err)
+
+		assert.Equal(t, test.DefaultParams, receivedParams)
+	})
+
+	t.Run("uses custom parameters when provided", func(t *testing.T) {
+		nat, err := New(context.Background(), cfg, logger, "test")
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := nat.Stop(context.Background())
+			require.NoError(t, err)
+		})
+
+		customParams := map[string]string{"value": "custom"}
+		nat.params = map[string]interface{}{
+			test.ID: customParams,
+		}
+
+		err = nat.Start(context.Background())
+		require.NoError(t, err)
+
+		assert.Equal(t, customParams, receivedParams)
+	})
+
+	t.Run("different test instances can have different parameters", func(t *testing.T) {
+		// Create two instances with different parameters
+		nat1, err := New(context.Background(), cfg, logger, "test1")
+		require.NoError(t, err)
+		nat1.params = map[string]interface{}{
+			test.ID: map[string]string{"value": "instance1"},
+		}
+
+		nat2, err := New(context.Background(), cfg, logger, "test2")
+		require.NoError(t, err)
+		nat2.params = map[string]interface{}{
+			test.ID: map[string]string{"value": "instance2"},
+		}
+
+		t.Cleanup(func() {
+			err := nat1.Stop(context.Background())
+			require.NoError(t, err)
+			err = nat2.Stop(context.Background())
+			require.NoError(t, err)
+		})
+
+		// Run first instance
+		err = nat1.Start(context.Background())
+		require.NoError(t, err)
+		assert.Equal(t, map[string]string{"value": "instance1"}, receivedParams)
+
+		// Run second instance
+		err = nat2.Start(context.Background())
+		require.NoError(t, err)
+		assert.Equal(t, map[string]string{"value": "instance2"}, receivedParams)
+	})
+
+	t.Run("results are properly recorded", func(t *testing.T) {
+		nat, err := New(context.Background(), cfg, logger, "test")
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := nat.Stop(context.Background())
+			require.NoError(t, err)
+		})
+		nat.params = make(map[string]interface{})
+
+		err = nat.Start(context.Background())
+		require.NoError(t, err)
+
+		require.Len(t, nat.results, 1)
+		assert.Equal(t, "test-with-params", nat.results[0].ID)
+		assert.Equal(t, "Test", nat.results[0].Type)
+		assert.True(t, nat.results[0].Passed)
+	})
+}

op-nat/suite.go

@@ -11,25 +11,31 @@ var _ Validator = &Suite{}
 
 // A Suite is a collection of tests.
 type Suite struct {
-	ID    string
-	Tests []Test
+	ID          string
+	Tests       []Test
+	TestsParams map[string]interface{}
 }
 
 // Run runs all the tests in the suite.
 // Returns the overall result of the suite and an error if any of the tests failed.
-func (s Suite) Run(ctx context.Context, log log.Logger, cfg Config) (ValidatorResult, error) {
+// Suite-specific params are passed in as `_` because we haven't implemented them yet.
+func (s Suite) Run(ctx context.Context, log log.Logger, cfg Config, _ interface{}) (ValidatorResult, error) {
 	log.Info("", "type", s.Type(), "id", s.Name())
 	allPassed := true
 	results := []ValidatorResult{}
 	var allErrors error
 	for _, test := range s.Tests {
-		res, err := test.Run(ctx, log, cfg)
+		// Get test-specific params
+		params := s.TestsParams[test.ID]
+
+		res, err := test.Run(ctx, log, cfg, params)
 		if err != nil || !res.Passed {
 			allPassed = false
 			allErrors = errors.Join(allErrors, err)
 		}
 		results = append(results, res)
 	}
+	log.Info("", "type", s.Type(), "id", s.Name(), "passed", allPassed, "error", allErrors)
 	return ValidatorResult{
 		ID:         s.ID,
 		Type:       s.Type(),

op-nat/suite_test.go

@@ -0,0 +1,65 @@
+package nat
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSuite(t *testing.T) {
+	t.Run("runs all tests in order", func(t *testing.T) {
+		executionOrder := []string{}
+
+		suite := &Suite{
+			Tests: []Test{
+				{
+					ID: "test1",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						executionOrder = append(executionOrder, "test1")
+						return true, nil
+					},
+				},
+				{
+					ID: "test2",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						executionOrder = append(executionOrder, "test2")
+						return true, nil
+					},
+				},
+			},
+		}
+
+		result, err := suite.Run(context.Background(), log.New(), Config{}, nil)
+
+		require.NoError(t, err)
+		assert.True(t, result.Passed)
+		assert.Equal(t, []string{"test1", "test2"}, executionOrder)
+	})
+
+	t.Run("fails if any test fails", func(t *testing.T) {
+		suite := &Suite{
+			Tests: []Test{
+				{
+					ID: "test1",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						return true, nil
+					},
+				},
+				{
+					ID: "test2",
+					Fn: func(ctx context.Context, log log.Logger, cfg Config, params interface{}) (bool, error) {
+						return false, nil
+					},
+				},
+			},
+		}
+
+		result, err := suite.Run(context.Background(), log.New(), Config{}, nil)
+
+		require.NoError(t, err)
+		assert.False(t, result.Passed)
+	})
+}