diff --git a/cmd/test.go b/cmd/test.go index df9c67375..dfc00901c 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -4,12 +4,14 @@ package cmd import ( "context" + "encoding/json" "fmt" "io" "net/http" "net/http/httptrace" "os" "os/signal" + "path/filepath" "slices" "sort" "strings" @@ -17,7 +19,6 @@ import ( "time" "unicode/utf8" - "github.com/pelletier/go-toml/v2" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/exp/maps" @@ -48,7 +49,7 @@ const ( ) type testConfig struct { - OutputToml string + OutputJSON string Quiet bool TestCases []string Timeout time.Duration @@ -67,7 +68,7 @@ func newTestCmd(cmds ...*cobra.Command) *cobra.Command { } func bindTestFlags(cmd *cobra.Command, config *testConfig) { - cmd.Flags().StringVar(&config.OutputToml, "output-toml", "", "File path to which output can be written in TOML format.") + cmd.Flags().StringVar(&config.OutputJSON, "output-json", "", "File path to which output can be written in JSON format.") cmd.Flags().StringSliceVar(&config.TestCases, "test-cases", nil, fmt.Sprintf("List of comma separated names of tests to be exeucted. Available tests are: %v", listTestCases(cmd))) cmd.Flags().DurationVar(&config.Timeout, "timeout", time.Hour, "Execution timeout for all tests.") cmd.Flags().BoolVar(&config.Quiet, "quiet", false, "Do not print test results to stdout.") @@ -117,8 +118,8 @@ func listTestCases(cmd *cobra.Command) []string { } func mustOutputToFileOnQuiet(cmd *cobra.Command) error { - if cmd.Flag("quiet").Changed && !cmd.Flag("output-toml").Changed { - return errors.New("on --quiet, an --output-toml is required") + if cmd.Flag("quiet").Changed && !cmd.Flag("output-json").Changed { + return errors.New("on --quiet, an --output-json is required") } return nil @@ -150,16 +151,15 @@ const ( categoryScoreC categoryScore = "C" ) -// toml fails on marshaling errors to string, so we wrap the errors and add custom marshal type testResultError struct{ error } type testResult struct { - Name string - Verdict testVerdict - Measurement string - Suggestion string - Error testResultError - IsAcceptable bool + Name string `json:"name"` + Verdict testVerdict `json:"verdict"` + Measurement string `json:"measurement,omitempty"` + Suggestion string `json:"suggestion,omitempty"` + Error testResultError `json:"error,omitempty"` + IsAcceptable bool `json:"-"` } func failedTestResult(testRes testResult, err error) testResult { @@ -198,10 +198,10 @@ type testCaseName struct { } type testCategoryResult struct { - CategoryName string - Targets map[string][]testResult - ExecutionTime Duration - Score categoryScore + CategoryName string `json:"category_name,omitempty"` + Targets map[string][]testResult `json:"targets,omitempty"` + ExecutionTime Duration `json:"execution_time,omitempty"` + Score categoryScore `json:"score,omitempty"` } func appendScore(cat []string, score []string) []string { @@ -213,15 +213,73 @@ func appendScore(cat []string, score []string) []string { return res } +type fileResult struct { + Peers testCategoryResult `json:"charon_peers,omitempty"` + Beacon testCategoryResult `json:"beacon_node,omitempty"` + Validator testCategoryResult `json:"validator_client,omitempty"` + MEV testCategoryResult `json:"mev,omitempty"` + Infra testCategoryResult `json:"infra,omitempty"` +} + func writeResultToFile(res testCategoryResult, path string) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o444) + // open or create a file + existingFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) if err != nil { return errors.Wrap(err, "create/open file") } - defer f.Close() - err = toml.NewEncoder(f).Encode(res) + defer existingFile.Close() + stat, err := existingFile.Stat() + if err != nil { + return errors.Wrap(err, "get file stat") + } + // read file contents or default to empty structure + var file fileResult + if stat.Size() == 0 { + file = fileResult{} + } else { + err = json.NewDecoder(existingFile).Decode(&file) + if err != nil { + return errors.Wrap(err, "decode fileResult from JSON") + } + } + + switch res.CategoryName { + case peersTestCategory: + file.Peers = res + case beaconTestCategory: + file.Beacon = res + case validatorTestCategory: + file.Validator = res + case mevTestCategory: + file.MEV = res + case infraTestCategory: + file.Infra = res + } + + // write data to temp file + tmpFile, err := os.CreateTemp(filepath.Dir(path), fmt.Sprintf("%v-tmp-*.json", filepath.Base(path))) + if err != nil { + return errors.Wrap(err, "create temp file") + } + defer tmpFile.Close() + err = tmpFile.Chmod(0o644) + if err != nil { + return errors.Wrap(err, "chmod temp file") + } + + fileContentJSON, err := json.Marshal(file) + if err != nil { + return errors.Wrap(err, "marshal fileResult to JSON") + } + + _, err = tmpFile.Write(fileContentJSON) + if err != nil { + return errors.Wrap(err, "write json to file") + } + + err = os.Rename(tmpFile.Name(), path) if err != nil { - return errors.Wrap(err, "encode testCategoryResult to TOML") + return errors.Wrap(err, "rename temp file") } return nil diff --git a/cmd/testbeacon.go b/cmd/testbeacon.go index e96b6ae8b..b5cb34231 100644 --- a/cmd/testbeacon.go +++ b/cmd/testbeacon.go @@ -180,6 +180,7 @@ func bindTestBeaconFlags(cmd *cobra.Command, config *testBeaconConfig, flagsPref cmd.Flags().BoolVar(&config.LoadTest, flagsPrefix+"load-test", false, "Enable load test, not advisable when testing towards external beacon nodes.") cmd.Flags().DurationVar(&config.LoadTestDuration, flagsPrefix+"load-test-duration", 5*time.Second, "Time to keep running the load tests in seconds. For each second a new continuous ping instance is spawned.") cmd.Flags().IntVar(&config.SimulationDuration, flagsPrefix+"simulation-duration-in-slots", slotsInEpoch, "Time to keep running the simulation in slots.") + cmd.Flags().StringVar(&config.SimulationFileDir, flagsPrefix+"simulation-file-dir", "./", "Time to keep running the simulation in slots.") cmd.Flags().BoolVar(&config.SimulationVerbose, flagsPrefix+"simulation-verbose", false, "Show results for each request and each validator.") mustMarkFlagRequired(cmd, flagsPrefix+"endpoints") } @@ -250,8 +251,8 @@ func runTestBeacon(ctx context.Context, w io.Writer, cfg testBeaconConfig) (err } } - if cfg.OutputToml != "" { - err = writeResultToFile(res, cfg.OutputToml) + if cfg.OutputJSON != "" { + err = writeResultToFile(res, cfg.OutputJSON) if err != nil { return err } @@ -270,9 +271,8 @@ func testAllBeacons(ctx context.Context, queuedTestCases []testCaseName, allTest group, _ := errgroup.WithContext(ctx) for _, endpoint := range conf.Endpoints { - currEndpoint := endpoint // TODO: can be removed after go1.22 version bump group.Go(func() error { - return testSingleBeacon(ctx, queuedTestCases, allTestCases, conf, currEndpoint, singleBeaconResCh) + return testSingleBeacon(ctx, queuedTestCases, allTestCases, conf, endpoint, singleBeaconResCh) }) } diff --git a/cmd/testbeacon_internal_test.go b/cmd/testbeacon_internal_test.go index 382b64e36..0c0897c9d 100644 --- a/cmd/testbeacon_internal_test.go +++ b/cmd/testbeacon_internal_test.go @@ -41,7 +41,7 @@ func TestBeaconTest(t *testing.T) { name: "default scenario", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -71,7 +71,7 @@ func TestBeaconTest(t *testing.T) { name: "connection refused", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -87,7 +87,7 @@ func TestBeaconTest(t *testing.T) { name: "timeout", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: 100 * time.Nanosecond, @@ -110,7 +110,7 @@ func TestBeaconTest(t *testing.T) { name: "quiet", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: true, TestCases: nil, Timeout: time.Minute, @@ -126,7 +126,7 @@ func TestBeaconTest(t *testing.T) { name: "unsupported test", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"notSupportedTest"}, Timeout: time.Minute, @@ -140,7 +140,7 @@ func TestBeaconTest(t *testing.T) { name: "custom test cases", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"ping"}, Timeout: time.Minute, @@ -163,7 +163,7 @@ func TestBeaconTest(t *testing.T) { name: "write to file", config: testBeaconConfig{ testConfig: testConfig{ - OutputToml: "./write-to-file-test.toml.tmp", + OutputJSON: "./write-to-file-test.json.tmp", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -196,7 +196,7 @@ func TestBeaconTest(t *testing.T) { } defer func() { if test.cleanup != nil { - test.cleanup(t, test.config.OutputToml) + test.cleanup(t, test.config.OutputJSON) } }() @@ -206,8 +206,8 @@ func TestBeaconTest(t *testing.T) { testWriteOut(t, test.expected, buf) } - if test.config.OutputToml != "" { - testWriteFile(t, test.expected, test.config.OutputToml) + if test.config.OutputJSON != "" { + testWriteFile(t, test.expected, test.config.OutputJSON) } }) } @@ -281,9 +281,9 @@ func TestBeaconTestFlags(t *testing.T) { expectedErr: "required flag(s) \"endpoints\" not set", }, { - name: "no output toml on quiet", + name: "no output json on quiet", args: []string{"beacon", "--endpoints=\"test.endpoint\"", "--quiet"}, - expectedErr: "on --quiet, an --output-toml is required", + expectedErr: "on --quiet, an --output-json is required", }, } diff --git a/cmd/testperformance.go b/cmd/testinfra.go similarity index 99% rename from cmd/testperformance.go rename to cmd/testinfra.go index c2160b676..cb0b2c945 100644 --- a/cmd/testperformance.go +++ b/cmd/testinfra.go @@ -166,8 +166,8 @@ func runTestInfra(ctx context.Context, w io.Writer, cfg testInfraConfig) (err er } } - if cfg.OutputToml != "" { - err = writeResultToFile(res, cfg.OutputToml) + if cfg.OutputJSON != "" { + err = writeResultToFile(res, cfg.OutputJSON) if err != nil { return err } diff --git a/cmd/testperformance_internal_test.go b/cmd/testinfra_internal_test.go similarity index 93% rename from cmd/testperformance_internal_test.go rename to cmd/testinfra_internal_test.go index fe98692f3..46ee70d72 100644 --- a/cmd/testperformance_internal_test.go +++ b/cmd/testinfra_internal_test.go @@ -31,7 +31,7 @@ func TestInfraTest(t *testing.T) { name: "default scenario", config: testInfraConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"availableMemory", "totalMemory", "internetLatency"}, Timeout: time.Minute, @@ -55,7 +55,7 @@ func TestInfraTest(t *testing.T) { name: "timeout", config: testInfraConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: 100 * time.Nanosecond, @@ -77,7 +77,7 @@ func TestInfraTest(t *testing.T) { name: "quiet", config: testInfraConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: true, TestCases: []string{"availableMemory", "totalMemory", "internetLatency"}, Timeout: time.Minute, @@ -101,7 +101,7 @@ func TestInfraTest(t *testing.T) { name: "unsupported test", config: testInfraConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"notSupportedTest"}, Timeout: time.Minute, @@ -118,7 +118,7 @@ func TestInfraTest(t *testing.T) { name: "custom test cases", config: testInfraConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"totalMemory"}, Timeout: time.Minute, @@ -140,7 +140,7 @@ func TestInfraTest(t *testing.T) { name: "write to file", config: testInfraConfig{ testConfig: testConfig{ - OutputToml: "./write-to-file-test.toml.tmp", + OutputJSON: "./write-to-file-test.json.tmp", Quiet: false, TestCases: []string{"availableMemory", "totalMemory", "internetLatency"}, Timeout: time.Minute, @@ -179,7 +179,7 @@ func TestInfraTest(t *testing.T) { } defer func() { if test.cleanup != nil { - test.cleanup(t, test.config.OutputToml) + test.cleanup(t, test.config.OutputJSON) } }() @@ -189,8 +189,8 @@ func TestInfraTest(t *testing.T) { testWriteOut(t, test.expected, buf) } - if test.config.OutputToml != "" { - testWriteFile(t, test.expected, test.config.OutputToml) + if test.config.OutputJSON != "" { + testWriteFile(t, test.expected, test.config.OutputJSON) } }) } @@ -227,9 +227,9 @@ func TestInfraTestFlags(t *testing.T) { expectedErr: "", }, { - name: "no output toml on quiet", + name: "no output json on quiet", args: []string{"infra", "--disk-io-block-size-kb=1", "--quiet"}, - expectedErr: "on --quiet, an --output-toml is required", + expectedErr: "on --quiet, an --output-json is required", }, } diff --git a/cmd/testmev.go b/cmd/testmev.go index b79955889..f640b8505 100644 --- a/cmd/testmev.go +++ b/cmd/testmev.go @@ -32,6 +32,7 @@ type testMEVConfig struct { testConfig Endpoints []string BeaconNodeEndpoint string + LoadTest bool LoadTestBlocks uint } @@ -65,15 +66,26 @@ func newTestMEVCmd(runFunc func(context.Context, io.Writer, testMEVConfig) error bindTestFlags(cmd, &config.testConfig) bindTestMEVFlags(cmd, &config, "") + wrapPreRunE(cmd, func(cmd *cobra.Command, _ []string) error { + loadTest := cmd.Flags().Lookup("load-test").Value.String() + beaconNodeEndpoint := cmd.Flags().Lookup("beacon-node-endpoint").Value.String() + + if loadTest == "true" && beaconNodeEndpoint == "" { + return errors.New("beacon-node-endpoint should be specified when load-test is") + } + + return nil + }) + return cmd } func bindTestMEVFlags(cmd *cobra.Command, config *testMEVConfig, flagsPrefix string) { cmd.Flags().StringSliceVar(&config.Endpoints, flagsPrefix+"endpoints", nil, "[REQUIRED] Comma separated list of one or more MEV relay endpoint URLs.") cmd.Flags().StringVar(&config.BeaconNodeEndpoint, flagsPrefix+"beacon-node-endpoint", "", "[REQUIRED] Beacon node endpoint URL used for block creation test.") + cmd.Flags().BoolVar(&config.LoadTest, flagsPrefix+"load-test", false, "Enable load test.") cmd.Flags().UintVar(&config.LoadTestBlocks, flagsPrefix+"load-test-blocks", 3, "Amount of blocks the 'createMultipleBlocks' test will create.") mustMarkFlagRequired(cmd, flagsPrefix+"endpoints") - mustMarkFlagRequired(cmd, flagsPrefix+"beacon-node-endpoint") } func supportedMEVTestCases() map[testCaseName]testCaseMEV { @@ -134,8 +146,8 @@ func runTestMEV(ctx context.Context, w io.Writer, cfg testMEVConfig) (err error) } } - if cfg.OutputToml != "" { - err = writeResultToFile(res, cfg.OutputToml) + if cfg.OutputJSON != "" { + err = writeResultToFile(res, cfg.OutputJSON) if err != nil { return err } diff --git a/cmd/testmev_internal_test.go b/cmd/testmev_internal_test.go index e69271219..5519904b9 100644 --- a/cmd/testmev_internal_test.go +++ b/cmd/testmev_internal_test.go @@ -43,7 +43,7 @@ func TestMEVTest(t *testing.T) { name: "default scenario", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -67,7 +67,7 @@ func TestMEVTest(t *testing.T) { name: "connection refused", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -97,7 +97,7 @@ func TestMEVTest(t *testing.T) { name: "timeout", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: 100 * time.Nanosecond, @@ -121,7 +121,7 @@ func TestMEVTest(t *testing.T) { name: "quiet", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: true, TestCases: nil, Timeout: time.Minute, @@ -151,7 +151,7 @@ func TestMEVTest(t *testing.T) { name: "unsupported test", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"notSupportedTest"}, Timeout: time.Minute, @@ -166,7 +166,7 @@ func TestMEVTest(t *testing.T) { name: "custom test cases", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"ping"}, Timeout: time.Minute, @@ -190,7 +190,7 @@ func TestMEVTest(t *testing.T) { name: "write to file", config: testMEVConfig{ testConfig: testConfig{ - OutputToml: "./write-to-file-test.toml.tmp", + OutputJSON: "./write-to-file-test.json.tmp", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -237,7 +237,7 @@ func TestMEVTest(t *testing.T) { } defer func() { if test.cleanup != nil { - test.cleanup(t, test.config.OutputToml) + test.cleanup(t, test.config.OutputJSON) } }() @@ -247,8 +247,8 @@ func TestMEVTest(t *testing.T) { testWriteOut(t, test.expected, buf) } - if test.config.OutputToml != "" { - testWriteFile(t, test.expected, test.config.OutputToml) + if test.config.OutputJSON != "" { + testWriteFile(t, test.expected, test.config.OutputJSON) } }) } @@ -270,18 +270,23 @@ func TestMEVTestFlags(t *testing.T) { }{ { name: "default scenario", - args: []string{"mev", "--endpoints=\"test.endpoint\"", "--beacon-node-endpoint=\"test.endpoint\""}, + args: []string{"mev", "--endpoints=\"test.endpoint\""}, expectedErr: "", }, { name: "no endpoints flag", args: []string{"mev"}, - expectedErr: "required flag(s) \"beacon-node-endpoint\", \"endpoints\" not set", + expectedErr: "required flag(s) \"endpoints\" not set", }, { - name: "no output toml on quiet", + name: "no output json on quiet", args: []string{"mev", "--endpoints=\"test.endpoint\"", "--quiet"}, - expectedErr: "on --quiet, an --output-toml is required", + expectedErr: "on --quiet, an --output-json is required", + }, + { + name: "no beacon node endpoint flag on load test", + args: []string{"mev", "--endpoints=\"test.endpoint\"", "--load-test"}, + expectedErr: "beacon-node-endpoint should be specified when load-test is", }, } diff --git a/cmd/testpeers.go b/cmd/testpeers.go index 987c448c5..a800768e5 100644 --- a/cmd/testpeers.go +++ b/cmd/testpeers.go @@ -228,8 +228,8 @@ func runTestPeers(ctx context.Context, w io.Writer, conf testPeersConfig) error } } - if conf.OutputToml != "" { - err = writeResultToFile(res, conf.OutputToml) + if conf.OutputJSON != "" { + err = writeResultToFile(res, conf.OutputJSON) if err != nil { return err } diff --git a/cmd/testpeers_internal_test.go b/cmd/testpeers_internal_test.go index 0d766c0f3..56d6d611f 100644 --- a/cmd/testpeers_internal_test.go +++ b/cmd/testpeers_internal_test.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "fmt" "io" "net/http" @@ -17,7 +18,6 @@ import ( k1 "github.com/decred/dcrd/dcrec/secp256k1/v4" "github.com/libp2p/go-libp2p/core/peer" - "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" @@ -50,7 +50,7 @@ func TestPeersTest(t *testing.T) { name: "default scenario", config: testPeersConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: 10 * time.Second, @@ -118,7 +118,7 @@ func TestPeersTest(t *testing.T) { name: "quiet", config: testPeersConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: true, TestCases: nil, Timeout: 3 * time.Second, @@ -162,7 +162,7 @@ func TestPeersTest(t *testing.T) { name: "unsupported test", config: testPeersConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"notSupportedTest"}, Timeout: 200 * time.Millisecond, @@ -185,7 +185,7 @@ func TestPeersTest(t *testing.T) { name: "custom test cases", config: testPeersConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"ping"}, Timeout: 200 * time.Millisecond, @@ -222,7 +222,7 @@ func TestPeersTest(t *testing.T) { name: "write to file", config: testPeersConfig{ testConfig: testConfig{ - OutputToml: "./write-to-file-test.toml.tmp", + OutputJSON: "./write-to-file-test.json.tmp", Quiet: false, Timeout: 3 * time.Second, }, @@ -289,7 +289,7 @@ func TestPeersTest(t *testing.T) { } defer func() { if test.cleanup != nil { - test.cleanup(t, conf.OutputToml) + test.cleanup(t, conf.OutputJSON) } }() @@ -299,8 +299,8 @@ func TestPeersTest(t *testing.T) { testWriteOut(t, test.expected, buf) } - if test.config.OutputToml != "" { - testWriteFile(t, test.expected, test.config.OutputToml) + if test.config.OutputJSON != "" { + testWriteFile(t, test.expected, test.config.OutputJSON) } }) } @@ -323,9 +323,9 @@ func TestPeersTestFlags(t *testing.T) { expectedErr: "--enrs, --cluster-lock-file-path or --cluster-definition-file-path must be specified.", }, { - name: "no output toml on quiet", + name: "no output json on quiet", args: []string{"peers", "--enrs=\"test.endpoint\"", "--quiet"}, - expectedErr: "on --quiet, an --output-toml is required", + expectedErr: "on --quiet, an --output-json is required", }, } @@ -379,14 +379,30 @@ func testWriteFile(t *testing.T, expectedRes testCategoryResult, path string) { t.Helper() file, err := os.ReadFile(path) require.NoError(t, err) - var res testCategoryResult - err = toml.Unmarshal(file, &res) + var res fileResult + err = json.Unmarshal(file, &res) require.NoError(t, err) - require.Equal(t, expectedRes.CategoryName, res.CategoryName) - require.Equal(t, len(expectedRes.Targets), len(res.Targets)) + var actualRes testCategoryResult + switch expectedRes.CategoryName { + case peersTestCategory: + actualRes = res.Peers + case beaconTestCategory: + actualRes = res.Beacon + case validatorTestCategory: + actualRes = res.Validator + case mevTestCategory: + actualRes = res.MEV + case infraTestCategory: + actualRes = res.Infra + default: + t.Error("unknown category") + } + + require.Equal(t, expectedRes.CategoryName, actualRes.CategoryName) + require.Equal(t, len(expectedRes.Targets), len(actualRes.Targets)) checkFinalScore := true - for targetName, testResults := range res.Targets { + for targetName, testResults := range actualRes.Targets { for idx, testRes := range testResults { // do not test verdicts based on measurements if expectedRes.Targets[targetName][idx].Verdict == testVerdictOk || expectedRes.Targets[targetName][idx].Verdict == testVerdictFail { @@ -406,7 +422,7 @@ func testWriteFile(t *testing.T, expectedRes testCategoryResult, path string) { } // check final score only if there are no tests based on actual measurement if checkFinalScore { - require.Equal(t, expectedRes.Score, res.Score) + require.Equal(t, expectedRes.Score, actualRes.Score) } } diff --git a/cmd/testvalidator.go b/cmd/testvalidator.go index 39d134e6e..57139a839 100644 --- a/cmd/testvalidator.go +++ b/cmd/testvalidator.go @@ -116,8 +116,8 @@ func runTestValidator(ctx context.Context, w io.Writer, cfg testValidatorConfig) } } - if cfg.OutputToml != "" { - err = writeResultToFile(res, cfg.OutputToml) + if cfg.OutputJSON != "" { + err = writeResultToFile(res, cfg.OutputJSON) if err != nil { return err } diff --git a/cmd/testvalidator_internal_test.go b/cmd/testvalidator_internal_test.go index 88ff1a8fb..9f4ab003f 100644 --- a/cmd/testvalidator_internal_test.go +++ b/cmd/testvalidator_internal_test.go @@ -42,7 +42,7 @@ func TestValidatorTest(t *testing.T) { name: "default scenario", config: testValidatorConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -66,7 +66,7 @@ func TestValidatorTest(t *testing.T) { name: "timeout", config: testValidatorConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: nil, Timeout: 100 * time.Nanosecond, @@ -88,7 +88,7 @@ func TestValidatorTest(t *testing.T) { name: "quiet", config: testValidatorConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: true, TestCases: nil, Timeout: time.Minute, @@ -112,7 +112,7 @@ func TestValidatorTest(t *testing.T) { name: "unsupported test", config: testValidatorConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"notSupportedTest"}, Timeout: time.Minute, @@ -129,7 +129,7 @@ func TestValidatorTest(t *testing.T) { name: "custom test cases", config: testValidatorConfig{ testConfig: testConfig{ - OutputToml: "", + OutputJSON: "", Quiet: false, TestCases: []string{"ping"}, Timeout: time.Minute, @@ -151,7 +151,7 @@ func TestValidatorTest(t *testing.T) { name: "write to file", config: testValidatorConfig{ testConfig: testConfig{ - OutputToml: "./write-to-file-test.toml.tmp", + OutputJSON: "./write-to-file-test.json.tmp", Quiet: false, TestCases: nil, Timeout: time.Minute, @@ -190,7 +190,7 @@ func TestValidatorTest(t *testing.T) { } defer func() { if test.cleanup != nil { - test.cleanup(t, test.config.OutputToml) + test.cleanup(t, test.config.OutputJSON) } }() @@ -200,8 +200,8 @@ func TestValidatorTest(t *testing.T) { testWriteOut(t, test.expected, buf) } - if test.config.OutputToml != "" { - testWriteFile(t, test.expected, test.config.OutputToml) + if test.config.OutputJSON != "" { + testWriteFile(t, test.expected, test.config.OutputJSON) } }) } @@ -238,9 +238,9 @@ func TestValidatorTestFlags(t *testing.T) { expectedErr: "", }, { - name: "no output toml on quiet", + name: "no output json on quiet", args: []string{"validator", "--validator-api-address=\"test.endpoint\"", "--quiet"}, - expectedErr: "on --quiet, an --output-toml is required", + expectedErr: "on --quiet, an --output-json is required", }, } diff --git a/go.mod b/go.mod index 6dc13d172..4f92370c6 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/libp2p/go-libp2p v0.33.2 github.com/libp2p/go-msgio v0.3.0 github.com/multiformats/go-multiaddr v0.14.0 - github.com/pelletier/go-toml/v2 v2.2.3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/protolambda/eth2-shuffle v1.1.0 @@ -160,6 +159,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pk910/dynamic-ssz v0.0.3 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index bbe69d01a..1ae8e6bc5 100644 --- a/go.sum +++ b/go.sum @@ -407,8 +407,8 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pk910/dynamic-ssz v0.0.3 h1:fCWzFowq9P6SYCc7NtJMkZcIHk+r5hSVD+32zVi6Aio= github.com/pk910/dynamic-ssz v0.0.3/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -526,6 +526,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=