diff --git a/.github/workflows/manual-e2e.yml b/.github/workflows/manual-e2e.yml index 899e4ba230..388a19f0f5 100644 --- a/.github/workflows/manual-e2e.yml +++ b/.github/workflows/manual-e2e.yml @@ -5,22 +5,99 @@ on: workflow_dispatch: jobs: - manual-integration-main: + happy-path-test: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 20 steps: - uses: actions/setup-go@v4 with: go-version: "1.20" - uses: actions/checkout@v3 - - name: Checkout LFS objects run: git lfs checkout - - name: Setup Go uses: actions/setup-go@v4 with: go-version: "1.20" # The Go version to download (if necessary) and use. - - - name: E2E tests - run: make test-e2e + - name: E2E happy-path test + run: go run ./tests/e2e/... --tc happy-path + changeover-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E changeover test + run: go run ./tests/e2e/... --tc changeover + democracy-reward-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E democracy-reward tests + run: go run ./tests/e2e/... --tc democracy-reward + democracy-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E democracy tests + run: go run ./tests/e2e/... --tc democracy + slash-throttle-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E slash-throttle tests + run: go run ./tests/e2e/... --tc slash-throttle + multiconsumer-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E multi-consumer tests + run: go run ./tests/e2e/... --tc multiconsumer diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index f69125e6b8..cd7f155e12 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -18,21 +18,111 @@ on: - cron: "0 3 * * *" jobs: - nightly-test: + happy-path-test: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 20 steps: - uses: actions/setup-go@v4 with: go-version: "1.20" - - uses: actions/checkout@v3 - - - name: E2E tests - run: make test-e2e + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E happy-path test + run: go run ./tests/e2e/... --tc happy-path + changeover-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E changeover test + run: go run ./tests/e2e/... --tc changeover + democracy-reward-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E democracy-reward tests + run: go run ./tests/e2e/... --tc democracy-reward + democracy-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E democracy tests + run: go run ./tests/e2e/... --tc democracy + slash-throttle-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E slash-throttle tests + run: go run ./tests/e2e/... --tc slash-throttle + multiconsumer-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.20" + - uses: actions/checkout@v3 + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" # The Go version to download (if necessary) and use. + - name: E2E multi-consumer tests + run: go run ./tests/e2e/... --tc multiconsumer nightly-test-fail: - needs: nightly-test + needs: + - happy-path-test + - changeover-test + - democracy-reward-test + - democracy-test + - slash-throttle-test + - multiconsumer-test if: ${{ failure() }} runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index edbacbc0e0..ac7c38f068 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ install: go.sum go install $(BUILD_FLAGS) ./cmd/interchain-security-sd # run all tests: unit, integration, diff, and E2E -test: - go test ./... && go run ./tests/e2e/... +test: + go test ./... && go run ./tests/e2e/... # run all unit tests test-unit: @@ -31,12 +31,12 @@ test-diff: # run only happy path E2E tests test-e2e-short: - go run ./tests/e2e/... --happy-path-only + go run ./tests/e2e/... --tc happy-path # run only happy path E2E tests with cometmock # this set of traces does not test equivocation but it does check downtime test-e2e-short-cometmock: - go run ./tests/e2e/... --cometmock-happy-path --use-cometmock --use-gorelayer + go run ./tests/e2e/... --tc happy-path-short --use-cometmock --use-gorelayer # run full E2E tests in sequence (including multiconsumer) test-e2e-multi-consumer: @@ -52,7 +52,7 @@ test-gaia-e2e: # run only happy path E2E tests using latest tagged gaia test-gaia-e2e-short: - go run ./tests/e2e/... --happy-path-only --use-gaia + go run ./tests/e2e/... --tc happy-path --use-gaia # run full E2E tests in parallel (including multiconsumer) using latest tagged gaia test-gaia-e2e-parallel: @@ -66,7 +66,7 @@ test-gaia-e2e-tagged: # run only happy path E2E tests using latest tagged gaia # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-short-tagged test-gaia-e2e-short-tagged: - go run ./tests/e2e/... --happy-path-only --use-gaia --gaia-tag $(GAIA_TAG) + go run ./tests/e2e/... --tc happy-path --use-gaia --gaia-tag $(GAIA_TAG) # run full E2E tests in parallel (including multiconsumer) using specific tagged version of gaia # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-parallel-tagged diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 58bb065c26..e9336422ae 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -15,13 +15,26 @@ import ( "github.com/kylelemons/godebug/pretty" ) +// The list of test cases to be executed +type TestSet []string + +func (t *TestSet) Set(value string) (err error) { + // Check and skip duplicates + for _, v := range *t { + if v == value { + return + } + } + *t = append(*t, value) + return +} + +func (t *TestSet) String() string { + return fmt.Sprint(*t) +} + var ( - verbose = flag.Bool("verbose", false, "turn verbose logging on/off") - happyPathOnly = flag.Bool("happy-path-only", false, "run happy path tests only") - cometmockCompatibleHappyPath = flag.Bool("cometmock-happy-path", false, `run cometmock compatible happy path tests only. -This is like the happy path, but skips steps -that involve starting or stopping nodes for the same chain outside of the chain setup or teardown. -This is suited for CometMock+Gorelayer testing`) + verbose = flag.Bool("verbose", false, "turn verbose logging on/off") includeMultiConsumer = flag.Bool("include-multi-consumer", false, "include multiconsumer tests in run") parallel = flag.Bool("parallel", false, "run all tests in parallel") localSdkPath = flag.String("local-sdk-path", "", @@ -35,59 +48,119 @@ var ( gaiaTag = flag.String("gaia-tag", "", "gaia tag to use - default is latest") ) -// runs E2E tests -// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk -// after building docker containers, all tests are run in parallel using their respective docker containers -func main() { - flag.Parse() - - if cometmockCompatibleHappyPath != nil && *cometmockCompatibleHappyPath { - fmt.Println("=============== running short happy path only ===============") - tr := DefaultTestRun() - tr.Run(cometmockCompatibleHappyPathSteps, *localSdkPath, *useGaia, *gaiaTag) - return - } - - if happyPathOnly != nil && *happyPathOnly { - fmt.Println("=============== running happy path only ===============") - tr := DefaultTestRun() - tr.Run(happyPathSteps, *localSdkPath, *useGaia, *gaiaTag) - return - } - - testRuns := []testRunWithSteps{ - {ChangeoverTestRun(), changeoverSteps}, - {DefaultTestRun(), happyPathSteps}, - {DemocracyTestRun(true), democracySteps}, - {DemocracyTestRun(false), rewardDenomConsumerSteps}, - {SlashThrottleTestRun(), slashThrottleSteps}, - } - if includeMultiConsumer != nil && *includeMultiConsumer { - testRuns = append(testRuns, testRunWithSteps{MultiConsumerTestRun(), multipleConsumers}) +var ( + testSelection TestSet + testMap map[string]*testRunWithSteps = map[string]*testRunWithSteps{ + "happy-path-short": { + testRun: DefaultTestRun(), steps: shortHappyPathSteps, + description: `This is like the happy path, but skips steps +that involve starting or stopping nodes for the same chain outside of the chain setup or teardown. +This is suited for CometMock+Gorelayer testing`, + }, + "happy-path": {testRun: DefaultTestRun(), steps: happyPathSteps, description: "happy path tests"}, + "changeover": {testRun: ChangeoverTestRun(), steps: changeoverSteps, description: "changeover tests"}, + "democracy-reward": {testRun: DemocracyTestRun(true), steps: democracySteps, description: "democracy tests allowing rewards"}, + "democracy": {testRun: DemocracyTestRun(false), steps: rewardDenomConsumerSteps, description: "democracy tests"}, + "slash-throttle": {testRun: SlashThrottleTestRun(), steps: slashThrottleSteps, description: "slash throttle tests"}, + "multiconsumer": {testRun: MultiConsumerTestRun(), steps: multipleConsumers, description: "multi consumer tests"}, } +) - start := time.Now() +func executeTests(tests []testRunWithSteps) (err error) { if parallel != nil && *parallel { fmt.Println("=============== running all tests in parallel ===============") - var wg sync.WaitGroup - for _, run := range testRuns { + } + + var wg sync.WaitGroup + for _, testCase := range tests { + if parallel != nil && *parallel { wg.Add(1) go func(run testRunWithSteps) { defer wg.Done() - tr := run.testRun - tr.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag) - }(run) + run.testRun.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag) + }(testCase) + } else { + log.Printf("=============== running %s ===============\n", testCase.testRun.name) + testCase.testRun.Run(testCase.steps, *localSdkPath, *useGaia, *gaiaTag) } + } + + if parallel != nil && *parallel { wg.Wait() - fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) - return + } + return +} + +func parseArguments() (err error) { + flag.Var(&testSelection, "tc", + fmt.Sprintf("Selection of test cases to be executed:\n%s,\n%s", + func() string { + var keys []string + for k, v := range testMap { + keys = append(keys, fmt.Sprintf("- %s : %s", k, v.description)) + } + return strings.Join(keys, "\n") + }(), + "Example: -tc multiconsumer -tc happy-path ")) + flag.Parse() + + // Enforce go-relayer in case of cometmock as hermes is not yet supported + if useCometmock != nil && *useCometmock && (useGorelayer == nil || !*useGorelayer) { + fmt.Println("Enforcing go-relayer as cometmock is requested") + if err = flag.Set("use-gorelayer", "true"); err != nil { + return + } + } + // check if specified test case exists + for _, tc := range testSelection { + if _, hasKey := testMap[tc]; !hasKey { + err := fmt.Errorf("unknown test case '%s'", tc) + return err + } + } + return +} + +func getTestCases(selection TestSet) (tests []testRunWithSteps) { + // Run default tests if no test cases were selected + if len(selection) == 0 { + selection = TestSet{ + "changeover", "happy-path", + "democracy-reward", "democracy", "slash-throttle", + } + if includeMultiConsumer != nil && *includeMultiConsumer { + selection = append(selection, "multiconsumer") + } + } + + // Get tests from selection + tests = []testRunWithSteps{} + for _, tc := range selection { + if _, exists := testMap[tc]; !exists { + log.Fatalf("Test case '%s' not found", tc) + } + tests = append(tests, *testMap[tc]) + } + return +} + +// runs E2E tests +// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk +// after building docker containers, all tests are run in parallel using their respective docker containers +func main() { + if err := parseArguments(); err != nil { + flag.Usage() + log.Fatalf("Error parsing command arguments %s\n", err) } - for _, run := range testRuns { - tr := run.testRun - tr.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag) + testCases := getTestCases(testSelection) + + start := time.Now() + err := executeTests(testCases) + if err != nil { + log.Fatalf("Test execution failed '%s'", err) } - fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) + log.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start)) } // Run sets up docker container and executes the steps in the test run. @@ -104,8 +177,9 @@ func (tr *TestRun) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag } type testRunWithSteps struct { - testRun TestRun - steps []Step + testRun TestRun + steps []Step + description string } func (tr *TestRun) runStep(step Step, verbose bool) { diff --git a/tests/e2e/steps.go b/tests/e2e/steps.go index 770ac45dda..b33d19783a 100644 --- a/tests/e2e/steps.go +++ b/tests/e2e/steps.go @@ -31,7 +31,7 @@ var happyPathSteps = concatSteps( stepsStopChain("consu", 4), // stop chain ) -var cometmockCompatibleHappyPathSteps = concatSteps( +var shortHappyPathSteps = concatSteps( stepsStartChains([]string{"consu"}, false), stepsDelegate("consu"), stepsUnbond("consu"),