Tests: 1175 ci refactor e2e (#1191)

* Refactor E2E Tests * Update docstring for short-happy-path The old docstring was outdated and since modified on main --------- Co-authored-by: Philip Offtermatt <[email protected]>
cosmos · Aug 11, 2023 · 64d860d · 64d860d
1 parent fbbac82
commit 64d860d
Show file tree

Hide file tree

Showing 5 changed files with 311 additions and 70 deletions.
diff --git a/.github/workflows/manual-e2e.yml b/.github/workflows/manual-e2e.yml
@@ -5,22 +5,99 @@ on:
   workflow_dispatch:
 
 jobs:
-  manual-integration-main:
+  happy-path-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 20
     steps:
       - uses: actions/setup-go@v4
         with:
           go-version: "1.20"
       - uses: actions/checkout@v3
-
       - name: Checkout LFS objects
         run: git lfs checkout
-
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
           go-version: "1.20" # The Go version to download (if necessary) and use.
-
-      - name: E2E tests
-        run: make test-e2e
+      - name: E2E happy-path test
+        run: go run ./tests/e2e/... --tc happy-path
+  changeover-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - uses: actions/checkout@v3
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E changeover test
+        run: go run ./tests/e2e/... --tc changeover
+  democracy-reward-test:
+      runs-on: ubuntu-latest
+      timeout-minutes: 20
+      steps:
+        - uses: actions/setup-go@v4
+          with:
+            go-version: "1.20"
+        - uses: actions/checkout@v3
+        - name: Checkout LFS objects
+          run: git lfs checkout
+        - name: Setup Go
+          uses: actions/setup-go@v4
+          with:
+            go-version: "1.20" # The Go version to download (if necessary) and use.
+        - name: E2E democracy-reward tests
+          run: go run ./tests/e2e/... --tc democracy-reward
+  democracy-test:
+      runs-on: ubuntu-latest
+      timeout-minutes: 20
+      steps:
+        - uses: actions/setup-go@v4
+          with:
+            go-version: "1.20"
+        - uses: actions/checkout@v3
+        - name: Checkout LFS objects
+          run: git lfs checkout
+        - name: Setup Go
+          uses: actions/setup-go@v4
+          with:
+            go-version: "1.20" # The Go version to download (if necessary) and use.
+        - name: E2E democracy tests
+          run: go run ./tests/e2e/... --tc democracy
+  slash-throttle-test:
+      runs-on: ubuntu-latest
+      timeout-minutes: 20
+      steps:
+        - uses: actions/setup-go@v4
+          with:
+            go-version: "1.20"
+        - uses: actions/checkout@v3
+        - name: Checkout LFS objects
+          run: git lfs checkout
+        - name: Setup Go
+          uses: actions/setup-go@v4
+          with:
+            go-version: "1.20" # The Go version to download (if necessary) and use.
+        - name: E2E slash-throttle tests
+          run: go run ./tests/e2e/... --tc slash-throttle
+  multiconsumer-test:
+      runs-on: ubuntu-latest
+      timeout-minutes: 20
+      steps:
+        - uses: actions/setup-go@v4
+          with:
+            go-version: "1.20"
+        - uses: actions/checkout@v3
+        - name: Checkout LFS objects
+          run: git lfs checkout
+        - name: Setup Go
+          uses: actions/setup-go@v4
+          with:
+            go-version: "1.20" # The Go version to download (if necessary) and use.
+        - name: E2E multi-consumer tests
+          run: go run ./tests/e2e/... --tc multiconsumer
diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml
@@ -18,21 +18,111 @@ on:
     - cron: "0 3 * * *"
 
 jobs:
-  nightly-test:
+  happy-path-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 20
     steps:
       - uses: actions/setup-go@v4
         with:
           go-version: "1.20"
-
       - uses: actions/checkout@v3
-
-      - name: E2E tests
-        run: make test-e2e
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E happy-path test
+        run: go run ./tests/e2e/... --tc happy-path
+  changeover-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - uses: actions/checkout@v3
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E changeover test
+        run: go run ./tests/e2e/... --tc changeover
+  democracy-reward-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - uses: actions/checkout@v3
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E democracy-reward tests
+        run: go run ./tests/e2e/... --tc democracy-reward
+  democracy-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - uses: actions/checkout@v3
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E democracy tests
+        run: go run ./tests/e2e/... --tc democracy
+  slash-throttle-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - uses: actions/checkout@v3
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E slash-throttle tests
+        run: go run ./tests/e2e/... --tc slash-throttle
+  multiconsumer-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+      - uses: actions/checkout@v3
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: E2E multi-consumer tests
+        run: go run ./tests/e2e/... --tc multiconsumer
 
   nightly-test-fail:
-    needs: nightly-test
+    needs:
+      - happy-path-test
+      - changeover-test
+      - democracy-reward-test
+      - democracy-test
+      - slash-throttle-test
+      - multiconsumer-test
     if: ${{ failure() }}
     runs-on: ubuntu-latest
     steps:

diff --git a/Makefile b/Makefile
@@ -10,8 +10,8 @@ install: go.sum
 		go install $(BUILD_FLAGS) ./cmd/interchain-security-sd
 
 # run all tests: unit, integration, diff, and E2E
-test: 
-	go test ./... && go run ./tests/e2e/... 
+test:
+	go test ./... && go run ./tests/e2e/...
 
 # run all unit tests
 test-unit:
@@ -31,12 +31,12 @@ test-diff:
 
 # run only happy path E2E tests
 test-e2e-short:
-	go run ./tests/e2e/... --happy-path-only
+	go run ./tests/e2e/... --tc happy-path
 
 # run only happy path E2E tests with cometmock
 # this set of traces does not test equivocation but it does check downtime
 test-e2e-short-cometmock:
-	go run ./tests/e2e/... --cometmock-happy-path --use-cometmock --use-gorelayer
+	go run ./tests/e2e/... --tc happy-path-short --use-cometmock --use-gorelayer
 
 # run full E2E tests in sequence (including multiconsumer)
 test-e2e-multi-consumer:
@@ -52,7 +52,7 @@ test-gaia-e2e:
 
 # run only happy path E2E tests using latest tagged gaia
 test-gaia-e2e-short:
-	go run ./tests/e2e/... --happy-path-only --use-gaia
+	go run ./tests/e2e/... --tc happy-path --use-gaia
 
 # run full E2E tests in parallel (including multiconsumer) using latest tagged gaia
 test-gaia-e2e-parallel:
@@ -66,7 +66,7 @@ test-gaia-e2e-tagged:
 # run only happy path E2E tests using latest tagged gaia
 # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-short-tagged
 test-gaia-e2e-short-tagged:
-	go run ./tests/e2e/... --happy-path-only --use-gaia --gaia-tag $(GAIA_TAG)
+	go run ./tests/e2e/... --tc happy-path --use-gaia --gaia-tag $(GAIA_TAG)
 
 # run full E2E tests in parallel (including multiconsumer) using specific tagged version of gaia
 # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-parallel-tagged