Refactor Tests

.github/workflows/automated-tests.yml

@@ -11,7 +11,7 @@ on:
       - release/v*
       - feat/*
 jobs:
-  Automated_Tests:
+  Unit_Integration_Tests:
     runs-on: ubuntu-latest
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -28,5 +28,20 @@ jobs:
         run: make proto-check
       - name: Unit, integration and difference tests
         run: go test ./...
+  E2E_Tests:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: Proto Check
+        run: make proto-check
       - name: E2E tests
         run: make test-e2e-short

.github/workflows/e2e-happy-path.yml

@@ -0,0 +1,30 @@
+name: E2E HappyPath Tests
+on:
+  push:
+    branches:
+      - main
+      - release/v*
+      - feat/*
+  pull_request:
+    branches:
+      - main
+      - release/v*
+      - feat/*
+jobs:
+  e2e-happy-path:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Checkout LFS objects
+        run: git lfs checkout
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20" # The Go version to download (if necessary) and use.
+      - name: Proto Check
+        run: make proto-check
+      - name: E2E tests
+        run: make test-e2e-short

tests/e2e/main.go

@@ -15,6 +15,24 @@ import (
 	"github.com/kylelemons/godebug/pretty"
 )
 
+// The list of test cases to be executed
+type TestSet []string
+
+func (t *TestSet) Set(value string) (err error) {
+	// Check and skip duplicates
+	for _, v := range *t {
+		if v == value {
+			return
+		}
+	}
+	*t = append(*t, value)
+	return
+}
+
+func (t *TestSet) String() string {
+	return fmt.Sprint(*t)
+}
+
 var (
 	verbose            = flag.Bool("verbose", false, "turn verbose logging on/off")
 	happyPathOnly      = flag.Bool("happy-path-only", false, "run happy path tests only")
@@ -36,59 +54,122 @@ var (
 	gaiaTag = flag.String("gaia-tag", "", "gaia tag to use - default is latest")
 )
 
-// runs E2E tests
-// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk
-// after building docker containers, all tests are run in parallel using their respective docker containers
-func main() {
+var (
+	testSelection TestSet
+	testMap       map[string]*testRunWithSteps = map[string]*testRunWithSteps{
+		"happy-path-short": {testRun: DefaultTestRun(), steps: shortHappyPathSteps,
+			description: `run abridged happy path tests only.
+		This is like the happy path, but skips steps
+		that involve starting or stopping nodes for the same chain outside of the chain setup or teardown.
+		In particular, this skips steps related to downtime and double signing.
+		This is suited for CometMock+Gorelayer testing`},
+		"happy-path":            {testRun: DefaultTestRun(), steps: happyPathSteps, description: "happy path tests"},
+		"happy-path-softoptout": {testRun: DefaultTestRun(), steps: happyPathSoftOptOutSteps, description: "happy path with soft opt-out downtime"},
+		"changeover":            {testRun: ChangeoverTestRun(), steps: changeoverSteps, description: "changeover tests"},
+		"democracy-reward":      {testRun: DemocracyTestRun(true), steps: democracySteps, description: "democracy tests allowing rewards"},
+		"democracy":             {testRun: DemocracyTestRun(false), steps: rewardDenomConsumerSteps, description: "democracy tests"}, //TODO: clarify why rewardsteps are with arg "reward=false" ???
+		"slash-throttle":        {testRun: SlashThrottleTestRun(), steps: slashThrottleSteps, description: "slash throttle tests"},
+		"multiconsumer":         {testRun: MultiConsumerTestRun(), steps: multipleConsumers, description: "multi-consumer tests"},
+	}
+)
+
+func executeTests(tests []testRunWithSteps) (err error) {
+	if parallel != nil && *parallel {
+		fmt.Println("=============== running all tests in parallel ===============")
+	}
+
+	var wg sync.WaitGroup
+	for _, testCase := range tests {
+		if parallel != nil && *parallel {
+			wg.Add(1)
+			go func(run testRunWithSteps) {
+				defer wg.Done()
+				run.testRun.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag)
+			}(testCase)
+		} else {
+			log.Printf("=============== running %s ===============\n", testCase.testRun.name)
+			testCase.testRun.Run(testCase.steps, *localSdkPath, *useGaia, *gaiaTag)
+		}
+	}
+
+	if parallel != nil && *parallel {
+		wg.Wait()
+	}
+	return
+}
+
+func parseArguments() (err error) {
+	flag.Var(&testSelection, "tc",
+		fmt.Sprintf("Selection of test cases to be executed:\n%s,\n%s",
+			func() string {
+				var keys []string
+				for k, v := range testMap {
+					keys = append(keys, fmt.Sprintf("- %s : %s", k, v.description))
+				}
+				return strings.Join(keys, "\n")
+			}(),
+			"Example: -tc multiconsumer -tc happy-path "))
 	flag.Parse()
 
+	// check if specified test case exists
+	for _, tc := range testSelection {
+		if _, hasKey := testMap[tc]; !hasKey {
+			err := fmt.Errorf("unknown test case '%s'", tc)
+			return err
+		}
+	}
+	return
+}
+
+func getTestCases(selection TestSet) (tests []testRunWithSteps) {
+
 	if shortHappyPathOnly != nil && *shortHappyPathOnly {
-		fmt.Println("=============== running short happy path only ===============")
-		tr := DefaultTestRun()
-		tr.Run(shortHappyPathSteps, *localSdkPath, *useGaia, *gaiaTag)
+		tests = append(tests, *testMap["happy-path-short"])
 		return
 	}
 
 	if happyPathOnly != nil && *happyPathOnly {
-		fmt.Println("=============== running happy path only ===============")
-		tr := DefaultTestRun()
-		tr.Run(happyPathSteps, *localSdkPath, *useGaia, *gaiaTag)
+		tests = append(tests, *testMap["happy-path"])
 		return
 	}
 
-	testRuns := []testRunWithSteps{
-		{ChangeoverTestRun(), changeoverSteps},
-		{DefaultTestRun(), happyPathSteps},
-		{DemocracyTestRun(true), democracySteps},
-		{DemocracyTestRun(false), rewardDenomConsumerSteps},
-		{SlashThrottleTestRun(), slashThrottleSteps},
-	}
-	if includeMultiConsumer != nil && *includeMultiConsumer {
-		testRuns = append(testRuns, testRunWithSteps{MultiConsumerTestRun(), multipleConsumers})
+	// Run default tests if no test cases were selected
+	if len(selection) == 0 {
+		selection = TestSet{"changeover", "happy-path",
+			"democracy-reward", "democracy", "slash-throttle"}
+		if includeMultiConsumer != nil && *includeMultiConsumer {
+			selection = append(selection, "multiconsumer")
+		}
 	}
 
-	start := time.Now()
-	if parallel != nil && *parallel {
-		fmt.Println("=============== running all tests in parallel ===============")
-		var wg sync.WaitGroup
-		for _, run := range testRuns {
-			wg.Add(1)
-			go func(run testRunWithSteps) {
-				defer wg.Done()
-				tr := run.testRun
-				tr.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag)
-			}(run)
+	// Get tests from selection
+	tests = []testRunWithSteps{}
+	for _, tc := range selection {
+		if _, exists := testMap[tc]; !exists {
+			log.Fatalf("Test case '%s' not found", tc)
 		}
-		wg.Wait()
-		fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
-		return
+		tests = append(tests, *testMap[tc])
+	}
+	return
+}
+
+// runs E2E tests
+// all docker containers are built sequentially to avoid race conditions when using local cosmos-sdk
+// after building docker containers, all tests are run in parallel using their respective docker containers
+func main() {
+	if err := parseArguments(); err != nil {
+		flag.Usage()
+		log.Fatalf("Error parsing command arguments %s\n", err)
 	}
 
-	for _, run := range testRuns {
-		tr := run.testRun
-		tr.Run(run.steps, *localSdkPath, *useGaia, *gaiaTag)
+	testCases := getTestCases(testSelection)
+
+	start := time.Now()
+	err := executeTests(testCases)
+	if err != nil {
+		log.Fatalf("Test execution failed '%s'", err)
 	}
-	fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
+	log.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
 }
 
 // Run sets up docker container and executes the steps in the test run.
@@ -105,8 +186,9 @@ func (tr *TestRun) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag
 }
 
 type testRunWithSteps struct {
-	testRun TestRun
-	steps   []Step
+	testRun     TestRun
+	steps       []Step
+	description string
 }
 
 func (tr *TestRun) runStep(step Step, verbose bool) {

tests/e2e/steps.go

@@ -31,6 +31,23 @@ var happyPathSteps = concatSteps(
 	stepsStopChain("consu", 4),                     // stop chain
 )
 
+var happyPathSoftOptOutSteps = concatSteps(
+	stepsStartChains([]string{"consu"}, false),
+	stepsDelegate("consu"),
+	stepsAssignConsumerKeyOnStartedChain("consu", "bob"),
+	stepsUnbond("consu"),
+	stepsCancelUnbond("consu"),
+	stepsRedelegateForOptOut("consu"),
+	stepsDowntimeWithOptOut("consu"),
+	stepsRedelegate("consu"),
+	stepsRejectEquivocationProposal("consu", 2),   // prop to tombstone bob is rejected
+	stepsDoubleSignOnProviderAndConsumer("consu"), // carol double signs on provider, bob double signs on consumer
+	stepsSubmitEquivocationProposal("consu", 2),   // now prop to tombstone bob is submitted and accepted
+	stepsStartRelayer(),
+	stepsConsumerRemovalPropNotPassing("consu", 3), // submit removal prop but vote no on it - chain should stay
+	stepsStopChain("consu", 4),                     // stop chain
+)
+
 var shortHappyPathSteps = concatSteps(
 	stepsStartChains([]string{"consu"}, false),
 	stepsDelegate("consu"),