test: fix e2e tests broken in various ways (#2078)

* Fix failing compatiblity tests * Improve identification of closest ICS version for a git reference * fix e2e tests * Fix e2e query command for rewards * Update tests/e2e/config.go Co-authored-by: Simon Noetzlin <[email protected]> --------- Co-authored-by: Simon Noetzlin <[email protected]>
cosmos · Jul 24, 2024 · 706fb44 · 706fb44
1 parent f8662ac
commit 706fb44
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 27 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ vendor/
 build/
 .vscode
 .idea
+__debug_*
 
 # docusaurus versioned docs created during build
 docs/versions.json

diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go
@@ -1000,10 +1000,10 @@ func (tr Chain) addChainToGorelayer(
 	}
 
 	addChainCommand := tr.target.ExecCommand("rly", "chains", "add", "--file", chainConfigFileName, string(ChainId))
-	e2e.ExecuteCommand(addChainCommand, "add chain")
+	e2e.ExecuteCommand(addChainCommand, "add chain", verbose)
 
 	keyRestoreCommand := tr.target.ExecCommand("rly", "keys", "restore", string(ChainId), "default", tr.testConfig.validatorConfigs[action.Validator].Mnemonic)
-	e2e.ExecuteCommand(keyRestoreCommand, "restore keys")
+	e2e.ExecuteCommand(keyRestoreCommand, "restore keys", verbose)
 }
 
 func (tr Chain) addChainToHermes(
@@ -1109,7 +1109,7 @@ func (tr Chain) addIbcConnectionGorelayer(
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
 	pathConfigCommand := tr.target.ExecCommand("bash", "-c", bashCommand)
-	e2e.ExecuteCommand(pathConfigCommand, "add path config")
+	e2e.ExecuteCommand(pathConfigCommand, "add path config", verbose)
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
 	newPathCommand := tr.target.ExecCommand("rly",
@@ -1120,20 +1120,20 @@ func (tr Chain) addIbcConnectionGorelayer(
 		"--file", pathConfigFileName,
 	)
 
-	e2e.ExecuteCommand(newPathCommand, "new path")
+	e2e.ExecuteCommand(newPathCommand, "new path", verbose)
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
 	newClientsCommand := tr.target.ExecCommand("rly", "transact", "clients", pathName)
 
-	e2e.ExecuteCommand(newClientsCommand, "new clients")
+	e2e.ExecuteCommand(newClientsCommand, "new clients", verbose)
 
 	tr.waitBlocks(action.ChainA, 1, 10*time.Second)
 	tr.waitBlocks(action.ChainB, 1, 10*time.Second)
 
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
 	newConnectionCommand := tr.target.ExecCommand("rly", "transact", "connection", pathName)
 
-	e2e.ExecuteCommand(newConnectionCommand, "new connection")
+	e2e.ExecuteCommand(newConnectionCommand, "new connection", verbose)
 
 	tr.waitBlocks(action.ChainA, 1, 10*time.Second)
 	tr.waitBlocks(action.ChainB, 1, 10*time.Second)
@@ -1300,7 +1300,7 @@ func (tr Chain) addIbcChannelGorelayer(
 		"--order", action.Order,
 		"--debug",
 	)
-	e2e.ExecuteCommand(cmd, "addChannel")
+	e2e.ExecuteCommand(cmd, "addChannel", verbose)
 }
 
 func (tr Chain) addIbcChannelHermes(
@@ -1373,16 +1373,16 @@ func (tr Chain) transferChannelComplete(
 		log.Fatal("transferChannelComplete is not implemented for rly")
 	}
 
-	chanOpenTryCmd := tr.target.ExecCommand("hermes",
-		"tx", "chan-open-try",
+	chanOpenTryCmd := tr.target.ExecCommand("hermes", "tx", "chan-open-try",
 		"--dst-chain", string(tr.testConfig.chainConfigs[action.ChainB].ChainId),
 		"--src-chain", string(tr.testConfig.chainConfigs[action.ChainA].ChainId),
 		"--dst-connection", "connection-"+fmt.Sprint(action.ConnectionA),
 		"--dst-port", action.PortB,
 		"--src-port", action.PortA,
 		"--src-channel", "channel-"+fmt.Sprint(action.ChannelA),
 	)
-	e2e.ExecuteCommand(chanOpenTryCmd, "transferChanOpenTry")
+
+	e2e.ExecuteCommand(chanOpenTryCmd, "transferChanOpenTry", verbose)
 
 	chanOpenAckCmd := tr.target.ExecCommand("hermes",
 		"tx", "chan-open-ack",
@@ -1395,7 +1395,7 @@ func (tr Chain) transferChannelComplete(
 		"--src-channel", "channel-"+fmt.Sprint(action.ChannelB),
 	)
 
-	e2e.ExecuteCommand(chanOpenAckCmd, "transferChanOpenAck")
+	e2e.ExecuteCommand(chanOpenAckCmd, "transferChanOpenAck", verbose)
 
 	chanOpenConfirmCmd := tr.target.ExecCommand("hermes",
 		"tx", "chan-open-confirm",
@@ -1407,7 +1407,7 @@ func (tr Chain) transferChannelComplete(
 		"--dst-channel", "channel-"+fmt.Sprint(action.ChannelB),
 		"--src-channel", "channel-"+fmt.Sprint(action.ChannelA),
 	)
-	e2e.ExecuteCommand(chanOpenConfirmCmd, "transferChanOpenConfirm")
+	e2e.ExecuteCommand(chanOpenConfirmCmd, "transferChanOpenConfirm", verbose)
 }
 
 type RelayPacketsAction struct {

diff --git a/tests/e2e/config.go b/tests/e2e/config.go
@@ -128,15 +128,24 @@ func (tr *TestConfig) Initialize() {
 // Note: if no matching version is found an empty string is returned
 func getIcsVersion(reference string) string {
 	icsVersion := ""
-	if reference == "" {
+
+	if reference == "" || reference == VLatest {
 		return icsVersion
 	}
+
 	if semver.IsValid(reference) {
 		// remove build suffix
 		return semver.Canonical(reference)
 	}
 
-	for _, tag := range []string{"v2.0.0", "v2.4.0", "v2.4.0-lsm", "v3.1.0", "v3.2.0", "v3.3.0", "v4.0.0", "v4.1.1", "v4.1.1-lsm"} {
+	// List of all tags matching vX.Y.Z or vX.Y.Z-lsm in ascending order
+	cmd := exec.Command("git", "tag", "-l", "--sort", "v:refname", "v*.?", "v*.?-lsm", "v*.??", "v*.??-lsm")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		panic(fmt.Sprintf("Error getting sorted tag list from git: %s", err.Error()))
+	}
+	icsVersions := strings.Split(string(out), "\n")
+	for _, tag := range icsVersions {
 		//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments
 		cmd := exec.Command("git", "merge-base", "--is-ancestor", reference, tag)
 		out, err := cmd.CombinedOutput()

diff --git a/tests/e2e/state.go b/tests/e2e/state.go
@@ -295,7 +295,7 @@ func (tr Chain) curlJsonRPCRequest(method, params, address string) {
 	cmd := tr.target.ExecCommand("bash", "-c", fmt.Sprintf(cmd_template, method, params, address))
 
 	verbosity := false
-	e2e.ExecuteCommandWithVerbosity(cmd, "curlJsonRPCRequest", verbosity)
+	e2e.ExecuteCommand(cmd, "curlJsonRPCRequest", verbosity)
 }
 
 func uintPtr(i uint) *uint {
@@ -357,8 +357,8 @@ func (tr Commands) GetReward(chain ChainID, validator ValidatorID, blockHeight u
 
 	binaryName := tr.chainConfigs[chain].BinaryName
 	cmd := tr.target.ExecCommand(binaryName,
-		"query", "distribution", "delegation-total-rewards",
-		"--delegator-address", delAddresss,
+		"query", "distribution", "rewards",
+		delAddresss,
 		`--height`, fmt.Sprint(blockHeight),
 		`--node`, tr.GetQueryNode(chain),
 		`-o`, `json`,
@@ -367,6 +367,7 @@ func (tr Commands) GetReward(chain ChainID, validator ValidatorID, blockHeight u
 	bz, err := cmd.CombinedOutput()
 
 	if err != nil {
+		log.Println("running cmd: ", cmd)
 		log.Fatal("failed getting rewards: ", err, "\n", string(bz))
 	}
 

diff --git a/tests/e2e/steps_compatibility.go b/tests/e2e/steps_compatibility.go
@@ -8,6 +8,7 @@ import (
 
 	gov "github.com/cosmos/cosmos-sdk/x/gov/types/v1"
 	clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types"
+	providertypes "github.com/cosmos/interchain-security/v5/x/ccv/provider/types"
 )
 
 func compstepStartProviderChain() []Step {
@@ -98,7 +99,7 @@ func compstepsStartConsumerChain(consumerName string, proposalIndex, chainIndex
 				ConsumerPubkey:  getDefaultValidators()[ValidatorID("carol")].ConsumerValPubKey,
 				ReconfigureNode: false,
 				ExpectError:     true,
-				ExpectedError:   "a validator has assigned the consumer key already: consumer key is already in use by a validator",
+				ExpectedError:   providertypes.ErrConsumerKeyInUse.Error(),
 			},
 			State: State{
 				ChainID(consumerName): ChainState{

diff --git a/tests/e2e/testlib/utils.go b/tests/e2e/testlib/utils.go
@@ -7,9 +7,7 @@ import (
 	"os/exec"
 )
 
-var verbose *bool //TODO: remove hack
-
-func ExecuteCommandWithVerbosity(cmd *exec.Cmd, cmdName string, verbose bool) {
+func ExecuteCommand(cmd *exec.Cmd, cmdName string, verbose bool) {
 	if verbose {
 		fmt.Println(cmdName+" cmd:", cmd.String())
 	}
@@ -36,8 +34,3 @@ func ExecuteCommandWithVerbosity(cmd *exec.Cmd, cmdName string, verbose bool) {
 		log.Fatal(err)
 	}
 }
-
-// Executes a command with verbosity specified by CLI flag
-func ExecuteCommand(cmd *exec.Cmd, cmdName string) {
-	ExecuteCommandWithVerbosity(cmd, cmdName, *verbose)
-}
diff --git a/tests/e2e/v4/state.go b/tests/e2e/v4/state.go
@@ -538,7 +538,7 @@ func (tr Commands) curlJsonRPCRequest(method, params, address string) {
 	cmd := tr.Target.ExecCommand("bash", "-c", fmt.Sprintf(cmd_template, method, params, address))
 
 	verbosity := false
-	e2e.ExecuteCommandWithVerbosity(cmd, "curlJsonRPCRequest", verbosity)
+	e2e.ExecuteCommand(cmd, "curlJsonRPCRequest", verbosity)
 }
 
 // GetClientFrozenHeight returns the frozen height for a client with the given client ID