test: Add backtransform functionality to genesis transform cli (#1534)

* Add backtransform functionality to genesis transform cli

* Addressed comments from review

* Fix issue with consumer v2.x compatibility e2e tests

app/consumer/genesis.go

@@ -5,15 +5,16 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/maps"
 
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/version"
-
 	consumerTypes "github.com/cosmos/interchain-security/v3/x/ccv/consumer/types"
 	"github.com/cosmos/interchain-security/v3/x/ccv/types"
 )
@@ -27,42 +28,66 @@ import (
 // object provided to it during init.
 type GenesisState map[string]json.RawMessage
 
-// Migration of consumer genesis content as it is exported from a provider version v1,2,3
+// Map of supported versions for consumer genesis transformation
+type IcsVersion string
+
+const (
+	v2_x   IcsVersion = "v2.x"
+	v3_0_x IcsVersion = "v3.0.x"
+	v3_1_x IcsVersion = "v3.1.x"
+	v3_2_x IcsVersion = "v3.2.x"
+	v3_3_x IcsVersion = "v3.3.x"
+	v4_x_x IcsVersion = "v4.x"
+)
+
+var TransformationVersions map[string]IcsVersion = map[string]IcsVersion{
+	"v2.x":   v2_x,
+	"v3.0.x": v3_0_x,
+	"v3.1.x": v3_1_x,
+	"v3.2.x": v3_2_x,
+	"v3.3.x": v3_3_x,
+	"v4.x":   v4_x_x,
+}
+
+// Transformation of consumer genesis content as it is exported from a provider version v1,2,3
 // to a format readable by current consumer implementation.
-func transform(jsonRaw []byte, ctx client.Context) (json.RawMessage, error) {
+func transformToNew(jsonRaw []byte, ctx client.Context) (json.RawMessage, error) {
 	// v1,2,3 uses deprecated fields of GenesisState type
 	oldConsumerGenesis := consumerTypes.GenesisState{}
 	err := ctx.Codec.UnmarshalJSON(jsonRaw, &oldConsumerGenesis)
 	if err != nil {
 		return nil, fmt.Errorf("reading consumer genesis data failed: %s", err)
 	}
 
-	// some sanity checks for v2 transformation
-	if len(oldConsumerGenesis.Provider.InitialValSet) > 0 {
-		return nil, fmt.Errorf("invalid source version. Unexpected element 'provider.initial_val_set'")
+	initialValSet := oldConsumerGenesis.InitialValSet
+	// transformation from >= v3.3.x
+	if len(initialValSet) == 0 {
+		initialValSet = oldConsumerGenesis.Provider.InitialValSet
 	}
 
-	if oldConsumerGenesis.Provider.ClientState != nil {
-		return nil, fmt.Errorf("invalid source version. Unexpected element 'provider.client_state'")
+	clientState := oldConsumerGenesis.ProviderClientState
+	if clientState == nil {
+		clientState = oldConsumerGenesis.Provider.ClientState
 	}
 
-	if oldConsumerGenesis.Provider.ConsensusState != nil {
-		return nil, fmt.Errorf("invalid source version. Unexpected element 'provider.consensus_state'")
+	consensusState := oldConsumerGenesis.ProviderConsensusState
+	if consensusState == nil {
+		consensusState = oldConsumerGenesis.Provider.ConsensusState
 	}
 
 	// Use DefaultRetryDelayPeriod if not set
 	if oldConsumerGenesis.Params.RetryDelayPeriod == 0 {
 		oldConsumerGenesis.Params.RetryDelayPeriod = types.DefaultRetryDelayPeriod
 	}
 
-	// Version 2 of provider genesis data fills up deprecated fields
-	// ProviderClientState, ConsensusState and InitialValSet
+	// Versions before v3.3.x of provider genesis data fills up deprecated fields
+	// ProviderClientState, ConsensusState and InitialValSet in type GenesisState
 	newGenesis := types.ConsumerGenesisState{
 		Params: oldConsumerGenesis.Params,
 		Provider: types.ProviderInfo{
-			ClientState:    oldConsumerGenesis.ProviderClientState,
-			ConsensusState: oldConsumerGenesis.ProviderConsensusState,
-			InitialValSet:  oldConsumerGenesis.InitialValSet,
+			ClientState:    clientState,
+			ConsensusState: consensusState,
+			InitialValSet:  initialValSet,
 		},
 		NewChain: oldConsumerGenesis.NewChain,
 	}
@@ -74,19 +99,210 @@ func transform(jsonRaw []byte, ctx client.Context) (json.RawMessage, error) {
 	return newJson, nil
 }
 
+// Transformation of consumer genesis content as it is exported by current provider version
+// to a format supported by consumer version v3.3.x
+func transformToV33(jsonRaw []byte, ctx client.Context) ([]byte, error) {
+	// v1,2,3 uses deprecated fields of GenesisState type
+	srcConGen := consumerTypes.GenesisState{}
+	err := ctx.Codec.UnmarshalJSON(jsonRaw, &srcConGen)
+	if err != nil {
+		return nil, fmt.Errorf("reading consumer genesis data failed: %s", err)
+	}
+
+	// Remove retry_delay_period from 'params'
+	params, err := ctx.Codec.MarshalJSON(&srcConGen.Params)
+	if err != nil {
+		return nil, err
+	}
+	tmp := map[string]json.RawMessage{}
+	if err := json.Unmarshal(params, &tmp); err != nil {
+		return nil, fmt.Errorf("unmarshalling 'params' failed: %v", err)
+	}
+	_, exists := tmp["retry_delay_period"]
+	if exists {
+		delete(tmp, "retry_delay_period")
+	}
+	params, err = json.Marshal(tmp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Marshal GenesisState and patch 'params' value
+	result, err := ctx.Codec.MarshalJSON(&srcConGen)
+	if err != nil {
+		return nil, err
+	}
+	genState := map[string]json.RawMessage{}
+	if err := json.Unmarshal(result, &genState); err != nil {
+		return nil, fmt.Errorf("unmarshalling 'GenesisState' failed: %v", err)
+	}
+	genState["params"] = params
+
+	result, err = json.Marshal(genState)
+	if err != nil {
+		return nil, fmt.Errorf("marshalling transformation result failed: %v", err)
+	}
+	return result, nil
+}
+
+// Transformation of consumer genesis content as it is exported from current provider version
+// to a format readable by consumer implementation of version v2.x
+// Use removePreHashKey to remove prehash_key_before_comparison from result.
+func transformToV2(jsonRaw []byte, ctx client.Context, removePreHashKey bool) (json.RawMessage, error) {
+
+	// populate deprecated fields of GenesisState used by version v2.x
+	srcConGen := consumerTypes.GenesisState{}
+	err := ctx.Codec.UnmarshalJSON(jsonRaw, &srcConGen)
+	if err != nil {
+		return nil, fmt.Errorf("reading consumer genesis data failed: %s", err)
+	}
+
+	// remove retry_delay_period from 'params' if present (introduced in v4.x)
+	params, err := ctx.Codec.MarshalJSON(&srcConGen.Params)
+	if err != nil {
+		return nil, err
+	}
+	paramsMap := map[string]json.RawMessage{}
+	if err := json.Unmarshal(params, &paramsMap); err != nil {
+		return nil, fmt.Errorf("unmarshalling 'params' failed: %v", err)
+	}
+	_, exists := paramsMap["retry_delay_period"]
+	if exists {
+		delete(paramsMap, "retry_delay_period")
+	}
+	params, err = json.Marshal(paramsMap)
+	if err != nil {
+		return nil, err
+	}
+
+	// marshal GenesisState and patch 'params' value
+	result, err := ctx.Codec.MarshalJSON(&srcConGen)
+	if err != nil {
+		return nil, err
+	}
+	genState := map[string]json.RawMessage{}
+	if err := json.Unmarshal(result, &genState); err != nil {
+		return nil, fmt.Errorf("unmarshalling 'GenesisState' failed: %v", err)
+	}
+	genState["params"] = params
+
+	provider, err := ctx.Codec.MarshalJSON(&srcConGen.Provider)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshalling 'Provider' failed: %v", err)
+	}
+	providerMap := map[string]json.RawMessage{}
+	if err := json.Unmarshal(provider, &providerMap); err != nil {
+		return nil, fmt.Errorf("unmarshalling 'provider' failed: %v", err)
+	}
+
+	// patch .initial_val_set form .provider.initial_val_set if needed
+	if len(srcConGen.Provider.InitialValSet) > 0 {
+		valSet, exists := providerMap["initial_val_set"]
+		if !exists {
+			return nil, fmt.Errorf("'initial_val_set' not found in provider data")
+		}
+		_, exists = genState["initial_val_set"]
+		if exists {
+			genState["initial_val_set"] = valSet
+		}
+	}
+
+	// patch .provider_consensus_state from provider.consensus_state if needed
+	if srcConGen.Provider.ConsensusState != nil {
+		valSet, exists := providerMap["consensus_state"]
+		if !exists {
+			return nil, fmt.Errorf("'consensus_state' not found in provider data")
+		}
+		_, exists = genState["provider_consensus_state"]
+		if exists {
+			genState["provider_consensus_state"] = valSet
+		}
+	}
+
+	// patch .provider_client_state from provider.client_state if needed
+	if srcConGen.Provider.ClientState != nil {
+		clientState, exists := providerMap["client_state"]
+		if !exists {
+			return nil, fmt.Errorf("'client_state' not found in provider data")
+		}
+		_, exists = genState["provider_client_state"]
+		if exists {
+			genState["provider_client_state"] = clientState
+		}
+	}
+
+	// delete .provider entry (introduced in v3.3.x)
+	delete(genState, "provider")
+
+	// Marshall final result
+	result, err = json.Marshal(genState)
+	if err != nil {
+		return nil, fmt.Errorf("marshalling transformation result failed: %v", err)
+	}
+
+	if removePreHashKey {
+		// remove all `prehash_key_before_comparison` entries not supported in v2.x (see ics23)
+		re := regexp.MustCompile(`,\s*"prehash_key_before_comparison"\s*:\s*(false|true)`)
+		result = re.ReplaceAll(result, []byte{})
+	}
+	return result, nil
+}
+
+// transformGenesis transforms ccv consumer genesis data to the specified target version
+// Returns the transformed data or an error in case the transformation failed or the format is not supported by current implementation
+func transformGenesis(ctx client.Context, targetVersion IcsVersion, jsonRaw []byte) (json.RawMessage, error) {
+	var newConsumerGenesis json.RawMessage = nil
+	var err error = nil
+
+	switch targetVersion {
+	// v2.x, v3.0-v3.2 share same consumer genesis type
+	case v2_x:
+		newConsumerGenesis, err = transformToV2(jsonRaw, ctx, true)
+	case v3_0_x, v3_1_x, v3_2_x:
+		// same as v2 replacement without need of `prehash_key_before_comparison` removal
+		newConsumerGenesis, err = transformToV2(jsonRaw, ctx, false)
+	case v3_3_x:
+		newConsumerGenesis, err = transformToV33(jsonRaw, ctx)
+	case v4_x_x:
+		newConsumerGenesis, err = transformToNew(jsonRaw, ctx)
+	default:
+		err = fmt.Errorf("unsupported target version '%s'. Run %s --help",
+			targetVersion, version.AppName)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("transformation failed: %v", err)
+	}
+	return newConsumerGenesis, err
+}
+
 // Transform a consumer genesis json file exported from a given ccv provider version
-// to a consumer genesis json format supported by current ccv consumer version.
+// to a consumer genesis json format supported by current ccv consumer version or v2.x
+// This allows user to patch consumer genesis of
+//   - current implementation from exports of provider of < v3.3.x
+//   - v2.x from exports of provider >= v3.2.x
+//
 // Result will be written to defined output.
 func TransformConsumerGenesis(cmd *cobra.Command, args []string) error {
-	sourceFile := args[0]
 
+	sourceFile := args[0]
 	jsonRaw, err := os.ReadFile(filepath.Clean(sourceFile))
 	if err != nil {
 		return err
 	}
 
 	clientCtx := client.GetClientContextFromCmd(cmd)
-	newConsumerGenesis, err := transform(jsonRaw, clientCtx)
+	version, err := cmd.Flags().GetString("to")
+	if err != nil {
+		return fmt.Errorf("error getting targetVersion %v", err)
+	}
+	targetVersion, exists := TransformationVersions[version]
+	if !exists {
+		return fmt.Errorf("unsupported target version '%s'", version)
+	}
+
+	// try to transform data to target format
+	newConsumerGenesis, err := transformGenesis(clientCtx, targetVersion, jsonRaw)
 	if err != nil {
 		return err
 	}
@@ -114,17 +330,25 @@ func NewDefaultGenesisState(cdc codec.JSONCodec) GenesisState {
 // provider version v1,v2 or v3 to a JSON format supported by this consumer version.
 func GetConsumerGenesisTransformCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "transform [genesis-file]",
-		Short: "Transform CCV consumer genesis from an older provider version not supporting current format",
+		Use:   "transform [-to version] genesis-file",
+		Short: "Transform CCV consumer genesis data exported to a specific target format",
 		Long: strings.TrimSpace(
-			fmt.Sprintf(`Transform the consumer genesis file from a provider version v1,v2 or v3 to a version supported by this consumer. Result is printed to STDOUT.
+			fmt.Sprintf(`
+Transform the consumer genesis data exported from a provider version v1,v2, v3, v4 to a specified consumer target version.
+The result is printed to STDOUT.
+
+Note: Content to be transformed is not the consumer genesis file itself but the exported content from provider chain which is used to patch the consumer genesis file!
 
 Example:
-$ %s transform /path/to/ccv_consumer_genesis.json`, version.AppName),
+$ %s transform /path/to/ccv_consumer_genesis.json
+$ %s --to v2.x transform /path/to/ccv_consumer_genesis.json
+`, version.AppName, version.AppName),
 		),
-		Args: cobra.ExactArgs(1),
+		Args: cobra.RangeArgs(1, 2),
 		RunE: TransformConsumerGenesis,
 	}
-
+	cmd.Flags().String("to", string(v4_x_x),
+		fmt.Sprintf("target version for consumer genesis. Supported versions %s",
+			maps.Keys(TransformationVersions)))
 	return cmd
 }