cosmos · shaspitz · Oct 6, 2023 · Sep 6, 2023 · Sep 14, 2023 · Sep 18, 2023
@@ -4,6 +4,7 @@
 
 Add an entry to the unreleased provider section whenever merging a PR to main that is not targeted at a specific release. These entries will eventually be included in a provider release.
 
+* (feat!) [#1230](https://github.com/cosmos/interchain-security/pull/1230) Throttle with retries provider changes.
 * (feature!) [#1280](https://github.com/cosmos/interchain-security/pull/1280) provider proposal for changing reward denoms
 * (feature!) [#1244](https://github.com/cosmos/interchain-security/pull/1244) Update the default consumer unbonding period to 2 weeks.
 * (deps) [#1259](https://github.com/cosmos/interchain-security/pull/1259) Bump [cosmos-sdk](https://github.com/cosmos/cosmos-sdk) to [v0.47.5](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.47.5).

@@ -7,6 +7,7 @@ option go_package = "github.com/cosmos/interchain-security/v3/x/ccv/consumer/typ
 import "gogoproto/gogo.proto";
 import "google/api/annotations.proto";
 import "interchain_security/ccv/consumer/v1/consumer.proto";
+import "interchain_security/ccv/v1/wire.proto";
 
 service Query {
   // ConsumerGenesis queries the genesis state needed to start a consumer chain
@@ -24,6 +25,11 @@ service Query {
   rpc QueryProviderInfo(QueryProviderInfoRequest) returns (QueryProviderInfoResponse) {
     option (google.api.http).get = "/interchain_security/ccv/consumer/provider-info";
   }
+
+  // QueryThrottleState returns on-chain state relevant to throttled consumer packets 
+  rpc QueryThrottleState(QueryThrottleStateRequest) returns (QueryThrottleStateResponse) {
+    option (google.api.http).get = "/interchain_security/ccv/consumer/throttle_state";
+  }
 }
 
 // NextFeeDistributionEstimate holds information about next fee distribution
@@ -65,6 +71,14 @@ message QueryProviderInfoResponse {
   ChainInfo provider = 2 [ (gogoproto.nullable) = false ];
 }
 
+message QueryThrottleStateRequest {}
+
+message QueryThrottleStateResponse {
+  SlashRecord slash_record = 1 [ (gogoproto.nullable) = true ];
+  repeated interchain_security.ccv.v1.ConsumerPacketData packet_data_queue = 2 [ (gogoproto.nullable) = false ];
+}
+
+
 message ChainInfo {
   string chainID = 1;
   string clientID = 2;

@@ -1,4 +1,4 @@
 syntax = "proto3";
 package interchain_security.ccv.provider.v1;

 option go_package = "github.com/cosmos/interchain-security/v3/x/ccv/provider/types";
@@ -10,7 +10,7 @@
 import "interchain_security/ccv/v1/shared_consumer.proto";
 import "interchain_security/ccv/v1/wire.proto";

 service Query {
  // ConsumerGenesis queries the genesis state needed to start a consumer chain
  // whose proposal has been accepted
  rpc QueryConsumerGenesis(QueryConsumerGenesisRequest)
@@ -65,14 +65,6 @@
         "/interchain_security/ccv/provider/throttle_state";
   }
 
-  // QueryThrottledConsumerPacketData returns a list of pending packet data
-  // instances (slash packet and vsc matured) for a single consumer chain
-  rpc QueryThrottledConsumerPacketData(QueryThrottledConsumerPacketDataRequest)
-      returns (QueryThrottledConsumerPacketDataResponse) {
-    option (google.api.http).get =
-        "/interchain_security/ccv/provider/pending_consumer_packets";
-  }
-
   // QueryRegisteredConsumerRewardDenoms returns a list of consumer reward
   // denoms that are registered
   rpc QueryRegisteredConsumerRewardDenoms(
@@ -141,7 +133,7 @@

 message QueryThrottleStateRequest {}

 message QueryThrottleStateResponse {
  // current slash_meter state
  int64 slash_meter = 1;
  // allowance of voting power units (int) that the slash meter is given per
@@ -151,35 +143,6 @@
   // full
   google.protobuf.Timestamp next_replenish_candidate = 3
       [ (gogoproto.stdtime) = true, (gogoproto.nullable) = false ];
-  // data relevant to currently throttled slash packets
-  repeated ThrottledSlashPacket packets = 4;
-}
-
-message QueryThrottledConsumerPacketDataRequest { string chain_id = 1; }
-
-message QueryThrottledConsumerPacketDataResponse {
-  string chain_id = 1;
-  uint64 size = 2;
-  repeated ThrottledPacketDataWrapper packetDataInstances = 3
-      [ (gogoproto.nullable) = false ];
-}
-
-// A query wrapper type for the global entry and data relevant to a throttled
-// slash packet.
-message ThrottledSlashPacket {
-  interchain_security.ccv.provider.v1.GlobalSlashEntry global_entry = 1
-      [ (gogoproto.nullable) = false ];
-  interchain_security.ccv.v1.SlashPacketData data = 2
-      [ (gogoproto.nullable) = false ];
-}
-
-// ThrottledPacketDataWrapper contains either SlashPacketData or
-// VSCMaturedPacketData
-message ThrottledPacketDataWrapper {
-  oneof data {
-    interchain_security.ccv.v1.SlashPacketData slash_packet = 1;
-    interchain_security.ccv.v1.VSCMaturedPacketData vsc_matured_packet = 2;
-  }
 }
 
 message QueryRegisteredConsumerRewardDenomsRequest {}

@@ -76,6 +76,10 @@ message ConsumerParams {
     // Provider-originated reward denoms. These are denoms coming from the
     // provider which are allowed to be used as rewards. e.g. "uatom"
     repeated string provider_reward_denoms = 12;
+
+    // The period after which a consumer can retry sending a throttled packet.
+    google.protobuf.Duration retry_delay_period = 13
+        [ (gogoproto.nullable) = false, (gogoproto.stdduration) = true ];
 }
 
 // ConsumerGenesisState defines the CCV consumer chain genesis state.

@@ -539,6 +539,7 @@ func (b *Builder) createConsumerGenesis(client *ibctmtypes.ClientState) *ccv.Con
 		"0", // disable soft opt-out
 		[]string{},
 		[]string{},
+		ccv.DefaultRetryDelayPeriod,
 	)
 	return ccv.NewInitialConsumerGenesisState(client, providerConsState, valUpdates, params)
 }

@@ -2045,50 +2045,53 @@ func (tr TestRun) assignConsumerPubKey(action assignConsumerPubKeyAction, verbos
 	tr.waitBlocks(ChainID("provi"), 2, 30*time.Second)
 }
 
-// slashThrottleDequeue polls slash queue sizes until nextQueueSize is achieved
-type slashThrottleDequeue struct {
-	Chain            ChainID
-	CurrentQueueSize int
-	NextQueueSize    int
-	// panic if Timeout is exceeded
+// slashMeterReplenishmentAction polls the slash meter on provider until value is achieved
+type slashMeterReplenishmentAction struct {
+	TargetValue int64
+	// panic if timeout is exceeded
 	Timeout time.Duration
 }
 
-func (tr TestRun) waitForSlashThrottleDequeue(
-	action slashThrottleDequeue,
+func (tr TestRun) waitForSlashMeterReplenishment(
+	action slashMeterReplenishmentAction,
 	verbose bool,
 ) {
 	timeout := time.Now().Add(action.Timeout)
-	initialGlobalQueueSize := int(tr.getGlobalSlashQueueSize())
+	initialSlashMeter := tr.getSlashMeter()
 
-	if initialGlobalQueueSize != action.CurrentQueueSize {
-		panic(fmt.Sprintf("wrong initial queue size: %d - expected global queue: %d\n", initialGlobalQueueSize, action.CurrentQueueSize))
+	if initialSlashMeter >= 0 {
+		panic(fmt.Sprintf("No need to wait for slash meter replenishment, current value: %d", initialSlashMeter))
 	}
+
 	for {
-		globalQueueSize := int(tr.getGlobalSlashQueueSize())
-		chainQueueSize := int(tr.getConsumerChainPacketQueueSize(action.Chain))
+		slashMeter := tr.getSlashMeter()
 		if verbose {
-			fmt.Printf("waiting for packed queue size to reach: %d - current: %d\n", action.NextQueueSize, globalQueueSize)
+			fmt.Printf("waiting for slash meter to be replenished, current value: %d\n", slashMeter)
 		}
 
-		// check if global queue size is equal to chain queue size
-		if globalQueueSize == chainQueueSize && globalQueueSize == action.NextQueueSize { //nolint:gocritic // this is the comparison that we want here.
+		// check if meter has reached target value
+		if slashMeter >= action.TargetValue {
 			break
 		}
 
 		if time.Now().After(timeout) {
-			panic(fmt.Sprintf("\n\n\nwaitForSlashThrottleDequeuemethod has timed out after: %s\n\n", action.Timeout))
+			panic(fmt.Sprintf("\n\nwaitForSlashMeterReplenishment has timed out after: %s\n\n", action.Timeout))
 		}
 
-		time.Sleep(500 * time.Millisecond)
+		tr.WaitTime(5 * time.Second)
 	}
-	// wair for 2 blocks to be created
-	// allowing the jailing to be incorporated into voting power
-	tr.waitBlocks(action.Chain, 2, time.Minute)
 }
 
-func uintPointer(i uint) *uint {
-	return &i
+type waitTimeAction struct {
+	Consumer ChainID
+	WaitTime time.Duration
+}
+
+func (tr TestRun) waitForTime(
+	action waitTimeAction,
+	verbose bool,
+) {
+	tr.WaitTime(action.WaitTime)
 }
 
 // GetPathNameForGorelayer returns the name of the path between two given chains used by Gorelayer.

@@ -186,7 +186,8 @@ func SlashThrottleTestRun() TestRun {
 					".app_state.slashing.params.signed_blocks_window = \"15\" | " +
 					".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " +
 					".app_state.slashing.params.downtime_jail_duration = \"60s\" | " +
-					".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"",
+					".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\" | " +
+					".app_state.ccvconsumer.params.retry_delay_period = \"30s\"",
 			},
 		},
 		tendermintConfigOverride: `s/timeout_commit = "5s"/timeout_commit = "1s"/;` +

@@ -254,8 +254,10 @@ func (tr *TestRun) runStep(step Step, verbose bool) {
 		tr.registerRepresentative(action, verbose)
 	case assignConsumerPubKeyAction:
 		tr.assignConsumerPubKey(action, verbose)
-	case slashThrottleDequeue:
-		tr.waitForSlashThrottleDequeue(action, verbose)
+	case slashMeterReplenishmentAction:
+		tr.waitForSlashMeterReplenishment(action, verbose)
+	case waitTimeAction:
+		tr.waitForTime(action, verbose)
 	case startRelayerAction:
 		tr.startRelayer(action, verbose)
 	case submitChangeRewardDenomsProposalAction:

@@ -26,8 +26,7 @@ type ChainState struct {
 	ConsumerChains                 *map[ChainID]bool
 	AssignedKeys                   *map[ValidatorID]string
 	ProviderKeys                   *map[ValidatorID]string // validatorID: validator provider key
-	ConsumerChainQueueSizes        *map[ChainID]uint
-	GlobalSlashQueueSize           *uint
+	ConsumerPendingPacketQueueSize *uint                   // Only relevant to consumer chains
 	RegisteredConsumerRewardDenoms *[]string
 }
 
@@ -168,24 +167,16 @@ func (tr TestRun) getChainState(chain ChainID, modelState ChainState) ChainState
 		chainState.ProviderKeys = &providerKeys
 	}
 
-	if modelState.GlobalSlashQueueSize != nil {
-		globalQueueSize := tr.getGlobalSlashQueueSize()
-		chainState.GlobalSlashQueueSize = &globalQueueSize
-	}
-
-	if modelState.ConsumerChainQueueSizes != nil {
-		consumerChainQueueSizes := map[ChainID]uint{}
-		for c := range *modelState.ConsumerChainQueueSizes {
-			consumerChainQueueSizes[c] = tr.getConsumerChainPacketQueueSize(c)
-		}
-		chainState.ConsumerChainQueueSizes = &consumerChainQueueSizes
-	}
-
 	if modelState.RegisteredConsumerRewardDenoms != nil {
 		registeredConsumerRewardDenoms := tr.getRegisteredConsumerRewardDenoms(chain)
 		chainState.RegisteredConsumerRewardDenoms = &registeredConsumerRewardDenoms
 	}
 
+	if modelState.ConsumerPendingPacketQueueSize != nil {
+		pendingPacketQueueSize := tr.getPendingPacketQueueSize(chain)
+		chainState.ConsumerPendingPacketQueueSize = &pendingPacketQueueSize
+	}
+
 	if *verbose {
 		log.Println("Done getting chain state:\n" + pretty.Sprint(chainState))
 	}
@@ -667,9 +658,10 @@ func (tr TestRun) getProviderAddressFromConsumer(consumerChain ChainID, validato
 	return addr
 }
 
-func (tr TestRun) getGlobalSlashQueueSize() uint {
+func (tr TestRun) getSlashMeter() int64 {
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[ChainID("provi")].BinaryName,
+	cmd := exec.Command("docker", "exec",
+		tr.containerConfig.InstanceName, tr.chainConfigs[ChainID("provi")].BinaryName,
 
 		"query", "provider", "throttle-state",
 		`--node`, tr.getQueryNode(ChainID("provi")),
@@ -680,33 +672,37 @@ func (tr TestRun) getGlobalSlashQueueSize() uint {
 		log.Fatal(err, "\n", string(bz))
 	}
 
-	packets := gjson.Get(string(bz), "packets").Array()
-	return uint(len(packets))
+	slashMeter := gjson.Get(string(bz), "slash_meter")
+	return slashMeter.Int()
 }
 
-func (tr TestRun) getConsumerChainPacketQueueSize(consumerChain ChainID) uint {
+func (tr TestRun) getRegisteredConsumerRewardDenoms(chain ChainID) []string {
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[ChainID("provi")].BinaryName,
+	cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[chain].BinaryName,
 
-		"query", "provider", "throttled-consumer-packet-data",
-		string(consumerChain),
-		`--node`, tr.getQueryNode(ChainID("provi")),
+		"query", "provider", "registered-consumer-reward-denoms",
+		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	)
 	bz, err := cmd.CombinedOutput()
 	if err != nil {
 		log.Fatal(err, "\n", string(bz))
 	}
 
-	size := gjson.Get(string(bz), "size").Uint()
-	return uint(size)
+	denoms := gjson.Get(string(bz), "denoms").Array()
+	rewardDenoms := make([]string, len(denoms))
+	for i, d := range denoms {
+		rewardDenoms[i] = d.String()
+	}
+
+	return rewardDenoms
 }
 
-func (tr TestRun) getRegisteredConsumerRewardDenoms(chain ChainID) []string {
+func (tr TestRun) getPendingPacketQueueSize(chain ChainID) uint {
 	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
 	cmd := exec.Command("docker", "exec", tr.containerConfig.InstanceName, tr.chainConfigs[chain].BinaryName,
 
-		"query", "provider", "registered-consumer-reward-denoms",
+		"query", "ccvconsumer", "throttle-state",
 		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	)
@@ -715,13 +711,12 @@ func (tr TestRun) getRegisteredConsumerRewardDenoms(chain ChainID) []string {
 		log.Fatal(err, "\n", string(bz))
 	}
 
-	denoms := gjson.Get(string(bz), "denoms").Array()
-	rewardDenoms := make([]string, len(denoms))
-	for i, d := range denoms {
-		rewardDenoms[i] = d.String()
+	if !gjson.ValidBytes(bz) {
+		panic("invalid json response from query ccvconsumer throttle-state: " + string(bz))
 	}
 
-	return rewardDenoms
+	packetData := gjson.Get(string(bz), "packet_data_queue").Array()
+	return uint(len(packetData))
 }
 
 func (tr TestRun) getValidatorNode(chain ChainID, validator ValidatorID) string {
@@ -772,3 +767,7 @@ func (tr TestRun) curlJsonRPCRequest(method, params, address string) {
 	verbosity := false
 	executeCommandWithVerbosity(cmd, "curlJsonRPCRequest", verbosity)
 }
+
+func uintPtr(i uint) *uint {
+	return &i
+}