feat(thorchain): add support for adding/duplicating validators (#1304)

chain/ethereum/foundry/anvil_chain.go

@@ -24,6 +24,9 @@ type AnvilChain struct {
 	*ethereum.EthereumChain
 
 	keystoreMap map[string]*NodeWallet
+
+	// Mutex for reading/writing keystoreMap (once wallet is created, it doesn't change)
+	MapAccess sync.Mutex
 }
 
 func NewAnvilChain(testName string, chainConfig ibc.ChainConfig, log *zap.Logger) *AnvilChain {
@@ -85,6 +88,8 @@ func (c *AnvilChain) CreateKey(ctx context.Context, keyName string) error {
 		return err
 	}
 
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	_, ok := c.keystoreMap[keyName]
 	if ok {
 		return fmt.Errorf("keyname (%s) already used", keyName)
@@ -122,6 +127,8 @@ func (c *AnvilChain) RecoverKey(ctx context.Context, keyName, mnemonic string) e
 	}
 
 	// This is needed for CreateKey() since that keystore path does not use the keyname
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	c.keystoreMap[keyName] = &NodeWallet{
 		keystore: path.Join(c.KeystoreDir(), keyName),
 	}
@@ -131,7 +138,9 @@ func (c *AnvilChain) RecoverKey(ctx context.Context, keyName, mnemonic string) e
 
 // Get address of account, cast to a string to use.
 func (c *AnvilChain) GetAddress(ctx context.Context, keyName string) ([]byte, error) {
+	c.MapAccess.Lock()
 	account, ok := c.keystoreMap[keyName]
+	c.MapAccess.Unlock()
 	if !ok {
 		return nil, fmt.Errorf("keyname (%s) not found", keyName)
 	}
@@ -168,7 +177,9 @@ func (c *AnvilChain) SendFundsWithNote(ctx context.Context, keyName string, amou
 		cmd = []string{"cast", "send", amount.Address, "--value", amount.Amount.String(), "--json"}
 	}
 
+	c.MapAccess.Lock()
 	account, ok := c.keystoreMap[keyName]
+	c.MapAccess.Unlock()
 	if !ok {
 		return "", fmt.Errorf("keyname (%s) not found", keyName)
 	}

chain/ethereum/foundry/forge.go

@@ -24,7 +24,9 @@ type ForgeScriptOpts struct {
 
 // Add private-key or keystore to cmd.
 func (c *AnvilChain) AddKey(cmd []string, keyName string) []string {
+	c.MapAccess.Lock()
 	account, ok := c.keystoreMap[keyName]
+	c.MapAccess.Unlock()
 	if !ok {
 		panic(fmt.Sprintf("Keyname (%s) not found", keyName))
 	}
@@ -77,7 +79,9 @@ func WriteConfigFile(configFile string, localContractRootDir string, solidityCon
 // Run "forge script"
 // see: https://book.getfoundry.sh/reference/forge/forge-script
 func (c *AnvilChain) ForgeScript(ctx context.Context, keyName string, opts ForgeScriptOpts) (stdout, stderr []byte, err error) {
+	c.MapAccess.Lock()
 	account, ok := c.keystoreMap[keyName]
+	c.MapAccess.Unlock()
 	if !ok {
 		return nil, nil, fmt.Errorf("keyname (%s) not found", keyName)
 	}

chain/ethereum/geth/geth_chain.go

@@ -26,6 +26,9 @@ type GethChain struct {
 
 	keynameToAccountMap map[string]*NodeWallet
 	nextAcctNum         int
+
+	// Mutex for reading/writing keynameToAccountMap (once wallet is created, it doesn't change)
+	MapAccess sync.Mutex
 }
 
 func NewGethChain(testName string, chainConfig ibc.ChainConfig, log *zap.Logger) *GethChain {
@@ -78,6 +81,8 @@ func (c *GethChain) JavaScriptExecTx(ctx context.Context, account *NodeWallet, j
 }
 
 func (c *GethChain) CreateKey(ctx context.Context, keyName string) error {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	_, ok := c.keynameToAccountMap[keyName]
 	if ok {
 		return fmt.Errorf("keyname (%s) already used", keyName)
@@ -106,6 +111,8 @@ EOF
 }
 
 func (c *GethChain) RecoverKey(ctx context.Context, keyName, mnemonic string) error {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	_, ok := c.keynameToAccountMap[keyName]
 	if ok {
 		return fmt.Errorf("keyname (%s) already used", keyName)
@@ -133,6 +140,8 @@ func (c *GethChain) RecoverKey(ctx context.Context, keyName, mnemonic string) er
 
 // Get address of account, cast to a string to use.
 func (c *GethChain) GetAddress(ctx context.Context, keyName string) ([]byte, error) {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	account, found := c.keynameToAccountMap[keyName]
 	if !found {
 		return nil, fmt.Errorf("GetAddress(): Keyname (%s) not found", keyName)
@@ -182,7 +191,9 @@ func (c *GethChain) SendFunds(ctx context.Context, keyName string, amount ibc.Wa
 }
 
 func (c *GethChain) SendFundsWithNote(ctx context.Context, keyName string, amount ibc.WalletAmount, note string) (string, error) {
+	c.MapAccess.Lock()
 	account, found := c.keynameToAccountMap[keyName]
+	c.MapAccess.Unlock()
 	if !found {
 		return "", fmt.Errorf("keyname (%s) not found", keyName)
 	}
@@ -205,7 +216,9 @@ func (c *GethChain) SendFundsWithNote(ctx context.Context, keyName string, amoun
 // DeployContract creates a new contract on-chain, returning the contract address
 // Constructor params are appended to the byteCode.
 func (c *GethChain) DeployContract(ctx context.Context, keyName string, abi []byte, byteCode []byte) (string, error) {
+	c.MapAccess.Lock()
 	account, found := c.keynameToAccountMap[keyName]
+	c.MapAccess.Unlock()
 	if !found {
 		return "", fmt.Errorf("SendFundsWithNote(): Keyname (%s) not found", keyName)
 	}

chain/thorchain/api_query.go

@@ -19,6 +19,28 @@ import (
 	"github.com/strangelove-ventures/interchaintest/v8/chain/thorchain/common"
 )
 
+// Generic query for routes not yet supported here.
+func (c *Thorchain) APIQuery(ctx context.Context, path string, args ...string) (any, error) {
+	url := fmt.Sprintf("%s/%s", c.GetAPIAddress(), path)
+	var res any
+	err := get(ctx, url, &res)
+	return res, err
+}
+
+func (c *Thorchain) APIGetNode(ctx context.Context, addr string) (OpenapiNode, error) {
+	url := fmt.Sprintf("%s/thorchain/node/%s", c.GetAPIAddress(), addr)
+	var node OpenapiNode
+	err := get(ctx, url, &node)
+	return node, err
+}
+
+func (c *Thorchain) APIGetNodes(ctx context.Context) ([]OpenapiNode, error) {
+	url := fmt.Sprintf("%s/thorchain/nodes", c.GetAPIAddress())
+	var nodes []OpenapiNode
+	err := get(ctx, url, &nodes)
+	return nodes, err
+}
+
 func (c *Thorchain) APIGetBalances(ctx context.Context, addr string) (common.Coins, error) {
 	url := fmt.Sprintf("%s/cosmos/bank/v1beta1/balances/%s", c.GetAPIAddress(), addr)
 	var balances struct {

chain/thorchain/module_thorchain.go

@@ -41,3 +41,37 @@ func (c *Thorchain) SetMimir(ctx context.Context, keyName string, key string, va
 	)
 	return err
 }
+
+func (tn *ChainNode) Bond(ctx context.Context, amount math.Int) error {
+	_, err := tn.ExecTx(ctx,
+		valKey, "thorchain", "deposit",
+		amount.String(), tn.Chain.Config().Denom,
+		fmt.Sprintf("bond:%s", tn.NodeAccount.NodeAddress),
+	)
+	return err
+}
+
+// Sets validator node keys, must be called by validator.
+func (tn *ChainNode) SetNodeKeys(ctx context.Context) error {
+	_, err := tn.ExecTx(ctx,
+		valKey, "thorchain", "set-node-keys",
+		tn.NodeAccount.PubKeySet.Secp256k1, tn.NodeAccount.PubKeySet.Ed25519, tn.NodeAccount.ValidatorConsPubKey,
+	)
+	return err
+}
+
+// Sets validator ip address, must be called by validator.
+func (tn *ChainNode) SetIPAddress(ctx context.Context) error {
+	_, err := tn.ExecTx(ctx,
+		valKey, "thorchain", "set-ip-address", tn.NodeAccount.IPAddress,
+	)
+	return err
+}
+
+// Sets validator's binary version.
+func (tn *ChainNode) SetVersion(ctx context.Context) error {
+	_, err := tn.ExecTx(ctx,
+		valKey, "thorchain", "set-version",
+	)
+	return err
+}

chain/thorchain/thorchain.go

@@ -137,6 +137,178 @@ func (c *Thorchain) Nodes() ChainNodes {
 	return append(c.Validators, c.FullNodes...)
 }
 
+// AddValidators adds new validators to the network, peering with the existing nodes.
+func (c *Thorchain) AddValidators(ctx context.Context, configFileOverrides map[string]any, inc int) error {
+	// Get peer string for existing nodes
+	peers := c.Nodes().PeerString(ctx)
+
+	// Get genesis.json
+	genbz, err := c.Validators[0].GenesisFileContent(ctx)
+	if err != nil {
+		return err
+	}
+
+	prevCount := c.NumValidators
+	c.NumValidators += inc
+	if err := c.initializeChainNodes(ctx, c.testName, c.getFullNode().DockerClient, c.getFullNode().NetworkID); err != nil {
+		return err
+	}
+
+	// Create full node, validator keys, and start up
+	var eg errgroup.Group
+	for i := prevCount; i < c.NumValidators; i++ {
+		eg.Go(func() error {
+			val := c.Validators[i]
+			if err := val.InitFullNodeFiles(ctx); err != nil {
+				return err
+			}
+			if err := val.SetPeers(ctx, peers); err != nil {
+				return err
+			}
+			if err := val.OverwriteGenesisFile(ctx, genbz); err != nil {
+				return err
+			}
+			for configFile, modifiedConfig := range configFileOverrides {
+				modifiedToml, ok := modifiedConfig.(testutil.Toml)
+				if !ok {
+					return fmt.Errorf("provided toml override for file %s is of type (%T). Expected (DecodedToml)", configFile, modifiedConfig)
+				}
+				if err := testutil.ModifyTomlConfigFile(
+					ctx,
+					val.logger(),
+					val.DockerClient,
+					val.TestName,
+					val.VolumeName,
+					configFile,
+					modifiedToml,
+				); err != nil {
+					return err
+				}
+			}
+			if err := val.CreateKey(ctx, valKey); err != nil {
+				return fmt.Errorf("failed to create key: %w", err)
+			}
+			if err := val.GetNodeAccount(ctx); err != nil {
+				return fmt.Errorf("failed to get node account info: %w", err)
+			}
+			if err := val.CreateNodeContainer(ctx); err != nil {
+				return err
+			}
+			return val.StartContainer(ctx)
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	// Fund validator address and register for next churn
+	decimalPow := int64(math.Pow10(int(*c.cfg.CoinDecimals)))
+	for i := prevCount; i < c.NumValidators; i++ {
+		// Fund validator from faucet
+		if err := c.SendFunds(ctx, "faucet", ibc.WalletAmount{
+			Address: c.Validators[i].NodeAccount.NodeAddress,
+			Amount:  sdkmath.NewInt(100).MulRaw(decimalPow), // 100e8 rune
+			Denom:   c.cfg.Denom,
+		}); err != nil {
+			return fmt.Errorf("failed to fund val %d, %w", i, err)
+		}
+
+		eg.Go(func() error {
+			val := c.Validators[i]
+			// thornode tx thorchain deposit 1e8 RUNE "bond:$NODE_ADDRESS"
+			// Bond 2 rune since the next 3 txs will deduct .02 rune/tx and we need > 1 rune bonded
+			if err := val.Bond(ctx, sdkmath.NewInt(2).MulRaw(decimalPow)); err != nil {
+				return fmt.Errorf("failed to set val %d node keys, %w", i, err)
+			}
+			// thornode tx thorchain set-node-keys "$NODE_PUB_KEY" "$NODE_PUB_KEY_ED25519" "$VALIDATOR"
+			if err := val.SetNodeKeys(ctx); err != nil {
+				return fmt.Errorf("failed to set val %d node keys, %w", i, err)
+			}
+			// thornode tx thorchain set-ip-address "192.168.0.10"
+			if err := val.SetIPAddress(ctx); err != nil {
+				return fmt.Errorf("failed to set val %d ip address, %w", i, err)
+			}
+			// thornode tx thorchain set-version
+			if err := val.SetVersion(ctx); err != nil {
+				return fmt.Errorf("failed to set val %d version, %w", i, err)
+			}
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	// start sidecar/bifrost
+	return c.StartAllValSidecars(ctx)
+}
+
+// AddDuplicateValidator spins up a duplicate validator node to test double signing.
+func (c *Thorchain) AddDuplicateValidator(ctx context.Context, configFileOverrides map[string]any, originalVal *ChainNode) (*ChainNode, error) {
+	// Get peer string for existing nodes
+	peers := c.Nodes().PeerString(ctx)
+
+	// Get genesis.json
+	genbz, err := c.Validators[0].GenesisFileContent(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	c.NumValidators += 1
+	if err := c.initializeChainNodes(ctx, c.testName, c.getFullNode().DockerClient, c.getFullNode().NetworkID); err != nil {
+		return nil, err
+	}
+
+	// Create full node, validator keys, and start up
+	val := c.Validators[c.NumValidators-1]
+	if err := val.InitFullNodeFiles(ctx); err != nil {
+		return nil, err
+	}
+	if err := val.SetPeers(ctx, peers); err != nil {
+		return nil, err
+	}
+	if err := val.OverwriteGenesisFile(ctx, genbz); err != nil {
+		return nil, err
+	}
+	for configFile, modifiedConfig := range configFileOverrides {
+		modifiedToml, ok := modifiedConfig.(testutil.Toml)
+		if !ok {
+			return nil, fmt.Errorf("provided toml override for file %s is of type (%T). Expected (DecodedToml)", configFile, modifiedConfig)
+		}
+		if err := testutil.ModifyTomlConfigFile(
+			ctx,
+			val.logger(),
+			val.DockerClient,
+			val.TestName,
+			val.VolumeName,
+			configFile,
+			modifiedToml,
+		); err != nil {
+			return nil, err
+		}
+	}
+	privValFile, err := originalVal.PrivValFileContent(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get priv_validator_key.json, %w", err)
+	}
+	if err := val.OverwritePrivValFile(ctx, privValFile); err != nil {
+		return nil, fmt.Errorf("failed to overwrite priv_validator_key.json, %w", err)
+	}
+	if err := val.RecoverKey(ctx, valKey, originalVal.ValidatorMnemonic); err != nil {
+		return nil, fmt.Errorf("failed to create key: %w", err)
+	}
+	val.ValidatorMnemonic = originalVal.ValidatorMnemonic
+	if err := val.GetNodeAccount(ctx); err != nil {
+		return nil, fmt.Errorf("failed to get node account info: %w", err)
+	}
+	if err := val.CreateNodeContainer(ctx); err != nil {
+		return nil, err
+	}
+	return val, val.StartContainer(ctx)
+}
+
 // AddFullNodes adds new fullnodes to the network, peering with the existing nodes.
 func (c *Thorchain) AddFullNodes(ctx context.Context, configFileOverrides map[string]any, inc int) error {
 	// Get peer string for existing nodes