Automatic gas in load tests (#12416)

* use experimental seth, use dynamic gas also when sending funds * fix a situation, when we lose transaction timeout setting for networks that are not overwritten * go mod tidy * add gas limit for Fiji, fix a situation when new networks were ignored * update Seth version * fix lints * fix lints * newer seth * newer Seth * use transfer fee not gas limit for sending funds, modify defaults for missing networks * use latest Seth that uses block headers not entire blocks * try new Seth config; more refund logic * use latest seth, fix fund return issue where if a retry was used, only funds from 1st node were returned * do not return, but continue, if one node has has less balance than tx costs on return * go mod tidy * validate seth config before creating k8s env in ocr soak test, better default tx timeout * init seth client, before staring ocr soak test * fix complile errors * go mod * use latest seth * couple of renames and streamlines * use Seth Network urls if provided, otherwise take url from evmnetwork * testconfig will now correctly use custom EVMNetwork * latest Seth; set gas limit to 0 in TOMLs, so that it can be estimated by the node; use urls_secret for Seth Network when set, otherwise use WS endpoints from EVMNetwork; update default TOMLs with new values * go mod * skip funds return for given CL node if balance is 0 * latest seth * gomodtidy + latest seth * fix load compile --------- Co-authored-by: davidcauchi <[email protected]>
smartcontractkit · Apr 9, 2024 · a3d5276 · a3d5276
1 parent 401d126
commit a3d5276
Show file tree

Hide file tree

Showing 17 changed files with 322 additions and 107 deletions.
diff --git a/integration-tests/actions/seth/actions.go b/integration-tests/actions/seth/actions.go
@@ -97,12 +97,17 @@ type FundsToSendPayload struct {
 	ToAddress  common.Address
 	Amount     *big.Int
 	PrivateKey *ecdsa.PrivateKey
-	GasLimit   *uint64
+	GasLimit   *int64
+	GasPrice   *big.Int
+	GasFeeCap  *big.Int
+	GasTipCap  *big.Int
+	TxTimeout  *time.Duration
 }
 
 // TODO: move to CTF?
 // SendFunds sends native token amount (expressed in human-scale) from address controlled by private key
-// to given address. If no gas limit is set, then network's default will be used.
+// to given address. You can override any or none of the following: gas limit, gas price, gas fee cap, gas tip cap.
+// Values that are not set will be estimated or taken from config.
 func SendFunds(logger zerolog.Logger, client *seth.Client, payload FundsToSendPayload) (*types.Receipt, error) {
 	fromAddress, err := privateKeyToAddress(payload.PrivateKey)
 	if err != nil {
@@ -117,38 +122,75 @@ func SendFunds(logger zerolog.Logger, client *seth.Client, payload FundsToSendPa
 	}
 
 	gasLimit := uint64(client.Cfg.Network.TransferGasFee)
+	gasPrice := big.NewInt(0)
+	gasFeeCap := big.NewInt(0)
+	gasTipCap := big.NewInt(0)
+
 	if payload.GasLimit != nil {
-		gasLimit = *payload.GasLimit
+		gasLimit = uint64(*payload.GasLimit)
 	}
 
-	var signedTx *types.Transaction
+	if client.Cfg.Network.EIP1559DynamicFees {
+		// if any of the dynamic fees are not set, we need to either estimate them or read them from config
+		if payload.GasFeeCap == nil || payload.GasTipCap == nil {
+			// estimatior or config reading happens here
+			txOptions := client.NewTXOpts(seth.WithGasLimit(gasLimit))
+			gasFeeCap = txOptions.GasFeeCap
+			gasTipCap = txOptions.GasTipCap
+		}
+
+		// override with payload values if they are set
+		if payload.GasFeeCap != nil {
+			gasFeeCap = payload.GasFeeCap
+		}
+
+		if payload.GasTipCap != nil {
+			gasTipCap = payload.GasTipCap
+		}
+	}
+
+	if !client.Cfg.Network.EIP1559DynamicFees {
+		if payload.GasPrice == nil {
+			txOptions := client.NewTXOpts((seth.WithGasLimit(gasLimit)))
+			gasPrice = txOptions.GasPrice
+		} else {
+			gasPrice = payload.GasPrice
+		}
+	}
+
+	var rawTx types.TxData
 
 	if client.Cfg.Network.EIP1559DynamicFees {
-		rawTx := &types.DynamicFeeTx{
+		rawTx = &types.DynamicFeeTx{
 			Nonce:     nonce,
 			To:        &payload.ToAddress,
 			Value:     payload.Amount,
 			Gas:       gasLimit,
-			GasFeeCap: big.NewInt(client.Cfg.Network.GasFeeCap),
-			GasTipCap: big.NewInt(client.Cfg.Network.GasTipCap),
+			GasFeeCap: gasFeeCap,
+			GasTipCap: gasTipCap,
 		}
-		signedTx, err = types.SignNewTx(payload.PrivateKey, types.NewLondonSigner(big.NewInt(client.ChainID)), rawTx)
 	} else {
-		rawTx := &types.LegacyTx{
+		rawTx = &types.LegacyTx{
 			Nonce:    nonce,
 			To:       &payload.ToAddress,
 			Value:    payload.Amount,
 			Gas:      gasLimit,
-			GasPrice: big.NewInt(client.Cfg.Network.GasPrice),
+			GasPrice: gasPrice,
 		}
-		signedTx, err = types.SignNewTx(payload.PrivateKey, types.NewEIP155Signer(big.NewInt(client.ChainID)), rawTx)
 	}
 
+	signedTx, err := types.SignNewTx(payload.PrivateKey, types.LatestSignerForChainID(big.NewInt(client.ChainID)), rawTx)
+
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to sign tx")
 	}
 
-	ctx, cancel = context.WithTimeout(ctx, client.Cfg.Network.TxnTimeout.Duration())
+	txTimeout := client.Cfg.Network.TxnTimeout.Duration()
+	if payload.TxTimeout != nil {
+		txTimeout = *payload.TxTimeout
+	}
+
+	ctx, cancel = context.WithTimeout(ctx, txTimeout)
 	defer cancel()
 	err = client.Client.SendTransaction(ctx, signedTx)
 	if err != nil {
@@ -162,9 +204,9 @@ func SendFunds(logger zerolog.Logger, client *seth.Client, payload FundsToSendPa
 		Str("Amount", conversions.WeiToEther(payload.Amount).String()).
 		Uint64("Nonce", nonce).
 		Uint64("Gas Limit", gasLimit).
-		Int64("Gas Price", client.Cfg.Network.GasPrice).
-		Int64("Gas Fee Cap", client.Cfg.Network.GasFeeCap).
-		Int64("Gas Tip Cap", client.Cfg.Network.GasTipCap).
+		Str("Gas Price", gasPrice.String()).
+		Str("Gas Fee Cap", gasFeeCap.String()).
+		Str("Gas Tip Cap", gasTipCap.String()).
 		Bool("Dynamic fees", client.Cfg.Network.EIP1559DynamicFees).
 		Msg("Sent funds")
 

diff --git a/integration-tests/actions/seth/refund.go b/integration-tests/actions/seth/refund.go
@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
@@ -19,7 +20,7 @@ import (
 
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 
-	"github.com/smartcontractkit/chainlink/integration-tests/client"
+	clClient "github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 )
 
@@ -103,7 +104,7 @@ func (r *InsufficientFundTransferRetrier) Retry(ctx context.Context, logger zero
 // by doubling the gas limit and retrying until reaching maxGasLimit
 type GasTooLowTransferRetrier struct {
 	nextRetrier TransactionRetrier
-	maxGasLimit uint64
+	maxGasLimit int64
 }
 
 func (r *GasTooLowTransferRetrier) Retry(ctx context.Context, logger zerolog.Logger, client *seth.Client, txErr error, payload FundsToSendPayload, currentAttempt int) error {
@@ -120,18 +121,18 @@ func (r *GasTooLowTransferRetrier) Retry(ctx context.Context, logger zerolog.Log
 	for txErr != nil && strings.Contains(txErr.Error(), GasTooLowErr) {
 		logger.Info().
 			Msg("Too low gas error detected, retrying with more gas")
-		var newGasLimit uint64
+		var newGasLimit int64
 		if payload.GasLimit != nil {
 			newGasLimit = *payload.GasLimit * 2
 		} else {
-			newGasLimit = uint64(client.Cfg.Network.TransferGasFee) * 2
+			newGasLimit = client.Cfg.Network.TransferGasFee * 2
 		}
 
 		logger.Debug().
 			Str("retier", "GasTooLowTransferRetrier").
-			Uint64("old gas limit", newGasLimit/2).
-			Uint64("new gas limit", newGasLimit).
-			Uint64("diff", newGasLimit).
+			Int64("old gas limit", newGasLimit/2).
+			Int64("new gas limit", newGasLimit).
+			Int64("diff", newGasLimit).
 			Msg("New gas limit to use")
 
 		payload.GasLimit = &newGasLimit
@@ -231,21 +232,21 @@ func (r *OvershotTransferRetrier) Retry(ctx context.Context, logger zerolog.Logg
 // ReturnFunds returns funds from the given chainlink nodes to the default network wallet. It will use a variety
 // of strategies to attempt to return funds, including retrying with less funds if the transaction fails due to
 // insufficient funds, and retrying with a higher gas limit if the transaction fails due to gas too low.
-func ReturnFunds(log zerolog.Logger, seth *seth.Client, chainlinkNodes []contracts.ChainlinkNodeWithKeysAndAddress) error {
-	if seth == nil {
+func ReturnFunds(log zerolog.Logger, sethClient *seth.Client, chainlinkNodes []contracts.ChainlinkNodeWithKeysAndAddress) error {
+	if sethClient == nil {
 		return fmt.Errorf("Seth client is nil, unable to return funds from chainlink nodes")
 	}
 	log.Info().Msg("Attempting to return Chainlink node funds to default network wallets")
-	if seth.Cfg.IsSimulatedNetwork() {
-		log.Info().Str("Network Name", seth.Cfg.Network.Name).
+	if sethClient.Cfg.IsSimulatedNetwork() {
+		log.Info().Str("Network Name", sethClient.Cfg.Network.Name).
 			Msg("Network is a simulated network. Skipping fund return.")
 		return nil
 	}
 
 	failedReturns := []common.Address{}
 
 	for _, chainlinkNode := range chainlinkNodes {
-		fundedKeys, err := chainlinkNode.ExportEVMKeysForChain(fmt.Sprint(seth.ChainID))
+		fundedKeys, err := chainlinkNode.ExportEVMKeysForChain(fmt.Sprint(sethClient.ChainID))
 		if err != nil {
 			return err
 		}
@@ -256,7 +257,7 @@ func ReturnFunds(log zerolog.Logger, seth *seth.Client, chainlinkNodes []contrac
 			}
 			// This can take up a good bit of RAM and time. When running on the remote-test-runner, this can lead to OOM
 			// issues. So we avoid running in parallel; slower, but safer.
-			decryptedKey, err := keystore.DecryptKey(keyToDecrypt, client.ChainlinkKeyPassword)
+			decryptedKey, err := keystore.DecryptKey(keyToDecrypt, clClient.ChainlinkKeyPassword)
 			if err != nil {
 				return err
 			}
@@ -268,24 +269,48 @@ func ReturnFunds(log zerolog.Logger, seth *seth.Client, chainlinkNodes []contrac
 			}
 			fromAddress := crypto.PubkeyToAddress(*publicKeyECDSA)
 
-			balance, err := seth.Client.BalanceAt(context.Background(), fromAddress, nil)
+			balance, err := sethClient.Client.BalanceAt(context.Background(), fromAddress, nil)
 			if err != nil {
 				return err
 			}
 
-			var totalGasCost *big.Int
-			if seth.Cfg.Network.EIP1559DynamicFees {
-				totalGasCost = new(big.Int).Mul(big.NewInt(0).SetInt64(seth.Cfg.Network.TransferGasFee), big.NewInt(0).SetInt64(seth.Cfg.Network.GasFeeCap))
+			if balance.Cmp(big.NewInt(0)) == 0 {
+				log.Info().
+					Str("Address", fromAddress.String()).
+					Msg("No balance to return. Skipping return.")
+			}
+
+			// if not set, it will be just set to empty string, which is okay as long as gas estimation is disabled
+			txPriority := sethClient.Cfg.Network.GasEstimationTxPriority
+			txTimeout := sethClient.Cfg.Network.TxnTimeout.Duration()
+
+			if sethClient.Cfg.IsExperimentEnabled(seth.Experiment_SlowFundsReturn) {
+				txPriority = "slow"
+				thirtyMinutes := time.Duration(30 * time.Minute)
+				txTimeout = thirtyMinutes
+			}
+
+			estimations := sethClient.CalculateGasEstimations(seth.GasEstimationRequest{
+				GasEstimationEnabled: sethClient.Cfg.Network.GasEstimationEnabled,
+				FallbackGasPrice:     sethClient.Cfg.Network.GasPrice,
+				FallbackGasFeeCap:    sethClient.Cfg.Network.GasFeeCap,
+				FallbackGasTipCap:    sethClient.Cfg.Network.GasTipCap,
+				Priority:             txPriority,
+			})
+
+			var maxTotalGasCost *big.Int
+			if sethClient.Cfg.Network.EIP1559DynamicFees {
+				maxTotalGasCost = new(big.Int).Mul(big.NewInt(0).SetInt64(sethClient.Cfg.Network.TransferGasFee), estimations.GasFeeCap)
 			} else {
-				totalGasCost = new(big.Int).Mul(big.NewInt(0).SetInt64(seth.Cfg.Network.TransferGasFee), big.NewInt(0).SetInt64(seth.Cfg.Network.GasPrice))
+				maxTotalGasCost = new(big.Int).Mul(big.NewInt(0).SetInt64(sethClient.Cfg.Network.TransferGasFee), estimations.GasPrice)
 			}
 
-			toSend := new(big.Int).Sub(balance, totalGasCost)
+			toSend := new(big.Int).Sub(balance, maxTotalGasCost)
 
 			if toSend.Cmp(big.NewInt(0)) <= 0 {
 				log.Warn().
 					Str("Address", fromAddress.String()).
-					Str("Estimated total cost", totalGasCost.String()).
+					Str("Estimated maximum total gas cost", maxTotalGasCost.String()).
 					Str("Balance", balance.String()).
 					Str("To send", toSend.String()).
 					Msg("Not enough balance to cover gas cost. Skipping return.")
@@ -294,12 +319,21 @@ func ReturnFunds(log zerolog.Logger, seth *seth.Client, chainlinkNodes []contrac
 				continue
 			}
 
-			payload := FundsToSendPayload{ToAddress: seth.Addresses[0], Amount: toSend, PrivateKey: decryptedKey.PrivateKey}
+			payload := FundsToSendPayload{
+				ToAddress:  sethClient.Addresses[0],
+				Amount:     toSend,
+				PrivateKey: decryptedKey.PrivateKey,
+				GasLimit:   &sethClient.Cfg.Network.TransferGasFee,
+				GasPrice:   estimations.GasPrice,
+				GasFeeCap:  estimations.GasFeeCap,
+				GasTipCap:  estimations.GasTipCap,
+				TxTimeout:  &txTimeout,
+			}
 
-			_, err = SendFunds(log, seth, payload)
+			_, err = SendFunds(log, sethClient, payload)
 			if err != nil {
-				handler := OvershotTransferRetrier{maxRetries: 10, nextRetrier: &InsufficientFundTransferRetrier{maxRetries: 10, nextRetrier: &GasTooLowTransferRetrier{maxGasLimit: uint64(seth.Cfg.Network.TransferGasFee * 10)}}}
-				err = handler.Retry(context.Background(), log, seth, err, payload, 0)
+				handler := OvershotTransferRetrier{maxRetries: 10, nextRetrier: &InsufficientFundTransferRetrier{maxRetries: 10, nextRetrier: &GasTooLowTransferRetrier{maxGasLimit: sethClient.Cfg.Network.TransferGasFee * 10}}}
+				err = handler.Retry(context.Background(), log, sethClient, err, payload, 0)
 				if err != nil {
 					log.Error().
 						Err(err).

diff --git a/integration-tests/chaos/ocr_chaos_test.go b/integration-tests/chaos/ocr_chaos_test.go
@@ -171,7 +171,8 @@ func TestOCRChaos(t *testing.T) {
 			network := networks.MustGetSelectedNetworkConfig(cfg.GetNetworkConfig())[0]
 			network = utils.MustReplaceSimulatedNetworkUrlWithK8(l, network, *testEnvironment)
 
-			sethCfg := utils.MergeSethAndEvmNetworkConfigs(l, network, *readSethCfg)
+			sethCfg, err := utils.MergeSethAndEvmNetworkConfigs(network, *readSethCfg)
+			require.NoError(t, err, "Error merging seth and evm network configs")
 			err = utils.ValidateSethNetworkConfig(sethCfg.Network)
 			require.NoError(t, err, "Error validating seth network config")
 			seth, err := seth.NewClientWithConfig(&sethCfg)

diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
@@ -322,7 +322,10 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 
 			if b.hasSeth {
 				readSethCfg := b.testConfig.GetSethConfig()
-				sethCfg := utils.MergeSethAndEvmNetworkConfigs(b.l, networkConfig, *readSethCfg)
+				sethCfg, err := utils.MergeSethAndEvmNetworkConfigs(networkConfig, *readSethCfg)
+				if err != nil {
+					return nil, err
+				}
 				err = utils.ValidateSethNetworkConfig(sethCfg.Network)
 				if err != nil {
 					return nil, err
@@ -421,7 +424,10 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 		if b.hasSeth {
 			b.te.sethClients = make(map[int64]*seth.Client)
 			readSethCfg := b.testConfig.GetSethConfig()
-			sethCfg := utils.MergeSethAndEvmNetworkConfigs(b.l, networkConfig, *readSethCfg)
+			sethCfg, err := utils.MergeSethAndEvmNetworkConfigs(networkConfig, *readSethCfg)
+			if err != nil {
+				return nil, err
+			}
 			err = utils.ValidateSethNetworkConfig(sethCfg.Network)
 			if err != nil {
 				return nil, err

diff --git a/integration-tests/experiments/gas_test.go b/integration-tests/experiments/gas_test.go
@@ -0,0 +1,39 @@
+package experiments
+
+import (
+	"testing"
+	"time"
+
+	"github.com/smartcontractkit/seth"
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
+	"github.com/smartcontractkit/chainlink-testing-framework/networks"
+	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
+	tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig"
+	"github.com/smartcontractkit/chainlink/integration-tests/utils"
+)
+
+func TestGasExperiment(t *testing.T) {
+	l := logging.GetTestLogger(t)
+	config, err := tc.GetConfig("Soak", tc.OCR)
+	require.NoError(t, err, "Error getting config")
+
+	network := networks.MustGetSelectedNetworkConfig(config.GetNetworkConfig())[0]
+	readSethCfg := config.GetSethConfig()
+	require.NotNil(t, readSethCfg, "Seth config shouldn't be nil")
+
+	sethCfg, err := utils.MergeSethAndEvmNetworkConfigs(network, *readSethCfg)
+	require.NoError(t, err, "Error merging seth and evm network configs")
+	err = utils.ValidateSethNetworkConfig(sethCfg.Network)
+	require.NoError(t, err, "Error validating seth network config")
+
+	seth, err := seth.NewClientWithConfig(&sethCfg)
+	require.NoError(t, err, "Error creating seth client")
+
+	for i := 0; i < 1; i++ {
+		_, err = contracts.DeployLinkTokenContract(l, seth)
+		require.NoError(t, err, "Error deploying LINK contract")
+		time.Sleep(2 * time.Second)
+	}
+}
diff --git a/integration-tests/go.mod b/integration-tests/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052
-	github.com/smartcontractkit/seth v0.1.2
+	github.com/smartcontractkit/seth v0.1.3
 	github.com/smartcontractkit/wasp v0.4.5
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
@@ -337,8 +337,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 // indirect
-	github.com/naoina/go-stringutil v0.1.0 // indirect
-	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect