From 7ffbfbf948a4c64d4b0c9bc295804654c2235f33 Mon Sep 17 00:00:00 2001 From: Anindita Ghosh <88458927+AnieeG@users.noreply.github.com> Date: Thu, 19 Sep 2024 08:21:26 -0700 Subject: [PATCH 1/7] CCIP-3428 Enabling ccip smoke test for testnet (#14452) * enabling test for testnet * home chain selector logic * testnet deployment * remove unwanted * lint fix * add readme * fix typo * fix * make timer rely on test deadline * update comment * update comment --- integration-tests/actions/actions.go | 7 +- integration-tests/actions/refund.go | 8 + .../ccip-tests/testsetups/test_env.go | 3 + .../contracts/contract_models.go | 6 +- integration-tests/deployment/README.md | 6 +- .../deployment/ccip/add_chain_test.go | 3 +- .../deployment/ccip/add_lane_test.go | 3 +- .../ccip/changeset/2_initial_deploy_test.go | 3 +- .../deployment/ccip/test_assertions.go | 14 +- .../deployment/ccip/test_helpers.go | 25 +-- .../deployment/devenv/.sample.env | 22 ++- integration-tests/deployment/devenv/README.md | 45 +++++ .../deployment/devenv/build_env.go | 185 +++++++++++++++--- integration-tests/deployment/devenv/chain.go | 60 ++---- integration-tests/deployment/devenv/don.go | 78 ++++---- .../deployment/devenv/environment.go | 7 +- integration-tests/deployment/environment.go | 5 +- .../deployment/memory/environment.go | 3 - .../docker/test_env/test_env_builder.go | 39 ++-- integration-tests/smoke/ccip_test.go | 3 +- integration-tests/testconfig/ccip/ccip.toml | 6 + integration-tests/testconfig/ccip/config.go | 7 + .../ccip/overrides/sepolia_avax_binance.toml | 55 ++++++ 23 files changed, 426 insertions(+), 167 deletions(-) create mode 100644 integration-tests/deployment/devenv/README.md create mode 100644 integration-tests/testconfig/ccip/overrides/sepolia_avax_binance.toml diff --git a/integration-tests/actions/actions.go b/integration-tests/actions/actions.go index c420f8e6729..198fa8e0dc8 100644 --- a/integration-tests/actions/actions.go +++ b/integration-tests/actions/actions.go @@ -25,6 +25,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" "github.com/smartcontractkit/chainlink-testing-framework/lib/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/conversions" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" ethContracts "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum" "github.com/smartcontractkit/chainlink/integration-tests/wrappers" @@ -285,7 +286,7 @@ func fundChainlinkNodesAtAnyKey( return err } - fromAddress, err := privateKeyToAddress(privateKey) + fromAddress, err := PrivateKeyToAddress(privateKey) if err != nil { return err } @@ -336,7 +337,7 @@ type FundsToSendPayload struct { // to given address. You can override any or none of the following: gas limit, gas price, gas fee cap, gas tip cap. // Values that are not set will be estimated or taken from config. func SendFunds(logger zerolog.Logger, client *seth.Client, payload FundsToSendPayload) (*types.Receipt, error) { - fromAddress, err := privateKeyToAddress(payload.PrivateKey) + fromAddress, err := PrivateKeyToAddress(payload.PrivateKey) if err != nil { return nil, err } @@ -910,7 +911,7 @@ func deployAnyOCRv1Contracts( return ocrInstances, nil } -func privateKeyToAddress(privateKey *ecdsa.PrivateKey) (common.Address, error) { +func PrivateKeyToAddress(privateKey *ecdsa.PrivateKey) (common.Address, error) { publicKey := privateKey.Public() publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) if !ok { diff --git a/integration-tests/actions/refund.go b/integration-tests/actions/refund.go index 1835d9a04a4..bcdf6a380d8 100644 --- a/integration-tests/actions/refund.go +++ b/integration-tests/actions/refund.go @@ -234,6 +234,14 @@ func (r *OvershotTransferRetrier) Retry(ctx context.Context, logger zerolog.Logg // of strategies to attempt to return funds, including retrying with less funds if the transaction fails due to // insufficient funds, and retrying with a higher gas limit if the transaction fails due to gas too low. func ReturnFundsFromNodes(log zerolog.Logger, client *seth.Client, chainlinkNodes []contracts.ChainlinkNodeWithKeysAndAddress) error { + var keyExporters []contracts.ChainlinkKeyExporter + for _, node := range chainlinkNodes { + keyExporters = append(keyExporters, node) + } + return ReturnFundsFromKeyExporterNodes(log, client, keyExporters) +} + +func ReturnFundsFromKeyExporterNodes(log zerolog.Logger, client *seth.Client, chainlinkNodes []contracts.ChainlinkKeyExporter) error { if client == nil { return fmt.Errorf("seth client is nil, unable to return funds from chainlink nodes") } diff --git a/integration-tests/ccip-tests/testsetups/test_env.go b/integration-tests/ccip-tests/testsetups/test_env.go index 4d968e83315..263d291453d 100644 --- a/integration-tests/ccip-tests/testsetups/test_env.go +++ b/integration-tests/ccip-tests/testsetups/test_env.go @@ -332,6 +332,9 @@ func DeployLocalCluster( // a func to start the CL nodes asynchronously deployCL := func() error { noOfNodes := pointer.GetInt(testInputs.EnvInput.NewCLCluster.NoOfNodes) + if env.ClCluster == nil { + env.ClCluster = &test_env.ClCluster{} + } // if individual nodes are specified, then deploy them with specified configs if len(testInputs.EnvInput.NewCLCluster.Nodes) > 0 { for _, clNode := range testInputs.EnvInput.NewCLCluster.Nodes { diff --git a/integration-tests/contracts/contract_models.go b/integration-tests/contracts/contract_models.go index 46f10f06bb0..006ee5db6a8 100644 --- a/integration-tests/contracts/contract_models.go +++ b/integration-tests/contracts/contract_models.go @@ -151,9 +151,13 @@ type OffchainAggregatorData struct { type ChainlinkNodeWithKeysAndAddress interface { MustReadOCRKeys() (*client.OCRKeys, error) MustReadP2PKeys() (*client.P2PKeys, error) - ExportEVMKeysForChain(string) ([]*client.ExportedEVMKey, error) PrimaryEthAddress() (string, error) EthAddresses() ([]string, error) + ChainlinkKeyExporter +} + +type ChainlinkKeyExporter interface { + ExportEVMKeysForChain(string) ([]*client.ExportedEVMKey, error) } type ChainlinkNodeWithForwarder interface { diff --git a/integration-tests/deployment/README.md b/integration-tests/deployment/README.md index 1c2019b540b..000219c8aba 100644 --- a/integration-tests/deployment/README.md +++ b/integration-tests/deployment/README.md @@ -19,11 +19,11 @@ environments like testnet/mainnet. - In-memory environment for fast integration testing - EVM only -/deployment/docker +/deployment/devenv - Coming soon -- package name `docker` +- package name `devenv` - Docker environment for higher fidelity testing -- Support non-EVMs +- Support non-EVMs (yet to be implemented) /deployment/ccip - package name `ccipdeployment` diff --git a/integration-tests/deployment/ccip/add_chain_test.go b/integration-tests/deployment/ccip/add_chain_test.go index fdfbb6e69a6..9441f1a2da6 100644 --- a/integration-tests/deployment/ccip/add_chain_test.go +++ b/integration-tests/deployment/ccip/add_chain_test.go @@ -154,8 +154,9 @@ func TestAddChainInbound(t *testing.T) { // TODO: Send via all inbound lanes and use parallel helper // Now that the proposal has been executed we expect to be able to send traffic to this new 4th chain. - startBlock, err := e.Env.Chains[newChain].LatestBlockNum(testcontext.Get(t)) + latesthdr, err := e.Env.Chains[newChain].Client.HeaderByNumber(testcontext.Get(t), nil) require.NoError(t, err) + startBlock := latesthdr.Number.Uint64() seqNr := SendRequest(t, e.Env, state, initialDeploy[0], newChain, true) require.NoError(t, ConfirmExecWithSeqNr(t, e.Env.Chains[initialDeploy[0]], e.Env.Chains[newChain], state.Chains[newChain].OffRamp, &startBlock, seqNr)) diff --git a/integration-tests/deployment/ccip/add_lane_test.go b/integration-tests/deployment/ccip/add_lane_test.go index 63af3b69c45..d43526d8d49 100644 --- a/integration-tests/deployment/ccip/add_lane_test.go +++ b/integration-tests/deployment/ccip/add_lane_test.go @@ -50,8 +50,9 @@ func TestAddLane(t *testing.T) { require.Len(t, offRamps, 0) } } - startBlock, err := e.Env.Chains[to].LatestBlockNum(testcontext.Get(t)) + latesthdr, err := e.Env.Chains[to].Client.HeaderByNumber(testcontext.Get(t), nil) require.NoError(t, err) + startBlock := latesthdr.Number.Uint64() seqNum := SendRequest(t, e.Env, state, from, to, false) require.Equal(t, uint64(1), seqNum) require.NoError(t, ConfirmExecWithSeqNr(t, e.Env.Chains[from], e.Env.Chains[to], state.Chains[to].OffRamp, &startBlock, seqNum)) diff --git a/integration-tests/deployment/ccip/changeset/2_initial_deploy_test.go b/integration-tests/deployment/ccip/changeset/2_initial_deploy_test.go index 8de2c4617b4..90a8627f38d 100644 --- a/integration-tests/deployment/ccip/changeset/2_initial_deploy_test.go +++ b/integration-tests/deployment/ccip/changeset/2_initial_deploy_test.go @@ -62,8 +62,9 @@ func Test0002_InitialDeploy(t *testing.T) { if src == dest { continue } - block, err := destChain.LatestBlockNum(testcontext.Get(t)) + latesthdr, err := destChain.Client.HeaderByNumber(testcontext.Get(t), nil) require.NoError(t, err) + block := latesthdr.Number.Uint64() startBlocks[dest] = &block seqNum := ccipdeployment.SendRequest(t, e, state, src, dest, false) expectedSeqNum[dest] = seqNum diff --git a/integration-tests/deployment/ccip/test_assertions.go b/integration-tests/deployment/ccip/test_assertions.go index 2041c1a9779..02a10fff3e6 100644 --- a/integration-tests/deployment/ccip/test_assertions.go +++ b/integration-tests/deployment/ccip/test_assertions.go @@ -78,7 +78,15 @@ func ConfirmCommitWithExpectedSeqNumRange( } defer subscription.Unsubscribe() - timer := time.NewTimer(5 * time.Minute) + var duration time.Duration + deadline, ok := t.Deadline() + if ok { + // make this timer end a minute before so that we don't hit the deadline + duration = deadline.Sub(time.Now().Add(-1 * time.Minute)) + } else { + duration = 5 * time.Minute + } + timer := time.NewTimer(duration) defer timer.Stop() ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() @@ -97,8 +105,8 @@ func ConfirmCommitWithExpectedSeqNumRange( case subErr := <-subscription.Err(): return fmt.Errorf("subscription error: %w", subErr) case <-timer.C: - return fmt.Errorf("timed out waiting for commit report on chain selector %d from source selector %d expected seq nr range %s", - dest.Selector, src.Selector, expectedSeqNumRange.String()) + return fmt.Errorf("timed out after waiting %s duration for commit report on chain selector %d from source selector %d expected seq nr range %s", + duration.String(), dest.Selector, src.Selector, expectedSeqNumRange.String()) case report := <-sink: if len(report.Report.MerkleRoots) > 0 { // Check the interval of sequence numbers and make sure it matches diff --git a/integration-tests/deployment/ccip/test_helpers.go b/integration-tests/deployment/ccip/test_helpers.go index 4458a49abc7..330cbae1964 100644 --- a/integration-tests/deployment/ccip/test_helpers.go +++ b/integration-tests/deployment/ccip/test_helpers.go @@ -8,6 +8,8 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" + "github.com/ethereum/go-ethereum/common" chainsel "github.com/smartcontractkit/chain-selectors" "github.com/stretchr/testify/require" @@ -158,7 +160,9 @@ func SendRequest(t *testing.T, e deployment.Environment, state CCIPOnChainState, }, []uint64{dest}) require.NoError(t, err) require.True(t, it.Next()) - return it.Event.Message.Header.SequenceNumber + seqNum := it.Event.Message.Header.SequenceNumber + t.Logf("CCIP message sent from chain selector %d to chain selector %d tx %s seqNum %d", src, dest, tx.Hash().String(), seqNum) + return seqNum } // DeployedLocalDevEnvironment is a helper struct for setting up a local dev environment with docker @@ -179,15 +183,13 @@ func NewDeployedLocalDevEnvironment(t *testing.T, lggr logger.Logger) DeployedLo require.NotEmpty(t, envConfig.JDConfig, "jdUrl should not be empty") chains, err := devenv.NewChains(lggr, envConfig.Chains) require.NoError(t, err) - homeChainSel := uint64(0) - homeChainEVM := uint64(0) + // locate the home chain + homeChainSel := envConfig.HomeChainSelector + require.NotEmpty(t, homeChainSel, "homeChainSel should not be empty") + homeChainEVM, err := chainsel.ChainIdFromSelector(homeChainSel) + require.NoError(t, err) + require.NotEmpty(t, homeChainEVM, "homeChainEVM should not be empty") - // Say first chain is home chain. - for chainSel := range chains { - homeChainEVM, _ = chainsel.ChainIdFromSelector(chainSel) - homeChainSel = chainSel - break - } // deploy the capability registry ab, capReg, err := DeployCapReg(lggr, chains, homeChainSel) require.NoError(t, err) @@ -205,10 +207,9 @@ func NewDeployedLocalDevEnvironment(t *testing.T, lggr logger.Logger) DeployedLo require.NoError(t, err) require.NotNil(t, e) require.NotNil(t, don) - + zeroLogLggr := logging.GetTestLogger(t) // fund the nodes - require.NoError(t, don.FundNodes(ctx, deployment.E18Mult(10), e.Chains)) - + devenv.FundNodes(t, zeroLogLggr, testEnv, cfg, don.PluginNodes()) return DeployedLocalDevEnvironment{ Ab: ab, Env: *e, diff --git a/integration-tests/deployment/devenv/.sample.env b/integration-tests/deployment/devenv/.sample.env index 97d550079a9..f0505cf857a 100644 --- a/integration-tests/deployment/devenv/.sample.env +++ b/integration-tests/deployment/devenv/.sample.env @@ -2,4 +2,24 @@ E2E_JD_IMAGE= E2E_JD_VERSION= E2E_TEST_CHAINLINK_IMAGE=public.ecr.aws/w0i8p0z9/chainlink-ccip -E2E_TEST_CHAINLINK_VERSION=2.14.0-ccip1.5.0 \ No newline at end of file +E2E_TEST_CHAINLINK_VERSION=2.14.0-ccip1.5.0 + +# RPC Configuration +E2E_TEST_SEPOLIA_WALLET_KEY= +E2E_TEST_SEPOLIA_RPC_HTTP_URL_1= +E2E_TEST_SEPOLIA_RPC_HTTP_URL_2= +E2E_TEST_SEPOLIA_RPC_WS_URL_1= +E2E_TEST_SEPOLIA_RPC_WS_URL_2= + +E2E_TEST_AVALANCHE_FUJI_WALLET_KEY= +E2E_TEST_AVALANCHE_FUJI_RPC_HTTP_URL_1= +E2E_TEST_AVALANCHE_FUJI_RPC_HTTP_URL_2= +E2E_TEST_AVALANCHE_FUJI_RPC_WS_URL_1= +E2E_TEST_AVALANCHE_FUJI_RPC_WS_URL_2= + +E2E_TEST_BSC_TESTNET_WALLET_KEY= +E2E_TEST_BSC_TESTNET_RPC_HTTP_URL_1= +E2E_TEST_BSC_TESTNET_RPC_HTTP_URL_2= +E2E_TEST_BSC_TESTNET_RPC_WS_URL_1= +E2E_TEST_BSC_TESTNET_RPC_WS_URL_2= + diff --git a/integration-tests/deployment/devenv/README.md b/integration-tests/deployment/devenv/README.md new file mode 100644 index 00000000000..a1f264dfce2 --- /dev/null +++ b/integration-tests/deployment/devenv/README.md @@ -0,0 +1,45 @@ +## Overview + +This package is used to create ephemeral environment for local/CI testing. +It sets up an environment with local Docker containers running Chainlink nodes and a job distributor. +It can either create new simulated private Ethereum network containers or connect to existing testnets/mainnets. + +### Run Tests with Devenv + +The tests created with this environment are run as [end-to-end integration smoke tests](../../smoke). + +#### Setting Up Testconfig with Simulated Private Ethereum Network + +To run tests (e.g., [ccip-test](../../smoke/ccip_test.go)), +you need to set up the testconfig following the [testconfig setup instructions](../../testconfig/README.md). +The testconfig specifies the details of the different configurations to set up the environment and run the tests. +Generally, tests are run with the [default](../../testconfig/default.toml) config unless overridden by product-specific config. +For example, the [ccip-test](../../smoke/ccip_test.go) uses [ccip.toml](../../testconfig/ccip/ccip.toml) to specify +CCIP-specific test environment details. + +There are additional secret configuration parameters required by the `devenv` environment that are not stored in the testconfig. +These are read from environment variables. For example, Chainlink image, Job-Distributor image, etc. +All such environment variables are listed in the [sample.env](./.sample.env) file. +You can create a `.env` file in the same directory of the test and set the required environment variables. + +#### Setting Up Testconfig for Running Tests with Existing Testnet/Mainnet + +To run tests with existing testnet/mainnet, set up the testconfig with the details of the testnet/mainnet networks. +Following the integration-test [testconfig framework](../../testconfig/README.md#configuration-and-overrides), +create a new `overrides.toml` file with testnet/mainnet network details and place it under any location in the `integration-tests` directory. +By default, tests are run with private Ethereum network containers set up in the same Docker network as +the Chainlink nodes and job distributor. To run tests against existing testnet/mainnet, +set the `selected_network` field in the testconfig with the specific network names. + +For example, if running [ccip-smoke](../../smoke/ccip_test.go) tests with Sepolia, Avax, and Binance testnets, +copy the contents of [sepolia_avax_binance.toml](../../testconfig/ccip/overrides/sepolia_avax_binance.toml) +to the `overrides.toml` file. + +Before running the test, ensure that RPC and wallet secrets are set as environment variables. +Refer to the environment variables pattern in the [sample.env](./.sample.env) file to +provide necessary secrets applicable to the network you are running the tests against: +- `E2E_TEST__WALLET_KEY_` +- `E2E_TEST__RPC_HTTP_URL_` +- `E2E_TEST__RPC_WS_URL_` + +Now you are all set to run the tests with the existing testnet/mainnet. \ No newline at end of file diff --git a/integration-tests/deployment/devenv/build_env.go b/integration-tests/deployment/devenv/build_env.go index 0373cf0b214..61e677ca435 100644 --- a/integration-tests/deployment/devenv/build_env.go +++ b/integration-tests/deployment/devenv/build_env.go @@ -9,22 +9,30 @@ import ( "github.com/AlekSi/pointer" "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/rs/zerolog" chainselectors "github.com/smartcontractkit/chain-selectors" "github.com/stretchr/testify/require" "github.com/subosito/gotenv" + "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/conversions" + "github.com/smartcontractkit/chainlink-testing-framework/seth" + ctf_config "github.com/smartcontractkit/chainlink-testing-framework/lib/config" ctftestenv "github.com/smartcontractkit/chainlink-testing-framework/lib/docker/test_env" "github.com/smartcontractkit/chainlink-testing-framework/lib/docker/test_env/job_distributor" "github.com/smartcontractkit/chainlink-testing-framework/lib/networks" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" + "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testsetups" clclient "github.com/smartcontractkit/chainlink/integration-tests/client" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" "github.com/smartcontractkit/chainlink/integration-tests/deployment" "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + "github.com/smartcontractkit/chainlink/integration-tests/utils" "github.com/smartcontractkit/chainlink/v2/core/services/relay" ) @@ -43,18 +51,31 @@ func CreateDockerEnv(t *testing.T) ( cfg, err := tc.GetChainAndTestTypeSpecificConfig("Smoke", tc.CCIP) require.NoError(t, err, "Error getting config") + evmNetworks := networks.MustGetSelectedNetworkConfig(cfg.GetNetworkConfig()) + + // find out if the selected networks are provided with PrivateEthereumNetworks configs + // if yes, PrivateEthereumNetworkConfig will be used to create simulated private ethereum networks in docker environment var privateEthereumNetworks []*ctf_config.EthereumNetworkConfig - for _, network := range cfg.CCIP.PrivateEthereumNetworks { - privateEthereumNetworks = append(privateEthereumNetworks, network) + for _, name := range cfg.GetNetworkConfig().SelectedNetworks { + if network, exists := cfg.CCIP.PrivateEthereumNetworks[name]; exists { + privateEthereumNetworks = append(privateEthereumNetworks, network) + } } - env, err := test_env.NewCLTestEnvBuilder(). + + builder := test_env.NewCLTestEnvBuilder(). WithTestConfig(&cfg). WithTestInstance(t). - WithPrivateEthereumNetworks(privateEthereumNetworks). - WithStandardCleanup(). - Build() + WithStandardCleanup() + + // if private ethereum networks are provided, we will use them to create the test environment + // otherwise we will use the network URLs provided in the network config + if len(privateEthereumNetworks) > 0 { + builder = builder.WithPrivateEthereumNetworks(privateEthereumNetworks) + } + env, err := builder.Build() require.NoError(t, err, "Error building test environment") - chains := CreateChainConfigFromPrivateEthereumNetworks(t, env, cfg.CCIP.PrivateEthereumNetworks, cfg.GetNetworkConfig()) + + chains := CreateChainConfigFromNetworks(t, env, privateEthereumNetworks, cfg.GetNetworkConfig()) var jdConfig JDConfig // TODO : move this as a part of test_env setup with an input in testconfig @@ -89,9 +110,24 @@ func CreateDockerEnv(t *testing.T) ( } require.NotEmpty(t, jdConfig, "JD config is empty") + homeChainSelector, err := cfg.CCIP.GetHomeChainSelector() + require.NoError(t, err, "Error getting home chain selector") + homeChainID, err := chainselectors.ChainIdFromSelector(homeChainSelector) + require.NoError(t, err, "Error getting chain id from selector") + // verify if the home chain selector is valid + validHomeChain := false + for _, net := range evmNetworks { + if net.ChainID == int64(homeChainID) { + validHomeChain = true + break + } + } + require.True(t, validHomeChain, "Invalid home chain selector, chain not found in network config") + return &EnvironmentConfig{ - Chains: chains, - JDConfig: jdConfig, + Chains: chains, + JDConfig: jdConfig, + HomeChainSelector: homeChainSelector, }, env, cfg } @@ -107,12 +143,19 @@ func StartChainlinkNodes( ) error { evmNetworks := networks.MustGetSelectedNetworkConfig(cfg.GetNetworkConfig()) for i, net := range evmNetworks { - rpcProvider, err := env.GetRpcProvider(net.ChainID) - require.NoError(t, err, "Error getting rpc provider") - evmNetworks[i].HTTPURLs = rpcProvider.PrivateHttpUrls() - evmNetworks[i].URLs = rpcProvider.PrivateWsUrsl() + // if network is simulated, update the URLs with private chain RPCs in the docker test environment + // so that nodes can internally connect to the chain + if net.Simulated { + rpcProvider, err := env.GetRpcProvider(net.ChainID) + require.NoError(t, err, "Error getting rpc provider") + evmNetworks[i].HTTPURLs = rpcProvider.PrivateHttpUrls() + evmNetworks[i].URLs = rpcProvider.PrivateWsUrsl() + } } noOfNodes := pointer.GetInt(cfg.CCIP.CLNode.NoOfPluginNodes) + pointer.GetInt(cfg.CCIP.CLNode.NoOfBootstraps) + if env.ClCluster == nil { + env.ClCluster = &test_env.ClCluster{} + } var nodeInfo []NodeInfo for i := 1; i <= noOfNodes; i++ { if i <= pointer.GetInt(cfg.CCIP.CLNode.NoOfBootstraps) { @@ -172,17 +215,88 @@ func StartChainlinkNodes( InternalIP: n.API.InternalIP(), } } - envConfig.nodeInfo = nodeInfo return nil } -// CreateChainConfigFromPrivateEthereumNetworks creates a list of ChainConfig from the private ethereum networks created by the test environment. +// FundNodes sends funds to the chainlink nodes based on the provided test config +// It also sets up a clean-up function to return the funds back to the deployer account once the test is done +// It assumes that the chainlink nodes are already started and the account addresses for all chains are available +func FundNodes(t *testing.T, lggr zerolog.Logger, env *test_env.CLClusterTestEnv, cfg tc.TestConfig, nodes []Node) { + evmNetworks := networks.MustGetSelectedNetworkConfig(cfg.GetNetworkConfig()) + for i, net := range evmNetworks { + // if network is simulated, update the URLs with deployed chain RPCs in the docker test environment + if net.Simulated { + rpcProvider, err := env.GetRpcProvider(net.ChainID) + require.NoError(t, err, "Error getting rpc provider") + evmNetworks[i].HTTPURLs = rpcProvider.PublicHttpUrls() + evmNetworks[i].URLs = rpcProvider.PublicWsUrls() + } + } + t.Cleanup(func() { + for i := range evmNetworks { + // if simulated no need for balance return + if evmNetworks[i].Simulated { + continue + } + evmNetwork := evmNetworks[i] + sethClient, err := utils.TestAwareSethClient(t, cfg, &evmNetwork) + require.NoError(t, err, "Error getting seth client for network %s", evmNetwork.Name) + require.Greater(t, len(sethClient.PrivateKeys), 0, seth.ErrNoKeyLoaded) + var keyExporters []contracts.ChainlinkKeyExporter + for j := range nodes { + node := nodes[j] + keyExporters = append(keyExporters, &node) + } + if err := actions.ReturnFundsFromKeyExporterNodes(lggr, sethClient, keyExporters); err != nil { + lggr.Error().Err(err).Str("Network", evmNetwork.Name). + Msg("Error attempting to return funds from chainlink nodes to network's default wallet. " + + "Environment is left running so you can try manually!") + } + } + }) + for i := range evmNetworks { + evmNetwork := evmNetworks[i] + sethClient, err := utils.TestAwareSethClient(t, cfg, &evmNetwork) + require.NoError(t, err, "Error getting seth client for network %s", evmNetwork.Name) + require.Greater(t, len(sethClient.PrivateKeys), 0, seth.ErrNoKeyLoaded) + privateKey := sethClient.PrivateKeys[0] + for _, node := range nodes { + nodeAddr, ok := node.AccountAddr[uint64(evmNetwork.ChainID)] + require.True(t, ok, "Account address not found for chain %d", evmNetwork.ChainID) + fromAddress, err := actions.PrivateKeyToAddress(privateKey) + require.NoError(t, err, "Error getting address from private key") + amount := big.NewFloat(pointer.GetFloat64(cfg.Common.ChainlinkNodeFunding)) + toAddr := common.HexToAddress(nodeAddr) + receipt, err := actions.SendFunds(lggr, sethClient, actions.FundsToSendPayload{ + ToAddress: toAddr, + Amount: conversions.EtherToWei(amount), + PrivateKey: privateKey, + }) + require.NoError(t, err, "Error sending funds to node %s", node.Name) + require.NotNil(t, receipt, "Receipt is nil") + txHash := "(none)" + if receipt != nil { + txHash = receipt.TxHash.String() + } + lggr.Info(). + Str("From", fromAddress.Hex()). + Str("To", toAddr.String()). + Str("TxHash", txHash). + Str("Amount", amount.String()). + Msg("Funded Chainlink node") + } + } +} + +// CreateChainConfigFromNetworks creates a list of ChainConfig from the network config provided in test config. +// It either creates it from the private ethereum networks created by the test environment or from the +// network URLs provided in the network config ( if the network is a live testnet). // It uses the private keys from the network config to create the deployer key for each chain. -func CreateChainConfigFromPrivateEthereumNetworks( +func CreateChainConfigFromNetworks( t *testing.T, env *test_env.CLClusterTestEnv, - privateEthereumNetworks map[string]*ctf_config.EthereumNetworkConfig, + privateEthereumNetworks []*ctf_config.EthereumNetworkConfig, networkConfig *ctf_config.NetworkConfig, ) []ChainConfig { evmNetworks := networks.MustGetSelectedNetworkConfig(networkConfig) @@ -192,6 +306,29 @@ func CreateChainConfigFromPrivateEthereumNetworks( networkPvtKeys[net.ChainID] = net.PrivateKeys[0] } var chains []ChainConfig + // if private ethereum networks are not provided, we will create chains from the network URLs + if len(privateEthereumNetworks) == 0 { + for _, net := range evmNetworks { + chainId := net.ChainID + chainName, err := chainselectors.NameFromChainId(uint64(chainId)) + require.NoError(t, err, "Error getting chain name") + pvtKeyStr, exists := networkPvtKeys[chainId] + require.Truef(t, exists, "Private key not found for chain id %d", chainId) + pvtKey, err := crypto.HexToECDSA(pvtKeyStr) + require.NoError(t, err) + deployer, err := bind.NewKeyedTransactorWithChainID(pvtKey, big.NewInt(chainId)) + require.NoError(t, err) + chains = append(chains, ChainConfig{ + ChainID: uint64(chainId), + ChainName: chainName, + ChainType: "EVM", + WSRPCs: net.URLs, + HTTPRPCs: net.HTTPURLs, + DeployerKey: deployer, + }) + } + return chains + } for _, networkCfg := range privateEthereumNetworks { chainId := networkCfg.EthereumChainConfig.ChainID chainName, err := chainselectors.NameFromChainId(uint64(chainId)) @@ -205,14 +342,12 @@ func CreateChainConfigFromPrivateEthereumNetworks( deployer, err := bind.NewKeyedTransactorWithChainID(pvtKey, big.NewInt(int64(chainId))) require.NoError(t, err) chains = append(chains, ChainConfig{ - ChainID: uint64(chainId), - ChainName: chainName, - ChainType: "EVM", - WSRPCs: rpcProvider.PublicWsUrls(), - HTTPRPCs: rpcProvider.PublicHttpUrls(), - PrivateHTTPRPCs: rpcProvider.PrivateHttpUrls(), - PrivateWSRPCs: rpcProvider.PrivateWsUrsl(), - DeployerKey: deployer, + ChainID: uint64(chainId), + ChainName: chainName, + ChainType: "EVM", + WSRPCs: rpcProvider.PublicWsUrls(), + HTTPRPCs: rpcProvider.PublicHttpUrls(), + DeployerKey: deployer, }) } return chains diff --git a/integration-tests/deployment/devenv/chain.go b/integration-tests/deployment/devenv/chain.go index 6374a2c213d..e40bbc066fd 100644 --- a/integration-tests/deployment/devenv/chain.go +++ b/integration-tests/deployment/devenv/chain.go @@ -3,11 +3,9 @@ package devenv import ( "context" "fmt" - "math/big" "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/sethvargo/go-retry" @@ -20,14 +18,12 @@ import ( // ChainConfig holds the configuration for a with a deployer key which can be used to send transactions to the chain. type ChainConfig struct { - ChainID uint64 // chain id as per EIP-155, mainly applicable for EVM chains - ChainName string // name of the chain populated from chainselector repo - ChainType string // should denote the chain family. Acceptable values are EVM, COSMOS, SOLANA, STARKNET, APTOS etc - WSRPCs []string // websocket rpcs to connect to the chain - HTTPRPCs []string // http rpcs to connect to the chain - PrivateWSRPCs []string // applicable for private chains spun up with docker/K8s only so that nodes within same cluster can connect internally - PrivateHTTPRPCs []string // applicable for private chains spun up with docker/K8s only so that nodes within same cluster can connect internally - DeployerKey *bind.TransactOpts // key to send transactions to the chain + ChainID uint64 // chain id as per EIP-155, mainly applicable for EVM chains + ChainName string // name of the chain populated from chainselector repo + ChainType string // should denote the chain family. Acceptable values are EVM, COSMOS, SOLANA, STARKNET, APTOS etc + WSRPCs []string // websocket rpcs to connect to the chain + HTTPRPCs []string // http rpcs to connect to the chain + DeployerKey *bind.TransactOpts // key to send transactions to the chain } func NewChains(logger logger.Logger, configs []ChainConfig) (map[uint64]deployment.Chain, error) { @@ -52,10 +48,9 @@ func NewChains(logger logger.Logger, configs []ChainConfig) (map[uint64]deployme return nil, fmt.Errorf("failed to connect to chain %s", chainCfg.ChainName) } chains[selector] = deployment.Chain{ - Selector: selector, - Client: ec, - DeployerKey: chainCfg.DeployerKey, - LatestBlockNum: ec.BlockNumber, + Selector: selector, + Client: ec, + DeployerKey: chainCfg.DeployerKey, Confirm: func(tx *types.Transaction) (uint64, error) { var blockNumber uint64 if tx == nil { @@ -64,9 +59,13 @@ func NewChains(logger logger.Logger, configs []ChainConfig) (map[uint64]deployme err := retry.Do(context.Background(), retry.WithMaxDuration(3*time.Minute, retry.NewFibonacci(1*time.Second)), func(ctx context.Context) error { - receipt, err := ec.TransactionReceipt(ctx, tx.Hash()) + chainId, err := ec.ChainID(ctx) if err != nil { - return retry.RetryableError(fmt.Errorf("failed to get receipt: %w", err)) + return fmt.Errorf("failed to get chain id: %w", err) + } + receipt, err := bind.WaitMined(ctx, ec, tx) + if err != nil { + return retry.RetryableError(fmt.Errorf("failed to get receipt for chain %d: %w", chainId, err)) } if receipt != nil { blockNumber = receipt.BlockNumber.Uint64() @@ -90,32 +89,3 @@ func NewChains(logger logger.Logger, configs []ChainConfig) (map[uint64]deployme } return chains, nil } - -// TODO : Remove this when seth is integrated. -func FundAddress(ctx context.Context, from *bind.TransactOpts, to common.Address, amount *big.Int, c deployment.Chain) error { - nonce, err := c.Client.PendingNonceAt(ctx, from.From) - if err != nil { - return fmt.Errorf("failed to get nonce: %w", err) - } - gp, err := c.Client.SuggestGasPrice(ctx) - if err != nil { - return fmt.Errorf("failed to suggest gas price: %w", err) - } - rawTx := types.NewTx(&types.LegacyTx{ - Nonce: nonce, - GasPrice: gp, - Gas: 21000, - To: &to, - Value: amount, - }) - signedTx, err := from.Signer(from.From, rawTx) - if err != nil { - return fmt.Errorf("failed to sign tx: %w", err) - } - err = c.Client.SendTransaction(ctx, signedTx) - if err != nil { - return fmt.Errorf("failed to send tx: %w", err) - } - _, err = c.Confirm(signedTx) - return err -} diff --git a/integration-tests/deployment/devenv/don.go b/integration-tests/deployment/devenv/don.go index 663f4c3329a..ab64eab5c5e 100644 --- a/integration-tests/deployment/devenv/don.go +++ b/integration-tests/deployment/devenv/don.go @@ -3,17 +3,14 @@ package devenv import ( "context" "fmt" - "math/big" "strconv" "strings" "github.com/AlekSi/pointer" - "github.com/ethereum/go-ethereum/common" "github.com/hashicorp/go-multierror" - chainselectors "github.com/smartcontractkit/chain-selectors" + "github.com/rs/zerolog" clclient "github.com/smartcontractkit/chainlink/integration-tests/client" - "github.com/smartcontractkit/chainlink/integration-tests/deployment" nodev1 "github.com/smartcontractkit/chainlink/integration-tests/deployment/jd/node/v1" "github.com/smartcontractkit/chainlink/integration-tests/deployment/jd/shared/ptypes" "github.com/smartcontractkit/chainlink/integration-tests/web/sdk/client" @@ -38,6 +35,18 @@ type DON struct { Nodes []Node } +func (don *DON) PluginNodes() []Node { + var pluginNodes []Node + for _, node := range don.Nodes { + for _, label := range node.labels { + if label.Key == NodeLabelKeyType && pointer.GetString(label.Value) == NodeLabelValuePlugin { + pluginNodes = append(pluginNodes, node) + } + } + } + return pluginNodes +} + func (don *DON) NodeIds() []string { var nodeIds []string for _, node := range don.Nodes { @@ -46,27 +55,6 @@ func (don *DON) NodeIds() []string { return nodeIds } -func (don *DON) FundNodes(ctx context.Context, amount *big.Int, chains map[uint64]deployment.Chain) error { - var err error - for sel, chain := range chains { - for _, node := range don.Nodes { - // if node is bootstrap, no need to fund it - if node.multiAddr != "" { - continue - } - accountAddr, ok := node.AccountAddr[sel] - if !ok { - err = multierror.Append(err, fmt.Errorf("node %s has no account address for chain %d", node.Name, sel)) - continue - } - if err1 := FundAddress(ctx, chain.DeployerKey, common.HexToAddress(accountAddr), amount, chain); err1 != nil { - err = multierror.Append(err, err1) - } - } - } - return err -} - func (don *DON) CreateSupportedChains(ctx context.Context, chains []ChainConfig) error { var err error for i, node := range don.Nodes { @@ -129,24 +117,30 @@ func NewNode(nodeInfo NodeInfo) (*Node, error) { Password: nodeInfo.CLConfig.Password, }) if err != nil { - return nil, fmt.Errorf("failed to create FMS client: %w", err) + return nil, fmt.Errorf("failed to create node graphql client: %w", err) + } + chainlinkClient, err := clclient.NewChainlinkClient(&nodeInfo.CLConfig, zerolog.Logger{}) + if err != nil { + return nil, fmt.Errorf("failed to create node rest client: %w", err) } return &Node{ - gqlClient: gqlClient, - Name: nodeInfo.Name, - adminAddr: nodeInfo.AdminAddr, + gqlClient: gqlClient, + restClient: chainlinkClient, + Name: nodeInfo.Name, + adminAddr: nodeInfo.AdminAddr, }, nil } type Node struct { - NodeId string // node id returned by job distributor after node is registered with it - JDId string // job distributor id returned by node after Job distributor is created in node - Name string // name of the node - AccountAddr map[uint64]string // chain selector to node's account address mapping for supported chains - gqlClient client.Client // graphql client to interact with the node - labels []*ptypes.Label // labels with which the node is registered with the job distributor - adminAddr string // admin address to send payments to, applicable only for non-bootstrap nodes - multiAddr string // multi address denoting node's FQN (needed for deriving P2PBootstrappers in OCR), applicable only for bootstrap nodes + NodeId string // node id returned by job distributor after node is registered with it + JDId string // job distributor id returned by node after Job distributor is created in node + Name string // name of the node + AccountAddr map[uint64]string // chain selector to node's account address mapping for supported chains + gqlClient client.Client // graphql client to interact with the node + restClient *clclient.ChainlinkClient // rest client to interact with the node + labels []*ptypes.Label // labels with which the node is registered with the job distributor + adminAddr string // admin address to send payments to, applicable only for non-bootstrap nodes + multiAddr string // multi address denoting node's FQN (needed for deriving P2PBootstrappers in OCR), applicable only for bootstrap nodes } // CreateCCIPOCRSupportedChains creates a JobDistributorChainConfig for the node. @@ -156,10 +150,6 @@ type Node struct { func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []ChainConfig) error { for _, chain := range chains { chainId := strconv.FormatUint(chain.ChainID, 10) - selector, err := chainselectors.SelectorFromChainId(chain.ChainID) - if err != nil { - return fmt.Errorf("failed to get selector from chain id %d: %w", chain.ChainID, err) - } accountAddr, err := n.gqlClient.FetchAccountAddress(ctx, chainId) if err != nil { return fmt.Errorf("failed to fetch account address for node %s: %w", n.Name, err) @@ -170,7 +160,7 @@ func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []ChainC if n.AccountAddr == nil { n.AccountAddr = make(map[uint64]string) } - n.AccountAddr[selector] = *accountAddr + n.AccountAddr[chain.ChainID] = *accountAddr peerID, err := n.gqlClient.FetchP2PPeerID(ctx) if err != nil { return fmt.Errorf("failed to fetch peer id for node %s: %w", n.Name, err) @@ -286,3 +276,7 @@ func (n *Node) SetUpAndLinkJobDistributor(ctx context.Context, jd JobDistributor n.JDId = id return nil } + +func (n *Node) ExportEVMKeysForChain(chainId string) ([]*clclient.ExportedEVMKey, error) { + return n.restClient.ExportEVMKeysForChain(chainId) +} diff --git a/integration-tests/deployment/devenv/environment.go b/integration-tests/deployment/devenv/environment.go index a62f7f5e84f..f7513798e37 100644 --- a/integration-tests/deployment/devenv/environment.go +++ b/integration-tests/deployment/devenv/environment.go @@ -14,9 +14,10 @@ const ( ) type EnvironmentConfig struct { - Chains []ChainConfig - nodeInfo []NodeInfo - JDConfig JDConfig + Chains []ChainConfig + HomeChainSelector uint64 + nodeInfo []NodeInfo + JDConfig JDConfig } func NewEnvironment(ctx context.Context, lggr logger.Logger, config EnvironmentConfig) (*deployment.Environment, *DON, error) { diff --git a/integration-tests/deployment/environment.go b/integration-tests/deployment/environment.go index 8d8fc909a93..0f3c85a3627 100644 --- a/integration-tests/deployment/environment.go +++ b/integration-tests/deployment/environment.go @@ -45,9 +45,8 @@ type Chain struct { Selector uint64 Client OnchainClient // Note the Sign function can be abstract supporting a variety of key storage mechanisms (e.g. KMS etc). - DeployerKey *bind.TransactOpts - LatestBlockNum func(ctx context.Context) (uint64, error) - Confirm func(tx *types.Transaction) (uint64, error) + DeployerKey *bind.TransactOpts + Confirm func(tx *types.Transaction) (uint64, error) } type Environment struct { diff --git a/integration-tests/deployment/memory/environment.go b/integration-tests/deployment/memory/environment.go index 5ae94464940..409e8d3a816 100644 --- a/integration-tests/deployment/memory/environment.go +++ b/integration-tests/deployment/memory/environment.go @@ -41,9 +41,6 @@ func NewMemoryChains(t *testing.T, numChains int) map[uint64]deployment.Chain { Selector: sel, Client: chain.Backend, DeployerKey: chain.DeployerKey, - LatestBlockNum: func(ctx context.Context) (uint64, error) { - return chain.Backend.Blockchain().CurrentBlock().Number.Uint64(), nil - }, Confirm: func(tx *types.Transaction) (uint64, error) { if tx == nil { return 0, fmt.Errorf("tx was nil, nothing to confirm") diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index a26c72a853b..bd3a458165c 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -24,6 +24,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/lib/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/lib/testsummary" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/osutil" + "github.com/smartcontractkit/chainlink/integration-tests/types/config/node" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" ) @@ -409,29 +410,29 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider b.te.EVMNetworks = append(b.te.EVMNetworks, &networkConfig) } + if b.clNodesCount > 0 { + dereferrencedEvms := make([]blockchain.EVMNetwork, 0) + for _, en := range b.te.EVMNetworks { + dereferrencedEvms = append(dereferrencedEvms, *en) + } - dereferrencedEvms := make([]blockchain.EVMNetwork, 0) - for _, en := range b.te.EVMNetworks { - dereferrencedEvms = append(dereferrencedEvms, *en) - } - - nodeConfigInToml := b.testConfig.GetNodeConfig() + nodeConfigInToml := b.testConfig.GetNodeConfig() - nodeConfig, _, err := node.BuildChainlinkNodeConfig( - dereferrencedEvms, - nodeConfigInToml.BaseConfigTOML, - nodeConfigInToml.CommonChainConfigTOML, - nodeConfigInToml.ChainConfigTOMLByChainID, - ) - if err != nil { - return nil, err - } + nodeConfig, _, err := node.BuildChainlinkNodeConfig( + dereferrencedEvms, + nodeConfigInToml.BaseConfigTOML, + nodeConfigInToml.CommonChainConfigTOML, + nodeConfigInToml.ChainConfigTOMLByChainID, + ) + if err != nil { + return nil, err + } - err = b.te.StartClCluster(nodeConfig, b.clNodesCount, b.secretsConfig, b.testConfig, b.clNodesOpts...) - if err != nil { - return nil, err + err = b.te.StartClCluster(nodeConfig, b.clNodesCount, b.secretsConfig, b.testConfig, b.clNodesOpts...) + if err != nil { + return nil, err + } } - b.te.isSimulatedNetwork = true return b.te, nil diff --git a/integration-tests/smoke/ccip_test.go b/integration-tests/smoke/ccip_test.go index d6c65f3c0e6..759e8eac1ec 100644 --- a/integration-tests/smoke/ccip_test.go +++ b/integration-tests/smoke/ccip_test.go @@ -73,8 +73,9 @@ func Test0002_InitialDeployOnLocal(t *testing.T) { if src == dest { continue } - block, err := destChain.LatestBlockNum(testcontext.Get(t)) + latesthdr, err := destChain.Client.HeaderByNumber(testcontext.Get(t), nil) require.NoError(t, err) + block := latesthdr.Number.Uint64() startBlocks[dest] = &block seqNum := ccipdeployment.SendRequest(t, e, state, src, dest, false) expectedSeqNum[dest] = seqNum diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index cc8d82f2468..3a27d0cd54d 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -1,3 +1,7 @@ +[Common] +# chainlink node funding in native token +chainlink_node_funding = 1 + [Network] selected_networks = ['SIMULATED_1', 'SIMULATED_2'] @@ -86,6 +90,8 @@ DeltaReconcile = '5s' """ [CCIP] +HomeChainSelector = '12922642891491394802' # for chain-2337 + [CCIP.CLNode] NoOfPluginNodes = 4 NoOfBootstraps = 1 diff --git a/integration-tests/testconfig/ccip/config.go b/integration-tests/testconfig/ccip/config.go index a5b168bec2f..b9d969fed1f 100644 --- a/integration-tests/testconfig/ccip/config.go +++ b/integration-tests/testconfig/ccip/config.go @@ -1,6 +1,8 @@ package ccip import ( + "strconv" + "github.com/AlekSi/pointer" ctfconfig "github.com/smartcontractkit/chainlink-testing-framework/lib/config" @@ -21,6 +23,7 @@ type Config struct { PrivateEthereumNetworks map[string]*ctfconfig.EthereumNetworkConfig `toml:",omitempty"` CLNode *NodeConfig `toml:",omitempty"` JobDistributorConfig JDConfig `toml:",omitempty"` + HomeChainSelector *string `toml:",omitempty"` } type NodeConfig struct { @@ -90,3 +93,7 @@ func (o *Config) GetJDDBVersion() string { } return dbversion } + +func (o *Config) GetHomeChainSelector() (uint64, error) { + return strconv.ParseUint(pointer.GetString(o.HomeChainSelector), 10, 64) +} diff --git a/integration-tests/testconfig/ccip/overrides/sepolia_avax_binance.toml b/integration-tests/testconfig/ccip/overrides/sepolia_avax_binance.toml new file mode 100644 index 00000000000..06af64d5d91 --- /dev/null +++ b/integration-tests/testconfig/ccip/overrides/sepolia_avax_binance.toml @@ -0,0 +1,55 @@ +[Common] +# chainlink node funding in native token +chainlink_node_funding = 2 + +[Logging] +test_log_collect = true + +[Logging.LogStream] +# supported targets: file, loki, in-memory. if empty no logs will be persisted +log_targets = ["loki"] + +[Network] +selected_networks = ['SEPOLIA', 'AVALANCHE_FUJI', 'BSC_TESTNET'] + +[Network.EVMNetworks.SEPOLIA] +evm_name = 'Sepolia Testnet' +evm_chain_id = 11155111 +evm_simulated = false +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '5m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_tag = true + +[Network.EVMNetworks.AVALANCHE_FUJI] +evm_name = 'Avalanche Fuji' +evm_chain_id = 43113 +evm_simulated = false +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '2m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_tag = true + +[Network.EVMNetworks.BSC_TESTNET] +evm_name = 'BSC Testnet' +evm_chain_id = 97 +evm_simulated = false +client_implementation = 'BSC' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '2m' +evm_minimum_confirmations = 3 +evm_gas_estimation_buffer = 0 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_tag = true + +[CCIP] +HomeChainSelector = '16015286601757825753' # for sepolia \ No newline at end of file From e9a9444dcd35af65a8fd8243c49a1e21cd316a17 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 19 Sep 2024 18:29:29 +0200 Subject: [PATCH 2/7] [TT-1345] option to use predeployed contracts in OCR tests (#13758) * working version * unify the config a bit, make it possible to use also pre-configured contracts * use Seth's contract loader * fix getting number of contracts to deploy from config * fix configuration logic in ocrv2 smoke/soak tests * uncomment ocr2 plugins * rename some methods, add contract-related information to readme * fix lints --- integration-tests/actions/actions.go | 110 +++++++---- integration-tests/actions/contracts.go | 23 +++ integration-tests/actions/ocr_helpers.go | 8 +- integration-tests/chaos/ocr_chaos_test.go | 4 +- .../contracts/ethereum_contracts.go | 59 +++--- integration-tests/crib/ocr_test.go | 24 ++- integration-tests/load/ocr/ocr_test.go | 8 +- integration-tests/load/ocr/vu.go | 14 +- integration-tests/smoke/forwarder_ocr_test.go | 11 +- .../smoke/forwarders_ocr2_test.go | 8 +- integration-tests/smoke/ocr2_test.go | 26 +-- integration-tests/smoke/ocr_test.go | 6 +- integration-tests/testconfig/README.md | 90 +++++++++ .../testconfig/forwarder_ocr/example.toml | 10 +- .../forwarder_ocr/forwarder_ocr.toml | 9 +- .../testconfig/forwarder_ocr2/example.toml | 22 ++- .../forwarder_ocr2/forwarder_ocr2.toml | 7 +- integration-tests/testconfig/ocr/example.toml | 7 +- integration-tests/testconfig/ocr/ocr.go | 186 +++++++++++++++++- integration-tests/testconfig/ocr/ocr.toml | 5 +- .../testconfig/ocr2/example.toml | 7 +- integration-tests/testconfig/ocr2/ocr2.go | 39 +--- integration-tests/testconfig/ocr2/ocr2.toml | 5 +- integration-tests/testconfig/testconfig.go | 6 + integration-tests/testsetups/ocr.go | 49 ++--- 25 files changed, 538 insertions(+), 205 deletions(-) create mode 100644 integration-tests/actions/contracts.go diff --git a/integration-tests/actions/actions.go b/integration-tests/actions/actions.go index 198fa8e0dc8..f864d9e7100 100644 --- a/integration-tests/actions/actions.go +++ b/integration-tests/actions/actions.go @@ -50,6 +50,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" "github.com/smartcontractkit/chainlink/integration-tests/client" + "github.com/smartcontractkit/chainlink/integration-tests/testconfig/ocr" "github.com/smartcontractkit/chainlink/integration-tests/types/config/node" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/link_token_interface" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/operator_factory" @@ -610,29 +611,48 @@ func TrackForwarder( Msg("Forwarder tracked") } -// DeployOCRv2Contracts deploys a number of OCRv2 contracts and configures them with defaults -func DeployOCRv2Contracts( +// SetupOCRv2Contracts deploys a number of OCRv2 contracts and configures them with defaults +func SetupOCRv2Contracts( l zerolog.Logger, seth *seth.Client, - numberOfContracts int, + ocrContractsConfig ocr.OffChainAggregatorsConfig, linkTokenAddress common.Address, transmitters []string, ocrOptions contracts.OffchainOptions, ) ([]contracts.OffchainAggregatorV2, error) { var ocrInstances []contracts.OffchainAggregatorV2 - for contractCount := 0; contractCount < numberOfContracts; contractCount++ { - ocrInstance, err := contracts.DeployOffchainAggregatorV2( - l, - seth, - linkTokenAddress, - ocrOptions, - ) - if err != nil { - return nil, fmt.Errorf("OCRv2 instance deployment have failed: %w", err) + + if ocrContractsConfig == nil { + return nil, fmt.Errorf("you need to pass non-nil OffChainAggregatorsConfig to setup OCR contracts") + } + + if !ocrContractsConfig.UseExistingOffChainAggregatorsContracts() { + for contractCount := 0; contractCount < ocrContractsConfig.NumberOfContractsToDeploy(); contractCount++ { + ocrInstance, err := contracts.DeployOffchainAggregatorV2( + l, + seth, + linkTokenAddress, + ocrOptions, + ) + if err != nil { + return nil, fmt.Errorf("OCRv2 instance deployment have failed: %w", err) + } + ocrInstances = append(ocrInstances, &ocrInstance) + if (contractCount+1)%ContractDeploymentInterval == 0 { // For large amounts of contract deployments, space things out some + time.Sleep(2 * time.Second) + } } - ocrInstances = append(ocrInstances, &ocrInstance) - if (contractCount+1)%ContractDeploymentInterval == 0 { // For large amounts of contract deployments, space things out some - time.Sleep(2 * time.Second) + } else { + for _, address := range ocrContractsConfig.OffChainAggregatorsContractsAddresses() { + ocrInstance, err := contracts.LoadOffchainAggregatorV2(l, seth, address) + if err != nil { + return nil, fmt.Errorf("OCRv2 instance loading have failed: %w", err) + } + ocrInstances = append(ocrInstances, &ocrInstance) + } + + if !ocrContractsConfig.ConfigureExistingOffChainAggregatorsContracts() { + return ocrInstances, nil } } @@ -781,7 +801,7 @@ func StartNewRound( func DeployOCRContractsForwarderFlow( logger zerolog.Logger, seth *seth.Client, - numberOfContracts int, + ocrContractsConfig ocr.OffChainAggregatorsConfig, linkTokenContractAddress common.Address, workerNodes []contracts.ChainlinkNodeWithKeysAndAddress, forwarderAddresses []common.Address, @@ -802,23 +822,23 @@ func DeployOCRContractsForwarderFlow( return forwarderAddresses, nil } - return deployAnyOCRv1Contracts(logger, seth, numberOfContracts, linkTokenContractAddress, workerNodes, transmitterPayeesFn, transmitterAddressesFn) + return setupAnyOCRv1Contracts(logger, seth, ocrContractsConfig, linkTokenContractAddress, workerNodes, transmitterPayeesFn, transmitterAddressesFn) } -// DeployOCRv1Contracts deploys and funds a certain number of offchain aggregator contracts -func DeployOCRv1Contracts( +// SetupOCRv1Contracts deploys and funds a certain number of offchain aggregator contracts or uses existing ones and returns a slice of contract wrappers. +func SetupOCRv1Contracts( logger zerolog.Logger, seth *seth.Client, - numberOfContracts int, + ocrContractsConfig ocr.OffChainAggregatorsConfig, linkTokenContractAddress common.Address, workerNodes []contracts.ChainlinkNodeWithKeysAndAddress, ) ([]contracts.OffchainAggregator, error) { transmitterPayeesFn := func() (transmitters []string, payees []string, err error) { transmitters = make([]string, 0) payees = make([]string, 0) - for _, node := range workerNodes { + for _, n := range workerNodes { var addr string - addr, err = node.PrimaryEthAddress() + addr, err = n.PrimaryEthAddress() if err != nil { err = fmt.Errorf("error getting node's primary ETH address: %w", err) return @@ -832,8 +852,8 @@ func DeployOCRv1Contracts( transmitterAddressesFn := func() ([]common.Address, error) { transmitterAddresses := make([]common.Address, 0) - for _, node := range workerNodes { - primaryAddress, err := node.PrimaryEthAddress() + for _, n := range workerNodes { + primaryAddress, err := n.PrimaryEthAddress() if err != nil { return nil, err } @@ -843,28 +863,48 @@ func DeployOCRv1Contracts( return transmitterAddresses, nil } - return deployAnyOCRv1Contracts(logger, seth, numberOfContracts, linkTokenContractAddress, workerNodes, transmitterPayeesFn, transmitterAddressesFn) + return setupAnyOCRv1Contracts(logger, seth, ocrContractsConfig, linkTokenContractAddress, workerNodes, transmitterPayeesFn, transmitterAddressesFn) } -func deployAnyOCRv1Contracts( +func setupAnyOCRv1Contracts( logger zerolog.Logger, seth *seth.Client, - numberOfContracts int, + ocrContractsConfig ocr.OffChainAggregatorsConfig, linkTokenContractAddress common.Address, workerNodes []contracts.ChainlinkNodeWithKeysAndAddress, getTransmitterAndPayeesFn func() ([]string, []string, error), getTransmitterAddressesFn func() ([]common.Address, error), ) ([]contracts.OffchainAggregator, error) { - // Deploy contracts var ocrInstances []contracts.OffchainAggregator - for contractCount := 0; contractCount < numberOfContracts; contractCount++ { - ocrInstance, err := contracts.DeployOffchainAggregator(logger, seth, linkTokenContractAddress, contracts.DefaultOffChainAggregatorOptions()) - if err != nil { - return nil, fmt.Errorf("OCR instance deployment have failed: %w", err) + + if ocrContractsConfig == nil { + return nil, fmt.Errorf("you need to pass non-nil OffChainAggregatorsConfig to setup OCR contracts") + } + + if !ocrContractsConfig.UseExistingOffChainAggregatorsContracts() { + // Deploy contracts + for contractCount := 0; contractCount < ocrContractsConfig.NumberOfContractsToDeploy(); contractCount++ { + ocrInstance, err := contracts.DeployOffchainAggregator(logger, seth, linkTokenContractAddress, contracts.DefaultOffChainAggregatorOptions()) + if err != nil { + return nil, fmt.Errorf("OCR instance deployment have failed: %w", err) + } + ocrInstances = append(ocrInstances, &ocrInstance) + if (contractCount+1)%ContractDeploymentInterval == 0 { // For large amounts of contract deployments, space things out some + time.Sleep(2 * time.Second) + } } - ocrInstances = append(ocrInstances, &ocrInstance) - if (contractCount+1)%ContractDeploymentInterval == 0 { // For large amounts of contract deployments, space things out some - time.Sleep(2 * time.Second) + } else { + // Load contract wrappers + for _, address := range ocrContractsConfig.OffChainAggregatorsContractsAddresses() { + ocrInstance, err := contracts.LoadOffChainAggregator(logger, seth, address) + if err != nil { + return nil, fmt.Errorf("OCR instance loading have failed: %w", err) + } + ocrInstances = append(ocrInstances, &ocrInstance) + } + + if !ocrContractsConfig.ConfigureExistingOffChainAggregatorsContracts() { + return ocrInstances, nil } } diff --git a/integration-tests/actions/contracts.go b/integration-tests/actions/contracts.go new file mode 100644 index 00000000000..1a50c4d7ba9 --- /dev/null +++ b/integration-tests/actions/contracts.go @@ -0,0 +1,23 @@ +package actions + +import ( + "github.com/rs/zerolog" + + "github.com/smartcontractkit/chainlink-testing-framework/seth" + + "github.com/smartcontractkit/chainlink/integration-tests/contracts" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" +) + +// LinkTokenContract returns a link token contract instance. Depending on test configuration, it either deploys a new one or uses an existing one. +func LinkTokenContract(l zerolog.Logger, sethClient *seth.Client, configWithLinkToken tc.LinkTokenContractConfig) (*contracts.EthereumLinkToken, error) { + if configWithLinkToken != nil && configWithLinkToken.UseExistingLinkTokenContract() { + linkAddress, err := configWithLinkToken.LinkTokenContractAddress() + if err != nil { + return nil, err + } + + return contracts.LoadLinkTokenContract(l, sethClient, linkAddress) + } + return contracts.DeployLinkTokenContract(l, sethClient) +} diff --git a/integration-tests/actions/ocr_helpers.go b/integration-tests/actions/ocr_helpers.go index 19cad817b75..0f6f65d1289 100644 --- a/integration-tests/actions/ocr_helpers.go +++ b/integration-tests/actions/ocr_helpers.go @@ -21,6 +21,8 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + "github.com/smartcontractkit/chainlink/integration-tests/testconfig/ocr" ) // This actions file often returns functions, rather than just values. These are used as common test helpers, and are @@ -229,13 +231,14 @@ func BuildNodeContractPairID(node contracts.ChainlinkNodeWithKeysAndAddress, ocr func SetupOCRv1Cluster( l zerolog.Logger, seth *seth.Client, + configWithLinkToken tc.LinkTokenContractConfig, workerNodes []*client.ChainlinkK8sClient, ) (common.Address, error) { err := FundChainlinkNodesFromRootAddress(l, seth, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(3)) if err != nil { return common.Address{}, err } - linkContract, err := contracts.DeployLinkTokenContract(l, seth) + linkContract, err := LinkTokenContract(l, seth, configWithLinkToken) if err != nil { return common.Address{}, err } @@ -246,11 +249,12 @@ func SetupOCRv1Feed( l zerolog.Logger, seth *seth.Client, lta common.Address, + ocrContractsConfig ocr.OffChainAggregatorsConfig, msClient *ctfClient.MockserverClient, bootstrapNode *client.ChainlinkK8sClient, workerNodes []*client.ChainlinkK8sClient, ) ([]contracts.OffchainAggregator, error) { - ocrInstances, err := DeployOCRv1Contracts(l, seth, 1, lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) + ocrInstances, err := SetupOCRv1Contracts(l, seth, ocrContractsConfig, lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) if err != nil { return nil, err } diff --git a/integration-tests/chaos/ocr_chaos_test.go b/integration-tests/chaos/ocr_chaos_test.go index 7ef03d98c8a..821d3cc48c0 100644 --- a/integration-tests/chaos/ocr_chaos_test.go +++ b/integration-tests/chaos/ocr_chaos_test.go @@ -179,13 +179,13 @@ func TestOCRChaos(t *testing.T) { }) ms := ctfClient.ConnectMockServer(testEnvironment) - linkContract, err := contracts.DeployLinkTokenContract(l, seth) + linkContract, err := actions.LinkTokenContract(l, seth, config.OCR) require.NoError(t, err, "Error deploying link token contract") err = actions.FundChainlinkNodesFromRootAddress(l, seth, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes), big.NewFloat(10)) require.NoError(t, err) - ocrInstances, err := actions.DeployOCRv1Contracts(l, seth, 1, common.HexToAddress(linkContract.Address()), contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) + ocrInstances, err := actions.SetupOCRv1Contracts(l, seth, config.OCR, common.HexToAddress(linkContract.Address()), contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) require.NoError(t, err) err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, ms, fmt.Sprint(seth.ChainID)) require.NoError(t, err) diff --git a/integration-tests/contracts/ethereum_contracts.go b/integration-tests/contracts/ethereum_contracts.go index c2f86286faf..288a36978a8 100644 --- a/integration-tests/contracts/ethereum_contracts.go +++ b/integration-tests/contracts/ethereum_contracts.go @@ -204,22 +204,17 @@ type EthereumOffchainAggregator struct { l zerolog.Logger } -func LoadOffchainAggregator(l zerolog.Logger, seth *seth.Client, contractAddress common.Address) (EthereumOffchainAggregator, error) { - abi, err := offchainaggregator.OffchainAggregatorMetaData.GetAbi() - if err != nil { - return EthereumOffchainAggregator{}, fmt.Errorf("failed to get OffChain Aggregator ABI: %w", err) - } - seth.ContractStore.AddABI("OffChainAggregator", *abi) - seth.ContractStore.AddBIN("OffChainAggregator", common.FromHex(offchainaggregator.OffchainAggregatorMetaData.Bin)) +func LoadOffChainAggregator(l zerolog.Logger, sethClient *seth.Client, contractAddress common.Address) (EthereumOffchainAggregator, error) { + loader := seth.NewContractLoader[offchainaggregator.OffchainAggregator](sethClient) + instance, err := loader.LoadContract("LinkToken", contractAddress, offchainaggregator.OffchainAggregatorMetaData.GetAbi, offchainaggregator.NewOffchainAggregator) - ocr, err := offchainaggregator.NewOffchainAggregator(contractAddress, wrappers.MustNewWrappedContractBackend(nil, seth)) if err != nil { - return EthereumOffchainAggregator{}, fmt.Errorf("failed to instantiate OCR instance: %w", err) + return EthereumOffchainAggregator{}, fmt.Errorf("failed to instantiate OCR v2 instance: %w", err) } return EthereumOffchainAggregator{ - client: seth, - ocr: ocr, + client: sethClient, + ocr: instance, address: &contractAddress, l: l, }, nil @@ -357,10 +352,6 @@ func (o *EthereumOffchainAggregator) SetConfig( return err } - // fails with error setting OCR config for contract '0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82': both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified - // but we only have gasPrice set... It also fails with the same error when we enable EIP-1559 - // fails when we wait for it to be minted, inside the wrapper there's no error when we call it, so it must be something inside smart contract - // that's reverting it and maybe the error message is completely off _, err = o.client.Decode(o.ocr.SetConfig(o.client.NewTXOpts(), signers, transmitters, threshold, encodedConfigVersion, encodedConfig)) return err } @@ -584,36 +575,36 @@ type EthereumOffchainAggregatorV2 struct { l zerolog.Logger } -func LoadOffChainAggregatorV2(l zerolog.Logger, seth *seth.Client, contractAddress common.Address) (EthereumOffchainAggregatorV2, error) { - oAbi, err := ocr2aggregator.OCR2AggregatorMetaData.GetAbi() +func LoadOffchainAggregatorV2(l zerolog.Logger, seth *seth.Client, address common.Address) (EthereumOffchainAggregatorV2, error) { + contractAbi, err := ocr2aggregator.OCR2AggregatorMetaData.GetAbi() if err != nil { - return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to get OffChain Aggregator ABI: %w", err) + return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to get OffChain Aggregator v2 ABI: %w", err) } - seth.ContractStore.AddABI("OffChainAggregatorV2", *oAbi) + seth.ContractStore.AddABI("OffChainAggregatorV2", *contractAbi) seth.ContractStore.AddBIN("OffChainAggregatorV2", common.FromHex(ocr2aggregator.OCR2AggregatorMetaData.Bin)) - ocr2, err := ocr2aggregator.NewOCR2Aggregator(contractAddress, seth.Client) + ocr2, err := ocr2aggregator.NewOCR2Aggregator(address, seth.Client) if err != nil { - return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to instantiate OCR instance: %w", err) + return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to instantiate OCRv2 instance: %w", err) } return EthereumOffchainAggregatorV2{ client: seth, contract: ocr2, - address: &contractAddress, + address: &address, l: l, }, nil } func DeployOffchainAggregatorV2(l zerolog.Logger, seth *seth.Client, linkTokenAddress common.Address, offchainOptions OffchainOptions) (EthereumOffchainAggregatorV2, error) { - oAbi, err := ocr2aggregator.OCR2AggregatorMetaData.GetAbi() + contractAbi, err := ocr2aggregator.OCR2AggregatorMetaData.GetAbi() if err != nil { - return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to get OffChain Aggregator ABI: %w", err) + return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to get OffChain Aggregator v2 ABI: %w", err) } - seth.ContractStore.AddABI("OffChainAggregatorV2", *oAbi) + seth.ContractStore.AddABI("OffChainAggregatorV2", *contractAbi) seth.ContractStore.AddBIN("OffChainAggregatorV2", common.FromHex(ocr2aggregator.OCR2AggregatorMetaData.Bin)) - ocrDeploymentData2, err := seth.DeployContract(seth.NewTXOpts(), "OffChainAggregatorV2", *oAbi, common.FromHex(ocr2aggregator.OCR2AggregatorMetaData.Bin), + ocrDeploymentData2, err := seth.DeployContract(seth.NewTXOpts(), "OffChainAggregatorV2", *contractAbi, common.FromHex(ocr2aggregator.OCR2AggregatorMetaData.Bin), linkTokenAddress, offchainOptions.MinimumAnswer, offchainOptions.MaximumAnswer, @@ -624,12 +615,12 @@ func DeployOffchainAggregatorV2(l zerolog.Logger, seth *seth.Client, linkTokenAd ) if err != nil { - return EthereumOffchainAggregatorV2{}, fmt.Errorf("OCR instance deployment have failed: %w", err) + return EthereumOffchainAggregatorV2{}, fmt.Errorf("OCRv2 instance deployment have failed: %w", err) } ocr2, err := ocr2aggregator.NewOCR2Aggregator(ocrDeploymentData2.Address, wrappers.MustNewWrappedContractBackend(nil, seth)) if err != nil { - return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to instantiate OCR instance: %w", err) + return EthereumOffchainAggregatorV2{}, fmt.Errorf("failed to instantiate OCRv2 instance: %w", err) } return EthereumOffchainAggregatorV2{ @@ -772,22 +763,16 @@ func DeployLinkTokenContract(l zerolog.Logger, client *seth.Client) (*EthereumLi } func LoadLinkTokenContract(l zerolog.Logger, client *seth.Client, address common.Address) (*EthereumLinkToken, error) { - linkABI, err := link_token_interface.LinkTokenMetaData.GetAbi() - if err != nil { - return &EthereumLinkToken{}, fmt.Errorf("failed to get LinkToken ABI: %w", err) - } + loader := seth.NewContractLoader[link_token_interface.LinkToken](client) + instance, err := loader.LoadContract("LinkToken", address, link_token_interface.LinkTokenMetaData.GetAbi, link_token_interface.NewLinkToken) - client.ContractStore.AddABI("LinkToken", *linkABI) - client.ContractStore.AddBIN("LinkToken", common.FromHex(link_token_interface.LinkTokenMetaData.Bin)) - - linkToken, err := link_token_interface.NewLinkToken(address, wrappers.MustNewWrappedContractBackend(nil, client)) if err != nil { return &EthereumLinkToken{}, fmt.Errorf("failed to instantiate LinkToken instance: %w", err) } return &EthereumLinkToken{ client: client, - instance: linkToken, + instance: instance, address: address, l: l, }, nil diff --git a/integration-tests/crib/ocr_test.go b/integration-tests/crib/ocr_test.go index 69804e24d77..215734c318e 100644 --- a/integration-tests/crib/ocr_test.go +++ b/integration-tests/crib/ocr_test.go @@ -8,25 +8,30 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/havoc" "github.com/smartcontractkit/chainlink-testing-framework/lib/client" + "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" + "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/contracts" - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + ocr_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/ocr" ) // TestCRIBChaos an example of how we can run chaos tests with havoc and core CRIB func TestCRIBChaos(t *testing.T) { l := logging.GetTestLogger(t) + config, err := tc.GetConfig([]string{"Crib"}, tc.OCR) + require.NoError(t, err) sethClient, msClient, bootstrapNode, workerNodes, _, err := ConnectRemote() require.NoError(t, err) - lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) + lta, err := actions.SetupOCRv1Cluster(l, sethClient, config.OCR, workerNodes) require.NoError(t, err) - ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) + ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, config.OCR, msClient, bootstrapNode, workerNodes) require.NoError(t, err) err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient) @@ -41,11 +46,12 @@ func TestCRIBChaos(t *testing.T) { 1*time.Second, os.Getenv("CRIB_NAMESPACE"), ) + require.NoError(t, err, "Error rebooting CL namespace") ch.Create(context.Background()) ch.AddListener(havoc.NewChaosLogger(l)) t.Cleanup(func() { err := ch.Delete(context.Background()) - require.NoError(t, err) + require.NoError(t, err, "Error deleting chaos") }) require.Eventually(t, func() bool { err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V1OffChainAgrregatorToOffChainAggregatorWithRounds(ocrInstances), 5*time.Minute) @@ -65,9 +71,15 @@ func TestCRIBRPCChaos(t *testing.T) { sethClient, msClient, bootstrapNode, workerNodes, vars, err := ConnectRemote() require.NoError(t, err) - lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) + ocrConfig := &ocr_config.Config{ + Contracts: &ocr_config.Contracts{ + ShouldBeUsed: ptr.Ptr(false), + }, + } + + lta, err := actions.SetupOCRv1Cluster(l, sethClient, ocrConfig, workerNodes) require.NoError(t, err) - ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) + ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, ocrConfig, msClient, bootstrapNode, workerNodes) require.NoError(t, err) err = actions.SetAllAdapterResponsesToTheSameValue(10, ocrInstances, workerNodes, msClient) diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index 281d2da2bb1..55b683a0a74 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -31,9 +31,9 @@ func TestOCRLoad(t *testing.T) { sethClient, msClient, bootstrapNode, workerNodes, _, err := crib.ConnectRemote() require.NoError(t, err) - lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) + lta, err := actions.SetupOCRv1Cluster(l, sethClient, config.OCR, workerNodes) require.NoError(t, err) - ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) + ocrInstances, err := actions.SetupOCRv1Feed(l, sethClient, lta, config.OCR, msClient, bootstrapNode, workerNodes) require.NoError(t, err) cfg := config.OCR @@ -64,7 +64,7 @@ func TestOCRVolume(t *testing.T) { sethClient, msClient, bootstrapNode, workerNodes, _, err := crib.ConnectRemote() require.NoError(t, err) - lta, err := actions.SetupOCRv1Cluster(l, sethClient, workerNodes) + lta, err := actions.SetupOCRv1Cluster(l, sethClient, config.OCR, workerNodes) require.NoError(t, err) cfg := config.OCR @@ -77,7 +77,7 @@ func TestOCRVolume(t *testing.T) { LoadType: wasp.VU, CallTimeout: cfg.Volume.VerificationTimeout.Duration, Schedule: wasp.Plain(*cfg.Volume.Rate, cfg.Volume.TestDuration.Duration), - VU: NewVU(l, sethClient, *cfg.Volume.VURequestsPerUnit, cfg.Volume.RateLimitUnitDuration.Duration, lta, bootstrapNode, workerNodes, msClient), + VU: NewVU(l, sethClient, cfg, *cfg.Volume.VURequestsPerUnit, cfg.Volume.RateLimitUnitDuration.Duration, lta, bootstrapNode, workerNodes, msClient), Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), })) diff --git a/integration-tests/load/ocr/vu.go b/integration-tests/load/ocr/vu.go index aece9cb74b1..c337e338a82 100644 --- a/integration-tests/load/ocr/vu.go +++ b/integration-tests/load/ocr/vu.go @@ -8,18 +8,16 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/rs/zerolog" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - "go.uber.org/ratelimit" - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - client2 "github.com/smartcontractkit/chainlink-testing-framework/lib/client" + "github.com/smartcontractkit/chainlink-testing-framework/seth" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" + "github.com/smartcontractkit/chainlink/integration-tests/testconfig/ocr" ) // VU is a virtual user for the OCR load test @@ -37,11 +35,13 @@ type VU struct { msClient *client2.MockserverClient l zerolog.Logger ocrInstances []contracts.OffchainAggregator + config ocr.OffChainAggregatorsConfig } func NewVU( l zerolog.Logger, seth *seth.Client, + config ocr.OffChainAggregatorsConfig, rate int, rateUnit time.Duration, lta common.Address, @@ -60,6 +60,7 @@ func NewVU( msClient: msClient, bootstrapNode: bootstrapNode, workerNodes: workerNodes, + config: config, } } @@ -75,11 +76,12 @@ func (m *VU) Clone(_ *wasp.Generator) wasp.VirtualUser { msClient: m.msClient, bootstrapNode: m.bootstrapNode, workerNodes: m.workerNodes, + config: m.config, } } func (m *VU) Setup(_ *wasp.Generator) error { - ocrInstances, err := actions.DeployOCRv1Contracts(m.l, m.seth, 1, m.lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(m.workerNodes)) + ocrInstances, err := actions.SetupOCRv1Contracts(m.l, m.seth, m.config, m.lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(m.workerNodes)) if err != nil { return err } diff --git a/integration-tests/smoke/forwarder_ocr_test.go b/integration-tests/smoke/forwarder_ocr_test.go index 4441a592886..5e5fb1d40b6 100644 --- a/integration-tests/smoke/forwarder_ocr_test.go +++ b/integration-tests/smoke/forwarder_ocr_test.go @@ -60,8 +60,8 @@ func TestForwarderOCRBasic(t *testing.T) { _ = actions.ReturnFundsFromNodes(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(env.ClCluster.NodeAPIs())) }) - lt, err := contracts.DeployLinkTokenContract(l, sethClient) - require.NoError(t, err, "Deploying Link Token Contract shouldn't fail") + linkContract, err := actions.LinkTokenContract(l, sethClient, config.OCR) + require.NoError(t, err, "Error loading/deploying link token contract") fundingAmount := big.NewFloat(.05) l.Info().Str("ETH amount per node", fundingAmount.String()).Msg("Funding Chainlink nodes") @@ -69,7 +69,7 @@ func TestForwarderOCRBasic(t *testing.T) { require.NoError(t, err, "Error funding Chainlink nodes") operators, authorizedForwarders, _ := actions.DeployForwarderContracts( - t, sethClient, common.HexToAddress(lt.Address()), len(workerNodes), + t, sethClient, common.HexToAddress(linkContract.Address()), len(workerNodes), ) require.Equal(t, len(workerNodes), len(operators), "Number of operators should match number of worker nodes") @@ -81,11 +81,12 @@ func TestForwarderOCRBasic(t *testing.T) { require.NoError(t, err, "Accepting Authorize Receivers on Operator shouldn't fail") actions.TrackForwarder(t, sethClient, authorizedForwarders[i], workerNodes[i]) } + ocrInstances, err := actions.DeployOCRContractsForwarderFlow( l, sethClient, - 1, - common.HexToAddress(lt.Address()), + config.OCR, + common.HexToAddress(linkContract.Address()), contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(workerNodes), authorizedForwarders, ) diff --git a/integration-tests/smoke/forwarders_ocr2_test.go b/integration-tests/smoke/forwarders_ocr2_test.go index 219df9ce432..0cc7d9fafe4 100644 --- a/integration-tests/smoke/forwarders_ocr2_test.go +++ b/integration-tests/smoke/forwarders_ocr2_test.go @@ -61,8 +61,8 @@ func TestForwarderOCR2Basic(t *testing.T) { _ = actions.ReturnFundsFromNodes(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(env.ClCluster.NodeAPIs())) }) - lt, err := contracts.DeployLinkTokenContract(l, sethClient) - require.NoError(t, err, "Deploying Link Token Contract shouldn't fail") + linkContract, err := actions.LinkTokenContract(l, sethClient, config.OCR2) + require.NoError(t, err, "Error loading/deploying link token contract") fundingAmount := big.NewFloat(.05) l.Info().Str("ETH amount per node", fundingAmount.String()).Msg("Funding Chainlink nodes") @@ -70,7 +70,7 @@ func TestForwarderOCR2Basic(t *testing.T) { require.NoError(t, err, "Error funding Chainlink nodes") operators, authorizedForwarders, _ := actions.DeployForwarderContracts( - t, sethClient, common.HexToAddress(lt.Address()), len(workerNodes), + t, sethClient, common.HexToAddress(linkContract.Address()), len(workerNodes), ) require.Equal(t, len(workerNodes), len(operators), "Number of operators should match number of worker nodes") @@ -90,7 +90,7 @@ func TestForwarderOCR2Basic(t *testing.T) { } ocrOffchainOptions := contracts.DefaultOffChainAggregatorOptions() - ocrInstances, err := actions.DeployOCRv2Contracts(l, sethClient, 1, common.HexToAddress(lt.Address()), transmitters, ocrOffchainOptions) + ocrInstances, err := actions.SetupOCRv2Contracts(l, sethClient, config.OCR2, common.HexToAddress(linkContract.Address()), transmitters, ocrOffchainOptions) require.NoError(t, err, "Error deploying OCRv2 contracts with forwarders") ocrv2Config, err := actions.BuildMedianOCR2ConfigLocal(workerNodes, ocrOffchainOptions) diff --git a/integration-tests/smoke/ocr2_test.go b/integration-tests/smoke/ocr2_test.go index 3203037ae55..325c88f979a 100644 --- a/integration-tests/smoke/ocr2_test.go +++ b/integration-tests/smoke/ocr2_test.go @@ -103,11 +103,11 @@ func TestOCRv2JobReplacement(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - env, aggregatorContracts, sethClient := prepareORCv2SmokeTestEnv(t, defaultTestData(), l, 5) - nodeClients := env.ClCluster.NodeAPIs() + testEnv, aggregatorContracts, sethClient := prepareORCv2SmokeTestEnv(t, defaultTestData(), l, 5) + nodeClients := testEnv.ClCluster.NodeAPIs() bootstrapNode, workerNodes := nodeClients[0], nodeClients[1:] - err := env.MockAdapter.SetAdapterBasedIntValuePath("ocr2", []string{http.MethodGet, http.MethodPost}, 10) + err := testEnv.MockAdapter.SetAdapterBasedIntValuePath("ocr2", []string{http.MethodGet, http.MethodPost}, 10) require.NoError(t, err) err = actions.WatchNewOCRRound(l, sethClient, 2, contracts.V2OffChainAgrregatorToOffChainAggregatorWithRounds(aggregatorContracts), time.Minute*5) require.NoError(t, err, "Error watching for new OCR2 round") @@ -125,7 +125,7 @@ func TestOCRv2JobReplacement(t *testing.T) { err = actions.DeleteBridges(nodeClients) require.NoError(t, err) - err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, env.MockAdapter, "ocr2", 15, uint64(sethClient.ChainID), false, false) + err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, testEnv.MockAdapter, "ocr2", 15, uint64(sethClient.ChainID), false, false) require.NoError(t, err, "Error creating OCRv2 jobs") err = actions.WatchNewOCRRound(l, sethClient, 3, contracts.V2OffChainAgrregatorToOffChainAggregatorWithRounds(aggregatorContracts), time.Minute*3) @@ -170,8 +170,8 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, nodeClients := testEnv.ClCluster.NodeAPIs() bootstrapNode, workerNodes := nodeClients[0], nodeClients[1:] - linkContract, err := contracts.DeployLinkTokenContract(l, sethClient) - require.NoError(t, err, "Error deploying link token contract") + linkContract, err := actions.LinkTokenContract(l, sethClient, config.OCR2) + require.NoError(t, err, "Error loading/deploying link token contract") err = actions.FundChainlinkNodesFromRootAddress(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(*config.Common.ChainlinkNodeFunding)) require.NoError(t, err, "Error funding Chainlink nodes") @@ -191,18 +191,20 @@ func prepareORCv2SmokeTestEnv(t *testing.T, testData ocr2test, l zerolog.Logger, transmitters = append(transmitters, addr) } - ocrOffchainOptions := contracts.DefaultOffChainAggregatorOptions() - aggregatorContracts, err := actions.DeployOCRv2Contracts(l, sethClient, 1, common.HexToAddress(linkContract.Address()), transmitters, ocrOffchainOptions) + ocrOffChainOptions := contracts.DefaultOffChainAggregatorOptions() + aggregatorContracts, err := actions.SetupOCRv2Contracts(l, sethClient, config.OCR2, common.HexToAddress(linkContract.Address()), transmitters, ocrOffChainOptions) require.NoError(t, err, "Error deploying OCRv2 aggregator contracts") err = actions.CreateOCRv2JobsLocal(aggregatorContracts, bootstrapNode, workerNodes, testEnv.MockAdapter, "ocr2", 5, uint64(sethClient.ChainID), false, testData.chainReaderAndCodec) require.NoError(t, err, "Error creating OCRv2 jobs") - ocrv2Config, err := actions.BuildMedianOCR2ConfigLocal(workerNodes, ocrOffchainOptions) - require.NoError(t, err, "Error building OCRv2 config") + if !config.OCR2.UseExistingOffChainAggregatorsContracts() || (config.OCR2.UseExistingOffChainAggregatorsContracts() && config.OCR2.ConfigureExistingOffChainAggregatorsContracts()) { + ocrV2Config, err := actions.BuildMedianOCR2ConfigLocal(workerNodes, ocrOffChainOptions) + require.NoError(t, err, "Error building OCRv2 config") - err = actions.ConfigureOCRv2AggregatorContracts(ocrv2Config, aggregatorContracts) - require.NoError(t, err, "Error configuring OCRv2 aggregator contracts") + err = actions.ConfigureOCRv2AggregatorContracts(ocrV2Config, aggregatorContracts) + require.NoError(t, err, "Error configuring OCRv2 aggregator contracts") + } assertCorrectNodeConfiguration(t, l, clNodeCount, testData, testEnv) diff --git a/integration-tests/smoke/ocr_test.go b/integration-tests/smoke/ocr_test.go index cd9db0e9740..a19adb5c022 100644 --- a/integration-tests/smoke/ocr_test.go +++ b/integration-tests/smoke/ocr_test.go @@ -113,10 +113,10 @@ func prepareORCv1SmokeTestEnv(t *testing.T, l zerolog.Logger, firstRoundResult i _ = actions.ReturnFundsFromNodes(l, sethClient, contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(env.ClCluster.NodeAPIs())) }) - linkContract, err := contracts.DeployLinkTokenContract(l, sethClient) - require.NoError(t, err, "Error deploying link token contract") + linkContract, err := actions.LinkTokenContract(l, sethClient, config.OCR) + require.NoError(t, err, "Error loading/deploying link token contract") - ocrInstances, err := actions.DeployOCRv1Contracts(l, sethClient, 1, common.HexToAddress(linkContract.Address()), contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(workerNodes)) + ocrInstances, err := actions.SetupOCRv1Contracts(l, sethClient, config.OCR, common.HexToAddress(linkContract.Address()), contracts.ChainlinkClientToChainlinkNodeWithKeysAndAddress(workerNodes)) require.NoError(t, err, "Error deploying OCR contracts") err = actions.CreateOCRJobsLocal(ocrInstances, bootstrapNode, workerNodes, 5, env.MockAdapter, big.NewInt(sethClient.ChainID)) diff --git a/integration-tests/testconfig/README.md b/integration-tests/testconfig/README.md index c698281b76e..f6186580550 100644 --- a/integration-tests/testconfig/README.md +++ b/integration-tests/testconfig/README.md @@ -194,6 +194,96 @@ BumpMin = '100 gwei' For more examples see `example.toml` in product TOML configs like `testconfig/automation/example.toml`. If either ChainConfigTOMLByChainID or CommonChainConfigTOML is defined, it will override any defaults that Chainlink Node might have for the given network. Part of the configuration that defines blockchain node URLs is always dynamically generated based on the EVMNetwork configuration. Currently, all networks are treated as EVM networks. There's no way to provide Solana, Starknet, Cosmos or Aptos configuration yet. +### OCR tests contract config +In order to allow running OCR soak/load/smoke tests with already deployed contracts, we have provided an experimental feature for providing addresses of LINK token and OCR contracts in the TOML config. Additionally, user can choose, whether existing OCR contracts should be configured or not. +If no contract addresses are provided, the tests will deploy new contracts. + +The feature is highly configurable and it possible to use existing LINK token contract, but deploy new OCR contracts or vice versa. Both OCRv1 and OCRv2 contracts are supported. + +To use existing LINK and OCRv1 contracts, provide the following configuration in the TOML file: +```toml +[OCR.Contracts] +link_token = "0x88d1239894D9582f5849E5b5a964da9e5730f1E6" +offchain_aggregators = ["0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] +``` + +For OCRv2, provide the following configuration: +```toml +[OCR2.Contracts] +link_token = "0x88d1239894D9582f5849E5b5a964da9e5730f1E6" +offchain_aggregators = ["0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] +``` + +If you want to disable them, you can set `use = false` or remove the addresses from the configuration. + +If you want to use existing OCRv1 contract, without configuring it, you can set `configure = false` in the configuration: +```toml +[OCR.Contracts] +link_token = "0x88d1239894D9582f5849E5b5a964da9e5730f1E6" +offchain_aggregators = ["0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] + +# notice that this address needs to match the one in offchain_aggregators +[OCR.Contracts.Settings."0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] +configure = false +``` + +Be aware that using multiple existing OCR contracts, but configuring only some of them is not supported. This is not a valid configuration: +```toml +[OCR.Contracts] +link_token = "0x88d1239894D9582f5849E5b5a964da9e5730f1E6" +offchain_aggregators = ["0xc1ce3815d6e7f3705265c2577F1342344752A5Eb", "0x2f4FA21fCd917C448C160caafEC874032F404c08"] + +# notice that this address needs to match the one in offchain_aggregators +[OCR.Contracts.Settings."0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] +configure = false + +# if setting for a given address is not present, we assume it should be used and configured +# so in this case "0x2f4FA21fCd917C448C160caafEC874032F404c08" will be evaluated as configure = true, +# but "0xc1ce3815d6e7f3705265c2577F1342344752A5Eb" is set to configure = false. +# this will fail configuration validation +``` + +This, more explicit version is also invalid: +```toml +[OCR.Contracts] +link_token = "0x88d1239894D9582f5849E5b5a964da9e5730f1E6" +offchain_aggregators = ["0xc1ce3815d6e7f3705265c2577F1342344752A5Eb", "0x2f4FA21fCd917C448C160caafEC874032F404c08"] + +# notice that this address needs to match the one in offchain_aggregators +[OCR.Contracts.Settings."0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] +configure = false + +[OCR.Contracts.Settings."0x2f4FA21fCd917C448C160caafEC874032F404c08"] +configure = true +``` + +Similarly, this one is also invalid: +```toml +[OCR.Contracts] +link_token = "0x88d1239894D9582f5849E5b5a964da9e5730f1E6" +offchain_aggregators = ["0xc1ce3815d6e7f3705265c2577F1342344752A5Eb", "0x2f4FA21fCd917C448C160caafEC874032F404c08"] + +# notice that this address needs to match the one in offchain_aggregators +[OCR.Contracts.Settings."0xc1ce3815d6e7f3705265c2577F1342344752A5Eb"] +use = false + +[OCR.Contracts.Settings."0x2f4FA21fCd917C448C160caafEC874032F404c08"] +use = true +``` + +There are no settings available for LINK token contract. + +Last, but not least, when deploying new OCR contracts you need to provide their number. For example: +```toml +# for OCRv1 +[OCR.Common] +number_of_contracts=2 + +# for OCRv2 +[OCR2.Common] +number_of_contracts=2 +``` + ### Setting env vars for Chainlink Node To set env vars for Chainlink Node use `WithCLNodeOptions()` and `WithNodeEnvVars()` when building a test environment. Example: diff --git a/integration-tests/testconfig/forwarder_ocr/example.toml b/integration-tests/testconfig/forwarder_ocr/example.toml index 0b762299af1..75143d7b77f 100644 --- a/integration-tests/testconfig/forwarder_ocr/example.toml +++ b/integration-tests/testconfig/forwarder_ocr/example.toml @@ -31,7 +31,7 @@ bearer_token_secret="bearer_token" base_url="http://grafana.url" # url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard dashboard_url="/d/your-dashboard" -# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model +# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model dashboard_uid="dashboard-uid-to-annotate" bearer_token_secret="my-awesome-token" @@ -143,6 +143,9 @@ BumpPercent = 20 BumpMin = '100 gwei' """ +[OCR.Common] +number_of_contracts=1 + # load test specific configuration [Load.OCR] [Load.OCR.Common] @@ -162,9 +165,8 @@ chainlink_node_funding = 100 [Soak.OCR] [Soak.OCR.Common] +number_of_contracts=2 test_duration="15m" [Soak.OCR.Soak] -ocr_version="1" -number_of_contracts=2 -time_between_rounds="1m" \ No newline at end of file +time_between_rounds="1m" diff --git a/integration-tests/testconfig/forwarder_ocr/forwarder_ocr.toml b/integration-tests/testconfig/forwarder_ocr/forwarder_ocr.toml index 8fa0fa5db25..68ed21404f3 100644 --- a/integration-tests/testconfig/forwarder_ocr/forwarder_ocr.toml +++ b/integration-tests/testconfig/forwarder_ocr/forwarder_ocr.toml @@ -57,7 +57,10 @@ MinContractPayment = 0 [Transactions] ForwardersEnabled = true - """ +""" + +[OCR.Common] +number_of_contracts=1 # load test specific configuration [Load.OCR] @@ -92,8 +95,8 @@ chainlink_node_funding = 0.5 [Soak.OCR] [Soak.OCR.Common] +number_of_contracts=2 test_duration = "15m" [Soak.OCR.Soak] -number_of_contracts = 2 -time_between_rounds = "1m" +time_between_rounds="1m" diff --git a/integration-tests/testconfig/forwarder_ocr2/example.toml b/integration-tests/testconfig/forwarder_ocr2/example.toml index b3bc45d270c..4941c49a983 100644 --- a/integration-tests/testconfig/forwarder_ocr2/example.toml +++ b/integration-tests/testconfig/forwarder_ocr2/example.toml @@ -31,7 +31,7 @@ bearer_token_secret="bearer_token" base_url="http://grafana.url" # url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard dashboard_url="/d/your-dashboard" -# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model +# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model dashboard_uid="dashboard-uid-to-annotate" bearer_token_secret="my-awesome-token" @@ -143,12 +143,15 @@ BumpPercent = 20 BumpMin = '100 gwei' """ +[OCR2.Common] +number_of_contracts=1 + # load test specific configuration -[Load.OCR] -[Load.OCR.Common] +[Load.OCR2] +[Load.OCR2.Common] eth_funds = 3 -[Load.OCR.Load] +[Load.OCR2.Load] test_duration = "3m" rate_limit_unit_duration = "1m" rate = 3 @@ -160,11 +163,10 @@ ea_change_interval = "5s" [Soak.Common] chainlink_node_funding = 100 -[Soak.OCR] -[Soak.OCR.Common] +[Soak.OCR2] +[Soak.OCR2.Common] +number_of_contracts=2 test_duration="15m" -[Soak.OCR.Soak] -ocr_version="1" -number_of_contracts=2 -time_between_rounds="1m" \ No newline at end of file +[Soak.OCR2.Soak] +time_between_rounds="1m" diff --git a/integration-tests/testconfig/forwarder_ocr2/forwarder_ocr2.toml b/integration-tests/testconfig/forwarder_ocr2/forwarder_ocr2.toml index 3f2a8610a83..76d5695a8b6 100644 --- a/integration-tests/testconfig/forwarder_ocr2/forwarder_ocr2.toml +++ b/integration-tests/testconfig/forwarder_ocr2/forwarder_ocr2.toml @@ -53,7 +53,10 @@ MinContractPayment = 0 [Transactions] ForwardersEnabled = true - """ +""" + +[OCR2.Common] +number_of_contracts=1 # load test specific configuration [Load.OCR2.Common] @@ -85,10 +88,10 @@ ea_change_interval = "5s" chainlink_node_funding = 1 [Soak.OCR2.Common] +number_of_contracts=2 test_duration="15m" [Soak.OCR2.Soak] -number_of_contracts=2 time_between_rounds="1m" diff --git a/integration-tests/testconfig/ocr/example.toml b/integration-tests/testconfig/ocr/example.toml index 92262241dff..26f0dd5a84e 100644 --- a/integration-tests/testconfig/ocr/example.toml +++ b/integration-tests/testconfig/ocr/example.toml @@ -31,7 +31,7 @@ bearer_token_secret="bearer_token" base_url="http://grafana.url" # url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard dashboard_url="/d/your-dashboard" -# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model +# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model dashboard_uid="dashboard-uid-to-annotate" bearer_token_secret="my-awesome-token" @@ -142,6 +142,9 @@ BumpPercent = 20 BumpMin = '100 gwei' """ +[OCR.Common] +number_of_contracts=1 + # load test specific configuration [Load.OCR] [Load.OCR.Common] @@ -161,8 +164,8 @@ chainlink_node_funding = 100 [Soak.OCR] [Soak.OCR.Common] +number_of_contracts=2 test_duration="15m" [Soak.OCR.Soak] -number_of_contracts=2 time_between_rounds="1m" diff --git a/integration-tests/testconfig/ocr/ocr.go b/integration-tests/testconfig/ocr/ocr.go index d8250d407fa..240fd2afeaa 100644 --- a/integration-tests/testconfig/ocr/ocr.go +++ b/integration-tests/testconfig/ocr/ocr.go @@ -2,15 +2,19 @@ package ocr import ( "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" "github.com/smartcontractkit/chainlink-testing-framework/lib/blockchain" ) type Config struct { - Soak *SoakConfig `toml:"Soak"` - Load *Load `toml:"Load"` - Volume *Volume `toml:"Volume"` - Common *Common `toml:"Common"` + Soak *SoakConfig `toml:"Soak"` + Load *Load `toml:"Load"` + Volume *Volume `toml:"Volume"` + Common *Common `toml:"Common"` + Contracts *Contracts `toml:"Contracts"` } func (o *Config) Validate() error { @@ -29,15 +33,24 @@ func (o *Config) Validate() error { return err } } + if o.Contracts != nil { + if err := o.Contracts.Validate(); err != nil { + return err + } + } return nil } type Common struct { - ETHFunds *int `toml:"eth_funds"` - TestDuration *blockchain.StrDuration `toml:"test_duration"` + NumberOfContracts *int `toml:"number_of_contracts"` + ETHFunds *int `toml:"eth_funds"` + TestDuration *blockchain.StrDuration `toml:"test_duration"` } func (o *Common) Validate() error { + if o.NumberOfContracts != nil && *o.NumberOfContracts < 1 { + return errors.New("when number_of_contracts is set, it must be greater than 0") + } if o.ETHFunds != nil && *o.ETHFunds < 0 { return errors.New("eth_funds must be set and cannot be negative") } @@ -117,16 +130,169 @@ func (o *Volume) Validate() error { } type SoakConfig struct { - NumberOfContracts *int `toml:"number_of_contracts"` TimeBetweenRounds *blockchain.StrDuration `toml:"time_between_rounds"` } func (o *SoakConfig) Validate() error { - if o.NumberOfContracts == nil || *o.NumberOfContracts <= 1 { - return errors.New("number_of_contracts must be set and be greater than 1") - } if o.TimeBetweenRounds == nil || o.TimeBetweenRounds.Duration == 0 { return errors.New("time_between_rounds must be set and be a positive integer") } return nil } + +// For more information on the configuration of contracts, see https://smartcontract-it.atlassian.net/wiki/spaces/TT/pages/828407894/Contracts+addresses+in+TOML+convention +type Contracts struct { + ShouldBeUsed *bool `toml:"use"` + LinkTokenAddress *string `toml:"link_token"` + OffChainAggregatorAddresses []string `toml:"offchain_aggregators"` + Settings map[string]ContractSetting `toml:"Settings"` +} + +func (o *Contracts) Validate() error { + if o.LinkTokenAddress != nil && !common.IsHexAddress(*o.LinkTokenAddress) { + return errors.New("link_token must be a valid ethereum address") + } + if o.OffChainAggregatorAddresses != nil { + allEnabled := make(map[bool]int) + allConfigure := make(map[bool]int) + for _, address := range o.OffChainAggregatorAddresses { + if !common.IsHexAddress(address) { + return fmt.Errorf("offchain_aggregators must be valid ethereum addresses, but %s is not", address) + } + + if v, ok := o.Settings[address]; ok { + if v.ShouldBeUsed != nil { + allEnabled[*v.ShouldBeUsed]++ + } else { + allEnabled[true]++ + } + if v.Configure != nil { + allConfigure[*v.Configure]++ + } else { + allConfigure[true]++ + } + } + } + + if allEnabled[true] > 0 && allEnabled[false] > 0 { + return errors.New("either all or none offchain_aggregators must be used") + } + + if allConfigure[true] > 0 && allConfigure[false] > 0 { + return errors.New("either all or none offchain_aggregators must be configured") + } + } + + return nil +} + +func (o *Config) UseExistingContracts() bool { + if o.Contracts == nil { + return false + } + + if o.Contracts.ShouldBeUsed != nil { + return *o.Contracts.ShouldBeUsed + } + + return false +} + +func (o *Config) LinkTokenContractAddress() (common.Address, error) { + if o.Contracts != nil && o.Contracts.LinkTokenAddress != nil { + return common.HexToAddress(*o.Contracts.LinkTokenAddress), nil + } + + return common.Address{}, errors.New("link token address must be set") +} + +func (o *Config) UseExistingLinkTokenContract() bool { + if !o.UseExistingContracts() { + return false + } + + if o.Contracts.LinkTokenAddress == nil { + return false + } + + if len(o.Contracts.Settings) == 0 { + return true + } + + if v, ok := o.Contracts.Settings[*o.Contracts.LinkTokenAddress]; ok { + return v.ShouldBeUsed != nil && *v.ShouldBeUsed + } + + return true +} + +type ContractSetting struct { + ShouldBeUsed *bool `toml:"use"` + Configure *bool `toml:"configure"` +} + +type OffChainAggregatorsConfig interface { + OffChainAggregatorsContractsAddresses() []common.Address + UseExistingOffChainAggregatorsContracts() bool + ConfigureExistingOffChainAggregatorsContracts() bool + NumberOfContractsToDeploy() int +} + +func (o *Config) UseExistingOffChainAggregatorsContracts() bool { + if !o.UseExistingContracts() { + return false + } + + if len(o.Contracts.OffChainAggregatorAddresses) == 0 { + return false + } + + if len(o.Contracts.Settings) == 0 { + return true + } + + for _, address := range o.Contracts.OffChainAggregatorAddresses { + if v, ok := o.Contracts.Settings[address]; ok { + return v.ShouldBeUsed != nil && *v.ShouldBeUsed + } + } + + return true +} + +func (o *Config) OffChainAggregatorsContractsAddresses() []common.Address { + var ocrInstanceAddresses []common.Address + if !o.UseExistingOffChainAggregatorsContracts() { + return ocrInstanceAddresses + } + + for _, address := range o.Contracts.OffChainAggregatorAddresses { + ocrInstanceAddresses = append(ocrInstanceAddresses, common.HexToAddress(address)) + } + + return ocrInstanceAddresses +} + +func (o *Config) ConfigureExistingOffChainAggregatorsContracts() bool { + if !o.UseExistingOffChainAggregatorsContracts() { + return true + } + + for _, address := range o.Contracts.OffChainAggregatorAddresses { + for maybeOcrAddress, setting := range o.Contracts.Settings { + if maybeOcrAddress == address { + return setting.Configure != nil && *setting.Configure + } + } + } + + return true +} + +func (o *Config) NumberOfContractsToDeploy() int { + if o.Common != nil && o.Common.NumberOfContracts != nil { + return *o.Common.NumberOfContracts + } + + return 0 +} diff --git a/integration-tests/testconfig/ocr/ocr.toml b/integration-tests/testconfig/ocr/ocr.toml index 17ee4d7b687..36cded0b85c 100644 --- a/integration-tests/testconfig/ocr/ocr.toml +++ b/integration-tests/testconfig/ocr/ocr.toml @@ -43,6 +43,9 @@ Enabled = true ListenAddresses = ['0.0.0.0:6690'] """ +[OCR.Common] +number_of_contracts=1 + # load test specific configuration [Load.OCR] [Load.OCR.Common] @@ -77,9 +80,9 @@ chainlink_node_funding = 0.5 [Soak.OCR] [Soak.OCR.Common] test_duration="15m" +number_of_contracts=2 [Soak.OCR.Soak] -number_of_contracts=2 time_between_rounds="1m" # Soak test configuration with Geth reorg below finality with FinalityTagEnabled=false diff --git a/integration-tests/testconfig/ocr2/example.toml b/integration-tests/testconfig/ocr2/example.toml index 36e3105f219..624c3b77752 100644 --- a/integration-tests/testconfig/ocr2/example.toml +++ b/integration-tests/testconfig/ocr2/example.toml @@ -31,7 +31,7 @@ bearer_token_secret="bearer_token" base_url="http://grafana.url" # url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard dashboard_url="/d/your-dashboard" -# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model +# Grafana dashboard uid to annotate. Find it in Dashboard Settings -> JSON Model dashboard_uid="dashboard-uid-to-annotate" bearer_token_secret="my-awesome-token" @@ -142,6 +142,9 @@ BumpPercent = 20 BumpMin = '100 gwei' """ +[OCR2.Common] +number_of_contracts=1 + # load test specific configuration [Load.OCR2] [Load.OCR2.Common] @@ -161,8 +164,8 @@ chainlink_node_funding = 100 [Soak.OCR2] [Soak.OCR.Common] +number_of_contracts=2 test_duration="15m" [Soak.OCR2.Soak] -number_of_contracts=2 time_between_rounds="1m" diff --git a/integration-tests/testconfig/ocr2/ocr2.go b/integration-tests/testconfig/ocr2/ocr2.go index 1e7f034e043..60169e944fa 100644 --- a/integration-tests/testconfig/ocr2/ocr2.go +++ b/integration-tests/testconfig/ocr2/ocr2.go @@ -1,14 +1,13 @@ package ocr2 import ( - "errors" - - "github.com/smartcontractkit/chainlink-testing-framework/lib/blockchain" + "github.com/smartcontractkit/chainlink/integration-tests/testconfig/ocr" ) type Config struct { - Soak *SoakConfig `toml:"Soak"` - Common *Common `toml:"Common"` + Soak *ocr.SoakConfig `toml:"Soak"` + Common *ocr.Common `toml:"Common"` + Contracts *ocr.Contracts `toml:"Contracts"` } func (o *Config) Validate() error { @@ -22,32 +21,10 @@ func (o *Config) Validate() error { return err } } - return nil -} - -type Common struct { - ETHFunds *int `toml:"eth_funds"` - TestDuration *blockchain.StrDuration `toml:"test_duration"` -} - -func (o *Common) Validate() error { - if o.ETHFunds != nil && *o.ETHFunds < 0 { - return errors.New("eth_funds must be set and cannot be negative") - } - return nil -} - -type SoakConfig struct { - NumberOfContracts *int `toml:"number_of_contracts"` - TimeBetweenRounds *blockchain.StrDuration `toml:"time_between_rounds"` -} - -func (o *SoakConfig) Validate() error { - if o.NumberOfContracts == nil || *o.NumberOfContracts <= 1 { - return errors.New("number_of_contracts must be set and be greater than 1") - } - if o.TimeBetweenRounds == nil || o.TimeBetweenRounds.Duration == 0 { - return errors.New("time_between_rounds must be set and be a positive integer") + if o.Contracts != nil { + if err := o.Contracts.Validate(); err != nil { + return err + } } return nil } diff --git a/integration-tests/testconfig/ocr2/ocr2.toml b/integration-tests/testconfig/ocr2/ocr2.toml index ad195913bd0..62d92574ea8 100644 --- a/integration-tests/testconfig/ocr2/ocr2.toml +++ b/integration-tests/testconfig/ocr2/ocr2.toml @@ -43,6 +43,9 @@ Enabled = true ListenAddresses = ['0.0.0.0:6690'] """ +[OCR2.Common] +number_of_contracts=1 + # load test specific configuration [Load.OCR2] [Load.OCR2.Common] @@ -76,8 +79,8 @@ chainlink_node_funding = 0.5 [Soak.OCR2] [Soak.OCR2.Common] +number_of_contracts=2 test_duration="15m" [Soak.OCR2.Soak] -number_of_contracts=2 time_between_rounds="1m" diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index fb692c56a75..545818e3348 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/barkimedes/go-deepcopy" + "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" @@ -77,6 +78,11 @@ type CCIPTestConfig interface { GetCCIPConfig() *ccip_config.Config } +type LinkTokenContractConfig interface { + LinkTokenContractAddress() (common.Address, error) + UseExistingLinkTokenContract() bool +} + const ( E2E_TEST_DATA_STREAMS_URL_ENV = "E2E_TEST_DATA_STREAMS_URL" E2E_TEST_DATA_STREAMS_USERNAME_ENV = "E2E_TEST_DATA_STREAMS_USERNAME" diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index 3653b9d5fb9..73b142b6297 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -281,8 +281,8 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { o.mockServer = ctf_client.ConnectMockServer(o.testEnvironment) require.NoError(o.t, err, "Creating mockserver clients shouldn't fail") - linkContract, err := contracts.DeployLinkTokenContract(o.log, sethClient) - require.NoError(o.t, err, "Error deploying LINK contract") + linkContract, err := actions.LinkTokenContract(o.log, sethClient, ocrTestConfig.GetActiveOCRConfig()) + require.NoError(o.t, err, "Error loading/deploying link token contract") // Fund Chainlink nodes, excluding the bootstrap node o.log.Info().Float64("ETH amount per node", *o.Config.Common.ChainlinkNodeFunding).Msg("Funding Chainlink nodes") @@ -290,7 +290,6 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { require.NoError(o.t, err, "Error funding Chainlink nodes") var forwarders []common.Address - if o.OperatorForwarderFlow { var operators []common.Address operators, forwarders, _ = actions.DeployForwarderContracts( @@ -311,17 +310,17 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { o.ocrV1Instances, err = actions.DeployOCRContractsForwarderFlow( o.log, o.seth, - *o.Config.GetActiveOCRConfig().Soak.NumberOfContracts, + o.Config.GetActiveOCRConfig(), common.HexToAddress(linkContract.Address()), contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), forwarders, ) require.NoError(o.t, err, "Error deploying OCR Forwarder contracts") } else { - o.ocrV1Instances, err = actions.DeployOCRv1Contracts( + o.ocrV1Instances, err = actions.SetupOCRv1Contracts( o.log, sethClient, - *o.Config.GetActiveOCRConfig().Soak.NumberOfContracts, + o.Config.GetActiveOCRConfig(), common.HexToAddress(linkContract.Address()), contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(o.workerNodes), ) @@ -343,19 +342,22 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { } ocrOffchainOptions := contracts.DefaultOffChainAggregatorOptions() - o.ocrV2Instances, err = actions.DeployOCRv2Contracts( + o.ocrV2Instances, err = actions.SetupOCRv2Contracts( o.log, o.seth, - *ocrTestConfig.GetActiveOCRConfig().Soak.NumberOfContracts, + ocrTestConfig.GetActiveOCRConfig(), common.HexToAddress(linkContract.Address()), transmitters, ocrOffchainOptions, ) require.NoError(o.t, err, "Error deploying OCRv2 contracts") - contractConfig, err := actions.BuildMedianOCR2Config(o.workerNodes, ocrOffchainOptions) - require.NoError(o.t, err, "Error building median config") - err = actions.ConfigureOCRv2AggregatorContracts(contractConfig, o.ocrV2Instances) - require.NoError(o.t, err, "Error configuring OCRv2 aggregator contracts") + + if !ocrTestConfig.GetActiveOCRConfig().UseExistingOffChainAggregatorsContracts() || (ocrTestConfig.GetActiveOCRConfig().UseExistingOffChainAggregatorsContracts() && ocrTestConfig.GetActiveOCRConfig().ConfigureExistingOffChainAggregatorsContracts()) { + contractConfig, err := actions.BuildMedianOCR2Config(o.workerNodes, ocrOffchainOptions) + require.NoError(o.t, err, "Error building median config") + err = actions.ConfigureOCRv2AggregatorContracts(contractConfig, o.ocrV2Instances) + require.NoError(o.t, err, "Error configuring OCRv2 aggregator contracts") + } } if o.OCRVersion == "1" { @@ -399,7 +401,7 @@ func (o *OCRSoakTest) Run() { o.log.Info(). Str("Test Duration", o.Config.GetActiveOCRConfig().Common.TestDuration.Duration.Truncate(time.Second).String()). - Int("Number of OCR Contracts", *config.GetActiveOCRConfig().Soak.NumberOfContracts). + Int("Number of OCR Contracts", *config.GetActiveOCRConfig().Common.NumberOfContracts). Str("OCR Version", o.OCRVersion). Msg("Starting OCR Soak Test") @@ -530,7 +532,7 @@ func (o *OCRSoakTest) LoadState() error { if testState.OCRVersion == "1" { o.ocrV1Instances = make([]contracts.OffchainAggregator, len(testState.OCRContractAddresses)) for i, addr := range testState.OCRContractAddresses { - instance, err := contracts.LoadOffchainAggregator(o.log, o.seth, common.HexToAddress(addr)) + instance, err := contracts.LoadOffChainAggregator(o.log, o.seth, common.HexToAddress(addr)) if err != nil { return fmt.Errorf("failed to instantiate OCR instance: %w", err) } @@ -539,7 +541,7 @@ func (o *OCRSoakTest) LoadState() error { } else if testState.OCRVersion == "2" { o.ocrV2Instances = make([]contracts.OffchainAggregatorV2, len(testState.OCRContractAddresses)) for i, addr := range testState.OCRContractAddresses { - instance, err := contracts.LoadOffChainAggregatorV2(o.log, o.seth, common.HexToAddress(addr)) + instance, err := contracts.LoadOffchainAggregatorV2(o.log, o.seth, common.HexToAddress(addr)) if err != nil { return err } @@ -561,7 +563,7 @@ func (o *OCRSoakTest) Resume() { Str("Time Left", o.timeLeft.String()). Msg("Resuming OCR Soak Test") - ocrAddresses := make([]common.Address, *o.Config.GetActiveOCRConfig().Soak.NumberOfContracts) + ocrAddresses := make([]common.Address, *o.Config.GetActiveOCRConfig().Common.NumberOfContracts) if o.OCRVersion == "1" { for i, ocrInstance := range o.ocrV1Instances { @@ -1021,12 +1023,12 @@ func (o *OCRSoakTest) collectEvents() error { // ensureValues ensures that all values needed to run the test are present func (o *OCRSoakTest) ensureInputValues() error { - ocrConfig := o.Config.GetActiveOCRConfig().Soak + ocrConfig := o.Config.GetActiveOCRConfig() if o.OCRVersion != "1" && o.OCRVersion != "2" { return fmt.Errorf("OCR version must be 1 or 2, found %s", o.OCRVersion) } - if ocrConfig.NumberOfContracts != nil && *ocrConfig.NumberOfContracts <= 0 { - return fmt.Errorf("number of OCR contracts must be set and greater than 0, found %d", ocrConfig.NumberOfContracts) + if ocrConfig.Common.NumberOfContracts != nil && *ocrConfig.Common.NumberOfContracts <= 0 { + return fmt.Errorf("number of OCR contracts must be set and greater than 0, found %d", ocrConfig.Common.NumberOfContracts) } if o.Config.Common.ChainlinkNodeFunding != nil && *o.Config.Common.ChainlinkNodeFunding <= 0 { return fmt.Errorf("chainlink node funding must be greater than 0, found %f", *o.Config.Common.ChainlinkNodeFunding) @@ -1034,11 +1036,12 @@ func (o *OCRSoakTest) ensureInputValues() error { if o.Config.GetActiveOCRConfig().Common.TestDuration != nil && o.Config.GetActiveOCRConfig().Common.TestDuration.Duration <= time.Minute { return fmt.Errorf("test duration must be greater than 1 minute, found %s", o.Config.GetActiveOCRConfig().Common.TestDuration) } - if ocrConfig.TimeBetweenRounds != nil && ocrConfig.TimeBetweenRounds.Duration >= time.Hour { - return fmt.Errorf("time between rounds must be less than 1 hour, found %s", ocrConfig.TimeBetweenRounds) + soakConfig := ocrConfig.Soak + if soakConfig.TimeBetweenRounds != nil && soakConfig.TimeBetweenRounds.Duration >= time.Hour { + return fmt.Errorf("time between rounds must be less than 1 hour, found %s", soakConfig.TimeBetweenRounds) } - if ocrConfig.TimeBetweenRounds != nil && ocrConfig.TimeBetweenRounds.Duration < time.Second*30 { - return fmt.Errorf("time between rounds must be greater or equal to 30 seconds, found %s", ocrConfig.TimeBetweenRounds) + if soakConfig.TimeBetweenRounds != nil && soakConfig.TimeBetweenRounds.Duration < time.Second*30 { + return fmt.Errorf("time between rounds must be greater or equal to 30 seconds, found %s", soakConfig.TimeBetweenRounds) } return nil From 0aa59850e08538be11e1b53406e786d15299ec98 Mon Sep 17 00:00:00 2001 From: Balamurali Gopalswami <167726375+b-gopalswami@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:32:47 -0400 Subject: [PATCH 3/7] CCIP-3420: Fix IsRequestTriggeredWithinTimeframe (#1445) (#14498) ## Motivation The prior approach to find if there is traffic in a lane was not detecting the transactions properly. It could be due to the `CCIPSendRequestedWatcher` captures the event after it is initiated not any transaction before that. ## Solution The revised approach is to use the FilterCCIPSendRequested by a past derived block number and see if there is any traffic. The block number is derived by this formula: `filterFromBlock = latestBlockNumber - (SkipRequestIfAnotherRequestTriggeredWithin/avgBlockTime)` By this approach, we will be able to find traffic better. https://smartcontract-it.atlassian.net/browse/CCIP-3420 --- .../ccip-tests/actions/ccip_helpers.go | 44 +++++++++++++++++++ .../ccip-tests/load/ccip_loadgen.go | 15 ++++++- .../ccip-tests/testconfig/ccip.go | 3 ++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go index 06334560e05..da2e870f3ec 100644 --- a/integration-tests/ccip-tests/actions/ccip_helpers.go +++ b/integration-tests/ccip-tests/actions/ccip_helpers.go @@ -1620,6 +1620,8 @@ func (sourceCCIP *SourceCCIPModule) AssertSendRequestedLogFinalized( return finalizedAt, finalizedBlockNum.Uint64(), nil } +// IsRequestTriggeredWithinTimeframe monitors for live events occurring within the specified timeframe. +// Live events refer to those that are triggered after subscribing to the CCIP Send Requested event. func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe *commonconfig.Duration) *time.Time { if timeframe == nil { return nil @@ -1644,6 +1646,48 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe return foundAt } +// IsPastRequestTriggeredWithinTimeframe determines the average block time and calculates the block numbers +// within the specified timeframe. It then uses FilterCCIPSendRequested to identify the past events. +func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(ctx context.Context, timeframe *commonconfig.Duration) (*time.Time, error) { + if timeframe == nil { + return nil, nil + } + //var foundAt *time.Time + latestBlock, err := sourceCCIP.Common.ChainClient.LatestBlockNumber(ctx) + if err != nil { + return nil, fmt.Errorf("error while getting latest source block number. Error: %w", err) + } + avgBlockTime, err := sourceCCIP.Common.ChainClient.AvgBlockTime(ctx) + if err != nil { + return nil, fmt.Errorf("error while getting average source block time. Error: %w", err) + } + filterFromBlock := latestBlock - uint64(timeframe.Duration()/avgBlockTime) + + onRampContract, err := evm_2_evm_onramp.NewEVM2EVMOnRamp(common.HexToAddress(sourceCCIP.OnRamp.EthAddress.Hex()), + sourceCCIP.Common.ChainClient.Backend()) + if err != nil { + return nil, fmt.Errorf("error while on ramp contract. Error: %w", err) + } + iterator, err := onRampContract.FilterCCIPSendRequested(&bind.FilterOpts{ + Start: filterFromBlock, + }) + if err != nil { + return nil, fmt.Errorf("error while filtering CCIP send requested starting block number: %d. Error: %w", filterFromBlock, err) + } + defer func() { + _ = iterator.Close() + }() + if iterator.Next() { + hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(iterator.Event.Raw.BlockNumber))) + if err != nil { + return nil, fmt.Errorf("error getting header for block: %d, Error: %w", iterator.Event.Raw.BlockNumber, err) + } + return pointer.ToTime(hdr.Timestamp), nil + } + + return nil, nil +} + func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested( lggr *zerolog.Logger, txHash string, diff --git a/integration-tests/ccip-tests/load/ccip_loadgen.go b/integration-tests/ccip-tests/load/ccip_loadgen.go index bc7627b4e08..3ce770d31bc 100644 --- a/integration-tests/ccip-tests/load/ccip_loadgen.go +++ b/integration-tests/ccip-tests/load/ccip_loadgen.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" + "github.com/AlekSi/pointer" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -222,7 +224,18 @@ func (c *CCIPE2ELoad) CCIPMsg() (router.ClientEVM2AnyMessage, *testreporters.Req func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response { res := &wasp.Response{} sourceCCIP := c.Lane.Source - recentRequestFoundAt := sourceCCIP.IsRequestTriggeredWithinTimeframe(c.SkipRequestIfAnotherRequestTriggeredWithin) + var recentRequestFoundAt *time.Time + var err error + // Use IsPastRequestTriggeredWithinTimeframe to check for any historical CCIP send request events + // within the specified timeframe for the first message. Subsequently, use the watcher method to monitor + // and detect any new events as they occur. + if c.CurrentMsgSerialNo.Load() == int64(1) { + recentRequestFoundAt, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin) + require.NoError(c.t, err, "error while filtering past requests") + } else { + recentRequestFoundAt = sourceCCIP.IsRequestTriggeredWithinTimeframe(c.SkipRequestIfAnotherRequestTriggeredWithin) + } + if recentRequestFoundAt != nil { c.Lane.Logger. Info(). diff --git a/integration-tests/ccip-tests/testconfig/ccip.go b/integration-tests/ccip-tests/testconfig/ccip.go index 76e00ecf496..0a53ee18732 100644 --- a/integration-tests/ccip-tests/testconfig/ccip.go +++ b/integration-tests/ccip-tests/testconfig/ccip.go @@ -242,6 +242,9 @@ func (l *LoadProfile) Validate() error { if l.TestDuration == nil || l.TestDuration.Duration().Minutes() == 0 { return fmt.Errorf("test duration should be set") } + if l.SkipRequestIfAnotherRequestTriggeredWithin != nil && l.TimeUnit.Duration() < l.SkipRequestIfAnotherRequestTriggeredWithin.Duration() { + return fmt.Errorf("SkipRequestIfAnotherRequestTriggeredWithin should be set below the TimeUnit duration") + } return nil } From d2a01ca51bb4a7654d2ceb4f5c25f2ca2de3df11 Mon Sep 17 00:00:00 2001 From: Oliver Townsend <133903322+ogtownsend@users.noreply.github.com> Date: Thu, 19 Sep 2024 09:54:04 -0700 Subject: [PATCH 4/7] Bring KMS client and multiclient over to chainlink (#14484) * Bring KMS client and multiclient over to chainlink * Pull seth client into multiclient * remove toml tags * Keep multiclient simple * Config validation for kms * Rename to ws, fix test * Changeset * Downgrade seth * Lint * More lint --------- Co-authored-by: AnieeG Co-authored-by: connorwstein --- .changeset/sour-ears-wink.md | 5 + .../ccip/ccip_integration_tests/helpers.go | 2 - integration-tests/deployment/evm_kmsclient.go | 233 ++++++++++++++++++ .../deployment/evm_kmsclient_test.go | 27 ++ integration-tests/deployment/multiclient.go | 124 ++++++++++ .../deployment/multiclient_test.go | 36 +++ integration-tests/go.mod | 2 +- 7 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 .changeset/sour-ears-wink.md create mode 100644 integration-tests/deployment/evm_kmsclient.go create mode 100644 integration-tests/deployment/evm_kmsclient_test.go create mode 100644 integration-tests/deployment/multiclient.go create mode 100644 integration-tests/deployment/multiclient_test.go diff --git a/.changeset/sour-ears-wink.md b/.changeset/sour-ears-wink.md new file mode 100644 index 00000000000..d77fd548c51 --- /dev/null +++ b/.changeset/sour-ears-wink.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#internal KMS client for deployment diff --git a/core/capabilities/ccip/ccip_integration_tests/helpers.go b/core/capabilities/ccip/ccip_integration_tests/helpers.go index 25baddfb48e..4670333e391 100644 --- a/core/capabilities/ccip/ccip_integration_tests/helpers.go +++ b/core/capabilities/ccip/ccip_integration_tests/helpers.go @@ -406,7 +406,6 @@ func (h *homeChain) AddNodes( p2pIDs [][32]byte, capabilityIDs [][32]byte, ) { - // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail sortP2PIDS(p2pIDs) var nodeParams []kcr.CapabilitiesRegistryNodeParams for _, p2pID := range p2pIDs { @@ -430,7 +429,6 @@ func AddChainConfig( p2pIDs [][32]byte, f uint8, ) ccip_config.CCIPConfigTypesChainConfigInfo { - // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail sortP2PIDS(p2pIDs) // First Add ChainConfig that includes all p2pIDs as readers encodedExtraChainConfig, err := chainconfig.EncodeChainConfig(chainconfig.ChainConfig{ diff --git a/integration-tests/deployment/evm_kmsclient.go b/integration-tests/deployment/evm_kmsclient.go new file mode 100644 index 00000000000..07af77523c8 --- /dev/null +++ b/integration-tests/deployment/evm_kmsclient.go @@ -0,0 +1,233 @@ +package deployment + +import ( + "bytes" + "context" + "crypto/ecdsa" + "encoding/asn1" + "encoding/hex" + "fmt" + "math/big" + + "github.com/aws/aws-sdk-go/aws/session" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/secp256k1" +) + +var ( + secp256k1N = crypto.S256().Params().N + secp256k1HalfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) +) + +// See https://docs.aws.amazon.com/kms/latest/APIReference/API_GetPublicKey.html#API_GetPublicKey_ResponseSyntax +// and https://datatracker.ietf.org/doc/html/rfc5280 for why we need to unpack the KMS public key. +type asn1SubjectPublicKeyInfo struct { + AlgorithmIdentifier asn1AlgorithmIdentifier + SubjectPublicKey asn1.BitString +} + +type asn1AlgorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier + Parameters asn1.ObjectIdentifier +} + +// See https://aws.amazon.com/blogs/database/part2-use-aws-kms-to-securely-manage-ethereum-accounts/ for why we +// need to manually prep the signature for Ethereum. +type asn1ECDSASig struct { + R asn1.RawValue + S asn1.RawValue +} + +// TODO: Mockery gen then test with a regular eth key behind the interface. +type KMSClient interface { + GetPublicKey(input *kms.GetPublicKeyInput) (*kms.GetPublicKeyOutput, error) + Sign(input *kms.SignInput) (*kms.SignOutput, error) +} + +type KMS struct { + KmsDeployerKeyId string + KmsDeployerKeyRegion string + AwsProfileName string +} + +func NewKMSClient(config KMS) (KMSClient, error) { + if config.KmsDeployerKeyId == "" { + return nil, fmt.Errorf("KMS key ID is required") + } + if config.KmsDeployerKeyRegion == "" { + return nil, fmt.Errorf("KMS key region is required") + } + var awsSessionFn AwsSessionFn + if config.AwsProfileName != "" { + awsSessionFn = awsSessionFromProfileFn + } else { + awsSessionFn = awsSessionFromEnvVarsFn + } + return kms.New(awsSessionFn(config)), nil +} + +type EVMKMSClient struct { + Client KMSClient + KeyID string +} + +func NewEVMKMSClient(client KMSClient, keyID string) *EVMKMSClient { + return &EVMKMSClient{ + Client: client, + KeyID: keyID, + } +} + +func (c *EVMKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { + ecdsaPublicKey, err := c.GetECDSAPublicKey() + if err != nil { + return nil, err + } + + pubKeyBytes := secp256k1.S256().Marshal(ecdsaPublicKey.X, ecdsaPublicKey.Y) + keyAddr := crypto.PubkeyToAddress(*ecdsaPublicKey) + if chainID == nil { + return nil, fmt.Errorf("chainID is required") + } + signer := types.LatestSignerForChainID(chainID) + + signerFn := func(address common.Address, tx *types.Transaction) (*types.Transaction, error) { + if address != keyAddr { + return nil, bind.ErrNotAuthorized + } + + txHashBytes := signer.Hash(tx).Bytes() + + mType := kms.MessageTypeDigest + algo := kms.SigningAlgorithmSpecEcdsaSha256 + signOutput, err := c.Client.Sign( + &kms.SignInput{ + KeyId: &c.KeyID, + SigningAlgorithm: &algo, + MessageType: &mType, + Message: txHashBytes, + }) + if err != nil { + return nil, fmt.Errorf("failed to call kms.Sign() on transaction: %w", err) + } + + ethSig, err := kmsToEthSig(signOutput.Signature, pubKeyBytes, txHashBytes) + if err != nil { + return nil, fmt.Errorf("failed to convert KMS signature to Ethereum signature: %w", err) + } + + return tx.WithSignature(signer, ethSig) + } + + return &bind.TransactOpts{ + From: keyAddr, + Signer: signerFn, + Context: ctx, + }, nil +} + +// GetECDSAPublicKey retrieves the public key from KMS and converts it to its ECDSA representation. +func (c *EVMKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { + getPubKeyOutput, err := c.Client.GetPublicKey(&kms.GetPublicKeyInput{ + KeyId: aws.String(c.KeyID), + }) + if err != nil { + return nil, fmt.Errorf("can not get public key from KMS for KeyId=%s: %w", c.KeyID, err) + } + + var asn1pubKeyInfo asn1SubjectPublicKeyInfo + _, err = asn1.Unmarshal(getPubKeyOutput.PublicKey, &asn1pubKeyInfo) + if err != nil { + return nil, fmt.Errorf("can not parse asn1 public key for KeyId=%s: %w", c.KeyID, err) + } + + pubKey, err := crypto.UnmarshalPubkey(asn1pubKeyInfo.SubjectPublicKey.Bytes) + if err != nil { + return nil, fmt.Errorf("can not unmarshal public key bytes: %w", err) + } + return pubKey, nil +} + +func kmsToEthSig(kmsSig, ecdsaPubKeyBytes, hash []byte) ([]byte, error) { + var asn1Sig asn1ECDSASig + _, err := asn1.Unmarshal(kmsSig, &asn1Sig) + if err != nil { + return nil, err + } + + rBytes := asn1Sig.R.Bytes + sBytes := asn1Sig.S.Bytes + + // Adjust S value from signature to match Eth standard. + // See: https://aws.amazon.com/blogs/database/part2-use-aws-kms-to-securely-manage-ethereum-accounts/ + // "After we extract r and s successfully, we have to test if the value of s is greater than secp256k1n/2 as + // specified in EIP-2 and flip it if required." + sBigInt := new(big.Int).SetBytes(sBytes) + if sBigInt.Cmp(secp256k1HalfN) > 0 { + sBytes = new(big.Int).Sub(secp256k1N, sBigInt).Bytes() + } + + return recoverEthSignature(ecdsaPubKeyBytes, hash, rBytes, sBytes) +} + +// See: https://aws.amazon.com/blogs/database/part2-use-aws-kms-to-securely-manage-ethereum-accounts/ +func recoverEthSignature(expectedPublicKeyBytes, txHash, r, s []byte) ([]byte, error) { + rsSig := append(padTo32Bytes(r), padTo32Bytes(s)...) + ethSig := append(rsSig, []byte{0}...) + + recoveredPublicKeyBytes, err := crypto.Ecrecover(txHash, ethSig) + if err != nil { + return nil, fmt.Errorf("failing to call Ecrecover: %w", err) + } + + if hex.EncodeToString(recoveredPublicKeyBytes) != hex.EncodeToString(expectedPublicKeyBytes) { + ethSig = append(rsSig, []byte{1}...) + recoveredPublicKeyBytes, err = crypto.Ecrecover(txHash, ethSig) + if err != nil { + return nil, fmt.Errorf("failing to call Ecrecover: %w", err) + } + + if hex.EncodeToString(recoveredPublicKeyBytes) != hex.EncodeToString(expectedPublicKeyBytes) { + return nil, fmt.Errorf("can not reconstruct public key from sig") + } + } + + return ethSig, nil +} + +func padTo32Bytes(buffer []byte) []byte { + buffer = bytes.TrimLeft(buffer, "\x00") + for len(buffer) < 32 { + zeroBuf := []byte{0} + buffer = append(zeroBuf, buffer...) + } + return buffer +} + +type AwsSessionFn func(config KMS) *session.Session + +var awsSessionFromEnvVarsFn = func(config KMS) *session.Session { + return session.Must( + session.NewSession(&aws.Config{ + Region: aws.String(config.KmsDeployerKeyRegion), + CredentialsChainVerboseErrors: aws.Bool(true), + })) +} + +var awsSessionFromProfileFn = func(config KMS) *session.Session { + return session.Must( + session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Profile: config.AwsProfileName, + Config: aws.Config{ + Region: aws.String(config.KmsDeployerKeyRegion), + CredentialsChainVerboseErrors: aws.Bool(true), + }, + })) +} diff --git a/integration-tests/deployment/evm_kmsclient_test.go b/integration-tests/deployment/evm_kmsclient_test.go new file mode 100644 index 00000000000..8a889a56067 --- /dev/null +++ b/integration-tests/deployment/evm_kmsclient_test.go @@ -0,0 +1,27 @@ +package deployment + +import ( + "encoding/hex" + "testing" + + "github.com/test-go/testify/require" +) + +func TestKMSToEthSigConversion(t *testing.T) { + kmsSigBytes, err := hex.DecodeString("304402206168865941bafcae3a8cf8b26edbb5693d62222b2e54d962c1aabbeaddf33b6802205edc7f597d2bf2d1eaa14fc514a6202bafcffe52b13ae3fec00674d92a874b73") + require.NoError(t, err) + ecdsaPublicKeyBytes, err := hex.DecodeString("04a735e9e3cb526f83be23b03f1f5ae7788a8654e3f0fcfb4f978290de07ebd47da30eeb72e904fdd4a81b46e320908ff4345e119148f89c1f04674c14a506e24b") + require.NoError(t, err) + txHashBytes, err := hex.DecodeString("a2f037301e90f58c084fe4bec2eef14b26e620d6b6cb46051037d03b29ab7d9a") + require.NoError(t, err) + expectedEthSignBytes, err := hex.DecodeString("6168865941bafcae3a8cf8b26edbb5693d62222b2e54d962c1aabbeaddf33b685edc7f597d2bf2d1eaa14fc514a6202bafcffe52b13ae3fec00674d92a874b7300") + require.NoError(t, err) + + actualEthSig, err := kmsToEthSig( + kmsSigBytes, + ecdsaPublicKeyBytes, + txHashBytes, + ) + require.NoError(t, err) + require.Equal(t, expectedEthSignBytes, actualEthSig) +} diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go new file mode 100644 index 00000000000..02a18f760df --- /dev/null +++ b/integration-tests/deployment/multiclient.go @@ -0,0 +1,124 @@ +package deployment + +import ( + "context" + "fmt" + "math/big" + "time" + + "github.com/avast/retry-go/v4" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/pkg/errors" +) + +const ( + RPC_DEFAULT_RETRY_ATTEMPTS = 10 + RPC_DEFAULT_RETRY_DELAY = 1000 * time.Millisecond +) + +type RetryConfig struct { + Attempts uint + Delay time.Duration +} + +func defaultRetryConfig() RetryConfig { + return RetryConfig{ + Attempts: RPC_DEFAULT_RETRY_ATTEMPTS, + Delay: RPC_DEFAULT_RETRY_DELAY, + } +} + +type RPC struct { + WSURL string + // TODO: http fallback needed for some networks? +} + +// MultiClient should comply with the OnchainClient interface +var _ OnchainClient = &MultiClient{} + +type MultiClient struct { + *ethclient.Client + Backups []*ethclient.Client + RetryConfig RetryConfig +} + +func NewMultiClient(rpcs []RPC, opts ...func(client *MultiClient)) (*MultiClient, error) { + if len(rpcs) == 0 { + return nil, fmt.Errorf("No RPCs provided, need at least one") + } + var mc MultiClient + clients := make([]*ethclient.Client, 0, len(rpcs)) + for _, rpc := range rpcs { + client, err := ethclient.Dial(rpc.WSURL) + if err != nil { + return nil, errors.Wrapf(err, "failed to dial %s", rpc.WSURL) + } + clients = append(clients, client) + } + mc.Client = clients[0] + mc.Backups = clients[1:] + mc.RetryConfig = defaultRetryConfig() + + for _, opt := range opts { + opt(&mc) + } + return &mc, nil +} + +func (mc *MultiClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { + var receipt *types.Receipt + err := mc.retryWithBackups(func(client *ethclient.Client) error { + var err error + receipt, err = client.TransactionReceipt(ctx, txHash) + return err + }) + return receipt, err +} + +func (mc *MultiClient) SendTransaction(ctx context.Context, tx *types.Transaction) error { + return mc.retryWithBackups(func(client *ethclient.Client) error { + return client.SendTransaction(ctx, tx) + }) +} + +func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { + var code []byte + err := mc.retryWithBackups(func(client *ethclient.Client) error { + var err error + code, err = client.CodeAt(ctx, account, blockNumber) + return err + }) + return code, err +} + +func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address) (uint64, error) { + var count uint64 + err := mc.retryWithBackups(func(client *ethclient.Client) error { + var err error + count, err = client.NonceAt(ctx, account, nil) + return err + }) + return count, err +} + +func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error { + var err error + for _, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) { + err2 := retry.Do(func() error { + err = op(client) + if err != nil { + // TODO: logger? + fmt.Printf("Error %v with client %v\n", err, client) + return err + } + return nil + }, retry.Attempts(mc.RetryConfig.Attempts), retry.Delay(mc.RetryConfig.Delay)) + if err2 == nil { + return nil + } + fmt.Printf("Client %v failed, trying next client\n", client) + } + return errors.Wrapf(err, "All backup clients %v failed", mc.Backups) +} diff --git a/integration-tests/deployment/multiclient_test.go b/integration-tests/deployment/multiclient_test.go new file mode 100644 index 00000000000..a3176691c0c --- /dev/null +++ b/integration-tests/deployment/multiclient_test.go @@ -0,0 +1,36 @@ +package deployment + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMultiClient(t *testing.T) { + // Expect an error if no RPCs supplied. + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusOK) + _, err := writer.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":true}`)) + require.NoError(t, err) + })) + defer s.Close() + _, err := NewMultiClient([]RPC{}) + require.Error(t, err) + + // Expect defaults to be set if not provided. + mc, err := NewMultiClient([]RPC{{WSURL: s.URL}}) + require.NoError(t, err) + assert.Equal(t, mc.RetryConfig.Attempts, uint(RPC_DEFAULT_RETRY_ATTEMPTS)) + assert.Equal(t, mc.RetryConfig.Delay, RPC_DEFAULT_RETRY_DELAY) + + // Expect second client to be set as backup. + mc, err = NewMultiClient([]RPC{ + {WSURL: s.URL}, + {WSURL: s.URL}, + }) + require.NoError(t, err) + require.Equal(t, len(mc.Backups), 1) +} diff --git a/integration-tests/go.mod b/integration-tests/go.mod index c8304390a00..d7d460801a3 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -11,6 +11,7 @@ require ( github.com/Khan/genqlient v0.7.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/avast/retry-go/v4 v4.6.0 + github.com/aws/aws-sdk-go v1.45.25 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a github.com/cli/go-gh/v2 v2.0.0 @@ -107,7 +108,6 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect - github.com/aws/aws-sdk-go v1.45.25 // indirect github.com/aws/aws-sdk-go-v2 v1.30.4 // indirect github.com/aws/aws-sdk-go-v2/config v1.27.28 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.28 // indirect From 3a89dceab79217880625f7af75db0d798cf79488 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko <34754799+dhaidashenko@users.noreply.github.com> Date: Thu, 19 Sep 2024 19:03:34 +0200 Subject: [PATCH 5/7] use tx in insertLogsWithinTx (#14361) --- .changeset/sixty-cougars-mix.md | 5 +++++ core/chains/evm/logpoller/orm.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/sixty-cougars-mix.md diff --git a/.changeset/sixty-cougars-mix.md b/.changeset/sixty-cougars-mix.md new file mode 100644 index 00000000000..a641ce19762 --- /dev/null +++ b/.changeset/sixty-cougars-mix.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Use tx in insertLogsWithinTx #internal diff --git a/core/chains/evm/logpoller/orm.go b/core/chains/evm/logpoller/orm.go index 5ab7db4eb02..e40fb80f108 100644 --- a/core/chains/evm/logpoller/orm.go +++ b/core/chains/evm/logpoller/orm.go @@ -390,7 +390,7 @@ func (o *DSORM) insertLogsWithinTx(ctx context.Context, logs []Log, tx sqlutil.D (:evm_chain_id, :log_index, :block_hash, :block_number, :block_timestamp, :address, :event_sig, :topics, :tx_hash, :data, NOW()) ON CONFLICT DO NOTHING` - _, err := o.ds.NamedExecContext(ctx, query, logs[start:end]) + _, err := tx.NamedExecContext(ctx, query, logs[start:end]) if err != nil { if pkgerrors.Is(err, context.DeadlineExceeded) && batchInsertSize > 500 { // In case of DB timeouts, try to insert again with a smaller batch upto a limit From 014a9f51df269d6461ce76c9418ce1977e04e3ac Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 19 Sep 2024 13:54:38 -0400 Subject: [PATCH 6/7] Fix CCIP Load Test Faulty Fund Return (#14499) * Stop fund return on nil client * DEBUG * Remove debug --- integration-tests/actions/actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/actions/actions.go b/integration-tests/actions/actions.go index f864d9e7100..e395c9a251a 100644 --- a/integration-tests/actions/actions.go +++ b/integration-tests/actions/actions.go @@ -714,7 +714,7 @@ func TeardownSuite( l.Warn().Msgf("Error deleting jobs %+v", err) } - if chainlinkNodes != nil { + if chainlinkNodes != nil && chainClient != nil { if err := ReturnFundsFromNodes(l, chainClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes)); err != nil { // This printed line is required for tests that use real funds to propagate the failure // out to the system running the test. Do not remove From 7b324cad8f46ab740ba84c889eca684ed1c87799 Mon Sep 17 00:00:00 2001 From: Domino Valdano Date: Thu, 19 Sep 2024 11:02:10 -0700 Subject: [PATCH 7/7] Fix data race in TestLogPoller_Replay (#14431) * Add RWMutex around global head var * Use atomic.Pointer instead of RWMutex --- .../evm/logpoller/log_poller_internal_test.go | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/core/chains/evm/logpoller/log_poller_internal_test.go b/core/chains/evm/logpoller/log_poller_internal_test.go index 448710b93f3..ca1bd72dd6c 100644 --- a/core/chains/evm/logpoller/log_poller_internal_test.go +++ b/core/chains/evm/logpoller/log_poller_internal_test.go @@ -7,6 +7,7 @@ import ( "math/big" "strings" "sync" + "sync/atomic" "testing" "time" @@ -287,12 +288,14 @@ func TestLogPoller_Replay(t *testing.T) { db := pgtest.NewSqlxDB(t) orm := NewORM(chainID, db, lggr) - head := evmtypes.Head{Number: 4} + var head atomic.Pointer[evmtypes.Head] + head.Store(&evmtypes.Head{Number: 4}) + events := []common.Hash{EmitterABI.Events["Log1"].ID} log1 := types.Log{ Index: 0, BlockHash: common.Hash{}, - BlockNumber: uint64(head.Number), + BlockNumber: uint64(head.Load().Number), Topics: events, Address: addr, TxHash: common.HexToHash("0x1234"), @@ -301,8 +304,7 @@ func TestLogPoller_Replay(t *testing.T) { ec := evmclimocks.NewClient(t) ec.On("HeadByNumber", mock.Anything, mock.Anything).Return(func(context.Context, *big.Int) (*evmtypes.Head, error) { - headCopy := head - return &headCopy, nil + return head.Load(), nil }) ec.On("FilterLogs", mock.Anything, mock.Anything).Return([]types.Log{log1}, nil).Once() ec.On("ConfiguredChainID").Return(chainID, nil) @@ -318,9 +320,9 @@ func TestLogPoller_Replay(t *testing.T) { headTracker := htMocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) headTracker.On("LatestAndFinalizedBlock", mock.Anything).Return(func(ctx context.Context) (*evmtypes.Head, *evmtypes.Head, error) { - headCopy := head - finalized := &evmtypes.Head{Number: headCopy.Number - lpOpts.FinalityDepth} - return &headCopy, finalized, nil + h := head.Load() + finalized := &evmtypes.Head{Number: h.Number - lpOpts.FinalityDepth} + return h, finalized, nil }) lp := NewLogPoller(orm, ec, lggr, headTracker, lpOpts) @@ -394,7 +396,7 @@ func TestLogPoller_Replay(t *testing.T) { var wg sync.WaitGroup defer func() { wg.Wait() }() ec.On("FilterLogs", mock.Anything, mock.Anything).Once().Return([]types.Log{log1}, nil).Run(func(args mock.Arguments) { - head = evmtypes.Head{Number: 4} + head.Store(&evmtypes.Head{Number: 4}) wg.Add(1) go func() { defer wg.Done() @@ -421,7 +423,7 @@ func TestLogPoller_Replay(t *testing.T) { ec.On("FilterLogs", mock.Anything, mock.Anything).Return([]types.Log{log1}, nil).Maybe() // in case task gets delayed by >= 100ms - head = evmtypes.Head{Number: 5} + head.Store(&evmtypes.Head{Number: 5}) t.Cleanup(lp.reset) servicetest.Run(t, lp) @@ -448,7 +450,7 @@ func TestLogPoller_Replay(t *testing.T) { go func() { defer close(done) - head = evmtypes.Head{Number: 4} // Restore latest block to 4, so this matches the fromBlock requested + head.Store(&evmtypes.Head{Number: 4}) // Restore latest block to 4, so this matches the fromBlock requested select { case lp.replayStart <- 4: case <-ctx.Done(): @@ -469,7 +471,7 @@ func TestLogPoller_Replay(t *testing.T) { ec.On("FilterLogs", mock.Anything, mock.Anything).Return([]types.Log{log1}, nil) t.Cleanup(lp.reset) - head = evmtypes.Head{Number: 5} // Latest block must be > lastProcessed in order for SaveAndPollLogs() to call FilterLogs() + head.Store(&evmtypes.Head{Number: 5}) // Latest block must be > lastProcessed in order for SaveAndPollLogs() to call FilterLogs() servicetest.Run(t, lp) select { @@ -482,7 +484,8 @@ func TestLogPoller_Replay(t *testing.T) { // ReplayAsync should return as soon as replayStart is received t.Run("ReplayAsync success", func(t *testing.T) { t.Cleanup(lp.reset) - head = evmtypes.Head{Number: 5} + + head.Store(&evmtypes.Head{Number: 5}) ec.On("FilterLogs", mock.Anything, mock.Anything).Return([]types.Log{log1}, nil) mockBatchCallContext(t, ec) servicetest.Run(t, lp) @@ -496,7 +499,7 @@ func TestLogPoller_Replay(t *testing.T) { ctx := testutils.Context(t) t.Cleanup(lp.reset) servicetest.Run(t, lp) - head = evmtypes.Head{Number: 4} + head.Store(&evmtypes.Head{Number: 4}) anyErr := pkgerrors.New("async error") observedLogs.TakeAll() @@ -528,7 +531,8 @@ func TestLogPoller_Replay(t *testing.T) { err := lp.orm.DeleteLogsAndBlocksAfter(ctx, 0) require.NoError(t, err) - err = lp.orm.InsertBlock(ctx, head.Hash, head.Number, head.Timestamp, head.Number) + h := head.Load() + err = lp.orm.InsertBlock(ctx, h.Hash, h.Number, h.Timestamp, h.Number) require.NoError(t, err) ec.On("FilterLogs", mock.Anything, mock.Anything).Return([]types.Log{log1}, nil)