CCIP-3461: Optimize mainnet scheduled test

.github/workflows/ccip-live-network-tests.yml

@@ -15,8 +15,8 @@ on:
         required: false
         type: choice
         options:
-          - 'load'
           - 'smoke'
+          - 'load'
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
@@ -41,19 +41,21 @@ env:
   E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
   # Default private key test secret loaded from Github Secret as only security team has access to it.
   # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets.
-  E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_ARBITRUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_AVALANCHE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_BLAST_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_CELO_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_METIS_ANDROMEDA_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_OPTIMISM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_POLYGON_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_BSC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+
 
 jobs:
   build-chainlink:
@@ -133,8 +135,8 @@ jobs:
       matrix:
         config: [mainnet.toml]
     needs: [ build-chainlink, build-test-image ]
-    # if the event is a scheduled event or the test type is load and no previous job failed
-    if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }}
+    # if the event is a workflow_dispatch event and the test type is load and no previous job failed
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'load' && !contains(needs.*.result, 'failure') }}
     permissions:
       issues: read
       checks: write
@@ -207,8 +209,8 @@ jobs:
           RR_CPU: 4
           DETACH_RUNNER: true
           TEST_TRIGGERED_BY: ccip-load-test-ci
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}          
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
           E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
         with:
           test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -233,8 +235,8 @@ jobs:
     environment: integration
     runs-on: ubuntu-latest
     needs: [ build-chainlink, build-test-image ]
-    # if the event is a scheduled event or the test type is load and no previous job failed
-    if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }}
+    # if the event is a scheduled event or the test type is smoke and no previous job failed
+    if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'smoke') && !contains(needs.*.result, 'failure') }}
     permissions:
       issues: read
       checks: write
@@ -276,6 +278,12 @@ jobs:
             echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
             echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
           fi
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+             BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }})
+             echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
+             echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
+             echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV
+           fi
       - name: step summary
         shell: bash
         run: |
@@ -301,11 +309,11 @@ jobs:
           RR_MEM: 8Gi
           RR_CPU: 4
           TEST_TRIGGERED_BY: ccip-smoke-test-ci
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}                    
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
           E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
         with:
-          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
+          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 3h -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           # Other default test secrets loaded from dotenv Github Secret.
           test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }}

integration-tests/ccip-tests/actions/ccip_helpers.go

@@ -1648,44 +1648,48 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe
 
 // IsPastRequestTriggeredWithinTimeframe determines the average block time and calculates the block numbers
 // within the specified timeframe. It then uses FilterCCIPSendRequested to identify the past events.
-func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(ctx context.Context, timeframe *commonconfig.Duration) (*time.Time, error) {
+func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(
+	ctx context.Context,
+	timeframe *commonconfig.Duration,
+) (latestEvent *types.Log, err error) {
 	if timeframe == nil {
 		return nil, nil
 	}
 	//var foundAt *time.Time
-	latestBlock, err := sourceCCIP.Common.ChainClient.LatestBlockNumber(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error while getting latest source block number. Error: %w", err)
+	latestBlock, err1 := sourceCCIP.Common.ChainClient.LatestBlockNumber(ctx)
+	if err1 != nil {
+		return nil, fmt.Errorf("error while getting latest source block number. Error: %w", err1)
 	}
-	avgBlockTime, err := sourceCCIP.Common.ChainClient.AvgBlockTime(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error while getting average source block time. Error: %w", err)
+	avgBlockTime, err1 := sourceCCIP.Common.ChainClient.AvgBlockTime(ctx)
+	if err1 != nil {
+		return nil, fmt.Errorf("error while getting average source block time. Error: %w", err1)
 	}
 	filterFromBlock := latestBlock - uint64(timeframe.Duration()/avgBlockTime)
 
-	onRampContract, err := evm_2_evm_onramp.NewEVM2EVMOnRamp(common.HexToAddress(sourceCCIP.OnRamp.EthAddress.Hex()),
+	onRampContract, err1 := evm_2_evm_onramp.NewEVM2EVMOnRamp(common.HexToAddress(sourceCCIP.OnRamp.EthAddress.Hex()),
 		sourceCCIP.Common.ChainClient.Backend())
-	if err != nil {
-		return nil, fmt.Errorf("error while on ramp contract. Error: %w", err)
+	if err1 != nil {
+		return nil, fmt.Errorf("error while on ramp contract. Error: %w", err1)
 	}
-	iterator, err := onRampContract.FilterCCIPSendRequested(&bind.FilterOpts{
+	iterator, err1 := onRampContract.FilterCCIPSendRequested(&bind.FilterOpts{
 		Start: filterFromBlock,
 	})
-	if err != nil {
-		return nil, fmt.Errorf("error while filtering CCIP send requested starting block number: %d. Error: %w", filterFromBlock, err)
+	if err1 != nil {
+		return nil, fmt.Errorf("error while filtering CCIP send requested starting block number: %d. Error: %w", filterFromBlock, err1)
 	}
 	defer func() {
-		_ = iterator.Close()
+		err = iterator.Close()
 	}()
-	if iterator.Next() {
-		hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(iterator.Event.Raw.BlockNumber)))
-		if err != nil {
-			return nil, fmt.Errorf("error getting header for block: %d, Error: %w", iterator.Event.Raw.BlockNumber, err)
+	lastBlockNumber := uint64(0)
+	latestEvent = nil
+	for iterator.Next() {
+		blockNum := iterator.Event.Raw.BlockNumber
+		if blockNum > lastBlockNumber {
+			lastBlockNumber = blockNum
+			latestEvent = &iterator.Event.Raw
 		}
-		return pointer.ToTime(hdr.Timestamp), nil
 	}
-
-	return nil, nil
+	return latestEvent, nil
 }
 
 func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested(

integration-tests/ccip-tests/load/ccip_loadgen.go

@@ -226,13 +226,19 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response {
 	res := &wasp.Response{}
 	sourceCCIP := c.Lane.Source
 	var recentRequestFoundAt *time.Time
+	var latestEvent *types.Log
 	var err error
 	// Use IsPastRequestTriggeredWithinTimeframe to check for any historical CCIP send request events
 	// within the specified timeframe for the first message. Subsequently, use the watcher method to monitor
 	// and detect any new events as they occur.
 	if c.CurrentMsgSerialNo.Load() == int64(1) {
-		recentRequestFoundAt, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin)
+		latestEvent, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin)
 		require.NoError(c.t, err, "error while filtering past requests")
+		if latestEvent != nil {
+			hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(latestEvent.BlockNumber)))
+			require.NoError(c.t, err, "error while getting header by block number")
+			recentRequestFoundAt = pointer.ToTime(hdr.Timestamp)
+		}
 	} else {
 		recentRequestFoundAt = sourceCCIP.IsRequestTriggeredWithinTimeframe(c.SkipRequestIfAnotherRequestTriggeredWithin)
 	}

integration-tests/ccip-tests/load/ccip_multicall_loadgen.go

@@ -91,7 +91,7 @@ func NewMultiCallLoadGenerator(testCfg *testsetups.CCIPTestConfig, lanes []*acti
 			testCfg.Test, lane, testCfg.TestGroupInput.PhaseTimeout.Duration(),
 			100000,
 			testCfg.TestGroupInput.LoadProfile.MsgProfile, 0,
-			testCfg.TestGroupInput.LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin,
+			testCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin,
 		)
 		ccipLoad.BeforeAllCall()
 		m.E2ELoads[fmt.Sprintf("%s-%s", lane.SourceNetworkName, lane.DestNetworkName)] = ccipLoad

integration-tests/ccip-tests/load/helper.go

@@ -276,7 +276,7 @@ func (l *LoadArgs) TriggerLoadByLane() {
 		ccipLoad := NewCCIPLoad(
 			l.TestCfg.Test, lane, l.TestCfg.TestGroupInput.PhaseTimeout.Duration(),
 			100000, l.TestCfg.TestGroupInput.LoadProfile.MsgProfile, sendMaxData,
-			l.TestCfg.TestGroupInput.LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin,
+			l.TestCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin,
 		)
 		ccipLoad.BeforeAllCall()
 		// if it's not multicall set the tokens to nil to free up some space,

integration-tests/ccip-tests/smoke/ccip_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/core/types"
+
 	"github.com/AlekSi/pointer"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
@@ -54,20 +56,40 @@ func TestSmokeCCIPForBidirectionalLane(t *testing.T) {
 		require.NoError(t, setUpOutput.TearDown())
 	})
 
-	// Create test definitions for each lane.
 	var tests []testDefinition
-	for _, lane := range setUpOutput.Lanes {
-		tests = append(tests, testDefinition{
-			testName: fmt.Sprintf("CCIP message transfer from network %s to network %s",
-				lane.ForwardLane.SourceNetworkName, lane.ForwardLane.DestNetworkName),
-			lane: lane.ForwardLane,
-		})
-		if lane.ReverseLane != nil {
+	lookBackDuration := TestCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin
+	var recentTxFound *types.Log
+	var err error
+
+	addLanesToTest := func(lane *actions.CCIPLane) {
+		// Create test definitions for given lane if no previous request has been triggered within the specified timeframe.
+		// By default, the timeframe is set to nil. To define a timeframe, assign a duration to the variable
+		// SkipRequestIfAnotherRequestTriggeredWithin.
+		if lookBackDuration != nil {
+			recentTxFound, err = lane.Source.IsPastRequestTriggeredWithinTimeframe(lane.Context, lookBackDuration)
+			require.NoError(t, err, "error while finding recent request for lane network %s to network %s",
+				lane.SourceNetworkName, lane.DestNetworkName)
+		}
+		if recentTxFound == nil {
 			tests = append(tests, testDefinition{
 				testName: fmt.Sprintf("CCIP message transfer from network %s to network %s",
-					lane.ReverseLane.SourceNetworkName, lane.ReverseLane.DestNetworkName),
-				lane: lane.ReverseLane,
+					lane.SourceNetworkName, lane.DestNetworkName),
+				lane: lane,
 			})
+		} else {
+			log.Info().
+				Str("TX", recentTxFound.TxHash.Hex()).
+				Uint64("Block Number", recentTxFound.BlockNumber).
+				Str("Source", lane.SourceNetworkName).
+				Str("Dest", lane.DestNetworkName).
+				Msgf("Lane Skipped. Recent request found within %v minutes.", lookBackDuration.Duration().Minutes())
+		}
+	}
+	for _, lane := range setUpOutput.Lanes {
+		addLanesToTest(lane.ForwardLane)
+		if lane.ReverseLane != nil {
+			recentTxFound = nil
+			addLanesToTest(lane.ReverseLane)
 		}
 	}
 

integration-tests/ccip-tests/testconfig/README.md

@@ -657,6 +657,13 @@ Specifies the OCR parameters for the execute job. This is only valid if the test
 
 Specifies the value for the `InflightExpiry` in commit job's offchain config. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment).
 
+### CCIP.Groups.[testgroup].SkipRequestIfAnotherRequestTriggeredWithin
+
+If there is CCIP Send requested event present within this duration, the test will skip sending another 
+request during load run or avoid sending request in smoke test in that lane. For Example,
+if `SkipRequestIfAnotherRequestTriggeredWithin` is set to `40m`, and a request is triggered at 0th second, the test will skip sending another request for another 40m.
+This particular field is used to avoid sending transaction when there is traffic already in that lane.
+
 ### CCIP.Groups.[testgroup].OffRampConfig
 
 Specifies the offramp configuration for the execution job. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment).
@@ -736,11 +743,6 @@ Specifies the duration network delay used for `NetworkChaos` experiment. This is
 
 If there are multiple chaos experiments, this specifies the duration to wait between each chaos experiment. This is only valid if the test is run on k8s and not on [existing deployments](#ccipgroupstestgroupexistingdeployment).
 
-#### CCIP.Groups.[testgroup].LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin
-
-If a request is triggered within this duration, the test will skip sending another request during load run. For Example, if `SkipRequestIfAnotherRequestTriggeredWithin` is set to `40m`, and a request is triggered at 0th second, the test will skip sending another request for another 40m.
-This particular field is used to avoid sending multiple requests in a short duration during load run.
-
 #### CCIP.Groups.[testgroup].LoadProfile.OptimizeSpace
 
 This is used internally to optimize memory usage during load run. If set to true, after the initial lane set up is over the test will discard the lane config to save memory.