CCIP-3461: Optimize mainnet soak test (#1458)

### Motivation Optimize the resource and fund consumption of the mainnet soak test. Currently, the mainnet soak test runs every six hours in Kubernetes, executing one CCIP transaction per hour for a duration of five hours across 28 bidirectional lanes. This setup is designed to provide consistent observability data on the CCIP mainnet, allowing us to differentiate between service outages and quiet periods. However, these tests are inefficient, consuming Kubernetes resources and significant mainnet funds. https://smartcontract-it.atlassian.net/browse/CCIP-3461 ### Ideas: Discussion [thread](https://chainlink-core.slack.com/archives/C02PBKHCP28/p1726678958525029) is initiated with o11y team and decided that it's not required to create txs for every hour instead create one tx if there are no activity for last 24h. Convert to smoke test as we are planning to fire only one request instead of Soak test. Converting to smoke will elevate the K8 resource consumption as the test will run using github runner. ### Solution - [x] 1. Modify the pipeline to run it as smoke test - [x] 2. Add traffic check to smoke test - [x] 3. Add new set of additional 21 lanes and resulted in total of 49 lanes. (i.e 98 unique lanes) - [x] 5. Add RPCs, Wallet key for new lanes to test secrets - [ ] 6. Load funds for new lanes - [x] 7. Schedule pipeline to run for every 6hrs ### Key outcomes: Present transaction count: 1tx * (28 * 2)lanes * 24hrs = 1344 After this change: 1tx * (49 * 2)lanes = 98 which is close to 92% reduction with additional 42 lanes coverage. As per the last [analysis](https://docs.google.com/document/d/1MqXoIyGtGE9o2usgpWdIHQUMblKTyeAvoa06MEsExpc/edit?pli=1#heading=h.8kwk6enqa4w) on the cost, we spend around 84k per quarter. I expect the fund reduction close to 90% which will give saving close to 300k annually. --------- Co-authored-by: Brandon West <[email protected]>
smartcontractkit · Sep 26, 2024 · 374482e · 374482e
1 parent 390ee23
commit 374482e
Show file tree

Hide file tree

Showing 9 changed files with 425 additions and 166 deletions.
diff --git a/.github/workflows/ccip-live-network-tests.yml b/.github/workflows/ccip-live-network-tests.yml
@@ -15,8 +15,8 @@ on:
         required: false
         type: choice
         options:
-          - 'load'
           - 'smoke'
+          - 'load'
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
@@ -30,8 +30,6 @@ concurrency:
 env:
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
   CHAINLINK_VERSION: ${{ github.sha }}
-  CHAINLINK_TEST_VERSION: ${{ github.sha }}
-  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }}
   INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
   AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws
   E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
@@ -41,19 +39,21 @@ env:
   E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
   # Default private key test secret loaded from Github Secret as only security team has access to it.
   # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets.
-  E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_ARBITRUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_AVALANCHE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_BLAST_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_CELO_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_METIS_ANDROMEDA_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_OPTIMISM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
   E2E_TEST_POLYGON_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
-  E2E_TEST_BSC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+  E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
+
 
 jobs:
   build-chainlink:
@@ -99,6 +99,7 @@ jobs:
 
   build-test-image:
     environment: integration
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'load' }}
     permissions:
       id-token: write
       contents: read
@@ -133,8 +134,8 @@ jobs:
       matrix:
         config: [mainnet.toml]
     needs: [ build-chainlink, build-test-image ]
-    # if the event is a scheduled event or the test type is load and no previous job failed
-    if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }}
+    # if the event is a workflow_dispatch event and the test type is load and no previous job failed
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'load' && !contains(needs.*.result, 'failure') }}
     permissions:
       issues: read
       checks: write
@@ -147,6 +148,8 @@ jobs:
       SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
       TEST_LOG_LEVEL: info
       REF_NAME: ${{ github.head_ref || github.ref_name }}
+      CHAINLINK_TEST_VERSION: ${{ github.sha }}
+      ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }}
       ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests
 
     steps:
@@ -207,11 +210,11 @@ jobs:
           RR_CPU: 4
           DETACH_RUNNER: true
           TEST_TRIGGERED_BY: ccip-load-test-ci
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}          
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
           E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
         with:
-          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
+          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           # Other default test secrets loaded from dotenv Github Secret.
           test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }}
@@ -229,12 +232,12 @@ jobs:
           should_cleanup: false
 
   ccip-smoke-test:
-    name: CCIP smoke Test
+    name: CCIP smoke Test ${{ matrix.lanes.name }}
     environment: integration
     runs-on: ubuntu-latest
-    needs: [ build-chainlink, build-test-image ]
-    # if the event is a scheduled event or the test type is load and no previous job failed
-    if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }}
+    needs: [ build-chainlink ]
+    # if the event is a scheduled event or the test type is smoke and no previous job failed
+    if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'smoke') && !contains(needs.*.result, 'failure') }}
     permissions:
       issues: read
       checks: write
@@ -247,10 +250,69 @@ jobs:
       SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
       TEST_LOG_LEVEL: info
       REF_NAME: ${{ github.head_ref || github.ref_name }}
-      ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests
-
+    strategy:
+      fail-fast: false
+      matrix:
+        lanes:
+          - name: 'ARBITRUM_MAINNET'
+            pairs: 'ARBITRUM_MAINNET,BSC_MAINNET;ARBITRUM_MAINNET,OPTIMISM_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'AVALANCHE_MAINNET'
+            pairs: 'AVALANCHE_MAINNET,ARBITRUM_MAINNET;AVALANCHE_MAINNET,BASE_MAINNET;AVALANCHE_MAINNET,BSC_MAINNET;AVALANCHE_MAINNET,OPTIMISM_MAINNET;AVALANCHE_MAINNET,POLYGON_MAINNET;AVALANCHE_MAINNET,WEMIX_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'BASE_MAINNET'
+            pairs: 'BASE_MAINNET,ARBITRUM_MAINNET;BASE_MAINNET,BSC_MAINNET;BASE_MAINNET,OPTIMISM_MAINNET;BASE_MAINNET,POLYGON_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'BLAST_MAINNET'
+            pairs: 'BLAST_MAINNET,ARBITRUM_MAINNET;BLAST_MAINNET,BASE_MAINNET;BLAST_MAINNET,BSC_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'BSC_MAINNET'
+            pairs: 'BSC_MAINNET,OPTIMISM_MAINNET;BSC_MAINNET,POLYGON_MAINNET;BSC_MAINNET,WEMIX_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'ETHEREUM_MAINNET 1'
+            pairs: 'ETHEREUM_MAINNET,ARBITRUM_MAINNET;ETHEREUM_MAINNET,AVALANCHE_MAINNET;ETHEREUM_MAINNET,BASE_MAINNET;ETHEREUM_MAINNET,BLAST_MAINNET;ETHEREUM_MAINNET,BSC_MAINNET;ETHEREUM_MAINNET,CELO_MAINNET;ETHEREUM_MAINNET,GNOSIS_MAINNET;ETHEREUM_MAINNET,OPTIMISM_MAINNET;ETHEREUM_MAINNET,POLYGON_MAINNET;ETHEREUM_MAINNET,WEMIX_MAINNET'
+            enabled: true
+            phaseTimeout: 40m
+          - name: 'ETHEREUM_MAINNET 2'
+            pairs: 'ETHEREUM_MAINNET,METIS_ANDROMEDA;ETHEREUM_MAINNET,ZKSYNC_MAINNET'
+            enabled: true
+            phaseTimeout: 90m
+          - name: 'GNOSIS_MAINNET'
+            pairs: 'GNOSIS_MAINNET,ARBITRUM_MAINNET;GNOSIS_MAINNET,AVALANCHE_MAINNET;GNOSIS_MAINNET,BASE_MAINNET;GNOSIS_MAINNET,BSC_MAINNET;GNOSIS_MAINNET,OPTIMISM_MAINNET;GNOSIS_MAINNET,POLYGON_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'METIS_ANDROMEDA'
+            pairs: 'METIS_ANDROMEDA,ARBITRUM_MAINNET'
+            enabled: true
+            phaseTimeout: 60m
+          - name: 'MODE_MAINNET'
+            pairs: 'MODE_MAINNET,OPTIMISM_MAINNET;MODE_MAINNET,ARBITRUM_MAINNET;MODE_MAINNET,BASE_MAINNET;MODE_MAINNET,BSC_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'OPTIMISM_MAINNET'
+            pairs: 'OPTIMISM_MAINNET,POLYGON_MAINNET;OPTIMISM_MAINNET,WEMIX_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'POLYGON_MAINNET'
+            pairs: 'POLYGON_MAINNET,ARBITRUM_MAINNET;POLYGON_MAINNET,WEMIX_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'WEMIX_MAINNET'
+            pairs: 'WEMIX_MAINNET,ARBITRUM_MAINNET;WEMIX_MAINNET,KROMA_MAINNET'
+            enabled: true
+            phaseTimeout: 20m
+          - name: 'ZKSYNC_MAINNET'
+            pairs: 'ZKSYNC_MAINNET,ARBITRUM_MAINNET'
+            enabled: true
+            phaseTimeout: 90m
     steps:
       - name: Collect Metrics
+        if: ${{ matrix.lanes.enabled == true }}
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
         with:
@@ -261,10 +323,12 @@ jobs:
           this-job-name: CCIP Smoke Test
         continue-on-error: true
       - name: Checkout the repo
+        if: ${{ matrix.lanes.enabled == true }}
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           ref: ${{ env.REF_NAME }}
       - name: Prepare Base64 TOML override
+        if: ${{ matrix.lanes.enabled == true }}
         id: set_override_config
         shell: bash
         run: |
@@ -276,14 +340,20 @@ jobs:
             echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
             echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
           fi
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+             BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/mainnet.toml)
+             echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
+             echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
+             echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV
+           fi
       - name: step summary
+        if: ${{ matrix.lanes.enabled == true }}
         shell: bash
         run: |
           echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
           echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
       - name: Prepare Base64 TOML override for CCIP secrets
+        if: ${{ matrix.lanes.enabled == true }}
         uses: ./.github/actions/setup-create-base64-config-ccip
         id: setup_create_base64_config_ccip
         with:
@@ -293,19 +363,22 @@ jobs:
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
+        if: ${{ matrix.lanes.enabled == true }}
         env:
           TEST_SUITE: smoke
           TEST_ARGS: -test.timeout 900h
-          DETACH_RUNNER: true
+          DETACH_RUNNER: false
           DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
           RR_MEM: 8Gi
           RR_CPU: 4
           TEST_TRIGGERED_BY: ccip-smoke-test-ci
-          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
-          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}                    
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
           E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          OVERRIDE_NETWORK_PAIRS: ${{ matrix.lanes.pairs }}
+          OVERRIDE_PHASE_TIMEOUT: ${{ matrix.lanes.phaseTimeout }}
         with:
-          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
+          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 3h -count=1 -p 30 -json -run ^TestSmokeCCIPForGivenNetworkPairs$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           # Other default test secrets loaded from dotenv Github Secret.
           test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }}
@@ -314,7 +387,6 @@ jobs:
           go_mod_path: ./integration-tests/go.mod
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           triggered_by: ${{ env.TEST_TRIGGERED_BY }}
           artifacts_location: ./integration-tests/smoke/logs/payload_ccip.json
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go
@@ -1648,7 +1648,10 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe
 
 // IsPastRequestTriggeredWithinTimeframe determines the average block time and calculates the block numbers
 // within the specified timeframe. It then uses FilterCCIPSendRequested to identify the past events.
-func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(ctx context.Context, timeframe *commonconfig.Duration) (*time.Time, error) {
+func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(
+	ctx context.Context,
+	timeframe *commonconfig.Duration,
+) (*types.Log, error) {
 	if timeframe == nil {
 		return nil, nil
 	}
@@ -1675,17 +1678,21 @@ func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(ctx co
 		return nil, fmt.Errorf("error while filtering CCIP send requested starting block number: %d. Error: %w", filterFromBlock, err)
 	}
 	defer func() {
-		_ = iterator.Close()
+		iterErr := iterator.Close()
+		if iterErr != nil {
+			sourceCCIP.Common.Logger.Error().Err(iterErr).Msg("Error closing iterator")
+		}
 	}()
-	if iterator.Next() {
-		hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(iterator.Event.Raw.BlockNumber)))
-		if err != nil {
-			return nil, fmt.Errorf("error getting header for block: %d, Error: %w", iterator.Event.Raw.BlockNumber, err)
+	lastBlockNumber := uint64(0)
+	var latestEvent *types.Log
+	for iterator.Next() {
+		blockNum := iterator.Event.Raw.BlockNumber
+		if blockNum > lastBlockNumber {
+			lastBlockNumber = blockNum
+			latestEvent = &iterator.Event.Raw
 		}
-		return pointer.ToTime(hdr.Timestamp), nil
 	}
-
-	return nil, nil
+	return latestEvent, nil
 }
 
 func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested(

diff --git a/integration-tests/ccip-tests/load/ccip_loadgen.go b/integration-tests/ccip-tests/load/ccip_loadgen.go
@@ -226,13 +226,19 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response {
 	res := &wasp.Response{}
 	sourceCCIP := c.Lane.Source
 	var recentRequestFoundAt *time.Time
+	var latestEvent *types.Log
 	var err error
 	// Use IsPastRequestTriggeredWithinTimeframe to check for any historical CCIP send request events
 	// within the specified timeframe for the first message. Subsequently, use the watcher method to monitor
 	// and detect any new events as they occur.
 	if c.CurrentMsgSerialNo.Load() == int64(1) {
-		recentRequestFoundAt, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin)
+		latestEvent, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin)
 		require.NoError(c.t, err, "error while filtering past requests")
+		if latestEvent != nil {
+			hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(latestEvent.BlockNumber)))
+			require.NoError(c.t, err, "error while getting header by block number")
+			recentRequestFoundAt = pointer.ToTime(hdr.Timestamp)
+		}
 	} else {
 		recentRequestFoundAt = sourceCCIP.IsRequestTriggeredWithinTimeframe(c.SkipRequestIfAnotherRequestTriggeredWithin)
 	}

diff --git a/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go b/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go
@@ -91,7 +91,7 @@ func NewMultiCallLoadGenerator(testCfg *testsetups.CCIPTestConfig, lanes []*acti
 			testCfg.Test, lane, testCfg.TestGroupInput.PhaseTimeout.Duration(),
 			100000,
 			testCfg.TestGroupInput.LoadProfile.MsgProfile, 0,
-			testCfg.TestGroupInput.LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin,
+			testCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin,
 		)
 		ccipLoad.BeforeAllCall()
 		m.E2ELoads[fmt.Sprintf("%s-%s", lane.SourceNetworkName, lane.DestNetworkName)] = ccipLoad

diff --git a/integration-tests/ccip-tests/load/helper.go b/integration-tests/ccip-tests/load/helper.go
@@ -276,7 +276,7 @@ func (l *LoadArgs) TriggerLoadByLane() {
 		ccipLoad := NewCCIPLoad(
 			l.TestCfg.Test, lane, l.TestCfg.TestGroupInput.PhaseTimeout.Duration(),
 			100000, l.TestCfg.TestGroupInput.LoadProfile.MsgProfile, sendMaxData,
-			l.TestCfg.TestGroupInput.LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin,
+			l.TestCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin,
 		)
 		ccipLoad.BeforeAllCall()
 		// if it's not multicall set the tokens to nil to free up some space,