[AUTO-7258] Setup log trigger load test using wasp (#11267)

* add simple log upkeep counter contract * WIP- add automationv2_1 load test * update contract * working state * add config * try GHA * disable reads inside loadgen * updates * add specs * fix linkfunds * detach runner, add loglevel config * increase geth capacity * fix batching - test end FilterLogs * fix batching * fix log filter context and batching * increase upkeep funding, fix event count calc * increase GasLimitPerReport to 10.3M * Decrease PerformLockoutWindow to 80k * reduce sleep time - log filter batch * use ConcurrentEVMClients * increase RR CPU and Mem * add pyroscope, add slack notifications * add tag and image to test config * fix dashbord URL * contracts prettier * run lint * add new load test workflow * fix AddNetworksConfig * undo changes to benchmark test workflow * add load/automationv2_1 to build test image steps * lint
smartcontractkit · Nov 17, 2023 · 55d4d9c · 55d4d9c
1 parent 4eb170b
commit 55d4d9c
Show file tree

Hide file tree

Showing 15 changed files with 1,447 additions and 5 deletions.
diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml
@@ -15,7 +15,7 @@ inputs:
     required: false
   suites:
     description: The test suites to build into the image
-    default: chaos migration performance reorg smoke soak benchmark
+    default: chaos migration performance reorg smoke soak benchmark load/automationv2_1
     required: false
   base_image_tag:
     description: The test base image version to use, if not provided it will use the version from the ./integration-tests/go.mod file

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
@@ -6,7 +6,7 @@ on:
         description: Chainlink image version to use
         required: true
         type: string
-        default: 2.5.0
+        default: 2.6.0
       chainlinkImage:
         description: Chainlink image repo to use
         required: true
@@ -108,6 +108,7 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          suites: benchmark load/automationv2_1 chaos reorg
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
         env:

diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml
@@ -0,0 +1,105 @@
+name: Automation Load Test
+on:
+  workflow_dispatch:
+    inputs:
+      chainlinkVersion:
+        description: Chainlink image version to use
+        required: true
+        type: string
+        default: 2.6.0
+      chainlinkImage:
+        description: Chainlink image repo to use
+        required: true
+        type: string
+        default: public.ecr.aws/chainlink/chainlink
+      network:
+        description: Network to run tests on
+        required: true
+        type: choice
+        options:
+          - SIMULATED
+      TestInputs:
+        description: TestInputs
+        required: false
+        type: string
+      slackMemberID:
+        description: Notifies test results (Not your @)
+        required: true
+        default: U02Q14G80TY
+        type: string
+
+jobs:
+  automation_load:
+    environment: integration
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    name: ${{ inputs.network }} Automation Load Test
+    runs-on: ubuntu20.04-16cores-64GB
+    env:
+      SELECTED_NETWORKS: ${{ inputs.network }}
+      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
+      SLACK_CHANNEL: C03KJ5S7KEK
+      TEST_INPUTS: ${{ inputs.TestInputs }}
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      REF_NAME: ${{ github.head_ref || github.ref_name }}
+    steps:
+      - name: Setup Push Tag
+        shell: bash
+        run: |
+          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${{ inputs.chainlinkVersion }}\`" >>$GITHUB_STEP_SUMMARY
+          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
+
+      - name: Add mask
+        run: |
+          SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
+          echo ::add-mask::$SLACK_USER
+          echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.REF_NAME }}
+      - name: Build Test Image
+        uses: ./.github/actions/build-test-image
+        with:
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          suites: benchmark load/automationv2_1 chaos reorg
+      - name: Run Tests
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
+        env:
+          RR_CPU: 4000m
+          RR_MEM: 4Gi
+          DETACH_RUNNER: true
+          TEST_SUITE: automationv2_1
+          TEST_ARGS: -test.timeout 720h
+          ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
+          INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
+          PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+        with:
+          test_command_to_run: cd integration-tests && go test -timeout 1h -v -run TestLogTrigger ./load/automationv2_1 -count=1
+          test_download_vendor_packages_command: make gomod
+          cl_repo: ${{ inputs.chainlinkImage }}
+          cl_image_tag: ${{ inputs.chainlinkVersion }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          should_cleanup: false
+          go_mod_path: ./integration-tests/go.mod
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      - name: Collect Metrics
+        if: always()
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: ${{ inputs.network }} Automation Load Test
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -104,6 +104,7 @@ core/scripts/gateway @bolekk @pinebit
 /.github/workflows/performance-tests.yml @smartcontractkit/test-tooling-team
 /.github/workflows/automation-ondemand-tests.yml @smartcontractkit/keepers
 /.github/workflows/automation-benchmark-tests.yml @smartcontractkit/keepers
+/.github/workflows/automation-load-tests.yml @smartcontractkit/keepers
 
 /core/chainlink.Dockerfile @smartcontractkit/prodsec-public
 

diff --git a/contracts/scripts/native_solc_compile_all_automation b/contracts/scripts/native_solc_compile_all_automation
@@ -41,6 +41,7 @@ compileContract automation/v2_0/KeeperRegistryLogic2_0.sol
 compileContract automation/UpkeepTranscoder.sol
 compileContract automation/mocks/MockAggregatorProxy.sol
 compileContract automation/testhelpers/LogUpkeepCounter.sol
+compileContract automation/testhelpers/SimpleLogUpkeepCounter.sol
 
 compileContract automation/mocks/KeeperRegistrar1_2Mock.sol
 compileContract automation/mocks/KeeperRegistryCheckUpkeepGasUsageWrapper1_2Mock.sol

diff --git a/contracts/src/v0.8/automation/testhelpers/SimpleLogUpkeepCounter.sol b/contracts/src/v0.8/automation/testhelpers/SimpleLogUpkeepCounter.sol
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: MIT
+
+pragma solidity 0.8.6;
+
+import {ILogAutomation, Log} from "../interfaces/ILogAutomation.sol";
+
+contract SimpleLogUpkeepCounter is ILogAutomation {
+  event PerformingUpkeep(
+    address indexed from,
+    uint256 initialBlock,
+    uint256 lastBlock,
+    uint256 previousBlock,
+    uint256 counter,
+    uint256 timeToPerform
+  );
+
+  uint256 public lastBlock;
+  uint256 public previousPerformBlock;
+  uint256 public initialBlock;
+  uint256 public counter;
+  uint256 public timeToPerform;
+
+  constructor() {
+    previousPerformBlock = 0;
+    lastBlock = block.number;
+    initialBlock = 0;
+    counter = 0;
+  }
+
+  function checkLog(Log calldata log, bytes memory) external view override returns (bool, bytes memory) {
+    return (true, abi.encode(log));
+  }
+
+  function performUpkeep(bytes calldata performData) external override {
+    if (initialBlock == 0) {
+      initialBlock = block.number;
+    }
+    lastBlock = block.number;
+    counter = counter + 1;
+    previousPerformBlock = lastBlock;
+    Log memory log = abi.decode(performData, (Log));
+    timeToPerform = block.timestamp - log.timestamp;
+    emit PerformingUpkeep(tx.origin, initialBlock, lastBlock, previousPerformBlock, counter, timeToPerform);
+  }
+}