CCIP On-Demand Live Network Tests #237
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CCIP On-Demand Live Network Tests | |
| on: | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| base64_test_input : # base64 encoded toml for test input | |
| description: 'Base64 encoded toml test input' | |
| required: false | |
| slackMemberID: | |
| description: 'Slack member ID to notify' | |
| required: false | |
| test_type: | |
| description: 'Type of test to run' | |
| required: false | |
| type: choice | |
| options: | |
| - 'load' | |
| - 'smoke' | |
| test_secrets_override_key: | |
| description: 'Key to run tests with custom test secrets' | |
| required: false | |
| type: string | |
| # Only run 1 of this workflow at a time per PR | |
| concurrency: | |
| group: live-testnet-tests | |
| cancel-in-progress: true | |
| env: | |
| CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| CHAINLINK_VERSION: ${{ github.sha }} | |
| CHAINLINK_TEST_VERSION: ${{ github.sha }} | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} | |
| INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
| AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws | |
| jobs: | |
| build-chainlink: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Chainlink Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Check if image exists | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 | |
| with: | |
| repository: chainlink | |
| tag: ${{ env.CHAINLINK_VERSION }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Build Image | |
| if: steps.check-image.outputs.exists == 'false' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| with: | |
| cl_repo: smartcontractkit/chainlink-ccip | |
| cl_ref: ${{ env.CHAINLINK_VERSION }} | |
| push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-build-chainlink-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Chainlink Image | |
| continue-on-error: true | |
| build-test-image: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-build-test-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Build Test Image | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| tag: ${{ env.CHAINLINK_TEST_VERSION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| ccip-load-test: | |
| name: CCIP Load Test | |
| environment: integration | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| config: [mainnet.toml] | |
| needs: [ build-chainlink, build-test-image ] | |
| # if the event is a scheduled event or the test type is load and no previous job failed | |
| if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }} | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| env: | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
| SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
| TEST_LOG_LEVEL: info | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
| BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-load-tests | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: CCIP Load Test | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ env.REF_NAME }} | |
| - name: Prepare Base64 TOML override | |
| id: set_override_config | |
| shell: bash | |
| run: | | |
| # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets. | |
| BASE64_NETWORK_CONFIG=$(echo $BASE64_NETWORK_CONFIG | base64 -w 0 -d | sed -e 's/evm_key/${{ secrets.QA_SHARED_803C_KEY }}/g' | base64 -w 0) | |
| echo ::add-mask::$BASE64_NETWORK_CONFIG | |
| echo "BASE64_NETWORK_CONFIG=$BASE64_NETWORK_CONFIG" >> "$GITHUB_ENV" | |
| SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$SLACK_USER | |
| echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
| echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }}) | |
| echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
| echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
| echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV | |
| fi | |
| - name: step summary | |
| shell: bash | |
| run: | | |
| echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Prepare Base64 TOML override for CCIP secrets | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| id: setup_create_base64_config_ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| chainlinkVersion: ${{ env.CHAINLINK_VERSION }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 | |
| env: | |
| TEST_SUITE: load | |
| TEST_ARGS: -test.timeout 900h | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| RR_MEM: 8Gi | |
| RR_CPU: 4 | |
| DETACH_RUNNER: true | |
| TEST_TRIGGERED_BY: ccip-load-test-ci | |
| BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| with: | |
| test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| artifacts_location: ./integration-tests/load/logs/payload_ccip.json | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| should_cleanup: false | |
| DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} | |
| DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} | |
| DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} | |
| DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} | |
| DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} | |
| DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" | |
| ccip-smoke-test: | |
| name: CCIP smoke Test | |
| environment: integration | |
| runs-on: ubuntu-latest | |
| needs: [ build-chainlink, build-test-image ] | |
| # if the event is a scheduled event or the test type is load and no previous job failed | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }} | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| env: | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
| SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
| TEST_LOG_LEVEL: info | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
| BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-smoke-test | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: CCIP Smoke Test | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ env.REF_NAME }} | |
| - name: Prepare Base64 TOML override | |
| id: set_override_config | |
| shell: bash | |
| run: | | |
| BASE64_NETWORK_CONFIG=$(echo $BASE64_NETWORK_CONFIG | base64 -w 0 -d | sed -e 's/evm_key/${{ secrets.QA_SHARED_803C_KEY }}/g' | base64 -w 0) | |
| echo ::add-mask::$BASE64_NETWORK_CONFIG | |
| echo "BASE64_NETWORK_CONFIG=$BASE64_NETWORK_CONFIG" >> "$GITHUB_ENV" | |
| SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$SLACK_USER | |
| echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
| echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
| fi | |
| - name: step summary | |
| shell: bash | |
| run: | | |
| echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Prepare Base64 TOML override for CCIP secrets | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| id: setup_create_base64_config_ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| chainlinkVersion: ${{ env.CHAINLINK_VERSION }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 | |
| env: | |
| TEST_SUITE: smoke | |
| TEST_ARGS: -test.timeout 900h | |
| DETACH_RUNNER: true | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| RR_MEM: 8Gi | |
| RR_CPU: 4 | |
| TEST_TRIGGERED_BY: ccip-smoke-test-ci | |
| BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| with: | |
| test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| artifacts_location: ./integration-tests/smoke/logs/payload_ccip.json | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| cache_key_id: ccip-smoke-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| should_cleanup: false | |
| DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} | |
| DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} | |
| DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} | |
| DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} | |
| DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} | |
| DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" |