CCIP On-Demand Live Network Tests #265
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CCIP On-Demand Live Network Tests | |
| on: | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| base64_test_input : # base64 encoded toml for test input | |
| description: 'Base64 encoded toml test input' | |
| required: false | |
| slackMemberID: | |
| description: 'Slack member ID to notify' | |
| required: false | |
| test_type: | |
| description: 'Type of test to run' | |
| required: false | |
| type: choice | |
| options: | |
| - 'load' | |
| - 'smoke' | |
| test_secrets_override_key: | |
| description: 'Key to run tests with custom test secrets' | |
| required: false | |
| type: string | |
| # Only run 1 of this workflow at a time per PR | |
| concurrency: | |
| group: live-testnet-tests | |
| cancel-in-progress: true | |
| env: | |
| CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| CHAINLINK_VERSION: ${{ github.sha }} | |
| CHAINLINK_TEST_VERSION: ${{ github.sha }} | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} | |
| INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
| AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws | |
| E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| E2E_TEST_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} | |
| E2E_TEST_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} | |
| E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} | |
| E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} | |
| # Default private key test secret loaded from Github Secret as only security team has access to it. | |
| # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets. | |
| E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_ARBITRUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_AVALANCHE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_METIS_ANDROMEDA_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_OPTIMISM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_POLYGON_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| E2E_TEST_BSC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} | |
| jobs: | |
| build-chainlink: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Chainlink Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Check if image exists | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 | |
| with: | |
| repository: chainlink | |
| tag: ${{ env.CHAINLINK_VERSION }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Build Image | |
| if: steps.check-image.outputs.exists == 'false' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| with: | |
| cl_repo: smartcontractkit/chainlink-ccip | |
| cl_ref: ${{ env.CHAINLINK_VERSION }} | |
| push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-build-chainlink-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Chainlink Image | |
| continue-on-error: true | |
| build-test-image: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-build-test-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Build Test Image | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| tag: ${{ env.CHAINLINK_TEST_VERSION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| ccip-load-test: | |
| name: CCIP Load Test | |
| environment: integration | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| config: [mainnet.toml] | |
| needs: [ build-chainlink, build-test-image ] | |
| # if the event is a scheduled event or the test type is load and no previous job failed | |
| if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }} | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| env: | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
| SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
| TEST_LOG_LEVEL: info | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-load-tests | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: CCIP Load Test | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ env.REF_NAME }} | |
| - name: Prepare Base64 TOML override | |
| id: set_override_config | |
| shell: bash | |
| run: | | |
| SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$SLACK_USER | |
| echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
| echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }}) | |
| echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
| echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
| echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV | |
| fi | |
| - name: step summary | |
| shell: bash | |
| run: | | |
| echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Prepare Base64 TOML override for CCIP secrets | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| id: setup_create_base64_config_ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| chainlinkVersion: ${{ env.CHAINLINK_VERSION }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32 | |
| env: | |
| TEST_SUITE: load | |
| TEST_ARGS: -test.timeout 900h | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| RR_MEM: 8Gi | |
| RR_CPU: 4 | |
| DETACH_RUNNER: true | |
| TEST_TRIGGERED_BY: ccip-load-test-ci | |
| BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" | |
| with: | |
| test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| # Other default test secrets loaded from dotenv Github Secret. | |
| test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }} | |
| test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| artifacts_location: ./integration-tests/load/logs/payload_ccip.json | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| should_cleanup: false | |
| ccip-smoke-test: | |
| name: CCIP smoke Test | |
| environment: integration | |
| runs-on: ubuntu-latest | |
| needs: [ build-chainlink, build-test-image ] | |
| # if the event is a scheduled event or the test type is load and no previous job failed | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }} | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| env: | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
| SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
| TEST_LOG_LEVEL: info | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-on-demand-live-testnet-tests-smoke-test | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: CCIP Smoke Test | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ env.REF_NAME }} | |
| - name: Prepare Base64 TOML override | |
| id: set_override_config | |
| shell: bash | |
| run: | | |
| SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$SLACK_USER | |
| echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
| echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
| fi | |
| - name: step summary | |
| shell: bash | |
| run: | | |
| echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Prepare Base64 TOML override for CCIP secrets | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| id: setup_create_base64_config_ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| chainlinkVersion: ${{ env.CHAINLINK_VERSION }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32 | |
| env: | |
| TEST_SUITE: smoke | |
| TEST_ARGS: -test.timeout 900h | |
| DETACH_RUNNER: true | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| RR_MEM: 8Gi | |
| RR_CPU: 4 | |
| TEST_TRIGGERED_BY: ccip-smoke-test-ci | |
| BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
| E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| with: | |
| test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| # Other default test secrets loaded from dotenv Github Secret. | |
| test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }} | |
| test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| artifacts_location: ./integration-tests/smoke/logs/payload_ccip.json | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| cache_key_id: ccip-smoke-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| should_cleanup: false |