CCIP On-Demand Live Network Tests #229
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CCIP On-Demand Live Network Tests | |
on: | |
schedule: | |
- cron: '0 */6 * * *' | |
workflow_dispatch: | |
inputs: | |
base64_test_input : # base64 encoded toml for test input | |
description: 'Base64 encoded toml test input' | |
required: false | |
slackMemberID: | |
description: 'Slack member ID to notify' | |
required: false | |
test_type: | |
description: 'Type of test to run' | |
required: false | |
type: choice | |
options: | |
- 'load' | |
- 'smoke' | |
test_secrets_override_key: | |
description: 'Key to run tests with custom test secrets' | |
required: false | |
type: string | |
# Only run 1 of this workflow at a time per PR | |
concurrency: | |
group: live-testnet-tests | |
cancel-in-progress: true | |
env: | |
CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
CHAINLINK_VERSION: ${{ github.sha }} | |
CHAINLINK_TEST_VERSION: ${{ github.sha }} | |
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} | |
INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws | |
jobs: | |
build-chainlink: | |
environment: integration | |
permissions: | |
id-token: write | |
contents: read | |
name: Build Chainlink Image | |
runs-on: ubuntu20.04-16cores-64GB | |
steps: | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
- name: Check if image exists | |
id: check-image | |
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 | |
with: | |
repository: chainlink | |
tag: ${{ env.CHAINLINK_VERSION }} | |
AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
- name: Build Image | |
if: steps.check-image.outputs.exists == 'false' | |
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 | |
env: | |
GH_TOKEN: ${{ github.token }} | |
with: | |
cl_repo: smartcontractkit/chainlink-ccip | |
cl_ref: ${{ env.CHAINLINK_VERSION }} | |
push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.CHAINLINK_VERSION }} | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
- name: Collect Metrics | |
if: always() | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
with: | |
id: ccip-on-demand-live-testnet-tests-build-chainlink-image | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: Build Chainlink Image | |
continue-on-error: true | |
build-test-image: | |
environment: integration | |
permissions: | |
id-token: write | |
contents: read | |
name: Build Test Image | |
runs-on: ubuntu20.04-16cores-64GB | |
steps: | |
- name: Collect Metrics | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
with: | |
id: ccip-on-demand-live-testnet-tests-build-test-image | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: Build Test Image | |
continue-on-error: true | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
- name: Build Test Image | |
uses: ./.github/actions/build-test-image | |
with: | |
tag: ${{ env.CHAINLINK_TEST_VERSION }} | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
ccip-load-test: | |
name: CCIP Load Test | |
environment: integration | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
config: [mainnet.toml] | |
needs: [ build-chainlink, build-test-image ] | |
# if the event is a scheduled event or the test type is load and no previous job failed | |
if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }} | |
permissions: | |
issues: read | |
checks: write | |
pull-requests: write | |
id-token: write | |
contents: read | |
env: | |
CHAINLINK_ENV_USER: ${{ github.actor }} | |
SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
TEST_LOG_LEVEL: info | |
REF_NAME: ${{ github.head_ref || github.ref_name }} | |
ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} | |
steps: | |
- name: Collect Metrics | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
with: | |
id: ccip-on-demand-live-testnet-tests-load-tests | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: CCIP Load Test | |
continue-on-error: true | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
with: | |
ref: ${{ env.REF_NAME }} | |
- name: Prepare Base64 TOML override | |
id: set_override_config | |
shell: bash | |
run: | | |
# this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets. | |
BASE64_NETWORK_CONFIG=$(echo $BASE64_NETWORK_CONFIG | base64 -w 0 -d | sed -e 's/evm_key/${{ secrets.QA_SHARED_803C_KEY }}/g' | base64 -w 0) | |
echo ::add-mask::$BASE64_NETWORK_CONFIG | |
echo "BASE64_NETWORK_CONFIG=$BASE64_NETWORK_CONFIG" >> "$GITHUB_ENV" | |
SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
echo ::add-mask::$SLACK_USER | |
echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" | |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) | |
echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
fi | |
if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }}) | |
echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV | |
fi | |
- name: step summary | |
shell: bash | |
run: | | |
echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
- name: Prepare Base64 TOML override for CCIP secrets | |
uses: ./.github/actions/setup-create-base64-config-ccip | |
id: setup_create_base64_config_ccip | |
with: | |
runId: ${{ github.run_id }} | |
testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
chainlinkVersion: ${{ env.CHAINLINK_VERSION }} | |
logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
- name: Run Tests | |
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 | |
env: | |
TEST_SUITE: load | |
TEST_ARGS: -test.timeout 900h | |
DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
RR_MEM: 8Gi | |
RR_CPU: 4 | |
DETACH_RUNNER: true | |
TEST_TRIGGERED_BY: ccip-load-test-ci | |
BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
with: | |
test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt | |
test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} | |
token: ${{ secrets.GITHUB_TOKEN }} | |
go_mod_path: ./integration-tests/go.mod | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
artifacts_location: ./integration-tests/load/logs/payload_ccip.json | |
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} | |
cache_restore_only: "true" | |
should_cleanup: false | |
DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} | |
DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} | |
DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} | |
DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} | |
DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} | |
DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" | |
ccip-smoke-test: | |
name: CCIP smoke Test | |
environment: integration | |
runs-on: ubuntu-latest | |
needs: [ build-chainlink, build-test-image ] | |
# if the event is a scheduled event or the test type is load and no previous job failed | |
if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }} | |
permissions: | |
issues: read | |
checks: write | |
pull-requests: write | |
id-token: write | |
contents: read | |
env: | |
CHAINLINK_ENV_USER: ${{ github.actor }} | |
SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
TEST_LOG_LEVEL: info | |
REF_NAME: ${{ github.head_ref || github.ref_name }} | |
ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
BASE64_NETWORK_CONFIG: ${{ secrets.BASE64_NETWORK_CONFIG }} | |
steps: | |
- name: Collect Metrics | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
with: | |
id: ccip-on-demand-live-testnet-tests-smoke-test | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: CCIP Smoke Test | |
continue-on-error: true | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
with: | |
ref: ${{ env.REF_NAME }} | |
- name: Prepare Base64 TOML override | |
id: set_override_config | |
shell: bash | |
run: | | |
BASE64_NETWORK_CONFIG=$(echo $BASE64_NETWORK_CONFIG | base64 -w 0 -d | sed -e 's/evm_key/${{ secrets.QA_SHARED_803C_KEY }}/g' | base64 -w 0) | |
echo ::add-mask::$BASE64_NETWORK_CONFIG | |
echo "BASE64_NETWORK_CONFIG=$BASE64_NETWORK_CONFIG" >> "$GITHUB_ENV" | |
SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
echo ::add-mask::$SLACK_USER | |
echo "SLACK_USER=$SLACK_USER" >> "$GITHUB_ENV" | |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) | |
echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE | |
echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT | |
fi | |
- name: step summary | |
shell: bash | |
run: | | |
echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
echo "\`${{ env.CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
- name: Prepare Base64 TOML override for CCIP secrets | |
uses: ./.github/actions/setup-create-base64-config-ccip | |
id: setup_create_base64_config_ccip | |
with: | |
runId: ${{ github.run_id }} | |
testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
chainlinkVersion: ${{ env.CHAINLINK_VERSION }} | |
logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
- name: Run Tests | |
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 | |
env: | |
TEST_SUITE: smoke | |
TEST_ARGS: -test.timeout 900h | |
DETACH_RUNNER: true | |
DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
RR_MEM: 8Gi | |
RR_CPU: 4 | |
TEST_TRIGGERED_BY: ccip-smoke-test-ci | |
BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} | |
with: | |
test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt | |
test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
token: ${{ secrets.GITHUB_TOKEN }} | |
go_mod_path: ./integration-tests/go.mod | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
artifacts_location: ./integration-tests/smoke/logs/payload_ccip.json | |
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
cache_key_id: ccip-smoke-${{ env.MOD_CACHE_VERSION }} | |
cache_restore_only: "true" | |
should_cleanup: false | |
DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} | |
DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} | |
DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} | |
DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} | |
DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} | |
DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" |