Feat: Automated Program Upgrade Testing #2700
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e_tests_custom_cl | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| cl_branch_ref: | |
| description: Chainlink repo branch to integrate with | |
| required: true | |
| default: develop | |
| type: string | |
| env: | |
| CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| CONTRACT_ARTIFACTS_PATH: contracts/target/deploy | |
| MOD_CACHE_VERSION: 1 | |
| # Only run 1 of this workflow at a time per PR | |
| concurrency: | |
| group: integration-tests-solana-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check_test_compilation: | |
| name: Check integration test compilation | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| - name: Setup go | |
| uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 | |
| with: | |
| go-version-file: "go.mod" | |
| check-latest: true | |
| cache: true | |
| - run: cd ./integration-tests && go test -run=xxx ./... | |
| get_projectserum_version: | |
| name: Get ProjectSerum Version | |
| environment: integration | |
| runs-on: ubuntu-latest | |
| outputs: | |
| projectserum_version: ${{ steps.psversion.outputs.projectserum_version }} | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| - name: Get ProjectSerum Version | |
| id: psversion | |
| uses: ./.github/actions/projectserum_version | |
| contract-changes: | |
| name: check for contract changes from develop | |
| runs-on: ubuntu-latest | |
| outputs: | |
| changed: ${{ steps.changes.outputs.contracts == 'true' }} | |
| steps: | |
| - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 | |
| id: changes | |
| with: | |
| base: develop | |
| list-files: csv | |
| filters: | | |
| contracts: | |
| - 'contracts/**' | |
| solana-test-image-exists: | |
| environment: integration | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| name: Check If Solana Test Image Exists | |
| runs-on: ubuntu-latest | |
| outputs: | |
| exists: ${{ steps.check-image.outputs.exists }} | |
| steps: | |
| - name: Check if image exists | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19 | |
| with: | |
| repository: chainlink-solana-tests | |
| tag: ${{ github.sha }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| e2e_custom_build_artifacts: | |
| name: E2E Custom Build Artifacts | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest-32cores-128GB | |
| needs: [get_projectserum_version] | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| - name: Build contracts | |
| uses: ./.github/actions/build_contract_artifacts | |
| with: | |
| image: backpackapp/build | |
| image-version: ${{ needs.get_projectserum_version.outputs.projectserum_version }} | |
| e2e_custom_build_previous_artifacts: | |
| name: E2E Build Previous Release Artifacts | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest-32cores-128GB | |
| needs: [contract-changes,e2e_custom_build_artifacts, get_projectserum_version] | |
| steps: | |
| # TODO: make this if conditional control workflow execution | |
| - name: Changes to develop branch detected | |
| if: needs.contract-changes.outputs.changed | |
| run: | | |
| echo "changes detected!" | |
| - name: Checkout previous release | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| with: | |
| ref: v1.0.2 # pinned to latest deployed release | |
| # ref: v1.1.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: artifacts | |
| path: ./temp/artifacts | |
| - name: hash current artifacts | |
| id: current | |
| run: | | |
| cd ./temp/artifacts # need to be in directory for hashes to compare without path differences | |
| # shellcheck disable=SC2035 | |
| echo "hash=$(sha1sum * | sha1sum)" >> "$GITHUB_OUTPUT" | |
| - name: build contracts | |
| id: previous | |
| run: | | |
| # for use with 1.0.2 release | |
| docker run -d -v "$(pwd)":/repo --name build-container projectserum/build:v0.25.0 tail -f /dev/null | |
| # for use with 1.1.0 release and onward | |
| # docker run -d -v "$(pwd)":/repo --name build-container backpackapp/build:${{ needs.get_projectserum_version.outputs.projectserum_version }} tail -f /dev/null | |
| # build with keys | |
| docker exec build-container bash -c "\ | |
| export RUSTUP_HOME=\"/root/.rustup\" &&\ | |
| cd /repo &&\ | |
| ./scripts/programs-keys-gen.sh &&\ | |
| cd ./contracts &&\ | |
| anchor build &&\ | |
| chown -R $(id -u):$(id -g) /repo" | |
| # clean up the container | |
| docker stop build-container | |
| docker rm build-container | |
| # copy + save hash | |
| mkdir ./temp/artifacts/previous | |
| cp ${{ env.CONTRACT_ARTIFACTS_PATH }}/* ./temp/artifacts/previous | |
| cd ./temp/artifacts/previous # need to be in directory for hashes to compare without path differences | |
| # shellcheck disable=SC2035 | |
| echo "hash=$(sha1sum * | sha1sum)" >> "$GITHUB_OUTPUT" | |
| - name: compare hashes | |
| id: check | |
| run: | | |
| echo "previous: ${{ steps.previous.outputs.hash }}" | |
| echo "current: ${{ steps.current.outputs.hash }}" | |
| if [ "${{ steps.previous.outputs.hash }}" != "${{ steps.current.outputs.hash }}" ]; then | |
| echo "continue=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/upload-artifact@v4 | |
| if: steps.check.outputs.continue | |
| with: | |
| name: artifacts | |
| path: ./temp/artifacts/ # combine artifacts | |
| overwrite: true | |
| e2e_custom_build_custom_chainlink_image: | |
| name: E2E Custom Build Custom CL Image | |
| runs-on: ubuntu-latest-16cores-64GB | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Get core ref from PR body | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| comment="$(gh pr view https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }} --json body -q '.body')" | |
| # shellcheck disable=SC2086 | |
| core_ref="$(echo $comment | grep -oP 'core ref: \K\S+' || true)" | |
| if [ -n "$core_ref" ]; then | |
| echo "CUSTOM_CORE_REF=${core_ref}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Check if image exists | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| repository: chainlink | |
| tag: solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Build Image | |
| if: steps.check-image.outputs.exists == 'false' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| should_checkout: true | |
| cl_repo: smartcontractkit/chainlink | |
| cl_ref: ${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref }} | |
| dep_solana_sha: ${{ github.event.pull_request.head.sha }} | |
| push_tag: ${{ env.CL_ECR }}:solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| build_test_image: | |
| environment: integration | |
| if: contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image') | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu-latest | |
| needs: [e2e_custom_build_artifacts] | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@v1 | |
| with: | |
| basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Build Image | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| e2e_custom_run_smoke_tests: | |
| name: E2E Custom Run Smoke Tests | |
| environment: integration | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest-16cores-64GB | |
| needs: [e2e_custom_build_artifacts, e2e_custom_build_custom_chainlink_image, check_test_compilation] | |
| env: | |
| TEST_SUITE: smoke | |
| TEST_ARGS: -test.timeout 30m | |
| TEST_LOG_LEVEL: debug | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: solana-e2e-smoke | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: E2E Custom Run Smoke Tests | |
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
| continue-on-error: true | |
| - name: Get core ref from PR body | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| comment="$(gh pr view https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }} --json body -q '.body')" | |
| # shellcheck disable=SC2086 | |
| core_ref="$(echo $comment | grep -oP 'core ref: \K\S+' || true)" | |
| if [ -n "$core_ref" ]; then | |
| echo "CUSTOM_CORE_REF=${core_ref}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Checkout the repo | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| - name: Install Solana CLI # required for ensuring the local test validator is configured correctly | |
| run: ./scripts/install-solana-ci.sh | |
| - name: Install gauntlet | |
| uses: ./.github/actions/build-gauntlet | |
| - name: Generate config overrides | |
| run: | # https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/README.md | |
| cat << EOF > config.toml | |
| [ChainlinkImage] | |
| image="${{ env.CL_ECR }}" | |
| version="solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }}" | |
| [Common] | |
| user="${{ github.actor }}" | |
| internal_docker_repo = "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com" | |
| EOF | |
| # shellcheck disable=SC2002 | |
| BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) | |
| # shellcheck disable=SC2086 | |
| echo ::add-mask::$BASE64_CONFIG_OVERRIDE | |
| # shellcheck disable=SC2086 | |
| echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci= -singlepackage=true -hidepassingtests=false -hidepassinglogs=false | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| download_contract_artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} | |
| go_mod_path: ./integration-tests/go.mod | |
| cl_repo: ${{ env.CL_ECR }} | |
| cl_image_tag: solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| artifacts_name: smoke-test-logs | |
| artifacts_location: ./integration-tests/smoke/logs/ | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| cache_key_id: solana-e2e-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "false" | |
| e2e_program_upgrade_tests: | |
| name: E2E Program Upgrade Tests | |
| environment: integration | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest-16cores-64GB | |
| needs: [contract-changes,e2e_custom_build_previous_artifacts,e2e_custom_build_artifacts, e2e_custom_build_custom_chainlink_image, check_test_compilation] | |
| env: | |
| TEST_SUITE: smoke | |
| TEST_ARGS: -test.timeout 30m | |
| TEST_LOG_LEVEL: debug | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| # TODO: make this if conditional control workflow execution | |
| - name: Changes to develop branch detected | |
| if: needs.contract-changes.outputs.changed | |
| run: | | |
| echo "changes detected!" | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: solana-e2e-program-upgrade | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: E2E Program Upgrade Tests | |
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
| continue-on-error: true | |
| - name: Get core ref from PR body | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| comment="$(gh pr view https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }} --json body -q '.body')" | |
| # shellcheck disable=SC2086 | |
| core_ref="$(echo $comment | grep -oP 'core ref: \K\S+' || true)" | |
| if [ -n "$core_ref" ]; then | |
| echo "CUSTOM_CORE_REF=${core_ref}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Checkout the repo | |
| uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 | |
| - name: Install Solana CLI # required for ensuring the local test validator is configured correctly | |
| run: ./scripts/install-solana-ci.sh | |
| - name: Install gauntlet | |
| uses: ./.github/actions/build-gauntlet | |
| - name: Generate config overrides | |
| run: | # https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/README.md | |
| cat << EOF > config.toml | |
| [ChainlinkImage] | |
| image="${{ env.CL_ECR }}" | |
| version="solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }}" | |
| [Common] | |
| user="${{ github.actor }}" | |
| internal_docker_repo = "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com" | |
| EOF | |
| # shellcheck disable=SC2002 | |
| BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) | |
| # shellcheck disable=SC2086 | |
| echo ::add-mask::$BASE64_CONFIG_OVERRIDE | |
| # shellcheck disable=SC2086 | |
| echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV | |
| - name: Run Upgrade Test | |
| run: | | |
| echo "TODO: implement OCR2 upgrade test!" | |
| # uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| # with: | |
| # test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci= -singlepackage=true -hidepassingtests=false -hidepassinglogs=false | |
| # test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| # download_contract_artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} | |
| # go_mod_path: ./integration-tests/go.mod | |
| # cl_repo: ${{ env.CL_ECR }} | |
| # cl_image_tag: solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }} | |
| # token: ${{ secrets.GITHUB_TOKEN }} | |
| # aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| # artifacts_name: smoke-test-logs | |
| # artifacts_location: ./integration-tests/smoke/logs/ | |
| # QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| # QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| # QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| # cache_key_id: solana-e2e-${{ env.MOD_CACHE_VERSION }} | |
| # cache_restore_only: "false" |