Build & Integration Tests: chore/celo-upgrade #1539
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build & Integration Tests | |
| run-name: 'Build & Integration Tests: ${{ github.head_ref || github.ref_name }}' | |
| # Dockefile for the self-hosted runner: | |
| # https://github.com/celo-org/infrastructure/blob/master/terraform/root-modules/gcp/integration-tests-gke/files/github-arc/Dockerfile-monorepo | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| - 'release/**' | |
| concurrency: | |
| group: celo-monorepo-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash --login -eo pipefail {0} | |
| env: | |
| # Increment these to force cache rebuilding | |
| NODE_MODULE_CACHE_VERSION: 5 | |
| NODE_OPTIONS: '--max-old-space-size=4096' | |
| TERM: dumb | |
| GRADLE_OPTS: '-Dorg.gradle.daemon=false -Dorg.gradle.parallel=false -Dorg.gradle.configureondemand=true -Dorg.gradle.jvmargs="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError"' | |
| # Git Tag for contract release to use | |
| #RELEASE_TAG: ganache-v7-core-contracts.v9 | |
| RELEASE_TAG: core-contracts.v10 | |
| # CELO_BLOCKCHAIN_BRANCH_TO_TEST: master | |
| CELO_BLOCKCHAIN_BRANCH_TO_TEST: release/1.7.x | |
| # EXAMPLE on debug ssh step | |
| # - name: Setup tmate session | |
| # uses: mxschmitt/action-tmate@v3 | |
| # timeout-minutes: 20 | |
| # if: contains(matrix.command, 'common/') && false | |
| # with: | |
| # limit-access-to-actor: true | |
| jobs: | |
| install-dependencies: | |
| name: Install dependencies | |
| outputs: | |
| # Propagate more outputs if you need https://github.com/tj-actions/changed-files#outputs | |
| # Adding a initial comma so ',<path>' matches also for the first file | |
| all_modified_files: ',${{ steps.changed-files.outputs.all_modified_files }}' | |
| artifacts_to_cache: ${{ steps.get_artifacts_to_cache.outputs.artifacts_to_cache }} | |
| # runs-on: ubuntu-latest | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Restore node cache | |
| uses: actions/cache@v3 | |
| id: cache_node | |
| with: | |
| # We need to cache all the artifacts generated by yarn install+build | |
| # Update this list also in .github/actions/sync-workspace/action.yml with exactly the same list | |
| path: | | |
| node_modules | |
| packages/**/node_modules | |
| key: node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}- | |
| - name: Install yarn dependencies | |
| run: git config --global url."https://".insteadOf ssh:// && yarn install --frozen-lockfile --network-timeout 1000000 | |
| if: steps.cache_node.outputs.cache-hit != 'true' | |
| - name: Run yarn postinstall if cache hitted | |
| run: yarn run postinstall | |
| if: steps.cache_node.outputs.cache-hit == 'true' | |
| - name: Build packages | |
| run: yarn build --include-dependencies | |
| - name: Check licenses | |
| if: steps.cache_node.outputs.cache-hit != 'true' | |
| run: | | |
| yarn check-licenses | |
| # Get a list of submodules to ignore in the changed files check | |
| - name: Get submodules to ignore in changed files check | |
| id: get_submodules_to_ignore | |
| run: | | |
| # Get all the submodules paths | |
| submodules=$(git config --file .gitmodules --name-only --get-regexp path) | |
| # Remove the "submodule." prefix and ".path" suffix | |
| submodules=$(echo $submodules | sed 's/^submodule.//g') | |
| submodules=$(echo $submodules | sed 's/.path$//g') | |
| # Add a ! in front of each path to ignore it latter in the changed files check | |
| submodules=$(echo $submodules | sed 's/^/!/g') | |
| echo "submodules<<EOF" >> $GITHUB_OUTPUT | |
| echo "$submodules" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| # Get workdir local changes and fail if there are any change | |
| - name: Verify Changed files | |
| id: verify-changed-files | |
| uses: tj-actions/verify-changed-files@v17 | |
| with: | |
| fail-if-changed: 'true' | |
| fail-message: 'Files changed during build. Please build locally and commit the changes.' | |
| files: | | |
| **/* | |
| ${{ steps.get_submodules_to_ignore.outputs.submodules }} | |
| - run: | | |
| echo "${{ steps.verify-changed-files.outputs.changed_files }}" | |
| - name: Get the artifacts to cache | |
| id: get_artifacts_to_cache | |
| run: | | |
| artifacts_to_cache="$(git ls-files --others --ignored --exclude-standard | grep -v node_modules | grep -v .js.map)" | |
| echo "artifacts_to_cache<<EOF" >> $GITHUB_OUTPUT | |
| echo "$artifacts_to_cache" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| # We use cache to share the build artifacts between jobs (gh artifacts are too slow...) | |
| # For more context check https://github.com/actions/upload-artifact/issues/199 | |
| - name: Restore build artifacts cache | |
| uses: actions/cache@v3 | |
| id: cache_build_artifacts | |
| with: | |
| # We need to cache all the artifacts generated by yarn install+build | |
| # Update this list also in .github/actions/sync-workspace/action.yml with exactly the same list | |
| path: | | |
| ${{ steps.get_artifacts_to_cache.outputs.artifacts_to_cache }} | |
| key: code-${{ github.sha }} | |
| restore-keys: | | |
| code-${{ github.sha }} | |
| - name: Detect files changed in PR (or commit), and expose as output | |
| id: changed-files | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| # Using comma as separator to be able to easily match full paths (using ,<path>) | |
| separator: ',' | |
| # Checking if changed in the last 100 commits in PRs | |
| fetch_depth: '150' | |
| - run: echo ",${{ steps.changed-files.outputs.all_modified_files }}" | |
| lint-checks: | |
| name: Lint code | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| timeout-minutes: 30 | |
| needs: install-dependencies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Sync workspace | |
| uses: ./.github/actions/sync-workspace | |
| with: | |
| artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }} | |
| - run: yarn run prettify:diff | |
| - run: yarn run lint | |
| general_test: | |
| name: General jest test | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| needs: install-dependencies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Sync workspace | |
| uses: ./.github/actions/sync-workspace | |
| with: | |
| artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }} | |
| - name: Run Jest Tests | |
| run: | | |
| mkdir -p test-results/jest | |
| # Skipping packages that are tested in a specific job below | |
| yarn run lerna \ | |
| --ignore @celo/protocol \ | |
| --ignore @celo/celotool \ | |
| --ignore @celo/env-tests \ | |
| run test | |
| - name: Upload Jest Test Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Jest Test Results | |
| path: test-results/jest | |
| protocol-test-release: | |
| name: Protocol Test Release | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| timeout-minutes: 500 | |
| needs: [install-dependencies, lint-checks] | |
| if: | | |
| github.base_ref == 'master' || contains(github.base_ref, 'release') || contains(github.base_ref, 'production') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/protocol') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',package.json') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',yarn.lock') || | |
| false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Sync workspace | |
| uses: ./.github/actions/sync-workspace | |
| with: | |
| artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }} | |
| - name: Download protocol devchain artifact | |
| uses: dawidd6/action-download-artifact@v2 | |
| with: | |
| workflow: protocol-devchain.yml | |
| name: devchain-${{ env.RELEASE_TAG }} | |
| path: packages/protocol/.tmp/devchain | |
| - name: Copy DevChain and Build generated from released tag | |
| run: | | |
| BUILD_AND_DEVCHAIN_DIR=$(echo build/$(echo $RELEASE_TAG | sed -e 's/\//_/g')) | |
| (cp -r packages/protocol/.tmp/devchain packages/protocol/$BUILD_AND_DEVCHAIN_DIR) | |
| - name: Test against current release | |
| run: | | |
| echo "Comparing against $RELEASE_TAG" | |
| BUILD_AND_DEVCHAIN_DIR=$(echo build/$(echo $RELEASE_TAG | sed -e 's/\//_/g')) | |
| yarn --cwd packages/protocol ci:test-make-release -b $RELEASE_TAG -d $BUILD_AND_DEVCHAIN_DIR | |
| protocol-test-matrix: | |
| # Keeping name short because GitHub UI does not handle long names well | |
| name: ${{ matrix.name }} | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| timeout-minutes: 60 | |
| needs: [install-dependencies, lint-checks] | |
| if: | | |
| github.base_ref == 'master' || contains(github.base_ref, 'staging') || contains(github.base_ref, 'production') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/protocol') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/typescript') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',package.json') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',yarn.lock') || | |
| false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Protocol Release Snapshots | |
| command: | | |
| yarn --cwd packages/protocol test:release-snapshots | |
| if [[ $(git status packages/protocol/releaseData/versionReports --porcelain) ]]; then | |
| git --no-pager diff packages/protocol/releaseData/versionReports | |
| echo "There are git differences after generating release version report snapshots" | |
| echo "If these changes are intended, update the 'releaseData/versionReports' accordingly" | |
| exit 1 | |
| fi | |
| - name: Protocol Common tests | |
| command: | | |
| yarn --cwd packages/protocol test common/ | |
| - name: Protocol Compatibility | |
| command: | | |
| yarn --cwd packages/protocol test compatibility/ | |
| - name: Protocol Governance Validators | |
| command: | | |
| yarn --cwd packages/protocol test governance/validators/ | |
| - name: Protocol Governance Voting | |
| command: | | |
| yarn --cwd packages/protocol test governance/voting/ | |
| - name: Protocol scripts test | |
| command: | | |
| yarn --cwd packages/protocol test:scripts | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Sync workspace | |
| uses: ./.github/actions/sync-workspace | |
| with: | |
| rebuild-package: 'true' | |
| artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }} | |
| - name: Execute matrix command for test | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 40 | |
| max_attempts: 3 | |
| command: | | |
| ${{ matrix.command }} | |
| end-to-end-geth-matrix: | |
| # Keeping name short because GitHub UI does not handle long names well | |
| name: e2e ${{ matrix.name }} | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| timeout-minutes: 60 | |
| needs: [install-dependencies, lint-checks] | |
| if: | | |
| github.base_ref == 'master' || contains(github.base_ref, 'release') || contains(github.base_ref, 'production') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/celotool') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/protocol') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/dev-utils') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/typescript') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',package.json') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',yarn.lock') || | |
| false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Transfer test | |
| command: | | |
| set -e | |
| # Forcing to load go and rust paths | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_transfers.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: Blockchain Parameters test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_blockchain_parameters.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: Slashing test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_slashing.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: Governance test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_governance.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: Replica test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_replicas.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: Sync test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_sync.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: CIP35 eth compatibility test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| echo "Test is skipped because migrations somehow fail" | |
| # ./ci_test_cip35.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| - name: Validator order test | |
| command: | | |
| set -e | |
| export PATH="/usr/local/go/bin:$HOME/.cargo/bin:${PATH}" | |
| cd packages/celotool | |
| ./ci_test_validator_order.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Sync workspace | |
| uses: ./.github/actions/sync-workspace | |
| with: | |
| artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }} | |
| - name: Execute matrix command for test | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| command: | | |
| ${{ matrix.command }} | |
| # NOTE: This has not been fully tested as we don't have a license for certora | |
| certora-test: | |
| name: Certora test ${{ matrix.name }} | |
| runs-on: ['self-hosted', 'monorepo-node18'] | |
| timeout-minutes: 30 | |
| needs: [install-dependencies, lint-checks] | |
| # Disable as certora license is not active | |
| if: | | |
| false && ( | |
| github.base_ref == 'master' || contains(github.base_ref, 'release') || contains(github.base_ref, 'production') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, 'packages/protocol') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',package.json') || | |
| contains(needs.install-dependencies.outputs.all_modified_files, ',yarn.lock') | |
| ) | |
| env: | |
| CERTORAKEY: NOT_ACTIVE | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Locked Gold | |
| command: | | |
| cd packages/protocol | |
| ./specs/scripts/lockedgold.sh | |
| - name: Accounts | |
| command: | | |
| cd packages/protocol | |
| ./specs/scripts/accounts.sh | |
| - name: Accounts | |
| command: | | |
| cd packages/protocol | |
| ./specs/scripts/accountsPrivileged.sh | |
| - name: Accounts | |
| command: | | |
| cd packages/protocol | |
| ./specs/scripts/applyHarness.sh | |
| ./specs/scripts/governance.sh | |
| - name: Accounts | |
| command: | | |
| cd packages/protocol | |
| ./specs/scripts/reserve.sh | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Sync workspace | |
| uses: ./.github/actions/sync-workspace | |
| with: | |
| artifacts_to_cache: ${{ needs.install-dependencies.outputs.artifacts_to_cache }} | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '11' | |
| - name: Certora dependencies | |
| run: | | |
| echo "export PATH=$PATH:~/.local/bin" >> ~/.bashrc | |
| sudo apt-get update | |
| sudo apt-get install -y software-properties-common python3-pip | |
| pip3 install certora-cli | |
| wget https://github.com/ethereum/solidity/releases/download/v0.5.13/solc-static-linux | |
| chmod +x solc-static-linux | |
| sudo mv solc-static-linux /usr/bin/solc | |
| - name: Build and run command | |
| run: | | |
| ${{ matrix.command }} |