Deploy infrastructure for SPO tickers #4359
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'hotfix**' | |
| pull_request: | |
| types: [opened, reopened, synchronize] | |
| paths-ignore: # ignore docs only changes since they use a dedicated workflows: docs.yml | |
| - 'docs/**' | |
| - 'mithril-explorer/**' | |
| - '.github/workflows/docs.yml' | |
| branches-ignore: | |
| - 'hotfix**' # hotfix are handled by the push trigger | |
| concurrency: | |
| group: ci-build-test-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-ubuntu-X64: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| eras: ${{ steps.eras-test-lab.outputs.eras }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install stable toolchain, tools, and restore cache | |
| uses: ./.github/workflows/actions/toolchain-and-cache | |
| with: | |
| cache-version: 20.04-${{ secrets.CACHE_VERSION }} | |
| cargo-tools: cargo-deb | |
| # We separate the build in 2 steps as we want to avoid side effects with Rust feature unification. | |
| - name: Cargo build - Tooling | |
| shell: bash | |
| run: cargo build --features portable --release --workspace --exclude mithril-aggregator --exclude mithril-client --exclude mithril-signer --exclude mithril-stm | |
| - name: Build Mithril workspace & publish artifacts | |
| uses: ./.github/workflows/actions/build-upload-mithril-artifact | |
| with: | |
| build-args: --features bundle_openssl | |
| - name: Build Debian packages | |
| shell: bash | |
| run: | | |
| cargo deb --no-build -p mithril-aggregator | |
| cargo deb --no-build -p mithril-signer | |
| cargo deb --no-build -p mithril-client | |
| - name: Publish Debian packages | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mithril-deb-packages-${{ runner.os }}-${{ runner.arch }} | |
| path: target/debian/*.deb | |
| if-no-files-found: error | |
| - name: Publish End-to-end runner (${{ runner.os }}-${{ runner.arch }}) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }} | |
| path: target/release/mithril-end-to-end | |
| if-no-files-found: error | |
| - name: Prepare test lab eras | |
| id: eras-test-lab | |
| run: | | |
| ERAS=$(./target/release/mithril-aggregator era list --json) | |
| echo "Test Lab Eras: $ERAS" | |
| echo "eras=$ERAS" >> $GITHUB_OUTPUT | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ macos-12, windows-latest ] | |
| include: | |
| # Only build client on windows & mac | |
| - os: macos-12 | |
| build-args: -p mithril-client | |
| - os: windows-latest | |
| build-args: -p mithril-client | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install stable toolchain and restore cache | |
| uses: ./.github/workflows/actions/toolchain-and-cache | |
| with: | |
| cache-version: ${{ secrets.CACHE_VERSION }} | |
| - name: Build Mithril workspace & publish artifacts | |
| uses: ./.github/workflows/actions/build-upload-mithril-artifact | |
| with: | |
| build-args: --features bundle_openssl ${{ matrix.build-args }} | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ ubuntu-22.04, macos-12, windows-latest ] | |
| include: | |
| - os: ubuntu-22.04 | |
| test-args: --features portable --workspace | |
| # Only test client on windows & mac (since its the only binaries supported for those os for now) | |
| - os: macos-12 | |
| test-args: -p mithril-client | |
| - os: windows-latest | |
| test-args: -p mithril-client | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install stable toolchain, tools, and restore cache | |
| uses: ./.github/workflows/actions/toolchain-and-cache | |
| with: | |
| cache-version: ${{ secrets.CACHE_VERSION }} | |
| cargo-tools: cargo-nextest | |
| - name: Build tests | |
| run: cargo nextest run --no-run ${{ matrix.test-args }} | |
| - name: Run tests | |
| shell: bash | |
| run: | | |
| cargo nextest run --profile ci ${{ matrix.test-args }} | |
| - name: Rename junit file to include runner info | |
| shell: bash | |
| if: success() || failure() | |
| run: | | |
| mv target/nextest/ci/tests-result.junit.xml test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}.xml | |
| - name: Upload Tests Results | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }} | |
| path: | | |
| ./test-results-*.xml | |
| check: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Install stable toolchain, tools, and restore cache | |
| uses: ./.github/workflows/actions/toolchain-and-cache | |
| with: | |
| cache-version: ${{ secrets.CACHE_VERSION }} | |
| cargo-tools: cargo-sort clippy-sarif sarif-fmt | |
| - name: Clippy Check | |
| if: success() || failure() | |
| run: | | |
| cargo clippy \ | |
| --all-features --all-targets --no-deps --message-format=json \ | |
| | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt | |
| # Make this step fail if any warning has been found | |
| if [[ $(cat rust-clippy-results.sarif | jq '.runs[0].results') != "[]" ]]; then | |
| false | |
| fi | |
| - name: Upload clippy analysis results to GitHub | |
| if: success() || failure() | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: rust-clippy-results.sarif | |
| wait-for-processing: true | |
| - name: Cargo fmt | |
| if: success() || failure() | |
| shell: bash | |
| run: cargo fmt --check | |
| - name: Cargo sort | |
| if: success() || failure() | |
| shell: bash | |
| run: cargo sort -w -c | |
| - name: Dependency & Vulnerabilities Review | |
| if: github.event_name == 'pull_request' | |
| uses: actions/dependency-review-action@v3 | |
| with: | |
| base-ref: ${{ github.event.pull_request.base.sha || 'main' }} | |
| head-ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| run-test-lab: | |
| runs-on: ubuntu-22.04 | |
| needs: [ build-ubuntu-X64 ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }} | |
| run_id: [1,2,3] | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Download binaries | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-distribution-${{ runner.os }}-${{ runner.arch }} | |
| path: ./bin | |
| - name: Download rust test runner | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }} | |
| path: ./ | |
| - run: | | |
| chmod +x ./bin/mithril-aggregator | |
| chmod +x ./bin/mithril-client | |
| chmod +x ./bin/mithril-signer | |
| chmod +x ./mithril-end-to-end | |
| mkdir artifacts | |
| - name: Test | |
| run: ./mithril-end-to-end --bin-directory ./bin --work-directory=./artifacts --devnet-scripts-directory=./mithril-test-lab/mithril-devnet --mithril-era=${{ matrix.era }} | |
| - name: Upload E2E Tests Artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-era_${{ matrix.era }}-run_id_${{ matrix.run_id }} | |
| path: | | |
| ./artifacts/* | |
| # including node.sock makes the upload fails so exclude them: | |
| !./artifacts/**/node.sock | |
| # exclude cardano tools, saving ~50mb of data: | |
| !./artifacts/devnet/cardano-cli | |
| !./artifacts/devnet/cardano-node | |
| if-no-files-found: error | |
| publish-tests-results: | |
| if: success() || failure() | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - test | |
| steps: | |
| - name: Download Tests Results (${{ runner.os }}-${{ runner.arch }}) | |
| if: success() || failure() | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test-results-${{ runner.os }}-${{ runner.arch }} | |
| - name: Download Tests Results (macOS-X64) | |
| if: success() || failure() | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test-results-macOS-X64 | |
| - name: Download Tests Results (Windows-X64) | |
| if: success() || failure() | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test-results-Windows-X64 | |
| - name: Publish Unit Test Results | |
| if: success() || failure() | |
| uses: EnricoMi/publish-unit-test-result-action@v2 | |
| with: | |
| junit_files: ./**/test-results-*.xml | |
| docker-mithril: | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - build | |
| - check | |
| - test | |
| - run-test-lab | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| project: [ mithril-aggregator, mithril-client, mithril-signer ] | |
| permissions: | |
| contents: read | |
| packages: write | |
| env: | |
| PUSH_PACKAGES: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref)) }} | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository_owner }}/${{ matrix.project }} | |
| DOCKER_FILE: ./${{ matrix.project }}/Dockerfile.ci | |
| CONTEXT: . | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| unstable | |
| type=raw,value=${{ github.base_ref || github.ref_name }}-{{sha}} | |
| - name: Download built artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-distribution-${{ runner.os }}-${{ runner.arch }} | |
| path: ${{ matrix.project }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: ${{ env.CONTEXT }} | |
| file: ${{ env.DOCKER_FILE }} | |
| push: ${{ env.PUSH_PACKAGES }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| unstable-release: | |
| if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref)) | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - build | |
| - test | |
| - run-test-lab | |
| - check | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Prepare packaging | |
| run: mkdir package | |
| - name: Get short SHA | |
| id: slug | |
| run: echo "sha8=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT | |
| - name: Download built artifacts (Linux-X64) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-distribution-Linux-X64 | |
| path: ./package-Linux-X64 | |
| - name: Download Debian packages (Linux-X64) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-deb-packages-Linux-X64 | |
| path: ./package | |
| - name: Download built artifacts (macOS-X64) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-distribution-macOS-X64 | |
| path: ./package-macOS-X64 | |
| - name: Download built artifacts (Windows-X64) | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mithril-distribution-Windows-X64 | |
| path: ./package-Windows-X64 | |
| - name: Prepare distribution package | |
| uses: ./.github/workflows/actions/prepare-distribution | |
| with: | |
| version-name: unstable-${{ steps.slug.outputs.sha8 }} | |
| download-url-base: ${{ github.server_url }}/${{ github.repository }}/releases/download/unstable | |
| gpg-secret-key: ${{ secrets.GPG_SECRET_KEY }} | |
| compatibility-table: '{ "release-mainnet": "⛔", "release-preprod": "⛔", "pre-release-preview": "⛔", "testing-preview": "✔" }' | |
| - name: Update unstable release | |
| uses: marvinpinto/action-automatic-releases@latest | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| automatic_release_tag: unstable | |
| prerelease: true | |
| title: Unstable Development Builds | |
| files: package/* | |
| - name: Update unstable release body with release notes addon | |
| uses: tubone24/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TAG_NAME: unstable | |
| with: | |
| is_append_body: true | |
| body_path: ./release-notes-addon.txt | |
| deploy-testing: | |
| # Don't run on pull request from forks since they don't have access to all the needed secrets | |
| if: github.event_name == 'push' || (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| environment: [ testing-preview ] | |
| include: | |
| - environment: testing-preview | |
| environment_prefix: testing | |
| cardano_network: preview | |
| mithril_api_domain: api.mithril.network | |
| mithril_protocol_parameters: | | |
| { | |
| k = 2422 | |
| m = 20973 | |
| phi_f = 0.2 | |
| } | |
| mithril_signers: | | |
| { | |
| "1" = { | |
| type = "verified", | |
| pool_id = "", | |
| }, | |
| "2" = { | |
| type = "verified", | |
| pool_id = "", | |
| }, | |
| "3" = { | |
| type = "verified", | |
| pool_id = "", | |
| }, | |
| } | |
| terraform_backend_bucket: hydra-terraform-admin | |
| google_region: europe-west1 | |
| google_zone: europe-west1-b | |
| google_machine_type: e2-highmem-4 | |
| google_compute_instance_boot_disk_size: 200 | |
| google_compute_instance_data_disk_size: 250 | |
| environment: ${{ matrix.environment }} | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - docker-mithril | |
| defaults: | |
| run: | |
| working-directory: mithril-infra | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| - name: Get Docker image id | |
| run: echo "DOCKER_IMAGE_ID=${{ github.base_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV | |
| - name: ${{ env.DEPLOY == 'true' && 'Apply' || 'Plan' }} terraform infrastructure | |
| uses: ./.github/workflows/actions/deploy-terraform-infrastructure | |
| env: | |
| DEPLOY: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| with: | |
| dry_run: ${{ env.DEPLOY == 'true' && 'false' || 'true' }} | |
| terraform_backend_bucket: ${{ matrix.terraform_backend_bucket }} | |
| environment_prefix: ${{ matrix.environment_prefix }} | |
| environment: ${{ matrix.environment }} | |
| cardano_network: ${{ matrix.cardano_network }} | |
| google_region: ${{ matrix.google_region }} | |
| google_zone: ${{ matrix.google_zone }} | |
| google_machine_type: ${{ matrix.google_machine_type }} | |
| google_compute_instance_boot_disk_size: ${{ matrix.google_compute_instance_boot_disk_size }} | |
| google_compute_instance_data_disk_size: ${{ matrix.google_compute_instance_data_disk_size }} | |
| google_application_credentials: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} | |
| mithril_api_domain: ${{ matrix.mithril_api_domain }} | |
| mithril_image_id: ${{ env.DOCKER_IMAGE_ID }} | |
| mithril_protocol_parameters: ${{ toJSON(matrix.mithril_protocol_parameters) }} | |
| mithril_signers: ${{ toJSON(matrix.mithril_signers) }} | |
| mithril_genesis_secret_key: ${{ secrets.GENESIS_SECRET_KEY }} | |
| mithril_genesis_verification_key_url: ${{ vars.GENESIS_VERIFICATION_KEY_URL }} | |
| mithril_era_reader_address_url: ${{ vars.ERA_READER_ADDRESS_URL }} | |
| mithril_era_reader_verification_key_url: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }} | |
| mithril_era_reader_secret_key: ${{ secrets.ERA_READER_SECRET_KEY }} | |
| mithril_aggregator_cdn_cname: ${{ vars.AGGREGATOR_CDN_CNAME }} | |
| mithril_aggregator_snapshot_use_cdn_domain: ${{ vars.AGGREGATOR_USE_CDN_DOMAIN }} | |
| mithril_aggregator_snapshot_compression_algorithm: ${{ vars.AGGREGATOR_SNAPSHOT_COMPRESSION_ALGORITHM }} | |
| mithril_aggregator_zstandard_parameters_level: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_LEVEL }} | |
| mithril_aggregator_zstandard_parameters_workers: ${{ vars.AGGREGATOR_SNAPSHOT_ZSTANDARD_WORKERS }} | |
| mithril_aggregator_cexplorer_pools_url: ${{ vars.AGGREGATOR_CEXPLORER_POOLS_URL }} | |
| prometheus_auth_username: ${{ secrets.PROMETHEUS_AUTH_USERNAME }} | |
| prometheus_auth_password: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }} | |
| prometheus_ingest_host: ${{ vars.PROMETHEUS_INGEST_HOST }} | |
| prometheus_ingest_username: ${{ secrets.PROMETHEUS_INGEST_USERNAME }} | |
| prometheus_ingest_password: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }} | |
| loki_auth_username: ${{ secrets.LOKI_AUTH_USERNAME }} | |
| loki_auth_password: ${{ secrets.LOKI_AUTH_PASSWORD }} | |
| loki_ingest_host: ${{ vars.LOKI_INGEST_HOST }} | |
| loki_ingest_username: ${{ secrets.LOKI_INGEST_USERNAME }} | |
| loki_ingest_password: ${{ secrets.LOKI_INGEST_PASSWORD }} |