ci.yml

name: CI

on:
  push:
    branches:
      - 'main'
      - 'hotfix**'
  pull_request:
    types: [opened, reopened, synchronize]
    paths-ignore: # ignore docs only changes since they use a dedicated workflows: docs.yml
      - 'docs/**'
      - 'mithril-explorer/**'
      - '.github/workflows/docs.yml'
    branches-ignore:
      - 'hotfix**' # hotfix are handled by the push trigger

concurrency:
  group: ci-build-test-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-ubuntu-X64:
    runs-on: ubuntu-20.04
    outputs:
      eras: ${{ steps.eras-test-lab.outputs.eras }}
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: 20.04-${{ secrets.CACHE_VERSION }}
          cargo-tools: cargo-deb

      # We separate the build in 2 steps as we want to avoid side effects with Rust feature unification.
      - name: Cargo build - Tooling
        shell: bash
        run: cargo build --features portable --release --workspace --exclude mithril-aggregator --exclude mithril-client --exclude mithril-signer --exclude mithril-stm

      - name: Build Mithril workspace & publish artifacts
        uses: ./.github/workflows/actions/build-upload-mithril-artifact
        with:
          build-args: --features bundle_openssl

      - name: Build Debian packages
        shell: bash
        run: |
          cargo deb --no-build -p mithril-aggregator
          cargo deb --no-build -p mithril-signer
          cargo deb --no-build -p mithril-client
      
      - name: Publish Debian packages
        uses: actions/upload-artifact@v3
        with:
          name: mithril-deb-packages-${{ runner.os }}-${{ runner.arch }}
          path: target/debian/*.deb
          if-no-files-found: error

      - name: Publish End-to-end runner (${{ runner.os }}-${{ runner.arch }})
        uses: actions/upload-artifact@v3
        with:
          name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
          path: target/release/mithril-end-to-end
          if-no-files-found: error

      - name: Prepare test lab eras
        id: eras-test-lab
        run: |
          ERAS=$(./target/release/mithril-aggregator era list --json)
          echo "Test Lab Eras: $ERAS"
          echo "eras=$ERAS" >> $GITHUB_OUTPUT
  
  build:
    strategy:
      fail-fast: false
      matrix:
        os: [ macos-12, windows-latest ]
        
        include:
        # Only build client on windows & mac
        - os: macos-12
          build-args: -p mithril-client
        - os: windows-latest
          build-args: -p mithril-client 
    runs-on: ${{ matrix.os }}
    
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}
      
      - name: Build Mithril workspace & publish artifacts
        uses: ./.github/workflows/actions/build-upload-mithril-artifact
        with:
          build-args: --features bundle_openssl ${{ matrix.build-args }}
  
  test:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-22.04, macos-12, windows-latest ]
        
        include:
          - os: ubuntu-22.04
            test-args: --features portable --workspace
          # Only test client on windows & mac (since its the only binaries supported for those os for now)
          - os: macos-12
            test-args: -p mithril-client
          - os: windows-latest
            test-args: -p mithril-client
    
    runs-on: ${{ matrix.os }}
    
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}
          cargo-tools: cargo-nextest
      
      - name: Build tests
        run: cargo nextest run --no-run ${{ matrix.test-args }}

      - name: Run tests
        shell: bash
        run: |
          cargo nextest run --profile ci ${{ matrix.test-args }}
          
      - name: Rename junit file to include runner info
        shell: bash
        if: success() || failure()
        run: |
          mv target/nextest/ci/tests-result.junit.xml test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}.xml

      - name: Upload Tests Results
        uses: actions/upload-artifact@v3
        if: success() || failure()
        with:
          name: test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}
          path: |
            ./test-results-*.xml
  
  check:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Install stable toolchain, tools, and restore cache
        uses: ./.github/workflows/actions/toolchain-and-cache
        with:
          cache-version: ${{ secrets.CACHE_VERSION }}
          cargo-tools: cargo-sort clippy-sarif sarif-fmt

      - name: Clippy Check
        if: success() || failure()
        run: |
          cargo clippy \
            --all-features --all-targets --no-deps --message-format=json \
            | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt
          
          # Make this step fail if any warning has been found
          if [[ $(cat rust-clippy-results.sarif | jq '.runs[0].results') != "[]" ]]; then
            false
          fi

      - name: Upload clippy analysis results to GitHub
        if: success() || failure()
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: rust-clippy-results.sarif
          wait-for-processing: true

      - name: Cargo fmt
        if: success() || failure()
        shell: bash
        run: cargo fmt --check

      - name: Cargo sort
        if: success() || failure()
        shell: bash
        run: cargo sort -w -c
      
      - name: Dependency & Vulnerabilities Review
        uses: actions/dependency-review-action@v3
        with:
          base-ref: ${{ github.event.pull_request.base.sha || 'main' }}
          head-ref: ${{ github.event.pull_request.head.sha || github.ref }}

  run-test-lab:
    runs-on: ubuntu-22.04
    needs: [ build-ubuntu-X64 ]
    strategy:
      fail-fast: false
      matrix:
        era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }}
        run_id: [1,2,3]
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Download binaries
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
          path: ./bin

      - name: Download rust test runner
        uses: actions/download-artifact@v3
        with:
          name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
          path: ./

      - run: |
          chmod +x ./bin/mithril-aggregator
          chmod +x ./bin/mithril-client
          chmod +x ./bin/mithril-signer
          chmod +x ./mithril-end-to-end
          mkdir artifacts

      - name: Test
        run: ./mithril-end-to-end --bin-directory ./bin --work-directory=./artifacts --devnet-scripts-directory=./mithril-test-lab/mithril-devnet --mithril-era=${{ matrix.era }}

      - name: Upload E2E Tests Artifacts
        if:  ${{ failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-era_${{ matrix.era }}-run_id_${{ matrix.run_id }}
          path: |
            ./artifacts/*
            # including node.sock makes the upload fails so exclude them:
            !./artifacts/**/node.sock
            # exclude cardano tools, saving ~50mb of data:
            !./artifacts/devnet/cardano-cli
            !./artifacts/devnet/cardano-node
          if-no-files-found: error
  
  publish-tests-results:
    if: success() || failure()
    runs-on: ubuntu-22.04
    needs: 
      - test
    steps:
      - name: Download Tests Results (${{ runner.os }}-${{ runner.arch }})
        if: success() || failure()
        uses: actions/download-artifact@v3
        with:
          name: test-results-${{ runner.os }}-${{ runner.arch }}

      - name: Download Tests Results (macOS-X64)
        if: success() || failure()
        uses: actions/download-artifact@v3
        with:
          name: test-results-macOS-X64

      - name: Download Tests Results (Windows-X64)
        if: success() || failure()
        uses: actions/download-artifact@v3
        with:
          name: test-results-Windows-X64

      - name: Publish Unit Test Results
        if: success() || failure()
        uses: EnricoMi/publish-unit-test-result-action@v2
        with:
          junit_files: ./**/test-results-*.xml

  docker-mithril:
    runs-on: ubuntu-22.04
    needs:
      - build
      - check
      - test
      - run-test-lab
    strategy:
      fail-fast: false
      matrix:
        project: [ mithril-aggregator, mithril-client, mithril-signer ]
    
    permissions:
      contents: read
      packages: write

    env:
      PUSH_PACKAGES: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref)) }}
      REGISTRY: ghcr.io
      IMAGE_NAME: ${{ github.repository_owner }}/${{ matrix.project }}
      DOCKER_FILE: ./${{ matrix.project }}/Dockerfile.ci
      CONTEXT: .

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            unstable
            type=raw,value=${{ github.base_ref || github.ref_name }}-{{sha}}

      - name: Download built artifacts
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
          path: ${{ matrix.project }}

      - name: Build and push Docker image
        uses: docker/build-push-action@v3
        with:
          context: ${{ env.CONTEXT }}
          file: ${{ env.DOCKER_FILE }}
          push: ${{ env.PUSH_PACKAGES }}
          tags: ${{ steps.meta.outputs.tags }}

  unstable-release:
    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref))
    runs-on: ubuntu-22.04
    needs:
      - build
      - test
      - run-test-lab
      - check
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3
      
      - name: Prepare packaging
        run: mkdir package

      - name: Get short SHA
        id: slug
        run: echo "sha8=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
      
      - name: Download built artifacts (Linux-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-Linux-X64
          path: ./package-Linux-X64

      - name: Download Debian packages (Linux-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-deb-packages-Linux-X64
          path: ./package

      - name: Download built artifacts (macOS-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-macOS-X64
          path: ./package-macOS-X64

      - name: Download built artifacts (Windows-X64)
        uses: actions/download-artifact@v3
        with:
          name: mithril-distribution-Windows-X64
          path: ./package-Windows-X64

      - name: Prepare distribution package
        uses: ./.github/workflows/actions/prepare-distribution
        with:
          version-name: unstable-${{ steps.slug.outputs.sha8 }}
          download-url-base: ${{ github.server_url }}/${{ github.repository }}/releases/download/unstable
          gpg-secret-key: ${{ secrets.GPG_SECRET_KEY }}

      - name: Update unstable release
        uses: marvinpinto/action-automatic-releases@latest
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          automatic_release_tag: unstable
          prerelease: true
          title: Unstable Development Builds
          files: package/*

      - name: Update unstable release body with release notes addon
        uses: tubone24/update_release@v1.3.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG_NAME: unstable 
        with:
          is_append_body: true
          body_path: ./release-notes-addon.txt

  deploy-testing:
    # Don't run on pull request from forks since they don't have access to all the needed secrets
    if: github.event_name == 'push' || (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork)
    strategy:
      fail-fast: false
      matrix:
        environment: [ testing-preview ]
        include:
        - environment: testing-preview
          environment_prefix: testing
          cardano_network: preview
          mithril_api_domain: api.mithril.network
          mithril_protocol_parameters: |
            {
              k     = 2422
              m     = 20973
              phi_f = 0.2
            }
          mithril_signers: |
            {
              "1" = {
                type    = "verified",
                pool_id = "",
              },
              "2" = {
                type    = "verified",
                pool_id = "",
              },
              "3" = {
                type    = "verified",
                pool_id = "",
              },
            }
          terraform_backend_bucket: hydra-terraform-admin
          google_region: europe-west1
          google_zone: europe-west1-b
          google_machine_type: e2-highmem-4
          google_compute_instance_data_disk_size: 250
          
    runs-on: ubuntu-22.04

    needs:
      - docker-mithril

    environment: ${{ matrix.environment }}

    env:
      GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
      GENESIS_SECRET_KEY: ${{ secrets.GENESIS_SECRET_KEY }}
      GENESIS_VERIFICATION_KEY_URL: ${{ vars.GENESIS_VERIFICATION_KEY_URL }}
      ERA_READER_ADDRESS_URL: ${{ vars.ERA_READER_ADDRESS_URL }}
      ERA_READER_VERIFICATION_KEY_URL: ${{ vars.ERA_READER_VERIFICATION_KEY_URL }}
      ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
      PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
      PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
      LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
      LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}

    defaults:
      run:
        working-directory: mithril-infra

    steps:

    - name: Checkout sources
      uses: actions/checkout@v3

    - name: Get Docker image id
      run: echo "DOCKER_IMAGE_ID=${{ github.base_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV

    - name: Prepare service account credentials
      run: |
        echo '${{ env.GOOGLE_APPLICATION_CREDENTIALS}}' > ./google-application-credentials.json
        chmod u+x ./assets/tools/utils/google-credentials-public-key.sh
        ./assets/tools/utils/google-credentials-public-key.sh ./google-application-credentials.json ./assets/ssh_keys curry

    - name: Prepare terraform variables
      run: |
        cat > ./env.variables.tfvars << EOF
          environment_prefix                      = "${{ matrix.environment_prefix }}"
          cardano_network                         = "${{ matrix.cardano_network }}"
          google_region                           = "${{ matrix.google_region }}"
          google_zone                             = "${{ matrix.google_zone }}"
          google_machine_type                     = "${{ matrix.google_machine_type }}"
          google_compute_instance_data_disk_size  = "${{ matrix.google_compute_instance_data_disk_size }}"
          google_service_credentials_json_file    = "./google-application-credentials.json"
          mithril_api_domain                      = "${{ matrix.mithril_api_domain }}"
          mithril_image_id                        = "${{ env.DOCKER_IMAGE_ID }}"
          mithril_genesis_verification_key_url    = "${{ env.GENESIS_VERIFICATION_KEY_URL }}"
          mithril_genesis_secret_key              = "${{ env.GENESIS_SECRET_KEY }}"
          mithril_protocol_parameters             = ${{ matrix.mithril_protocol_parameters }}
          mithril_era_reader_adapter_type         = "cardano-chain"
          mithril_era_reader_address_url          = "${{ env.ERA_READER_ADDRESS_URL }}"
          mithril_era_reader_verification_key_url = "${{ env.ERA_READER_VERIFICATION_KEY_URL }}"
          mithril_era_reader_secret_key           = "${{ env.ERA_READER_SECRET_KEY }}"
          mithril_signers                         = ${{ matrix.mithril_signers }}
          prometheus_auth_username                = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
          prometheus_auth_password                = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
          loki_auth_username                      = "${{ env.LOKI_AUTH_USERNAME }}"
          loki_auth_password                      = "${{ env.LOKI_AUTH_PASSWORD }}"
        EOF
        terraform fmt ./env.variables.tfvars
        cat ./env.variables.tfvars

    - name: Setup Terraform
      uses: hashicorp/setup-terraform@v2
      with:
        terraform_wrapper: false

    - name: Init Terraform
      run: |
        GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform init -backend-config="bucket=${{ matrix.terraform_backend_bucket }}" -backend-config="prefix=terraform/mithril-${{ matrix.environment }}"

    - name: Check Terraform
      run: terraform fmt -check

    - name: Terraform Plan
      run: |
        GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform plan --var-file=./env.variables.tfvars

    - name: Terraform Apply
      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
      run: |
        GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform apply -auto-approve --var-file=./env.variables.tfvars

    - name: Cleanup
      run: |
        rm -f ./env.variables.tfvars
        rm -f ./google-application-credentials.json