mobilecoin-dev-cd.yaml

# Copyright (c) 2018-2022 The MobileCoin Foundation
#
# MobileCoin Core projects - Build, deploy to development.

name: Mobilecoin CD

env:
  CHART_REPO: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public
  DOCKER_ORG: mobilecoin
  RELEASE_3X_TAG: v3.0.0-dev

on:
  pull_request:
    branches:
    - 'release/**'
    paths-ignore:
    - '**.md'
  push:
    branches:
    - 'feature/**'
    tags:
    - 'v[0-9]+*'
    paths-ignore:
    - '**.md'

# don't run more than one at a time for a branch/tag
concurrency:
  group: mobilecoin-dev-cd-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

# Ignore dependabot. We just need to 'if' the top level jobs.
# Other jobs that 'need' these top level jobs will be skipped.
jobs:
############################################
# Generate environment information
############################################
  generate-metadata:
    if: "! startsWith(github.head_ref, 'dependabot/')"
    name: 👾 Environment Info 👾
    runs-on: [self-hosted, Linux, small]
    outputs:
      namespace: ${{ steps.meta.outputs.namespace }}
      tag: ${{ steps.meta.outputs.tag }}
      docker_tag: ${{ steps.meta.outputs.docker_tag }}
      docker_org: ${{ env.DOCKER_ORG }}
      chart_repo: ${{ env.CHART_REPO }}
      release_3x_tag: ${{ env.RELEASE_3X_TAG }}

    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Generate version metadata
      id: meta
      shell: bash
      run: |
        .internal-ci/util/metadata.sh

    - name: 👾 Print Environment Details 👾
      shell: bash
      env:
        CHART_REPO: ${{ env.CHART_REPO }}
        NAMESPACE: ${{ steps.meta.outputs.namespace }}
        VERSION: ${{ steps.meta.outputs.tag }}
      run: |
        .internal-ci/util/print_details.sh

#########################################
# Build binaries
#########################################
  build-rust-hardware-projects:
    needs:
    - generate-metadata
    runs-on: [self-hosted, Linux, large-cd]
    container:
      image: mobilecoin/rust-sgx-base:v0.0.23

    env:
      ENCLAVE_SIGNING_KEY_PATH: ${{ github.workspace }}/.tmp/enclave_signing.pem
      MINTING_TRUST_ROOT_PUBLIC_KEY_PEM: ${{ github.workspace }}/.tmp/minting_trust_root.public.pem
    steps:
    - name: Checkout
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      uses: actions/checkout@v3

    - name: Write environment values
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      env:
        ENCLAVE_SIGNING_KEY: ${{ secrets.DEV_ENCLAVE_SIGNING_KEY }}
        MINTING_TRUST_ROOT_PUBLIC: ${{ secrets.DEV_MINTING_TRUST_ROOT_PUBLIC }}
      run: |
        mkdir -p "${GITHUB_WORKSPACE}/.tmp"
        echo "${ENCLAVE_SIGNING_KEY}" > "${ENCLAVE_SIGNING_KEY_PATH}"
        echo "${MINTING_TRUST_ROOT_PUBLIC}" > "${MINTING_TRUST_ROOT_PUBLIC_KEY_PEM}"

    - name: Cache rust build binaries
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      id: rust_artifact_cache
      uses: ./.github/actions/mobilecoin-cache-rust-binaries
      with:
        cache_buster: ${{ secrets.CACHE_BUSTER }}

    - name: Cache cargo packages
      # We don't need cargo packages if we already have binaries.
      if: |
        steps.rust_artifact_cache.outputs.cache-hit != 'true' &&
        ! contains(github.event.head_commit.message, '[skip build]')
      uses: ./.github/actions/mobilecoin-cache-cargo-package
      with:
        cache_buster: ${{ secrets.CACHE_BUSTER }}

    - name: Build rust hardware projects
      if: |
        steps.rust_artifact_cache.outputs.cache-hit != 'true' &&
        ! contains(github.event.head_commit.message, '[skip build]')
      env:
        IAS_MODE: DEV
        SGX_MODE: HW
        RUST_BACKTRACE: full
        MOB_RELEASE: 1
        CONSENSUS_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }}
        LEDGER_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }}
        VIEW_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }}
        INGEST_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }}
      run: |
        git config --global --add safe.directory '*'
        cargo build --release \
          -p mc-admin-http-gateway \
          -p mc-consensus-mint-client \
          -p mc-consensus-service \
          -p mc-fog-distribution \
          -p mc-fog-ingest-client \
          -p mc-fog-ingest-server \
          -p mc-fog-ledger-server \
          -p mc-fog-report-cli \
          -p mc-fog-report-server \
          -p mc-fog-sql-recovery-db \
          -p mc-fog-test-client \
          -p mc-fog-view-server \
          -p mc-ledger-distribution \
          -p mc-ledger-from-archive \
          -p mc-ledger-migration \
          -p mc-mobilecoind \
          -p mc-mobilecoind-dev-faucet \
          -p mc-mobilecoind-json \
          -p mc-util-generate-sample-ledger \
          -p mc-util-grpc-admin-tool \
          -p mc-util-grpc-token-generator \
          -p mc-util-keyfile \
          -p mc-util-seeded-ed25519-key-gen \
          -p mc-watcher

    - name: Copy artifacts to cache
      if: |
        steps.rust_artifact_cache.outputs.cache-hit != 'true' &&
        ! contains(github.event.head_commit.message, '[skip build]')
      run: |
        mkdir -p rust_build_artifacts
        find target/release -maxdepth 1 -executable -type f -exec cp "{}" rust_build_artifacts/ \;
        find target/release -maxdepth 1 -name "*.signed.so" -exec cp "{}" rust_build_artifacts/ \;

    - name: Create css measurements
      if: |
        steps.rust_artifact_cache.outputs.cache-hit != 'true' &&
        ! contains(github.event.head_commit.message, '[skip build]')
      shell: bash
      run: |
        cd rust_build_artifacts
        for i in *.signed.so
        do
          css=$(echo -n "${i}" | sed -r 's/(.*)\.signed\.so/\1/')
          sgx_sign dump -enclave "${i}" -dumpfile /dev/null -cssfile "${css}.css"
        done

    - name: Check artifacts
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      run: |
        ls -alR rust_build_artifacts

    - name: Upload artifacts
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      uses: actions/upload-artifact@v3
      with:
        name: rust-binaries
        path: rust_build_artifacts/

  build-go-projects:
    runs-on: [self-hosted, Linux, small]
    needs:
    - generate-metadata
    container:
      image: golang:1.18.5
    steps:
    - name: Checkout
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      uses: actions/checkout@v3

    - name: Add protobuf-compiler
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      run: |
        apt update
        apt install -y protobuf-compiler zstd

    - name: Cache go build binaries
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      id: go_artifact_cache
      uses: ./.github/actions/mobilecoin-cache-go-binaries
      with:
        cache_buster: ${{ secrets.CACHE_BUSTER }}

    - name: Build go-grpc-gateway
      if: |
        steps.go_artifact_cache.outputs.cache-hit != 'true' &&
        ! contains(github.event.head_commit.message, '[skip build]')
      run: |
        cd go-grpc-gateway
        ./install_tools.sh
        ./build.sh
        mkdir -p ../go_build_artifacts
        cp grpc-proxy ../go_build_artifacts/

    - name: check artifacts
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      run: |
        ls -alR go_build_artifacts

    - name: Upload Artifacts
      if: "! contains(github.event.head_commit.message, '[skip build]')"
      uses: actions/upload-artifact@v3
      with:
        name: go-binaries
        path: go_build_artifacts/

########################################
# Create/Refresh base runtime image
########################################
  docker-base:
    runs-on: [self-hosted, Linux, small]
    needs:
    - generate-metadata
    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Generate Docker Tags
      id: docker_meta
      uses: docker/metadata-action@v4
      with:
        images: ${{ env.DOCKER_ORG }}/runtime-base
        flavor: |
          latest=true
        tags: |
          type=sha

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    - name: Publish to DockerHub
      id: docker_publish_dockerhub
      uses: docker/build-push-action@v4
      with:
        build-args: |
          REPO_ORG=${{ env.DOCKER_ORG }}
        context: .
        file: .internal-ci/docker/Dockerfile.runtime-base
        labels: ${{ steps.docker_meta.outputs.labels }}
        push: true
        tags: ${{ steps.docker_meta.outputs.tags }}

#########################################
# Build/Publish public artifacts
#########################################
  docker:
    runs-on: [self-hosted, Linux, small]
    needs:
    - build-go-projects
    - build-rust-hardware-projects
    - docker-base
    - generate-metadata
    strategy:
      matrix:
        image:
        - bootstrap-tools
        - fogingest
        - fog-ledger
        - fogreport
        - fog-test-client
        - fogview
        - go-grpc-gateway
        - node_hw
        - mobilecoind
        - watcher
    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Cache rust build binaries
      id: rust_artifact_cache
      uses: ./.github/actions/mobilecoin-cache-rust-binaries
      with:
        cache_buster: ${{ secrets.CACHE_BUSTER }}

    - name: Cache go build binaries
      id: go_artifact_cache
      uses: ./.github/actions/mobilecoin-cache-go-binaries
      with:
        cache_buster: ${{ secrets.CACHE_BUSTER }}

    - name: Generate Docker Tags
      id: docker_meta
      uses: docker/metadata-action@v4
      with:
        images: ${{ env.DOCKER_ORG }}/${{ matrix.image }}
        tags: ${{ needs.generate-metadata.outputs.docker_tag }}

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    - name: Get short SHA
      run: echo "GH_SHORT_SHA=sha-$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "${GITHUB_ENV}"

    - name: Publish to DockerHub
      id: docker_publish_dockerhub
      uses: docker/build-push-action@v4
      with:
        build-args: |
          REPO_ORG=${{ env.DOCKER_ORG }}
          BASE_TAG=${{ env.GH_SHORT_SHA }}
          RUST_BIN_PATH=rust_build_artifacts
          GO_BIN_PATH=go_build_artifacts
        cache-from: type=registry,ref=${{ env.DOCKER_ORG }}/${{ matrix.image }}:buildcache-${{ needs.generate-metadata.outputs.namespace }}
        cache-to: type=registry,ref=${{ env.DOCKER_ORG }}/${{ matrix.image }}:buildcache-${{ needs.generate-metadata.outputs.namespace }}
        context: .
        file: .internal-ci/docker/Dockerfile.${{ matrix.image }}
        labels: ${{ steps.docker_meta.outputs.labels }}
        push: true
        tags: ${{ steps.docker_meta.outputs.tags }}

  charts:
    runs-on: [self-hosted, Linux, small]
    needs:
    - docker
    - generate-metadata
    strategy:
      matrix:
        chart:
        - consensus-node
        - consensus-node-config
        - fog-ingest
        - fog-ingest-config
        - fog-services
        - fog-services-config
        - fog-test-client
        - mc-core-common-config
        - mc-core-dev-env-setup
        - mobilecoind
        - watcher
    steps:
    - name: Checkout
      if: "! contains(github.event.head_commit.message, '[skip charts]')"
      uses: actions/checkout@v3

    - name: Package and publish chart
      if: "! contains(github.event.head_commit.message, '[skip charts]')"
      uses: mobilecoinofficial/gha-k8s-toolbox@v1
      with:
        action: helm-publish
        chart_repo_username: ${{ secrets.HARBOR_USERNAME }}
        chart_repo_password: ${{ secrets.HARBOR_PASSWORD }}
        chart_repo: ${{ env.CHART_REPO }}
        chart_app_version: ${{ needs.generate-metadata.outputs.tag }}
        chart_version: ${{ needs.generate-metadata.outputs.tag }}
        chart_path: .internal-ci/helm/${{ matrix.chart }}

################################################
# Bootstrap namespace to v3.0.0-dev from backup
################################################
  bootstrap-v3-bv2:
    uses: ./.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml
    needs:
    - generate-metadata
    with:
      block_version: 2
      chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }}
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      version: ${{ needs.generate-metadata.outputs.release_3x_tag }}
    secrets: inherit

###############################################
# Deploy current version to namespace block v2
###############################################
  deploy-current-bv2-release:
    uses: ./.github/workflows/mobilecoin-workflow-dev-deploy.yaml
    needs:
    - bootstrap-v3-bv2
    - charts
    - generate-metadata
    with:
      block_version: 2
      chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }}
      docker_image_org: ${{ needs.generate-metadata.outputs.docker_org }}
      ingest_color: blue
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      version: ${{ needs.generate-metadata.outputs.tag }}
    secrets: inherit

  test-current-bv2-release:
    uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml
    needs:
    - deploy-current-bv2-release
    - generate-metadata
    with:
      fog_distribution: false
      ingest_color: blue
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      testing_block_v0: false
      testing_block_v2: true
      testing_block_v3: false
      generate_and_submit_mint_config_tx_uses_json: true
    secrets: inherit

#################################################
# Update current consensus to namespace block v3
#################################################
  update-current-to-bv3:
    uses: ./.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml
    needs:
    - test-current-bv2-release
    - generate-metadata
    with:
      block_version: 3
      chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }}
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      version: ${{ needs.generate-metadata.outputs.tag }}
    secrets: inherit

  test-current-bv3-release:
    uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml
    needs:
    - update-current-to-bv3
    - generate-metadata
    with:
      fog_distribution: false
      ingest_color: blue
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      testing_block_v0: false
      testing_block_v2: false
      testing_block_v3: true
      generate_and_submit_mint_config_tx_uses_json: true
    secrets: inherit

  mobilecoin-cd-complete:
    # Dummy step for a standard GHA Check that won't change when we update the tests.
    runs-on: [self-hosted, Linux, small]
    needs:
    - test-current-bv3-release
    steps:
      - name: CD is Complete
        run: 'true'

###############################################################
# Clean up deployments
###############################################################
# we keep feature/*
# run on tag
# run on pr to release/*
  cleanup-after-tag:
    if: github.ref_type == 'tag'
    needs:
    - test-current-bv3-release
    - generate-metadata
    uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml
    with:
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      delete_namespace: true
    secrets: inherit

  cleanup-after-pr-to-release-branch:
    if: github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/')
    needs:
    - test-current-bv3-release
    - generate-metadata
    uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml
    with:
      namespace: ${{ needs.generate-metadata.outputs.namespace }}
      delete_namespace: true
    secrets: inherit