build-ci-docker-images.yaml

# ## build-ci-docker-images
#
# Action to build and deploy Docker images from the CI Dockerfiles in the repo.
#
# ### Action Triggers
#
# The action is triggered by a push to the `main`/`master` or `testing` branch when it
# affects the CI `Dockerfile`s. It can also be triggered manually via the
# Actions web UI, the GH REST API or the GH CLI tool, e.g.
#
# gh workflow run build-ci-docker-images
#
# #### `push`
#
# When triggered by a push, it will dynamically determine the context and the
# targets for each `Dockerfile`, automatically removing some targets by default
# (`RE_TARGET_EXCLUDE` and `testimage`) and it will build a Docker image for
# each remaining target.
#
# For every image built it will push to the registry three image tags:
#   * a `sha` tag (the short hash of the commit that triggered the push)
#   * a `date` tag with the current date in ISO format (`YYYYMMDD`)
#   * a `latest` tag
#
# #### `workflow_dispatch` (manual)
#
# When triggered by hand, the workflow will run by default "like a `push` run
# in `dry-run` mode".
#
# That is, it will still dynamically determine the context and the targets for
# each `Dockerfile` and it will automatically remove `RE_TARGET_EXCLUDE` but:
#
# 1. it won't exclude `testimage`, rather it will have `RE_TARGET_INCLUDE` set
#    to only `testimage` by default (a very simple image to exercise the build
#    that doesn't take much compute)
#
# 2. it will only build `linux/amd64` by default (to further reduce testing
#    times and cost)
#
# 2. it will build the `testimage` but **it won't tag `latest` or push any of
#    the image tags to the registry**.
#
# These defaults can be overriden by setting the `workflow_dispatch` input
# variables: `RE_TARGET_EXCLUDE`, `RE_TARGET_INCLUDE`, `TAG_DATE`, `TAG_LATEST`
# and `PUSH`. E.g.:
#
# gh workflow run build-ci-docker-images \
#   -f RE_TARGET_INCLUDE=ubuntu2404 -f TAG_DATE=20241101

name: build-ci-docker-images

on:
  push:
    branches:
      - main
      - master
      - testing
    paths:
      # NOTE: keep in-sync with env.SPARSE_PATH
      - buildkite/docker/*/Dockerfile

  workflow_dispatch:
    inputs:
      RE_TARGET_EXCLUDE:
        description: |-
          Filter out Docker targets matching this extended regex pattern
        # NOTE: keep in-sync with env.RE_TARGET_EXCLUDE
        default: ^centos7|^ubuntu16|nojdk
      RE_TARGET_INCLUDE:
        description: |-
          Select Docker targets matching this extended regex pattern
        # NOTE: keep in-sync with env.TEST_IMAGE
        default: testimage
      PLATFORMS:
        default: linux/amd64
      TAG_DATE:
        description: Tag image date in ISO format (YYYYMMDD)
      TAG_LATEST:
        description: Tag image version as 'latest'
        default: false
      PUSH:
        description: Push images to registry
        default: false

env:
  REGISTRY: ghcr.io
  SPARSE_PATH: buildkite/docker
  TEST_IMAGE: testimage

  GH_EVENT_NAME: ${{ github.event_name }}
  GH_REF_NAME: ${{ github.ref_name }}
  GH_REPO: ${{ github.repository }}
  GH_SHA: ${{ github.sha }}
  GH_SHA_BEFORE: ${{ github.event.before }}

  RE_TARGET_EXCLUDE: ${{ inputs.RE_TARGET_EXCLUDE || '^centos7|^ubuntu16|nojdk' }}
  RE_TARGET_INCLUDE: ${{ inputs.RE_TARGET_INCLUDE }}
  PLATFORMS: ${{ inputs.PLATFORMS || 'linux/amd64,linux/arm64' }}
  TAG_DATE: ${{ inputs.TAG_DATE }}
  TAG_LATEST: ${{ inputs.TAG_LATEST }}
  PUSH: ${{ github.event_name == 'push' || inputs.PUSH }}

jobs:
  setup-targets:
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash --noprofile --norc -euo pipefail {0}

    outputs:
      targets: ${{ steps.define_targets.outputs.targets }}
      targets_length: ${{ steps.define_targets.outputs.targets_length }}

    steps:
      - name: Sparse checkout SPARSE_PATH
        uses: actions/checkout@v4
        with:
          sparse-checkout-cone-mode: false
          sparse-checkout: |-
            ${{ env.SPARSE_PATH }}

      - name: Get DOCKERFILES
        run: |-
          # NOTE: on pushing the first commit of a new branch or when running
          # manually via workflow_dispatch the before SHA will be null
          if [[ -z $GH_SHA_BEFORE ]]; then
            echo '>>>> ls'
            DOCKERFILES="$(ls $SPARSE_PATH/*/Dockerfile)"
          else
            echo '>>>> git diff'
            DOCKERFILES="$(
              git diff --name-only $GH_SHA_BEFORE $GH_SHA | grep Dockerfile || echo ""
            )"
            echo ">>> DOCKERFILES=@$DOCKERFILES@"
          fi

          # NOTE: git diff could be empty if e.g. we are force-pushing a stack
          # of commits where one or more commits do change Dockerfiles but
          # there's no change to any Dockerfile between the last commit and this
          # forced push. In such case, we will leave it empty and will handle
          # it in the next step

          DOCKERFILES_JSON="$(echo "$DOCKERFILES" | jq -R '.' | jq -sc '.' )"
          echo "DOCKERFILES_JSON=$DOCKERFILES_JSON"
          echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" >> $GITHUB_ENV

      - name: Define targets
        id: define_targets
        run: |-
          set -euo pipefail

          if [[ "$GH_EVENT_NAME" == "push" ]]; then
            RE_TARGET_EXCLUDE="$RE_TARGET_EXCLUDE|$TEST_IMAGE"
          fi

          DOCKERFILES=("$(echo "$DOCKERFILES_JSON" | jq -r 'join(" ")')")

          if [[ ${#DOCKERFILES[@]} -gt 0 ]]; then
            TARGETS_LS="$(
              grep -i '^FROM .* AS ' ${DOCKERFILES[@]} |
                awk '{print $NF}' |
                grep -E "$RE_TARGET_INCLUDE" |
                grep -vE "$RE_TARGET_EXCLUDE" |
                jq -R '.'
            )"
          else
            TARGETS_LS=""
          fi

          TARGETS="$(echo "$TARGETS_LS" | jq -sc '.')"
          TARGETS_LENGTH="$(echo "$TARGETS_LS" | jq -s 'length')"

          echo "targets=$TARGETS"
          echo "targets_length=$TARGETS_LENGTH"

          echo "targets=$TARGETS" >> "$GITHUB_OUTPUT"
          echo "targets_length=$TARGETS_LENGTH" >> "$GITHUB_OUTPUT"

  build-and-publish-docker-images:
    needs: setup-targets
    if: ${{ false && needs.setup-targets.outputs.targets_length > 0 }}

    runs-on: ubuntu-latest

    strategy:
      matrix:
        target: ${{ fromJson(needs.setup-targets.outputs.targets) }}

    defaults:
      run:
        shell: bash --noprofile --norc -euo pipefail {0}

    steps:
      - name: Sparse checkout SPARSE_PATH
        uses: actions/checkout@v4
        with:
          sparse-checkout-cone-mode: false
          sparse-checkout: |-
            ${{ env.SPARSE_PATH }}

      - name: Set up dynamic env
        env:
          MATRIX_TARGET: ${{ matrix.target }}
        run: |-
          declare -A TAGS

          TAGS[sha]="${GH_SHA::7}"

          TAGS[date]="$TAG_DATE"
          # set default date value if TAG_DATE is not set, is empty
          # or is an empty string
          if [[ -z "${TAGS[date]+isset}" || -z "${TAGS[date]// }" ]]; then
            TAGS[date]="$(date +%Y%m%d)"
          fi

          if [[ "$GH_EVENT_NAME" == "push" || "$TAG_LATEST" == "true" ]]; then
            TAGS[latest]="latest"
          fi

          if [[ "$REGISTRY" == "gcr.io" ]]; then
            IMAGE_PREFIX="bazel-public"
          elif [[ "$REGISTRY" == "ghcr.io" ]]; then
            IMAGE_PREFIX="$GH_REPO"
          else
            echo "Invalid registry: $REGISTRY"
            exit 1
          fi

          if [[ "$GH_REF_NAME" == "testing" ]]; then
            IMAGE_PREFIX="$IMAGE_PREFIX/testing"
          fi

          IMAGE_NAME="$REGISTRY/$IMAGE_PREFIX/$MATRIX_TARGET"

          DISTRO="${MATRIX_TARGET%%-*}"
          CONTEXT="$SPARSE_PATH/$DISTRO"

          {
            for tag in ${!TAGS[@]}; do
              echo "IMAGE_TAG_${tag^^}=$IMAGE_NAME:${TAGS[$tag]}"
            done

            echo "CONTEXT=$CONTEXT"
          } >> $GITHUB_ENV

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push
        uses: docker/build-push-action@v6
        with:
          context: ${{ env.CONTEXT }}
          target: ${{ matrix.target }}
          platforms: ${{ env.PLATFORMS }}
          tags: |-
            ${{ env.IMAGE_TAG_SHA }}
            ${{ env.IMAGE_TAG_DATE }}
            ${{ env.IMAGE_TAG_LATEST }}
          labels: |-
            org.opencontainers.image.source=${{ github.repositoryUrl }}
          push: ${{ env.PUSH }}