ci.yaml

name: CI
on:
  schedule:
    - cron: '0 */3 * * *'
  push:
    branches:
      - master
      - 'release-*'
    tags:
      - '**'
  pull_request:
    types:
      - opened
      - reopened
      - synchronize

jobs:
  define-scanner-job-matrix:
    outputs:
      matrix: ${{ steps.define-scanner-job-matrix.outputs.matrix }}
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Define the matrix for build jobs
      id: define-scanner-job-matrix
      run: |
        # If goarch is updated, be sure to update architectures in push-manifests below.
        matrix='{ "build_and_push": { "goos":["linux"], "goarch":["amd64", "arm64", "ppc64le", "s390x"] } }'

        jq <<< "$matrix"

        condensed="$(jq -c <<< "$matrix")"
        echo "matrix=$condensed" >> "$GITHUB_OUTPUT"

  pre-build-updater:
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - name: Cache Go dependencies
        uses: ./.github/actions/cache-go-dependencies

      - uses: ./.github/actions/handle-tagged-build

      - name: Build updater (amd64)
        run: make build-updater

      - name: Archive the build to preserve permissions
        run: tar -cvzf updater-build.tgz bin/updater

      - uses: actions/upload-artifact@v4
        with:
          name: updater-build
          path: updater-build.tgz

  pre-build-scanner:
    needs: define-scanner-job-matrix
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push }}
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - name: Cache Go dependencies
        uses: ./.github/actions/cache-go-dependencies

      - uses: ./.github/actions/handle-tagged-build

      - name: Build Scanner
        run: make GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} scanner-build-nodeps

      - name: Archive the build to preserve permissions
        run: tar -cvzf scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}.tgz image/scanner/bin/scanner

      - uses: actions/upload-artifact@v4
        with:
          name: scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}
          path: scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}.tgz

  style-check:
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - name: Cache Go dependencies
        uses: ./.github/actions/cache-go-dependencies

      - name: Run style checks (amd64)
        run: ./scripts/ci/jobs/style-checks.sh

  unit-tests:
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - name: Cache Go dependencies
        uses: ./.github/actions/cache-go-dependencies

      - name: Run unit tests (amd64)
        run: ./scripts/ci/jobs/unit-tests.sh

  db-integration-tests:
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - name: Cache Go dependencies
        uses: ./.github/actions/cache-go-dependencies

      - name: Run db integration tests (amd64)
        run: ./scripts/ci/jobs/db-integration-tests.sh

  generate-genesis-dump:
    # Run this job if it's not a PR or the PR contains the `generate-dumps-on-pr` label
    if: |
      github.event_name != 'pull_request' ||
      contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr')
    env:
      NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
      ROX_LEGACY_NVD_LOADER: true
    runs-on: ubuntu-latest
    needs:
      - pre-build-updater
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        with:
          name: updater-build

      - name: Unpack updater build
        run: |
          tar xvzf updater-build.tgz

      - name: genesis-dump
        run: |
          source ./scripts/ci/lib.sh
          generate_genesis_dump

      - uses: actions/upload-artifact@v4
        with:
          name: genesis-dump
          path: /tmp/genesis-dump/genesis-dump.zip

      - uses: actions/upload-artifact@v4
        with:
          name: vuln-dump
          path: /tmp/vuln-dump

  generate-db-dump:
    # Run this job if it's not a PR or the PR contains the `generate-dumps-on-pr` label
    if: |
      github.event_name != 'pull_request' ||
      contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr')
    runs-on: ubuntu-latest
    needs:
      - generate-genesis-dump
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        with:
          name: updater-build

      - name: Unpack updater build
        run: |
          tar xvzf updater-build.tgz

      - uses: actions/download-artifact@v4
        with:
          name: genesis-dump
          path: /tmp/genesis-dump

      - name: db-dump
        run: |
          source ./scripts/ci/lib.sh
          generate_db_dump

      - uses: actions/upload-artifact@v4
        with:
          name: db-dump
          path: /tmp/postgres/pg-definitions.sql.gz

  generate-scanner-bundle:
    # Run this job even if the generate-genesis-dump job was skipped, i.e., only skip this job if
    # generate-genesis-dump failed
    if: |
      always() &&
      (needs.generate-genesis-dump.result == 'success' || needs.generate-genesis-dump.result == 'skipped')
    runs-on: ubuntu-latest
    needs:
      - define-scanner-job-matrix
      - pre-build-scanner
      - generate-genesis-dump
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push }}
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        with:
          name: scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}

      - name: Unpack scanner build
        run: |
          tar xvzf scanner-build-${{ matrix.goos }}-${{ matrix.goarch }}.tgz

      - uses: actions/download-artifact@v4
        # Run this step if it's not a PR or the PR contains the `generate-dumps-on-pr` label
        # When this step is skipped `get_genesis_dump` will pull the vulnerability data from our GCS bucket
        if: |
          github.event_name != 'pull_request' ||
          contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr')
        with:
          name: vuln-dump
          path: /tmp/vuln-dump

      - name: Generate OSS notice
        run: make ossls-notice

      - name: Get genesis dump
        run: |
          source ./scripts/ci/lib.sh
          get_genesis_dump

      - name: Make bundle
        run: image/scanner/rhel/create-bundle.sh image/scanner image/scanner/rhel

      - name: Archive the bundle to preserve permissions
        run: tar -cvzf scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}.tgz image/scanner/rhel

      - uses: actions/upload-artifact@v4
        with:
          name: scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}
          path: scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}.tgz

  generate-scanner-db-bundle:
    # Run this job even if the generate-db-dump job was skipped, i.e., only skip this job if
    # generate-db-dump failed
    if: |
      always() &&
      (needs.generate-db-dump.result == 'success' || needs.generate-db-dump.result == 'skipped')
    runs-on: ubuntu-latest
    needs:
      - generate-db-dump
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        if: |
          github.event_name != 'pull_request' ||
          contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr')
        with:
          name: db-dump
          path: /tmp/postgres

      - name: Get db dump
        run: |
          source ./scripts/ci/lib.sh
          get_db_dump

      - name: Make db bundle
        run: image/db/rhel/create-bundle.sh image/db image/db/rhel

      - uses: actions/upload-artifact@v4
        with:
          name: scanner-db-bundle
          path: image/db/rhel

  build-images:
    # Run this job even if previous jobs were skipped, i.e., only skip this job if one of the previous jobs failed
    # or was cancelled
    if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
    env:
      QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
      QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
      QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
      QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
      QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
      QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
    runs-on: ubuntu-latest
    needs:
      - define-scanner-job-matrix
      - generate-scanner-bundle
      - generate-scanner-db-bundle
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push }}
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: ./.github/actions/handle-tagged-build

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - uses: actions/download-artifact@v4
        with:
          name: scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}

      - name: Unpack bundle
        run: |
          tar xvzf scanner-bundle-${{ matrix.goos }}-${{ matrix.goarch }}.tgz

      - uses: actions/download-artifact@v4
        with:
          name: scanner-db-bundle
          path: image/db/rhel

      - name: Build scanner image
        run: |
          docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner:"$(make --no-print-directory --quiet tag)" $(make GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} image-build-args) -f image/scanner/rhel/Dockerfile image/scanner/rhel

      - name: Build scanner-slim image
        run: |
          docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner-slim:"$(make --no-print-directory --quiet tag)" $(make GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} image-build-args) -f image/scanner/rhel/Dockerfile.slim image/scanner/rhel

      - name: Build scanner-db image
        run: |
          docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner-db:"$(make --no-print-directory --quiet tag)" $(make GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} db-image-build-args) -f image/db/rhel/Dockerfile image/db/rhel

      - name: Build scanner-db-slim image
        run: |
          docker buildx build --platform "${{ matrix.goos }}/${{ matrix.goarch }}" --load -t stackrox/scanner-db-slim:"$(make --no-print-directory --quiet tag)" $(make GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} db-image-build-args) -f image/db/rhel/Dockerfile.slim image/db/rhel

      - name: Docker login
        # Skip for external contributions.
        if: |
          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
        run: |
          docker login -u "${QUAY_RHACS_ENG_RO_USERNAME}" --password-stdin quay.io <<<"${QUAY_RHACS_ENG_RO_PASSWORD}"

      - name: Push images
        # Skip for external contributions.
        if: |
          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
        run: |
          source ./scripts/ci/lib.sh
          push_scanner_image_set ${{ matrix.goarch }}

  push-manifests:
    # Run this job even if previous jobs were skipped, i.e., only skip this job if one of the previous jobs failed
    # or was cancelled
    if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
    needs:
      - define-scanner-job-matrix
      - generate-scanner-bundle
      - generate-scanner-db-bundle
      - build-images
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    env:
      QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
      QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
      QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
      QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: ./.github/actions/handle-tagged-build

      - name: Push Scanner and ScannerDB image manifests
        # Skip for external contributions.
        if: |
          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
        run: |
          source ./scripts/ci/lib.sh

          # If this is updated, be sure to update goarch in define-scanner-job-matrix above.
          architectures="amd64,arm64,ppc64le,s390x"

          push_scanner_image_manifest_lists "$architectures"

  diff-dumps:
    # Run this job if:
    #   - it's running on the master branch OR
    #   - it's in a PR context and the PR contains the `generate-dumps-on-pr` label
    # Note that this doesn't run on tags
    if: |
      github.ref == 'refs/heads/master' ||
      (github.event_name == 'pull_request' &&
      contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr'))
    env:
      GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }}
      SCANNER_GCP_SERVICE_ACCOUNT_CREDS: ${{ secrets.SCANNER_GCP_SERVICE_ACCOUNT_CREDS }}
      GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}
    runs-on: ubuntu-latest
    needs:
      - generate-genesis-dump
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        with:
          name: updater-build

      - name: Unpack updater build
        run: |
          tar xvzf updater-build.tgz

      - uses: actions/download-artifact@v4
        with:
          name: genesis-dump
          path: /tmp/genesis-dump

      - name: diff-dumps
        run: ./scripts/ci/jobs/diff-dumps.sh

      - uses: actions/upload-artifact@v4
        with:
          name: diff-dumps-inspect
          path: /tmp/diff-dumps-inspect

      - uses: actions/upload-artifact@v4
        with:
          name: offline-dump
          path: /tmp/offline-dump/scanner-vuln-updates.zip

  # TODO(blugo): This job is for a very particular use case. If we want to officially support this image, we should
  # bake it into the rest of the build process instead of having it be its own bespoke job.
  # This job downloads the vulnerability data we push to the GCS bucket in a previous job and adds it into a container
  # image.
  build-and-push-vulnerabilities-image:
    env:
      QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
      QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
    runs-on: ubuntu-latest
    # Only run on master branch
    if: github.ref == 'refs/heads/master'
    needs:
      - diff-dumps
    steps:
      # Required for making the tag
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: actions/download-artifact@v4
        with:
          name: offline-dump

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Docker login
        # Skip for external contributions.
        if: |
          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
        run: |
          docker login -u "${QUAY_STACKROX_IO_RW_USERNAME}" --password-stdin quay.io <<<"${QUAY_STACKROX_IO_RW_PASSWORD}"

      - name: Build and save definitions images
        run: |
          set -e
          
          TAG="$(make tag)"
          QUAY_TAG_EXPIRATION=never
          DOCKER_ARGS=(
            "--build-arg" "LABEL_VERSION=${TAG}"
            "--build-arg" "LABEL_RELEASE=${TAG}"
            "--build-arg" "QUAY_TAG_EXPIRATION=${QUAY_TAG_EXPIRATION}"
          )
          BASE_IMAGE_TAG="quay.io/stackrox-io/vulnerabilities:latest"
          
          architectures=("amd64" "arm64" "ppc64le" "s390x")
          image_list=()
          for arch in "${architectures[@]}"; do
            arch_image_tag="${BASE_IMAGE_TAG}-${arch}"
          
            docker buildx build \
              --platform "linux/${arch}" \
              --load \
              --tag "${arch_image_tag}" \
              "${DOCKER_ARGS[@]}" \
              -f image/vulnerabilities/Dockerfile .
          
            image_list+=("${arch_image_tag}")
          done
          
          for image in "${image_list[@]}"; do
            docker push "${image}"
          done 
          docker manifest create "${BASE_IMAGE_TAG}" "${image_list[@]}"
          docker manifest push "${BASE_IMAGE_TAG}"

  upload-db-dump:
    # Only run this step on the master branch
    # Note that our scheduled jobs run on the master branch
    if: github.ref == 'refs/heads/master'
    env:
      GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}
    runs-on: ubuntu-latest
    needs:
      - generate-db-dump
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: master

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        with:
          name: db-dump
          path: /tmp/postgres

      - name: upload-db-dump
        run: ./scripts/ci/jobs/upload-db-dump.sh

  upload-dumps-for-downstream:
    # Only run this step on the master branch or any tags
    # Note that our scheduled jobs run on the master branch
    if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
    env:
      GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }}
    runs-on: ubuntu-latest
    needs:
      - generate-db-dump
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: ./.github/actions/job-preamble

      - uses: ./.github/actions/handle-tagged-build

      - uses: actions/download-artifact@v4
        with:
          name: vuln-dump
          path: /tmp/vuln-dump

      - uses: actions/download-artifact@v4
        with:
          name: db-dump
          path: /tmp/postgres

      - name: upload-dumps-for-downstream
        run: ./scripts/ci/jobs/upload-dumps-for-downstream.sh

  upload-dumps-for-embedding:
    # Only run this step on the master branch
    # Note that our scheduled jobs run on the master branch
    if: github.ref == 'refs/heads/master'
    env:
      GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}
    runs-on: ubuntu-latest
    needs:
      - generate-db-dump
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: ./.github/actions/job-preamble

      - uses: actions/download-artifact@v4
        with:
          name: vuln-dump
          path: /tmp/vuln-dump

      - name: upload-dumps-for-downstream
        run: ./scripts/ci/jobs/upload-dumps-for-embedding.sh

  send-notification:
    needs:
      - diff-dumps
    runs-on: ubuntu-latest
    if: failure()
    steps:
      - name: Send Slack notification on workflow failure
        run: |
          curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update offline vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }}