scanner-versioned-definitions-update.yaml

name: Scanner versioned vulnerabilities update

on:
  schedule:
  - cron: "30 */4 * * *"
  pull_request:
    types:
    - opened
    - reopened
    - synchronize
  workflow_dispatch:
    inputs:
      job:
        type: choice
        description: "Choose the NVD source"
        options:
        - nvd-api
        - nvd-feeds
        required: true
        default: nvd-api

jobs:
  parse-versions:
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns')
    runs-on: ubuntu-latest
    outputs:
      versions: ${{ steps.set-versions.outputs.versions }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Parse VULNERABILITY_BUNDLE_VERSION
      id: set-versions
      run: |
        EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
        echo "versions<<$EOF" >> "$GITHUB_OUTPUT"
        ./.github/workflows/scripts/scanner-get-released-tags.sh | tee -a "$GITHUB_OUTPUT"
        echo "$EOF" >> "$GITHUB_OUTPUT"

  prepare-environment:
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns')
    runs-on: ubuntu-latest
    outputs:
      manual_url: ${{ steps.set-manual-url.outputs.manual_url }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Download NVD
      run: |
        set -eu
        since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT')
        NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-api' }}
        case "$NVD_BUNDLE_TYPE" in
            nvd-api)
                nvd_file=nvd-api.zip
                ;;
            nvd-feeds)
                nvd_file=nvd-feeds.zip
                ;;
            *)
                echo >&2 "error: invalid NVD bundle type '$NVD_BUNDLE_TYPE'"
                exit 1
        esac
        url="https://definitions.stackrox.io/v4/nvd/$nvd_file"

        code=$(curl \
            -o nvd.zip \
            -w "%{http_code}" \
            -H "If-Modified-Since: $since_time" \
            "$url")

        echo "code: $code"
        echo "$code" | grep -q 200

    - name: Set MANUAL_URL
      id: set-manual-url
      run: |
        if [ "${{ github.event_name }}" == "schedule" ]; then
          # Scheduled workflows should use the master branch
          echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/master/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV"
        else
          # Determine the SHA to use
          if [ -z "${{ github.event.pull_request.head.sha }}" ]; then
            # Fallback to the current ref or SHA if it's not a pull request event
            sha=${{ github.sha }}
          else
            sha=${{ github.event.pull_request.head.sha }}
          fi

          echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/$sha/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV"
        fi

    - uses: ./.github/actions/upload-artifact-with-retry
      with:
        name: nvd
        path: nvd.zip
        if-no-files-found: error

  build-and-run:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    needs:
    - parse-versions
    - prepare-environment
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4
      volumes:
      # The updater makes heavy use of /tmp files.
      - /tmp:/tmp
      - /usr:/mnt/usr
      - /opt:/mnt/opt
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        include: ${{ fromJson(needs.parse-versions.outputs.versions) }}
    env:
      SCANNER_BUNDLE_VERSION: ${{ matrix.version }}
      ROX_GIT_REF: ${{ matrix.ref }}
    steps:
    - name: Free up disk space
      shell: bash
      run: |
        set +e
        set -x
        df -h
        for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do
          rm -rf "/mnt${delete:?}"
        done
        df -h

    - name: Checkout specific reference
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ env.ROX_GIT_REF }}

    - uses: ./.github/actions/job-preamble
      with:
        gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}

    - name: Download NVD
      uses: ./.github/actions/download-artifact-with-retry
      with:
        name: nvd
        path: .

    - name: Build updater
      run: |
        echo "Building updater for version ${{ env.SCANNER_BUNDLE_VERSION }} based on git ref ${{ env.ROX_GIT_REF }}..."
        make tag
        make -C scanner bin/updater

    - name: Create bundle output directory
      run: mkdir -p definitions/${{ env.SCANNER_BUNDLE_VERSION }}

    - name: Sanity check NVD zip
      run: |
        path="$PWD/nvd.zip"
        echo "checking contents of $path"
        unzip -l "$path"
        echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV"

    - name: Run Updater (single bundle)
      run: |
        scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${{ env.SCANNER_BUNDLE_VERSION }}"

    - name: Run updater (multi bundle)
      run: |
        scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles
        zip definitions/${{ env.SCANNER_BUNDLE_VERSION }}/vulnerabilities.zip bundles/*.json.zst

    - name: Upload definitions artifacts
      uses: actions/upload-artifact@v4
      with:
        name: artifact_${{ env.SCANNER_BUNDLE_VERSION }}
        path: definitions

  build-upload-pr-vulnerabilities:
    if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns')
    needs:
    - prepare-environment
    runs-on: ubuntu-latest
    container:
      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4
      volumes:
      # The updater makes heavy use of /tmp files.
      - /tmp:/tmp
      - /usr:/mnt/usr
      - /opt:/mnt/opt
    steps:
    - name: Free up disk space
      shell: bash
      run: |
        set +e
        set -x
        df -h
        for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do
          rm -rf "/mnt${delete:?}"
        done
        df -h

    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        ref: ${{ github.event.pull_request.head.sha }}

    - uses: ./.github/actions/job-preamble
      with:
        gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}

    - name: Authenticate with test GCS bucket
      if: github.event_name == 'pull_request'
      uses: google-github-actions/auth@v2
      with:
        credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}

    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v2

    - name: Download NVD
      uses: ./.github/actions/download-artifact-with-retry
      with:
        name: nvd
        path: .

    - name: Build updater
      run: |
        echo "Building updater for pull request ${{ env.PR_NAME }}..."
        make tag
        make -C scanner bin/updater

    - name: Sanity check NVD zip
      run: |
        path="$PWD/nvd.zip"
        echo "checking contents of $path"
        unzip -l "$path"
        echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV"

    - name: Create bundle output directory
      run: mkdir -p definitions/${{ github.event.pull_request.number }}

    - name: Run Updater (single bundle)
      run: |
        scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${{ github.event.pull_request.number }}"

    - name: Run updater (multi bundle)
      run: |
        scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles
        zip definitions/${{ github.event.pull_request.number }}/vulnerabilities.zip bundles/*.json.zst

    # PR owner is responsible for verifying vulnerability bundles are generated
    - name: Upload PR vulnerabilities
      run: |
        branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
        # Replace / with -, so the branch name isn't truncated when pushed to GCS.
        dir=${branch////-}
        mkdir -p "$dir"
        cp -r "definitions/${{ github.event.pull_request.number }}/." "$dir/"
        gsutil -m cp -r "$dir" "gs://scanner-v4-test/vulnerability-bundles/"

  upload-definitions:
    needs:
    - build-and-run
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    steps:
    # Checkout to run ./.github/actions/download-artifact-with-retry
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: ./.github/actions/download-artifact-with-retry
      with:
        # Only download vulnerability bundles.
        # This avoids accidentally downloading nvd.zip.
        pattern: "artifact_*"
        path: downloaded_artifacts

    - name: Move files artifacts
      run: rsync -av downloaded_artifacts/*/ definitions_files/

    - name: Authenticate with Google Cloud
      uses: google-github-actions/auth@v2
      with:
        credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }}

    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v2

    - name: Upload to GCS
      run: |
        ls -lr definitions_files
        for dir in definitions_files/*; do
          if [ -d "$dir" ]; then
            gsutil -m cp -r "$dir" gs://definitions.stackrox.io/v4/vulnerability-bundles/
          fi
        done
        gsutil cp -r gs://definitions.stackrox.io/v4/vulnerability-bundles/dev/* gs://definitions.stackrox.io/v4/vulnerability-bundles/1.0.0/

  send-notification:
    needs:
    - build-and-run
    - upload-definitions
    runs-on: ubuntu-latest
    if: ${{ failure() && github.ref_name == 'master' }}
    steps:
    - name: Send Slack notification on workflow failure
      run: |
        curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }}