Scanner versioned vulnerabilities update #392
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scanner versioned vulnerabilities update | |
| on: | |
| schedule: | |
| - cron: "30 */4 * * *" | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| workflow_dispatch: | |
| inputs: | |
| job: | |
| type: choice | |
| description: "Choose the NVD source" | |
| options: | |
| - nvd-api | |
| - nvd-feeds | |
| required: true | |
| default: nvd-api | |
| jobs: | |
| parse-versions: | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| versions: ${{ steps.set-versions.outputs.versions }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Parse VULNERABILITY_BUNDLE_VERSION | |
| id: set-versions | |
| run: | | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "versions<<$EOF" >> "$GITHUB_OUTPUT" | |
| ./.github/workflows/scripts/scanner-get-released-tags.sh | tee -a "$GITHUB_OUTPUT" | |
| echo "$EOF" >> "$GITHUB_OUTPUT" | |
| prepare-environment: | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| manual_url: ${{ steps.set-manual-url.outputs.manual_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download NVD | |
| run: | | |
| set -eu | |
| since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT') | |
| NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-api' }} | |
| case "$NVD_BUNDLE_TYPE" in | |
| nvd-api) | |
| nvd_file=nvd-api.zip | |
| ;; | |
| nvd-feeds) | |
| nvd_file=nvd-feeds.zip | |
| ;; | |
| *) | |
| echo >&2 "error: invalid NVD bundle type '$NVD_BUNDLE_TYPE'" | |
| exit 1 | |
| esac | |
| url="https://definitions.stackrox.io/v4/nvd/$nvd_file" | |
| code=$(curl \ | |
| -o nvd.zip \ | |
| -w "%{http_code}" \ | |
| -H "If-Modified-Since: $since_time" \ | |
| "$url") | |
| echo "code: $code" | |
| echo "$code" | grep -q 200 | |
| - name: Set MANUAL_URL | |
| id: set-manual-url | |
| run: | | |
| if [ "${{ github.event_name }}" == "schedule" ]; then | |
| # Scheduled workflows should use the master branch | |
| echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/master/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV" | |
| else | |
| # Determine the SHA to use | |
| if [ -z "${{ github.event.pull_request.head.sha }}" ]; then | |
| # Fallback to the current ref or SHA if it's not a pull request event | |
| sha=${{ github.sha }} | |
| else | |
| sha=${{ github.event.pull_request.head.sha }} | |
| fi | |
| echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/$sha/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV" | |
| fi | |
| - uses: ./.github/actions/upload-artifact-with-retry | |
| with: | |
| name: nvd | |
| path: nvd.zip | |
| if-no-files-found: error | |
| build-and-run: | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| needs: | |
| - parse-versions | |
| - prepare-environment | |
| runs-on: ubuntu-latest | |
| container: | |
| image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4 | |
| volumes: | |
| # The updater makes heavy use of /tmp files. | |
| - /tmp:/tmp | |
| - /usr:/mnt/usr | |
| - /opt:/mnt/opt | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| include: ${{ fromJson(needs.parse-versions.outputs.versions) }} | |
| env: | |
| SCANNER_BUNDLE_VERSION: ${{ matrix.version }} | |
| ROX_GIT_REF: ${{ matrix.ref }} | |
| steps: | |
| - name: Free up disk space | |
| shell: bash | |
| run: | | |
| set +e | |
| set -x | |
| df -h | |
| for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do | |
| rm -rf "/mnt${delete:?}" | |
| done | |
| df -h | |
| - name: Checkout specific reference | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ env.ROX_GIT_REF }} | |
| - uses: ./.github/actions/job-preamble | |
| with: | |
| gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} | |
| - name: Download NVD | |
| uses: ./.github/actions/download-artifact-with-retry | |
| with: | |
| name: nvd | |
| path: . | |
| - name: Build updater | |
| run: | | |
| echo "Building updater for version ${{ env.SCANNER_BUNDLE_VERSION }} based on git ref ${{ env.ROX_GIT_REF }}..." | |
| make tag | |
| make -C scanner bin/updater | |
| - name: Create bundle output directory | |
| run: mkdir -p definitions/${{ env.SCANNER_BUNDLE_VERSION }} | |
| - name: Sanity check NVD zip | |
| run: | | |
| path="$PWD/nvd.zip" | |
| echo "checking contents of $path" | |
| unzip -l "$path" | |
| echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV" | |
| - name: Run Updater (single bundle) | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${{ env.SCANNER_BUNDLE_VERSION }}" | |
| - name: Run updater (multi bundle) | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles | |
| zip definitions/${{ env.SCANNER_BUNDLE_VERSION }}/vulnerabilities.zip bundles/*.json.zst | |
| - name: Upload definitions artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: artifact_${{ env.SCANNER_BUNDLE_VERSION }} | |
| path: definitions | |
| build-upload-pr-vulnerabilities: | |
| if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') | |
| needs: | |
| - prepare-environment | |
| runs-on: ubuntu-latest | |
| container: | |
| image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4 | |
| volumes: | |
| # The updater makes heavy use of /tmp files. | |
| - /tmp:/tmp | |
| - /usr:/mnt/usr | |
| - /opt:/mnt/opt | |
| steps: | |
| - name: Free up disk space | |
| shell: bash | |
| run: | | |
| set +e | |
| set -x | |
| df -h | |
| for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do | |
| rm -rf "/mnt${delete:?}" | |
| done | |
| df -h | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - uses: ./.github/actions/job-preamble | |
| with: | |
| gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} | |
| - name: Authenticate with test GCS bucket | |
| if: github.event_name == 'pull_request' | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Download NVD | |
| uses: ./.github/actions/download-artifact-with-retry | |
| with: | |
| name: nvd | |
| path: . | |
| - name: Build updater | |
| run: | | |
| echo "Building updater for pull request ${{ env.PR_NAME }}..." | |
| make tag | |
| make -C scanner bin/updater | |
| - name: Sanity check NVD zip | |
| run: | | |
| path="$PWD/nvd.zip" | |
| echo "checking contents of $path" | |
| unzip -l "$path" | |
| echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV" | |
| - name: Create bundle output directory | |
| run: mkdir -p definitions/${{ github.event.pull_request.number }} | |
| - name: Run Updater (single bundle) | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${{ github.event.pull_request.number }}" | |
| - name: Run updater (multi bundle) | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles | |
| zip definitions/${{ github.event.pull_request.number }}/vulnerabilities.zip bundles/*.json.zst | |
| # PR owner is responsible for verifying vulnerability bundles are generated | |
| - name: Upload PR vulnerabilities | |
| run: | | |
| branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| # Replace / with -, so the branch name isn't truncated when pushed to GCS. | |
| dir=${branch////-} | |
| mkdir -p "$dir" | |
| cp -r "definitions/${{ github.event.pull_request.number }}/." "$dir/" | |
| gsutil -m cp -r "$dir" "gs://scanner-v4-test/vulnerability-bundles/" | |
| upload-definitions: | |
| needs: | |
| - build-and-run | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checkout to run ./.github/actions/download-artifact-with-retry | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/download-artifact-with-retry | |
| with: | |
| # Only download vulnerability bundles. | |
| # This avoids accidentally downloading nvd.zip. | |
| pattern: "artifact_*" | |
| path: downloaded_artifacts | |
| - name: Move files artifacts | |
| run: rsync -av downloaded_artifacts/*/ definitions_files/ | |
| - name: Authenticate with Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Upload to GCS | |
| run: | | |
| ls -lr definitions_files | |
| for dir in definitions_files/*; do | |
| if [ -d "$dir" ]; then | |
| gsutil -m cp -r "$dir" gs://definitions.stackrox.io/v4/vulnerability-bundles/ | |
| fi | |
| done | |
| gsutil cp -r gs://definitions.stackrox.io/v4/vulnerability-bundles/dev/* gs://definitions.stackrox.io/v4/vulnerability-bundles/1.0.0/ | |
| send-notification: | |
| needs: | |
| - build-and-run | |
| - upload-definitions | |
| runs-on: ubuntu-latest | |
| if: ${{ failure() && github.ref_name == 'master' }} | |
| steps: | |
| - name: Send Slack notification on workflow failure | |
| run: | | |
| curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} |