Skip to content

Scanner versioned vulnerabilities update #96

Scanner versioned vulnerabilities update

Scanner versioned vulnerabilities update #96

name: Scanner versioned vulnerabilities update
on:
schedule:
- cron: "30 */4 * * *"
pull_request:
types:
- opened
- reopened
- synchronize
workflow_dispatch:
inputs:
job:
type: choice
description: "Choose the NVD source"
options:
- nvd-api
- nvd-feeds
required: true
default: nvd-api
jobs:
parse-versions:
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns')
runs-on: ubuntu-latest
outputs:
versions: ${{ steps.set-versions.outputs.versions }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Parse VULNERABILITY_BUNDLE_VERSION
id: set-versions
run: |
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "versions<<$EOF" >> "$GITHUB_OUTPUT"
./.github/workflows/scripts/scanner-get-released-tags.sh | tee -a "$GITHUB_OUTPUT"
echo "$EOF" >> "$GITHUB_OUTPUT"
prepare-environment:
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns')
runs-on: ubuntu-latest
outputs:
manual_url: ${{ steps.set-manual-url.outputs.manual_url }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download NVD
run: |
set -eu
since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT')
NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-api' }}
case "$NVD_BUNDLE_TYPE" in
nvd-api)
nvd_file=nvd-api.zip
;;
nvd-feeds)
nvd_file=nvd-feeds.zip
;;
*)
echo >&2 "error: invalid NVD bundle type '$NVD_BUNDLE_TYPE'"
exit 1
esac
url="https://definitions.stackrox.io/v4/nvd/$nvd_file"
code=$(curl \
-o nvd.zip \
-w "%{http_code}" \
-H "If-Modified-Since: $since_time" \
"$url")
echo "code: $code"
echo "$code" | grep -q 200
- name: Set MANUAL_URL
id: set-manual-url
run: |
if [ "${{ github.event_name }}" == "schedule" ]; then
# Scheduled workflows should use the master branch
echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/master/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV"
else
# Determine the SHA to use
if [ -z "${{ github.event.pull_request.head.sha }}" ]; then
# Fallback to the current ref or SHA if it's not a pull request event
sha=${{ github.sha }}
else
sha=${{ github.event.pull_request.head.sha }}
fi
echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/$sha/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV"
fi
- uses: ./.github/actions/upload-artifact-with-retry
with:
name: nvd
path: nvd.zip
if-no-files-found: error
build-and-run:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
needs:
- parse-versions
- prepare-environment
runs-on: ubuntu-latest
container:
image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4
volumes:
# The updater makes heavy use of /tmp files.
- /tmp:/tmp
- /usr:/mnt/usr
- /opt:/mnt/opt
strategy:
fail-fast: false
max-parallel: 1
matrix:
include: ${{ fromJson(needs.parse-versions.outputs.versions) }}
env:
SCANNER_BUNDLE_VERSION: ${{ matrix.version }}
ROX_GIT_REF: ${{ matrix.ref }}
steps:
- name: Free up disk space
shell: bash
run: |
set +e
set -x
df -h
for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do
rm -rf "/mnt${delete:?}"
done
df -h
- name: Checkout specific reference
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ env.ROX_GIT_REF }}
- uses: ./.github/actions/job-preamble
with:
gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
- name: Download NVD
uses: ./.github/actions/download-artifact-with-retry
with:
name: nvd
path: .
- name: Build updater
run: |
echo "Building updater for version ${{ env.SCANNER_BUNDLE_VERSION }} based on git ref ${{ env.ROX_GIT_REF }}..."
make tag
make -C scanner bin/updater
- name: Create bundle output directory
run: mkdir -p definitions/${{ env.SCANNER_BUNDLE_VERSION }}
- name: Sanity check NVD zip
run: |
path="$PWD/nvd.zip"
echo "checking contents of $path"
unzip -l "$path"
echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV"
- name: Run Updater (single bundle)
run: |
scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${{ env.SCANNER_BUNDLE_VERSION }}"
- name: Run updater (multi bundle)
run: |
scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles
zip definitions/${{ env.SCANNER_BUNDLE_VERSION }}/vulnerabilities.zip bundles/*.json.zst
- name: Upload definitions artifacts
uses: actions/upload-artifact@v4
with:
name: artifact_${{ env.SCANNER_BUNDLE_VERSION }}
path: definitions
build-upload-pr-vulnerabilities:
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns')
needs:
- prepare-environment
runs-on: ubuntu-latest
container:
image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4
volumes:
# The updater makes heavy use of /tmp files.
- /tmp:/tmp
- /usr:/mnt/usr
- /opt:/mnt/opt
steps:
- name: Free up disk space
shell: bash
run: |
set +e
set -x
df -h
for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do
rm -rf "/mnt${delete:?}"
done
df -h
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- uses: ./.github/actions/job-preamble
with:
gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
- name: Authenticate with test GCS bucket
if: github.event_name == 'pull_request'
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
- name: Download NVD
uses: ./.github/actions/download-artifact-with-retry
with:
name: nvd
path: .
- name: Build updater
run: |
echo "Building updater for pull request ${{ env.PR_NAME }}..."
make tag
make -C scanner bin/updater
- name: Sanity check NVD zip
run: |
path="$PWD/nvd.zip"
echo "checking contents of $path"
unzip -l "$path"
echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV"
- name: Create bundle output directory
run: mkdir -p definitions/${{ github.event.pull_request.number }}
- name: Run Updater (single bundle)
run: |
scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${{ github.event.pull_request.number }}"
- name: Run updater (multi bundle)
run: |
scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles
zip definitions/${{ github.event.pull_request.number }}/vulnerabilities.zip bundles/*.json.zst
# PR owner is responsible for verifying vulnerability bundles are generated
- name: Upload PR vulnerabilities
run: |
branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
# Replace / with -, so the branch name isn't truncated when pushed to GCS.
dir=${branch////-}
mkdir -p "$dir"
cp -r "definitions/${{ github.event.pull_request.number }}/." "$dir/"
gsutil -m cp -r "$dir" "gs://scanner-v4-test/vulnerability-bundles/"
upload-definitions:
needs:
- build-and-run
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
# Checkout to run ./.github/actions/download-artifact-with-retry
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: ./.github/actions/download-artifact-with-retry
with:
# Only download vulnerability bundles.
# This avoids accidentally downloading nvd.zip.
pattern: "artifact_*"
path: downloaded_artifacts
- name: Move files artifacts
run: rsync -av downloaded_artifacts/*/ definitions_files/
- name: Authenticate with Google Cloud
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
- name: Upload to GCS
run: |
ls -lr definitions_files
for dir in definitions_files/*; do
if [ -d "$dir" ]; then
gsutil -m cp -r "$dir" gs://definitions.stackrox.io/v4/vulnerability-bundles/
fi
done
gsutil cp -r gs://definitions.stackrox.io/v4/vulnerability-bundles/dev/* gs://definitions.stackrox.io/v4/vulnerability-bundles/1.0.0/
send-notification:
needs:
- build-and-run
- upload-definitions
runs-on: ubuntu-latest
if: ${{ failure() && github.ref_name == 'master' }}
steps:
- name: Send Slack notification on workflow failure
run: |
curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }}