diff --git a/.github/actions/cache-go-dependencies/action.yaml b/.github/actions/cache-go-dependencies/action.yaml new file mode 100644 index 000000000..929a15972 --- /dev/null +++ b/.github/actions/cache-go-dependencies/action.yaml @@ -0,0 +1,29 @@ +name: Cache Go Dependencies +description: Cache Go Dependencies +runs: + using: composite + steps: + - name: Determine Go cache paths + id: cache-paths + run: | + echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Cache Go Dependencies + uses: actions/cache@v3 + with: + path: | + ${{ steps.cache-paths.outputs.GOMODCACHE }} + key: go-mod-v1-${{ hashFiles('**/go.sum') }} + + - name: Cache Go Build + uses: actions/cache@v3 + with: + path: | + ${{ steps.cache-paths.outputs.GOCACHE }} + key: go-build-v1-${{ github.job }}-${{ hashFiles('**/go.sum') }} + + - name: Download Go modules + run: make deps --always-make + shell: bash diff --git a/.github/actions/job-preamble/action.yaml b/.github/actions/job-preamble/action.yaml new file mode 100644 index 000000000..42f59f243 --- /dev/null +++ b/.github/actions/job-preamble/action.yaml @@ -0,0 +1,17 @@ +name: Job Preamble +description: Common steps for most jobs +runs: + using: composite + steps: + - name: Recover docker image cache space + run: | + df --si / + docker system prune --force --all + df --si / + shell: bash + + - name: Ignore dubious repository ownership + run: | + # Prevent fatal error "detected dubious ownership in repository" from recent git. + git config --global --add safe.directory "$(pwd)" + shell: bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..0626cf8e2 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,514 @@ +name: CI +on: + schedule: + - cron: '0 */3 * * *' + push: + branches: + - master + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + pre-build-updater: + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - name: Build updater + run: make build-updater + + - name: Archive the build to preserve permissions + run: tar -cvzf updater-build.tgz bin/updater + + - uses: actions/upload-artifact@v3 + with: + name: updater-build + path: updater-build.tgz + + pre-build-scanner: + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - name: Build Scanner + run: make scanner-build-nodeps + + - name: Archive the build to preserve permissions + run: tar -cvzf scanner-build.tgz image/scanner/bin/scanner + + - uses: actions/upload-artifact@v3 + with: + name: scanner-build + path: scanner-build.tgz + + style-check: + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - name: Run style checks + run: ./scripts/ci/jobs/style-checks.sh + + unit-tests: + runs-on: ubuntu-latest + needs: + - pre-build-scanner + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - uses: actions/download-artifact@v3 + with: + name: scanner-build + + - name: Unpack scanner build + run: | + tar xvzf scanner-build.tgz + + - name: Run unit tests + run: ./scripts/ci/jobs/unit-tests.sh + + db-integration-tests: + runs-on: ubuntu-latest + needs: + - pre-build-scanner + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - uses: actions/download-artifact@v3 + with: + name: scanner-build + + - name: Unpack scanner build + run: | + tar xvzf scanner-build.tgz + + - name: Run db integration tests + run: ./scripts/ci/jobs/db-integration-tests.sh + + generate-genesis-dump: + if: | + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') + env: + NVD_API_KEY: ${{ secrets.NVD_API_KEY }} + runs-on: ubuntu-latest + needs: + - pre-build-updater + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: updater-build + + - name: Unpack updater build + run: | + tar xvzf updater-build.tgz + + - name: genesis-dump + run: | + source ./scripts/ci/lib.sh + generate_genesis_dump + + - uses: actions/upload-artifact@v3 + with: + name: genesis-dump + path: /tmp/genesis-dump/genesis-dump.zip + + - uses: actions/upload-artifact@v3 + with: + name: vuln-dump + path: /tmp/vuln-dump + + generate-db-dump: + if: | + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') + runs-on: ubuntu-latest + needs: + - generate-genesis-dump + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: updater-build + + - name: Unpack updater build + run: | + tar xvzf updater-build.tgz + + - uses: actions/download-artifact@v3 + with: + name: genesis-dump + path: /tmp/genesis-dump + + - name: db-dump + run: | + source ./scripts/ci/lib.sh + generate_db_dump + + - uses: actions/upload-artifact@v3 + with: + name: db-dump + path: /tmp/postgres/pg-definitions.sql.gz + + generate-scanner-bundle: + runs-on: ubuntu-latest + needs: + - pre-build-scanner + - generate-genesis-dump + if: | + always() && + (needs.generate-genesis-dump.result == 'success' || needs.generate-genesis-dump.result == 'skipped') + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: scanner-build + + - name: Unpack scanner build + run: | + tar xvzf scanner-build.tgz + + - uses: actions/download-artifact@v3 + if: | + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') + with: + name: vuln-dump + path: /tmp/vuln-dump + + - name: Generate OSS notice + run: make ossls-notice + + - name: Get genesis dump + run: | + source ./scripts/ci/lib.sh + get_genesis_dump + + - name: Make bundle + run: image/scanner/rhel/create-bundle.sh image/scanner image/scanner/rhel + + - name: Archive the bundle to preserve permissions + run: tar -cvzf bundle.tgz image/scanner/rhel + + - uses: actions/upload-artifact@v3 + with: + name: scanner-bundle + path: bundle.tgz + + generate-scanner-db-bundle: + runs-on: ubuntu-latest + needs: + - generate-db-dump + if: | + always() && + (needs.generate-db-dump.result == 'success' || needs.generate-db-dump.result == 'skipped') + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + if: | + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') + with: + name: db-dump + path: /tmp/postgres + + - name: Get db dump + run: | + source ./scripts/ci/lib.sh + get_db_dump + + - name: Make db bundle + run: image/db/rhel/create-bundle.sh image/db image/db/rhel + + - uses: actions/upload-artifact@v3 + with: + name: scanner-db-bundle + path: image/db/rhel + + build-images: + env: + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + runs-on: ubuntu-latest + needs: + - generate-scanner-bundle + - generate-scanner-db-bundle + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: scanner-bundle + + - name: Unpack bundle + run: | + tar xvzf bundle.tgz + + - uses: actions/download-artifact@v3 + with: + name: scanner-db-bundle + path: image/db/rhel + + - name: Build scanner image + run: | + docker build -t scanner:"$(make --no-print-directory --quiet tag)" -f image/scanner/rhel/Dockerfile image/scanner/rhel + + - name: Build scanner-slim image + run: | + docker build -t scanner-slim:"$(make --no-print-directory --quiet tag)" -f image/scanner/rhel/Dockerfile.slim image/scanner/rhel + + - name: Build scanner-db image + run: | + docker build -t scanner-db:"$(make --no-print-directory --quiet tag)" -f image/db/rhel/Dockerfile image/db/rhel + + - name: Build scanner-db-slim image + run: | + docker build -t scanner-db-slim:"$(make --no-print-directory --quiet tag)" -f image/db/rhel/Dockerfile.slim image/db/rhel + + - name: Docker login + # Skip for external contributions. + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + run: | + docker login -u "${QUAY_RHACS_ENG_RO_USERNAME}" --password-stdin quay.io <<<"${QUAY_RHACS_ENG_RO_PASSWORD}" + + - name: Push images + # Skip for external contributions. + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + run: | + source ./scripts/ci/lib.sh + push_image_set + + diff-dumps: + if: | + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'generate-dumps-on-pr') + env: + GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + SCANNER_GCP_SERVICE_ACCOUNT_CREDS: ${{ secrets.SCANNER_GCP_SERVICE_ACCOUNT_CREDS }} + GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} + runs-on: ubuntu-latest + needs: + - generate-genesis-dump + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: updater-build + + - name: Unpack updater build + run: | + tar xvzf updater-build.tgz + + - uses: actions/download-artifact@v3 + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + with: + name: genesis-dump + path: /tmp/genesis-dump + + - name: diff-dumps + run: ./scripts/ci/jobs/diff-dumps.sh + + upload-db-dump: + # Only run on master branch + if: github.ref == 'refs/heads/master' + env: + GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} + runs-on: ubuntu-latest + needs: + - generate-db-dump + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: db-dump + path: /tmp/postgres + + - name: upload-db-dump + run: ./scripts/ci/jobs/upload-db-dump.sh + + upload-dumps-for-downstream: + # Only run on master branch + if: github.ref == 'refs/heads/master' + env: + GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + runs-on: ubuntu-latest + needs: + - generate-db-dump + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: vuln-dump + path: /tmp/vuln-dump + + - uses: actions/download-artifact@v3 + with: + name: db-dump + path: /tmp/postgres + + - name: upload-dumps-for-downstream + run: ./scripts/ci/jobs/upload-dumps-for-downstream.sh + + upload-dumps-for-embedding: + # Only run on master branch + if: github.ref == 'refs/heads/master' + env: + GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} + runs-on: ubuntu-latest + needs: + - generate-db-dump + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - uses: actions/download-artifact@v3 + with: + name: vuln-dump + path: /tmp/vuln-dump + + - name: upload-dumps-for-downstream + run: ./scripts/ci/jobs/upload-dumps-for-embedding.sh diff --git a/.github/workflows/sanity-check-vuln-updates.yaml b/.github/workflows/sanity-check-vuln-updates.yaml new file mode 100644 index 000000000..396a6b4b1 --- /dev/null +++ b/.github/workflows/sanity-check-vuln-updates.yaml @@ -0,0 +1,23 @@ +name: Vulnerability updates sanity check +on: + schedule: + - cron: '5 0,4,8,12,16,20 * * *' + +jobs: + sanity-check-vuln-updates: + env: + SLACK_WEBHOOK_ONCALL: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: sanity-check-vuln-updates + run: ./scripts/ci/jobs/sanity-check-vuln-updates.sh diff --git a/pkg/vulnloader/nvdloader/loader.go b/pkg/vulnloader/nvdloader/loader.go index 849af8255..b8889b77e 100644 --- a/pkg/vulnloader/nvdloader/loader.go +++ b/pkg/vulnloader/nvdloader/loader.go @@ -93,9 +93,9 @@ func (l *loader) DownloadFeedsToPath(outputDir string) error { // Rudimentary rate-limiting. // NVD limits users without an API key to roughly one call every 6 seconds. // With an API key, it is roughly one call every 0.6 seconds. - // At this time, the API key is inaccessible in CI, so we opt to play it safe and make a call every 10 seconds. - // As of writing there are ~216,000 vulnerabilities, so this whole process should take ~18 minutes. - time.Sleep(10 * time.Second) + // We'll play it safe and do one call every 3 seconds. + // As of writing there are ~216,000 vulnerabilities, so this whole process should take ~5.4 minutes. + time.Sleep(3 * time.Second) startIdx += apiResp.ResultsPerPage apiResp, err = query(fmt.Sprintf(urlFmt, startIdx)) @@ -128,6 +128,7 @@ func query(url string) (*apischema.CVEAPIJSON20, error) { if err != nil { return nil, fmt.Errorf("creating HTTP request: %w", err) } + req.Header.Set("apiKey", os.Getenv("NVD_API_KEY")) apiResp, err := queryWithBackoff(req) if err != nil { @@ -152,8 +153,8 @@ func queryWithBackoff(req *http.Request) (*apischema.CVEAPIJSON20, error) { } } log.Warnf("Failed query attempt %d for %s: %v", i, req.URL.String(), err) - // Wait some multiple of 10 seconds before next attempt. - time.Sleep(time.Duration(10*i) * time.Second) + // Wait some multiple of 3 seconds before next attempt. + time.Sleep(time.Duration(3*i) * time.Second) } return apiResp, err diff --git a/scripts/ci/jobs/db-integration-tests.sh b/scripts/ci/jobs/db-integration-tests.sh index bee2d1972..2f6cb4452 100755 --- a/scripts/ci/jobs/db-integration-tests.sh +++ b/scripts/ci/jobs/db-integration-tests.sh @@ -16,9 +16,11 @@ db_integration_tests() { make db-integration-tests || touch FAIL - info "Saving junit XML report" - make generate-junit-reports || touch FAIL - store_test_results junit-reports junit-reports + if is_OPENSHIFT_CI; then + info "Saving junit XML report" + make generate-junit-reports || touch FAIL + store_test_results junit-reports junit-reports + fi [[ ! -f FAIL ]] || die "DB integration tests failed" } diff --git a/scripts/ci/jobs/diff-dumps.sh b/scripts/ci/jobs/diff-dumps.sh index 37085817e..6c470c9ab 100755 --- a/scripts/ci/jobs/diff-dumps.sh +++ b/scripts/ci/jobs/diff-dumps.sh @@ -100,7 +100,7 @@ diff_dumps() { # These are not needed until later, but no reason to continue if these do not exist. require_environment "GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER" - require_environment "GCP_SERVICE_ACCOUNT_CREDS" + require_environment "SCANNER_GCP_SERVICE_ACCOUNT_CREDS" # Create diff dumps setup_gcp @@ -112,7 +112,7 @@ diff_dumps() { # Create offline dump create_offline_dump # Upload offline dump - setup_gcp "${GCP_SERVICE_ACCOUNT_CREDS}" + setup_gcp "${SCANNER_GCP_SERVICE_ACCOUNT_CREDS}" upload_offline_dump } diff --git a/scripts/ci/jobs/style-checks.sh b/scripts/ci/jobs/style-checks.sh index a24f2df04..3229eb744 100755 --- a/scripts/ci/jobs/style-checks.sh +++ b/scripts/ci/jobs/style-checks.sh @@ -11,10 +11,12 @@ style_checks() { make style || touch FAIL - info "Saving junit XML report" - mkdir -p junit-reports - cp -a report.xml "junit-reports/" || true - store_test_results junit-reports reports + if is_OPENSHIFT_CI; then + info "Saving junit XML report" + mkdir -p junit-reports + cp -a report.xml "junit-reports/" || true + store_test_results junit-reports reports + fi [[ ! -f FAIL ]] || die "Style checks failed" } diff --git a/scripts/ci/jobs/unit-tests.sh b/scripts/ci/jobs/unit-tests.sh index 64e22ab84..ebaaf3b44 100755 --- a/scripts/ci/jobs/unit-tests.sh +++ b/scripts/ci/jobs/unit-tests.sh @@ -13,9 +13,11 @@ unit_tests() { make unit-tests || touch FAIL - info "Saving junit XML report" - make generate-junit-reports || touch FAIL - store_test_results junit-reports junit-reports + if is_OPENSHIFT_CI; then + info "Saving junit XML report" + make generate-junit-reports || touch FAIL + store_test_results junit-reports junit-reports + fi [[ ! -f FAIL ]] || die "Unit tests failed" } diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh index c74e8f844..08eea2b1b 100755 --- a/scripts/ci/lib.sh +++ b/scripts/ci/lib.sh @@ -225,11 +225,11 @@ is_nightly_run() { } is_in_PR_context() { - if ! is_OPENSHIFT_CI; then - return 1 - elif [[ -n "${PULL_NUMBER:-}" ]]; then + if is_GITHUB_ACTIONS && [[ -n "${GITHUB_BASE_REF:-}" ]]; then return 0 - elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then + elif is_OPENSHIFT_CI && [[ -n "${PULL_NUMBER:-}" ]]; then + return 0 + elif is_OPENSHIFT_CI && [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then # bin, test-bin, images local pull_request pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number' 2>&1) || return 1 @@ -772,7 +772,7 @@ send_slack_notice_for_failures_on_merge() { } send_slack_notice_for_vuln_check_failure() { - if ! is_OPENSHIFT_CI; then + if ! is_OPENSHIFT_CI && ! is_GITHUB_ACTIONS; then return 0 fi @@ -819,6 +819,74 @@ send_slack_notice_for_vuln_check_failure() { curl -XPOST -d @- -H 'Content-Type: application/json' "$webhook_url" } +generate_genesis_dump() { + info "Generating genesis dump" + mkdir -p /tmp/genesis-dump + bin/updater generate-dump --out-file /tmp/genesis-dump/genesis-dump.zip + ls -lrt /tmp/genesis-dump + + info "Printing some stats" + bin/updater print-stats /tmp/genesis-dump/genesis-dump.zip + + info "Extracting dumps" + mkdir -p /tmp/vuln-dump + zip /tmp/genesis-dump/genesis-dump.zip 'nvd/*' --copy --out /tmp/vuln-dump/nvd-definitions.zip + zip /tmp/genesis-dump/genesis-dump.zip 'k8s/*' --copy --out /tmp/vuln-dump/k8s-definitions.zip + zip /tmp/genesis-dump/genesis-dump.zip 'istio/*' --copy --out /tmp/vuln-dump/istio-definitions.zip + zip /tmp/genesis-dump/genesis-dump.zip 'rhelv2/repository-to-cpe.json' --copy --out /tmp/vuln-dump/repo2cpe.zip +} + +get_genesis_dump() { + info "Retrieving Genesis dump" + + ls -lrt /tmp/vuln-dump || info "No local genesis dump" + + if is_in_PR_context && ! pr_has_label "generate-dumps-on-pr"; then + info "Label generate-dumps-on-pr not set. Pulling dumps from GCS bucket" + mkdir -p /tmp/vuln-dump + gsutil cp gs://stackrox-scanner-ci-vuln-dump/nvd-definitions.zip /tmp/vuln-dump/nvd-definitions.zip + gsutil cp gs://stackrox-scanner-ci-vuln-dump/k8s-definitions.zip /tmp/vuln-dump/k8s-definitions.zip + gsutil cp gs://stackrox-scanner-ci-vuln-dump/istio-definitions.zip /tmp/vuln-dump/istio-definitions.zip + gsutil cp gs://stackrox-scanner-ci-vuln-dump/repo2cpe.zip /tmp/vuln-dump/repo2cpe.zip + fi + + unzip -d image/scanner/dump /tmp/vuln-dump/nvd-definitions.zip + unzip -d image/scanner/dump /tmp/vuln-dump/k8s-definitions.zip + unzip -d image/scanner/dump /tmp/vuln-dump/istio-definitions.zip + unzip -d image/scanner/dump /tmp/vuln-dump/repo2cpe.zip +} + +generate_db_dump() { + info "Generating DB dump" + + groupadd -g 1001 pg + adduser pg -u 1001 -g 1001 -d /var/lib/postgresql -s /bin/sh + + # The PATH is not completely preserved, so set the PATH here to ensure postgres-related commands can be found. + runuser -l pg -c "PATH=$PATH $SCRIPTS_ROOT/scripts/ci/postgres.sh start_postgres" + + bin/updater load-dump --postgres-host 127.0.0.1 --postgres-port 5432 --dump-file /tmp/genesis-dump/genesis-dump.zip + + mkdir /tmp/postgres + pg_dump -U postgres postgres://127.0.0.1:5432 > /tmp/postgres/pg-definitions.sql + ls -lrt /tmp/postgres + gzip --best /tmp/postgres/pg-definitions.sql + ls -lrt /tmp/postgres +} + +get_db_dump() { + info "Retrieving DB dump" + + ls -lrt /tmp/postgres || info "No local DB dump" + + if is_in_PR_context && ! pr_has_label "generate-dumps-on-pr"; then + info "Label generate-dumps-on-pr not set. Pulling dumps from GCS bucket" + gsutil cp gs://stackrox-scanner-ci-vuln-dump/pg-definitions.sql.gz image/db/dump/definitions.sql.gz + else + cp /tmp/postgres/pg-definitions.sql.gz image/db/dump/definitions.sql.gz + fi +} + if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then if [[ "$#" -lt 1 ]]; then die "When invoked at the command line a method is required." diff --git a/scripts/ci/postgres.sh b/scripts/ci/postgres.sh index 177835bc0..f4ec6761d 100755 --- a/scripts/ci/postgres.sh +++ b/scripts/ci/postgres.sh @@ -8,15 +8,33 @@ source "$SCRIPTS_ROOT/scripts/lib.sh" set -euo pipefail -start_postgres() { - info "Starting Postgres" - +_start_postgres() { initdb "${HOME}/data" pg_ctl -D "${HOME}/data" -l logfile -o "-k /tmp" start export PGHOST=/tmp createuser -s postgres } +start_postgres() { + info "Starting Postgres" + + if [[ $(id -u) == 0 ]]; then + info "This function should not be run as root." + if is_CI; then + info "Running in CI. Creating a non-root user." + groupadd -g 1001 pg + adduser pg -u 1001 -g 1001 -d /var/lib/postgresql -s /bin/sh + + # The PATH is not completely preserved, so set the PATH here to ensure postgres-related commands can be found. + runuser -l pg -c "PATH=$PATH $SCRIPTS_ROOT/scripts/ci/postgres.sh _start_postgres" + else + die "Please re-run as a non-root user." + fi + else + _start_postgres + fi +} + if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then if [[ "$#" -lt 1 ]]; then die "When invoked at the command line a method is required." diff --git a/scripts/lib.sh b/scripts/lib.sh index 04862f625..43f83de25 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -52,6 +52,10 @@ is_OPENSHIFT_CI() { [[ "${OPENSHIFT_CI:-}" == "true" ]] } +is_GITHUB_ACTIONS() { + [[ -n "${GITHUB_ACTION:-}" ]] +} + is_darwin() { uname -a | grep -i darwin >/dev/null 2>&1 }