From aaf6df7d358fb61d35d613a54a32433752fa10ba Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 5 Sep 2024 18:11:22 -0400 Subject: [PATCH] refactor test cache workflow Signed-off-by: Alex Goodman --- .binny.yaml | 16 ++ .github/actions/bootstrap/action.yaml | 29 ++- .github/scripts/ci-check.sh | 11 -- .github/scripts/find_cache_paths.py | 126 +++++++++++++ .github/workflows/update-bootstrap-tools.yml | 1 - .github/workflows/validations.yaml | 74 +------- Makefile | 4 +- Taskfile.yaml | 169 ++++++++++++------ .../test/integration/test-fixtures/Makefile | 27 ++- go.mod | 1 + go.sum | 2 + .../executable/test-fixtures/Makefile | 24 +++ .../executable/test-fixtures/elf/Makefile | 25 ++- .../test-fixtures/shared-info/Makefile | 28 ++- .../cataloger/binary/test-fixtures/.gitignore | 1 - .../cataloger/binary/test-fixtures/Makefile | 31 +++- .../traefik/3.0.4/linux-riscv64/traefik | Bin 0 -> 352 bytes .../binary/test-fixtures/config.yaml | 1 + .../internal/config/binary_from_image.go | 8 +- .../internal/config/binary_from_image_test.go | 6 +- .../manager/internal/download_from_image.go | 48 +++-- .../internal/download_from_image_test.go | 30 ++-- .../manager/internal/list_entries.go | 2 +- .../cataloger/golang/test-fixtures/Makefile | 24 +++ .../golang/test-fixtures/archs/Makefile | 32 ++-- .../golang/test-fixtures/archs/src/build.sh | 5 +- .../pkg/cataloger/java/test-fixtures/Makefile | 24 +++ .../java/test-fixtures/jar-metadata/Makefile | 50 ++++-- .../java/test-fixtures/java-builds/Makefile | 29 +-- .../cataloger/kernel/test-fixtures/Makefile | 26 ++- .../cataloger/redhat/test-fixtures/Makefile | 39 ++-- test/cli/test-fixtures/Makefile | 28 ++- test/install/Makefile | 16 +- 33 files changed, 677 insertions(+), 260 deletions(-) delete mode 100755 .github/scripts/ci-check.sh create mode 100755 .github/scripts/find_cache_paths.py create mode 100644 syft/file/cataloger/executable/test-fixtures/Makefile create mode 100644 syft/pkg/cataloger/binary/test-fixtures/classifiers/snippets/traefik/3.0.4/linux-riscv64/traefik create mode 100644 syft/pkg/cataloger/golang/test-fixtures/Makefile create mode 100644 syft/pkg/cataloger/java/test-fixtures/Makefile diff --git a/.binny.yaml b/.binny.yaml index a9a37439333..cab909d2941 100644 --- a/.binny.yaml +++ b/.binny.yaml @@ -115,3 +115,19 @@ tools: method: github-release with: repo: cli/cli + + # used to upload test fixture cache + - name: oras + version: + want: v1.2.0 + method: github-release + with: + repo: oras-project/oras + + # used to upload test fixture cache + - name: yq + version: + want: v4.44.3 + method: github-release + with: + repo: mikefarah/yq \ No newline at end of file diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index bc771d3b508..2617ea52ad4 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -13,16 +13,15 @@ inputs: cache-key-prefix: description: "Prefix all cache keys with this value" required: true - default: "1ac8281053" - compute-fingerprints: - description: "Compute test fixture fingerprints" + default: "181053ac82" + download-test-fixture-cache: + description: "Download test fixture cache from OCI and github actions" required: true - default: "true" + default: "false" bootstrap-apt-packages: description: "Space delimited list of tools to install via apt" default: "libxml2-utils" - runs: using: "composite" steps: @@ -55,7 +54,23 @@ runs: DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }} - name: Create all cache fingerprints - if: inputs.compute-fingerprints == 'true' + id: fingerprint + if: inputs.download-test-fixture-cache == 'true' shell: bash - run: make fingerprints + run: | + make fingerprints + echo "fingerprint=$(make fingerprint)" | tee -a $GITHUB_OUTPUT + + - name: Restore ORAS cache from github actions + if: inputs.download-test-fixture-cache == 'true' + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + with: + path: ${{ github.workspace }}/.tool + key: | + ${{ inputs.cache-key-prefix }}-${{ runner.os }}-oras-cache-${{ steps.fingerprint.outputs.fingerprint }} + ${{ inputs.cache-key-prefix }}-${{ runner.os }}-oras-cache- + - name: Download test fixture cache + if: inputs.download-test-fixture-cache == 'true' + shell: bash + run: make download-test-fixture-cache diff --git a/.github/scripts/ci-check.sh b/.github/scripts/ci-check.sh deleted file mode 100755 index 0ab83a318ae..00000000000 --- a/.github/scripts/ci-check.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -red=$(tput setaf 1) -bold=$(tput bold) -normal=$(tput sgr0) - -# assert we are running in CI (or die!) -if [[ -z "$CI" ]]; then - echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}" - exit 1 -fi diff --git a/.github/scripts/find_cache_paths.py b/.github/scripts/find_cache_paths.py new file mode 100755 index 00000000000..40165117454 --- /dev/null +++ b/.github/scripts/find_cache_paths.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +import os +import glob +import sys +import json +import hashlib + + +IGNORED_PREFIXES = [] + + +def find_fingerprints_and_check_dirs(base_dir): + all_fingerprints = set(glob.glob(os.path.join(base_dir, '**', 'test*', '**', '*.fingerprint'), recursive=True)) + cache_dirs = set(glob.glob(os.path.join(base_dir, '**', 'test-fixtures', 'cache'), recursive=True)) + + all_fingerprints = {os.path.relpath(fp) for fp in all_fingerprints + if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)} + + if not all_fingerprints and not cache_dirs: + show("No .fingerprint files or cache directories found.") + exit(1) + + missing_content = [] + valid_paths = set() + fingerprint_contents = [] + + for fingerprint in all_fingerprints: + path = fingerprint.replace('.fingerprint', '') + + if not os.path.exists(path): + missing_content.append(path) + continue + + if not os.path.isdir(path): + valid_paths.add(path) + continue + + if os.listdir(path): + valid_paths.add(path) + else: + missing_content.append(path) + + with open(fingerprint, 'r') as f: + content = f.read().strip() + fingerprint_contents.append((fingerprint, content)) + + # add cache dirs to valid paths + for cache_dir in cache_dirs: + if os.path.isdir(cache_dir) and os.listdir(cache_dir): + valid_paths.add(os.path.relpath(cache_dir)) + else: + missing_content.append(os.path.relpath(cache_dir)) + + return sorted(valid_paths), missing_content, fingerprint_contents + + +def calculate_sha256(fingerprint_contents): + sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0]) + + concatenated_contents = ''.join(content for _, content in sorted_fingerprint_contents) + + sha256_hash = hashlib.sha256(concatenated_contents.encode()).hexdigest() + + return sha256_hash + + +def show(*s: str): + print(*s, file=sys.stderr) + + +# outputs a input hash and set of paths that should be cached, for example: +# { +# "input": "85ae27de8da54826dbb374e8a302b19c9ecc6361f1714dce88f2e45a60b58870", +# "paths": [ +# "cmd/syft/internal/test/integration/test-fixtures/cache", +# "syft/file/cataloger/executable/test-fixtures/elf/bin", +# "syft/file/cataloger/executable/test-fixtures/shared-info/bin", +# "syft/file/cataloger/filedigest/test-fixtures/cache", +# "syft/file/cataloger/filemetadata/test-fixtures/cache", +# "syft/file/cataloger/internal/test-fixtures/cache", +# "syft/format/internal/testutil/test-fixtures/cache", +# "syft/format/syftjson/test-fixtures/cache", +# "syft/internal/fileresolver/test-fixtures/cache", +# "syft/pkg/cataloger/binary/test-fixtures/cache", +# "syft/pkg/cataloger/binary/test-fixtures/classifiers/bin", +# "syft/pkg/cataloger/binary/test-fixtures/classifiers/dynamic", +# "syft/pkg/cataloger/debian/test-fixtures/cache", +# "syft/pkg/cataloger/golang/test-fixtures/archs/binaries", +# "syft/pkg/cataloger/golang/test-fixtures/cache", +# "syft/pkg/cataloger/java/test-fixtures/jar-metadata/cache", +# "syft/pkg/cataloger/java/test-fixtures/java-builds/packages", +# "syft/pkg/cataloger/kernel/test-fixtures/cache", +# "syft/pkg/cataloger/python/test-fixtures/cache", +# "syft/pkg/cataloger/redhat/test-fixtures/cache", +# "syft/pkg/cataloger/redhat/test-fixtures/rpms", +# "syft/pkg/cataloger/rust/test-fixtures/cache", +# "syft/source/test-fixtures/cache", +# "test/cli/test-fixtures/cache", +# "test/install/cache" +# ] +# } +# +# ...if the input is inconsistent then this script will exit with a non-zero status code instead. +def main(): + base_dir = '.' + valid_paths, missing_content, fingerprint_contents = find_fingerprints_and_check_dirs(base_dir) + + if missing_content: + show("The following paths are missing or have no content, but have corresponding .fingerprint files:") + for path in sorted(missing_content): + show(f"- {path}") + show("Please ensure these paths exist and have content if they are directories.") + exit(1) + + sha256_hash = calculate_sha256(fingerprint_contents) + + output = { + "input": sha256_hash, + "paths": sorted(valid_paths) + } + + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/update-bootstrap-tools.yml b/.github/workflows/update-bootstrap-tools.yml index 488a2cf5bc2..2e891e85b02 100644 --- a/.github/workflows/update-bootstrap-tools.yml +++ b/.github/workflows/update-bootstrap-tools.yml @@ -19,7 +19,6 @@ jobs: uses: ./.github/actions/bootstrap with: bootstrap-apt-packages: "" - compute-fingerprints: "false" go-dependencies: false - name: "Update tool versions" diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 669d8b8c5c4..67fc5cd1878 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -35,48 +35,8 @@ jobs: - name: Bootstrap environment uses: ./.github/actions/bootstrap - - - name: Restore file executable test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: syft/file/cataloger/executable/test-fixtures/elf/bin - key: ${{ runner.os }}-unit-file-executable-elf-cache-${{ hashFiles( 'syft/file/cataloger/executable/test-fixtures/elf/cache.fingerprint' ) }} - - - name: Restore file executable shared-info test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: syft/file/cataloger/executable/test-fixtures/shared-info/bin - key: ${{ runner.os }}-unit-file-executable-shared-info-cache-${{ hashFiles( 'syft/file/cataloger/executable/test-fixtures/shared-info/cache.fingerprint' ) }} - - - name: Restore Java test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages - key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint' ) }} - - - name: Restore RPM test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: syft/pkg/cataloger/redhat/test-fixtures/rpms - key: ${{ runner.os }}-unit-rpm-cache-${{ hashFiles( 'syft/pkg/cataloger/redhat/test-fixtures/rpms.fingerprint' ) }} - - - name: Restore go binary test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: syft/pkg/cataloger/golang/test-fixtures/archs/binaries - key: ${{ runner.os }}-unit-go-binaries-cache-${{ hashFiles( 'syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint' ) }} - - - name: Restore binary cataloger test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 with: - path: syft/pkg/cataloger/binary/test-fixtures/classifiers/bin - key: ${{ runner.os }}-unit-binary-cataloger-cache-${{ hashFiles( 'syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint' ) }} - - - name: Restore Kernel test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: syft/pkg/cataloger/kernel/test-fixtures/cache - key: ${{ runner.os }}-unit-kernel-cache-${{ hashFiles( 'syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint' ) }} + download-test-fixture-cache: true - name: Run unit tests run: make unit @@ -91,16 +51,12 @@ jobs: - name: Bootstrap environment uses: ./.github/actions/bootstrap + with: + download-test-fixture-cache: true - name: Validate syft output against the CycloneDX schema run: make validate-cyclonedx-schema - - name: Restore integration test cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: ${{ github.workspace }}/cmd/syft/internal/test/integration/test-fixtures/cache - key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('/cmd/syft/internal/test/integration/test-fixtures/cache.fingerprint') }} - - name: Run integration tests run: make integration @@ -143,6 +99,8 @@ jobs: - name: Bootstrap environment uses: ./.github/actions/bootstrap + with: + download-test-fixture-cache: true - name: Download snapshot build id: snapshot-cache @@ -162,13 +120,6 @@ jobs: - name: Run comparison tests (Linux) run: make compare-linux - - name: Restore install.sh test image cache - id: install-test-image-cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: ${{ github.workspace }}/test/install/cache - key: ${{ runner.os }}-install-test-image-cache-${{ hashFiles('test/install/cache.fingerprint') }} - - name: Load test image cache if: steps.install-test-image-cache.outputs.cache-hit == 'true' run: make install-test-cache-load @@ -196,8 +147,8 @@ jobs: uses: ./.github/actions/bootstrap with: bootstrap-apt-packages: "" - compute-fingerprints: "false" go-dependencies: false + download-test-fixture-cache: true - name: Download snapshot build id: snapshot-cache @@ -214,13 +165,6 @@ jobs: if: steps.snapshot-cache.outputs.cache-hit != 'true' run: echo "unable to download snapshots from previous job" && false - - name: Restore docker image cache for compare testing - id: mac-compare-testing-cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 - with: - path: image.tar - key: ${{ runner.os }}-${{ hashFiles('test/compare/mac.sh') }} - - name: Run comparison tests (Mac) run: make compare-mac @@ -238,12 +182,8 @@ jobs: - name: Bootstrap environment uses: ./.github/actions/bootstrap - - - name: Restore CLI test-fixture cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2 with: - path: ${{ github.workspace }}/test/cli/test-fixtures/cache - key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} + download-test-fixture-cache: true - name: Download snapshot build id: snapshot-cache diff --git a/Makefile b/Makefile index 9089ee6192c..2f1ae1f8e5b 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,8 @@ ci-bootstrap-go: # this is a bootstrapping catch-all, where if the target doesn't exist, we'll ensure the tools are installed and then try again %: - make $(TASK) - $(TASK) $@ + @make --silent $(TASK) + @$(TASK) $@ ## Shim targets ################################# diff --git a/Taskfile.yaml b/Taskfile.yaml index 17368566387..28eb0c09296 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -7,6 +7,11 @@ vars: # static file dirs TOOL_DIR: .tool TMP_DIR: .tmp + ORAS_CACHE: "{{ .TMP_DIR }}/oras-cache" + + # TOOLS + ORAS: "{{ .TOOL_DIR }}/oras" + YQ: "{{ .TOOL_DIR }}/yq" # used for changelog generation CHANGELOG: CHANGELOG.md @@ -213,10 +218,6 @@ tasks: # that the cache being restored with the correct binary will be rebuilt since the timestamps # and local checksums will not line up. deps: [tools, snapshot] - sources: - - "{{ .SNAPSHOT_BIN }}" - - ./test/cli/** - - ./**/*.go cmds: - cmd: "echo 'testing binary: {{ .SNAPSHOT_BIN }}'" silent: true @@ -230,8 +231,6 @@ tasks: test-utils: desc: Run tests for pipeline utils - sources: - - .github/scripts/labeler*.py cmds: - cmd: python .github/scripts/labeler_test.py @@ -240,8 +239,6 @@ tasks: benchmark: deps: [tmpdir] - sources: - - ./**/*.go generates: - "{{ .TMP_DIR }}/benchmark-main.txt" cmds: @@ -254,8 +251,6 @@ tasks: show-benchstat: deps: [benchmark, tmpdir] - sources: - - "{{ .TMP_DIR }}/benchstat.txt" cmds: - cmd: "cat {{ .TMP_DIR }}/benchstat.txt" silent: true @@ -264,55 +259,93 @@ tasks: ## Test-fixture-related targets ################################# fingerprints: - desc: Generate test fixture fingerprints + desc: Generate fingerprints for all non-docker test fixture + silent: true + # this will look for `test-fixtures/Makefile` and invoke the `fingerprint` target to calculate all cache input fingerprint files generates: - - cmd/syft/internal/test/integration/test-fixtures/cache.fingerprint - - syft/file/cataloger/executable/test-fixtures/elf/cache.fingerprint - - syft/file/cataloger/executable/test-fixtures/shared-info/cache.fingerprint - - syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint - - syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint - - syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint - - syft/pkg/cataloger/redhat/test-fixtures/rpms.fingerprint - - syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint + - '**/test-fixtures/**/*.fingerprint' - test/install/cache.fingerprint - - test/cli/test-fixtures/cache.fingerprint - cmds: - # for EXECUTABLE unit test fixtures - - "cd syft/file/cataloger/executable/test-fixtures/elf && make cache.fingerprint" - - "cd syft/file/cataloger/executable/test-fixtures/shared-info && make cache.fingerprint" - # for IMAGE integration test fixtures - - "cd cmd/syft/internal/test/integration/test-fixtures && make cache.fingerprint" - # for BINARY unit test fixtures - - "cd syft/pkg/cataloger/binary/test-fixtures && make cache.fingerprint" - # for JAVA BUILD unit test fixtures - - "cd syft/pkg/cataloger/java/test-fixtures/java-builds && make cache.fingerprint" - # for GO BINARY unit test fixtures - - "cd syft/pkg/cataloger/golang/test-fixtures/archs && make binaries.fingerprint" - # for RPM unit test fixtures - - "cd syft/pkg/cataloger/redhat/test-fixtures && make rpms.fingerprint" - # for Kernel unit test fixtures - - "cd syft/pkg/cataloger/kernel/test-fixtures && make cache.fingerprint" - # for INSTALL test fixtures - - "cd test/install && make cache.fingerprint" - # for CLI test fixtures - - "cd test/cli/test-fixtures && make cache.fingerprint" - - fixtures: - desc: Generate test fixtures - cmds: - - "cd syft/file/cataloger/executable/test-fixtures/elf && make" - - "cd syft/file/cataloger/executable/test-fixtures/shared-info && make" - - "cd syft/pkg/cataloger/java/test-fixtures/java-builds && make" - - "cd syft/pkg/cataloger/redhat/test-fixtures && make" - - "cd syft/pkg/cataloger/binary/test-fixtures && make" + cmds: + - | + BOLD='\033[1m' + YELLOW='\033[0;33m' + RESET='\033[0m' + + # Use a for loop with command substitution to avoid subshell issues + for dir in $(find . -type d -name 'test-fixtures'); do + if [ -f "$dir/Makefile" ]; then + echo -e "${YELLOW}${BOLD}calculating fingerprints in $dir${RESET}" + (make -C "$dir" fingerprint) + fi + done + echo -e "${BOLD}Generated all fixtures${RESET}" + + fingerprint: + desc: Generate single fingerprint for all non-docker test fixtures + silent: true + cmds: + - .github/scripts/find_cache_paths.py | jq -r '.input' + + refresh-fixtures: + desc: Clear and fetch all test fixture cache + aliases: + - fixtures + cmds: + - task: clean-cache + - task: download-test-fixture-cache + + build-fixtures: + desc: Generate all non-docker test fixtures + silent: true + # this will look for `test-fixtures/Makefile` and invoke the `fixtures` target to generate any and all test fixtures + cmds: + - | + BOLD='\033[1m' + YELLOW='\033[0;33m' + RESET='\033[0m' + + # Use a for loop with command substitution to avoid subshell issues + for dir in $(find . -type d -name 'test-fixtures'); do + if [ -f "$dir/Makefile" ]; then + echo -e "${YELLOW}${BOLD}generating fixtures in $dir${RESET}" + (make -C "$dir" fixtures) + fi + done + echo -e "${BOLD}Generated all fixtures${RESET}" + + download-test-fixture-cache: + desc: Download test fixture cache from ghcr.io + deps: [tools] + cmd: "ORAS_CACHE={{ .ORAS_CACHE }} {{ .ORAS }} pull ghcr.io/{{ .OWNER }}/{{ .PROJECT }}/test-fixture-cache:latest" + + upload-test-fixture-cache: + desc: Upload the test fixture cache to ghcr.io + deps: [tools, fingerprints] + silent: true + cmd: | + set -eu + + paths_json=$(python .github/scripts/find_cache_paths.py) + + oras_command="{{ .ORAS }} push ghcr.io/{{ .OWNER }}/{{ .PROJECT }}/test-fixture-cache:latest" + + paths=$(echo "$paths_json" | {{ .YQ }} -r '.paths[]') + for path in $paths; do + oras_command+=" $path" + done + + oras_command+=" --annotation org.opencontainers.image.source=https://github.com/{{ .OWNER }}/{{ .PROJECT }}" + + echo "Executing: $oras_command" + eval $oras_command show-test-image-cache: silent: true cmds: - - "echo '\nDocker daemon cache:'" + - "echo 'Docker daemon cache:'" - "docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}:{{`{{.Tag}}`}}' | grep stereoscope-fixture- | sort" - "echo '\nTar cache:'" - - 'find . -type f -wholename "**/test-fixtures/snapshot/*" | sort' + - 'find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" | sort' check-docker-cache: desc: Ensure docker caches aren't using too much disk space @@ -470,7 +503,16 @@ tasks: ci-check: # desc: "[CI only] Are you in CI?" cmds: - - cmd: .github/scripts/ci-check.sh + - cmd: | + red=$(tput setaf 1) + bold=$(tput bold) + normal=$(tput sgr0) + + # assert we are running in CI (or die!) + if [[ -z "$CI" ]]; then + echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}" + exit 1 + fi silent: true ci-release: @@ -502,8 +544,25 @@ tasks: - "rm -rf {{ .SNAPSHOT_DIR }}" - "rm -rf {{ .TMP_DIR }}/goreleaser.yaml" + clean-oras-cache: + desc: Remove all cache for oras commands + cmd: rm -rf {{ .ORAS_CACHE }} + clean-cache: - desc: Remove all docker cache and local image tar cache + desc: Remove all image tar cache, docker images, and ephemeral test fixtures cmds: - - 'find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" -delete' - - "docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}' | grep stereoscope-fixture- | awk '{print $1}' | uniq | xargs -r docker rmi --force" + - find . -type d -wholename "**/test-fixtures/cache" | xargs rm -rf + - docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}' | grep stereoscope-fixture- | awk '{print $1}' | uniq | xargs -r docker rmi --force + - | + BOLD='\033[1m' + YELLOW='\033[0;33m' + RESET='\033[0m' + + # Use a for loop with command substitution to avoid subshell issues + for dir in $(find . -type d -name 'test-fixtures'); do + if [ -f "$dir/Makefile" ]; then + echo -e "${YELLOW}${BOLD}deleting ephemeral test fixtures in $dir${RESET}" + (make -C "$dir" clean) + fi + done + echo -e "${BOLD}Deleted all ephemeral test fixtures${RESET}" diff --git a/cmd/syft/internal/test/integration/test-fixtures/Makefile b/cmd/syft/internal/test/integration/test-fixtures/Makefile index 2a75aa43616..5c04bf63dd6 100644 --- a/cmd/syft/internal/test/integration/test-fixtures/Makefile +++ b/cmd/syft/internal/test/integration/test-fixtures/Makefile @@ -1,6 +1,25 @@ # change these if you want CI to not use previous stored cache -INTEGRATION_CACHE_BUSTER := "894d8ca" +CACHE_BUSTER := "fd8e5cd" -.PHONY: cache.fingerprint -cache.fingerprint: - find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> cache.fingerprint +FINGERPRINT_FILE := cache.fingerprint + +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: + @echo "nothing to do" + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) + +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find image-* -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @echo "$(CACHE_BUSTER)" >> $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum + +# requirement 4: 'clean' goal to remove all generated test fixtures +.PHONY: clean +clean: + rm -f $(FINGERPRINT_FILE) diff --git a/go.mod b/go.mod index 8e6a7ea63dd..a94884192f0 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirec require ( github.com/BurntSushi/toml v1.4.0 + github.com/OneOfOne/xxhash v1.2.8 github.com/adrg/xdg v0.5.0 github.com/magiconair/properties v1.8.7 golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 diff --git a/go.sum b/go.sum index 0cc5381b240..dd8d600e70f 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,8 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= diff --git a/syft/file/cataloger/executable/test-fixtures/Makefile b/syft/file/cataloger/executable/test-fixtures/Makefile new file mode 100644 index 00000000000..656190af9d9 --- /dev/null +++ b/syft/file/cataloger/executable/test-fixtures/Makefile @@ -0,0 +1,24 @@ +ORANGE := \033[38;5;208m +BOLD := \033[1m +RESET := \033[0m +PWD := $(shell pwd) + +.DEFAULT_GOAL := default + +default: + @echo "$(ORANGE)$(BOLD)building test fixtures from $(PWD)$(RESET)" + @for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \ + if [ -f "$$dir/Makefile" ]; then \ + echo "$(ORANGE)• running make in $$dir$(RESET)"; \ + $(MAKE) -C $$dir; \ + fi; \ + done + +%: + @echo "$(ORANGE)$(BOLD)building test fixtures from $(PWD)$(RESET)" + @for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \ + if [ -f "$$dir/Makefile" ]; then \ + echo "$(ORANGE)• running make '$@' in $$dir$(RESET)"; \ + $(MAKE) -C $$dir $@; \ + fi; \ + done diff --git a/syft/file/cataloger/executable/test-fixtures/elf/Makefile b/syft/file/cataloger/executable/test-fixtures/elf/Makefile index 1cff6183e1e..2e98e8c6d99 100644 --- a/syft/file/cataloger/executable/test-fixtures/elf/Makefile +++ b/syft/file/cataloger/executable/test-fixtures/elf/Makefile @@ -1,8 +1,19 @@ BIN=./bin TOOL_IMAGE=localhost/syft-bin-build-tools:latest VERIFY_FILE=actual_verify +FINGERPRINT_FILE=$(BIN).fingerprint -all: build verify +ifndef BIN + $(error BIN is not set) +endif + +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: build verify + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) tools-check: @sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1) @@ -25,10 +36,14 @@ verify: tools debug: docker run -i --rm -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) bash -cache.fingerprint: - @find project Dockerfile Makefile -type f -exec md5sum {} + | awk '{print $1}' | sort | tee cache.fingerprint +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find project Dockerfile Makefile -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum +# requirement 4: 'clean' goal to remove all generated test fixtures clean: - rm -f $(BIN)/* + rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE) -.PHONY: build verify debug build-image build-bins clean dockerfile-check cache.fingerprint +.PHONY: tools tools-check build verify debug clean \ No newline at end of file diff --git a/syft/file/cataloger/executable/test-fixtures/shared-info/Makefile b/syft/file/cataloger/executable/test-fixtures/shared-info/Makefile index a3d5959c358..961ef66817d 100644 --- a/syft/file/cataloger/executable/test-fixtures/shared-info/Makefile +++ b/syft/file/cataloger/executable/test-fixtures/shared-info/Makefile @@ -1,8 +1,20 @@ BIN=./bin TOOL_IMAGE=localhost/syft-shared-info-build-tools:latest VERIFY_FILE=actual_verify +FINGERPRINT_FILE=$(BIN).fingerprint + +ifndef BIN + $(error BIN is not set) +endif + +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: build + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) -all: build tools-check: @sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1) @@ -10,16 +22,20 @@ tools: @(docker inspect $(TOOL_IMAGE) > /dev/null && make tools-check) || (docker build -t $(TOOL_IMAGE) . && sha256sum Dockerfile > Dockerfile.sha256) build: tools - mkdir -p $(BIN) + @mkdir -p $(BIN) docker run --platform linux/amd64 -i -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) make debug: docker run --platform linux/amd64 -i --rm -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) bash -cache.fingerprint: - @find project Dockerfile Makefile -type f -exec md5sum {} + | awk '{print $1}' | sort | tee cache.fingerprint +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find project Dockerfile Makefile -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum +# requirement 4: 'clean' goal to remove all generated test fixtures clean: - rm -f $(BIN)/* + rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE) -.PHONY: build verify debug build-image build-bins clean dockerfile-check cache.fingerprint +.PHONY: tools tools-check build debug clean diff --git a/syft/pkg/cataloger/binary/test-fixtures/.gitignore b/syft/pkg/cataloger/binary/test-fixtures/.gitignore index e1d59c126c8..4d4d11ec993 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/.gitignore +++ b/syft/pkg/cataloger/binary/test-fixtures/.gitignore @@ -1,6 +1,5 @@ classifiers/dynamic classifiers/bin -cache.fingerprint # allow for lb patterns (rust, pytho, php and more) !lib*.so diff --git a/syft/pkg/cataloger/binary/test-fixtures/Makefile b/syft/pkg/cataloger/binary/test-fixtures/Makefile index 3e8efed9468..e3429853cc1 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/Makefile +++ b/syft/pkg/cataloger/binary/test-fixtures/Makefile @@ -1,8 +1,14 @@ -.PHONY: default list download download-all cache.fingerprint +BIN=classifiers/bin +FINGERPRINT_FILE=$(BIN).fingerprint -.DEFAULT_GOAL := default -default: download +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: download + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: clean-fingerprint $(FINGERPRINT_FILE) list: ## list all managed binaries and snippets go run ./manager list @@ -16,14 +22,23 @@ download-all: ## download all managed binaries add-snippet: ## add a new snippet from an existing binary go run ./manager add-snippet -cache.fingerprint: ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss) - @cat ./config.yaml | sha256sum | awk '{print $$1}' | tee cache.fingerprint +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss) + @cat ./config.yaml | sha256sum | awk '{print $$1}' | tee $(FINGERPRINT_FILE) + +# requirement 4: 'clean' goal to remove all generated test fixtures +clean: ## clean up all downloaded binaries + rm -rf $(BIN) + +clean-fingerprint: ## clean up all legacy fingerprint files + find $(BIN) -name '*.fingerprint' -delete -clean: ## clean up all downloaded binaries - rm -rf ./classifiers/bin ## Halp! ################################# .PHONY: help help: - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' \ No newline at end of file + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' + +.PHONY: default list download download-all clean clean-fingerprint add-snippet fingerprint \ No newline at end of file diff --git a/syft/pkg/cataloger/binary/test-fixtures/classifiers/snippets/traefik/3.0.4/linux-riscv64/traefik b/syft/pkg/cataloger/binary/test-fixtures/classifiers/snippets/traefik/3.0.4/linux-riscv64/traefik new file mode 100644 index 0000000000000000000000000000000000000000..f361c692988ec8d3246272fd8a69b029bd6509c4 GIT binary patch literal 352 zcmY+8txf|$5P%zSYY-3=Vv{qdvVXU;Q$t8_NS6+l64t;$jEY4)Am?MXhB85 z5Yd}}62X>9IY)^sWKCd{%Sj_BnJ{__mL#FZU`lDUVyq|%9_~Bh!<)Qod2Z{vz2kk0 zjYh}MzpGKt;`HF<@o2vfJr-lLE)DDq-rE+=e|FdD;_dsNR~sH)m+v1J@O1X|x#48k JI~>e={{f-rVJ-jw literal 0 HcmV?d00001 diff --git a/syft/pkg/cataloger/binary/test-fixtures/config.yaml b/syft/pkg/cataloger/binary/test-fixtures/config.yaml index 15273d33d2f..2f4b8cdec34 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/config.yaml +++ b/syft/pkg/cataloger/binary/test-fixtures/config.yaml @@ -85,6 +85,7 @@ from-images: paths: - /usr/local/go/bin/go + # TODO: this is no longer available from dockerhub! (the snippet is vital) - version: 1.5.14 images: - ref: haproxy:1.5.14@sha256:3d57e3921cc84e860f764e863ce729dd0765e3d28d444775127bc42d68f98e10 diff --git a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go index f26ac3ae40a..dc558250186 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go +++ b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go @@ -1,11 +1,11 @@ package config import ( - "crypto/sha256" "fmt" "path/filepath" "strings" + "github.com/OneOfOne/xxhash" "gopkg.in/yaml.v3" ) @@ -68,13 +68,13 @@ func PlatformAsValue(platform string) string { return strings.ReplaceAll(platform, "/", "-") } -func (c BinaryFromImage) Fingerprint() string { +func (c BinaryFromImage) Digest() string { by, err := yaml.Marshal(c) if err != nil { panic(err) } - hasher := sha256.New() - hasher.Write(by) + hasher := xxhash.New64() + _, _ = hasher.Write(by) return fmt.Sprintf("%x", hasher.Sum(nil)) } diff --git a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go index 8d76d5a2b29..55bb3ee40c2 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go +++ b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go @@ -158,7 +158,7 @@ func TestPlatformAsValue(t *testing.T) { } } -func TestFingerprint(t *testing.T) { +func TestDigest(t *testing.T) { tests := []struct { name string binary BinaryFromImage @@ -179,13 +179,13 @@ func TestFingerprint(t *testing.T) { "path/to/test", }, }, - expected: "54ed081c07e4eba031afed4c04315cf96047822196473971be98d0769a0e3645", + expected: "fc25c48e3d2f01e3", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, tt.binary.Fingerprint()) + assert.Equal(t, tt.expected, tt.binary.Digest()) }) } } diff --git a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go index 32b9c83d6dd..1d5a667eabf 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go +++ b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go @@ -2,6 +2,7 @@ package internal import ( "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -14,6 +15,8 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui" ) +const digestFileSuffix = ".xxh64" + func DownloadFromImage(dest string, config config.BinaryFromImage) error { t := ui.Title{Name: config.Name(), Version: config.Version} t.Start() @@ -39,22 +42,22 @@ func DownloadFromImage(dest string, config config.BinaryFromImage) error { } func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool { - currentFingerprint := config.Fingerprint() + currentDigest := config.Digest() for _, path := range binaryPaths { - fingerprintPath := path + ".fingerprint" - if _, err := os.Stat(fingerprintPath); err != nil { + digestPath := path + digestFileSuffix + if _, err := os.Stat(digestPath); err != nil { // missing a fingerprint file means the download is stale return true } - writtenFingerprint, err := os.ReadFile(fingerprintPath) + writtenDigest, err := os.ReadFile(digestPath) if err != nil { // missing a fingerprint file means the download is stale return true } - if string(writtenFingerprint) != currentFingerprint { + if string(writtenDigest) != currentDigest { // the fingerprint file does not match the current fingerprint, so the download is stale return true } @@ -103,6 +106,12 @@ func pullDockerImage(imageReference, platform string) error { cmd := exec.Command("docker", "pull", "--platform", platform, imageReference) err := cmd.Run() if err != nil { + // attach stderr to output message + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { + err = fmt.Errorf("pull failed: %w:\n%s", err, exitErr.Stderr) + } + a.Done(err) return err } @@ -152,6 +161,12 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri cmd := exec.Command("docker", "create", "--name", containerName, image.Reference) if err = cmd.Run(); err != nil { + // attach stderr to output message + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { + err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr) + } + return err } @@ -162,7 +177,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri for i, destinationPath := range config.AllStorePathsForImage(image, destination) { path := config.PathsInImage[i] - if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Fingerprint()); err != nil { + if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Digest()); err != nil { return err } } @@ -170,7 +185,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri return nil } -func copyBinaryFromContainer(containerName, containerPath, destinationPath, fingerprint string) (err error) { +func copyBinaryFromContainer(containerName, containerPath, destinationPath, digest string) (err error) { a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)} a.Start() @@ -185,13 +200,24 @@ func copyBinaryFromContainer(containerName, containerPath, destinationPath, fing cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) //nolint:gosec // reason for gosec exception: this is for processing test fixtures only, not used in production if err := cmd.Run(); err != nil { + // attach stderr to output message + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { + err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr) + } + return err } - // capture fingerprint file - fingerprintPath := destinationPath + ".fingerprint" - if err := os.WriteFile(fingerprintPath, []byte(fingerprint), 0600); err != nil { - return fmt.Errorf("unable to write fingerprint file: %w", err) + // ensure permissions are 600 for destination + if err := os.Chmod(destinationPath, 0600); err != nil { + return fmt.Errorf("unable to set permissions on file %q: %w", destinationPath, err) + } + + // capture digest file + digestPath := destinationPath + digestFileSuffix + if err := os.WriteFile(digestPath, []byte(digest), 0600); err != nil { + return fmt.Errorf("unable to write digest file: %w", err) } return nil diff --git a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go index ca62ea5476a..fc097a7dbf6 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go +++ b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go @@ -14,35 +14,35 @@ import ( func TestIsDownloadStale(t *testing.T) { cases := []struct { - name string - fingerprint string - expected bool + name string + digest string + expected bool }{ { - name: "no fingerprint", - fingerprint: "", - expected: true, + name: "no digest", + digest: "", + expected: true, }, { - name: "fingerprint matches", - // this is the fingerprint for config in the loop body - fingerprint: "5177d458eaca031ea16fa707841043df2e31b89be6bae7ea41290aa32f0251a6", - expected: false, + name: "digest matches", + // this is the digest for config in the loop body + digest: "c9c8007f9c55c2f1", + expected: false, }, { - name: "fingerprint does not match", - fingerprint: "fingerprint", - expected: true, + name: "digest does not match", + digest: "bogus", + expected: true, }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { binaryPath := filepath.Join(t.TempDir(), "binary") - fh, err := os.Create(binaryPath + ".fingerprint") + fh, err := os.Create(binaryPath + digestFileSuffix) require.NoError(t, err) - fh.Write([]byte(tt.fingerprint)) + fh.Write([]byte(tt.digest)) require.NoError(t, fh.Close()) cfg := config.BinaryFromImage{ diff --git a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go index 7d6c2063045..9ecf254401e 100644 --- a/syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go +++ b/syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go @@ -170,7 +170,7 @@ func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) { func allFilePaths(root string) ([]string, error) { var paths []string err := filepath.Walk(root, func(path string, info os.FileInfo, _ error) error { - if info != nil && !info.IsDir() && !strings.HasSuffix(path, ".fingerprint") { + if info != nil && !info.IsDir() && !strings.HasSuffix(path, digestFileSuffix) { paths = append(paths, path) } return nil diff --git a/syft/pkg/cataloger/golang/test-fixtures/Makefile b/syft/pkg/cataloger/golang/test-fixtures/Makefile new file mode 100644 index 00000000000..656190af9d9 --- /dev/null +++ b/syft/pkg/cataloger/golang/test-fixtures/Makefile @@ -0,0 +1,24 @@ +ORANGE := \033[38;5;208m +BOLD := \033[1m +RESET := \033[0m +PWD := $(shell pwd) + +.DEFAULT_GOAL := default + +default: + @echo "$(ORANGE)$(BOLD)building test fixtures from $(PWD)$(RESET)" + @for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \ + if [ -f "$$dir/Makefile" ]; then \ + echo "$(ORANGE)• running make in $$dir$(RESET)"; \ + $(MAKE) -C $$dir; \ + fi; \ + done + +%: + @echo "$(ORANGE)$(BOLD)building test fixtures from $(PWD)$(RESET)" + @for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \ + if [ -f "$$dir/Makefile" ]; then \ + echo "$(ORANGE)• running make '$@' in $$dir$(RESET)"; \ + $(MAKE) -C $$dir $@; \ + fi; \ + done diff --git a/syft/pkg/cataloger/golang/test-fixtures/archs/Makefile b/syft/pkg/cataloger/golang/test-fixtures/archs/Makefile index 60eee7ff96a..c3bc1fb5f62 100644 --- a/syft/pkg/cataloger/golang/test-fixtures/archs/Makefile +++ b/syft/pkg/cataloger/golang/test-fixtures/archs/Makefile @@ -1,29 +1,39 @@ DESTINATION=binaries +FINGERPRINT_FILE=$(DESTINATION).fingerprint -all: $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64 +ifndef DESTINATION + $(error DESTINATION is not set) +endif + +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: $(DESTINATION) + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(DESTINATION).fingerprint + +$(DESTINATION): $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64 $(DESTINATION)/hello-mach-o-arm64: - mkdir -p $(DESTINATION) GOARCH=arm64 GOOS=darwin ./src/build.sh $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm: - mkdir -p $(DESTINATION) GOARCH=arm GOOS=linux ./src/build.sh $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le: - mkdir -p $(DESTINATION) GOARCH=ppc64le GOOS=linux ./src/build.sh $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64: - mkdir -p $(DESTINATION) GOARCH=amd64 GOOS=windows ./src/build.sh $(DESTINATION)/hello-win-amd64 -# we need a way to determine if CI should bust the test cache based on the source material -$(DESTINATION).fingerprint: clean - mkdir -p $(DESTINATION) - find src -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(DESTINATION).fingerprint - sha256sum $(DESTINATION).fingerprint +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find src -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum +# requirement 4: 'clean' goal to remove all generated test fixtures .PHONY: clean clean: - rm -f $(DESTINATION)/* + rm -rf $(DESTINATION) diff --git a/syft/pkg/cataloger/golang/test-fixtures/archs/src/build.sh b/syft/pkg/cataloger/golang/test-fixtures/archs/src/build.sh index 8a3919470b3..a740b7dba02 100755 --- a/syft/pkg/cataloger/golang/test-fixtures/archs/src/build.sh +++ b/syft/pkg/cataloger/golang/test-fixtures/archs/src/build.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash -set -uxe +set -ue # note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible) # note: gocache override is so we can run docker build not as root in a container without permission issues BINARY=$1 + +mkdir -p "$(dirname "$BINARY")" + CTRID=$(docker create -e GOOS="${GOOS}" -e GOARCH="${GOARCH}" -u "$(id -u):$(id -g)" -e GOCACHE=/tmp -w /src golang:1.17 go build -o main main.go) function cleanup() { diff --git a/syft/pkg/cataloger/java/test-fixtures/Makefile b/syft/pkg/cataloger/java/test-fixtures/Makefile new file mode 100644 index 00000000000..656190af9d9 --- /dev/null +++ b/syft/pkg/cataloger/java/test-fixtures/Makefile @@ -0,0 +1,24 @@ +ORANGE := \033[38;5;208m +BOLD := \033[1m +RESET := \033[0m +PWD := $(shell pwd) + +.DEFAULT_GOAL := default + +default: + @echo "$(ORANGE)$(BOLD)building test fixtures from $(PWD)$(RESET)" + @for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \ + if [ -f "$$dir/Makefile" ]; then \ + echo "$(ORANGE)• running make in $$dir$(RESET)"; \ + $(MAKE) -C $$dir; \ + fi; \ + done + +%: + @echo "$(ORANGE)$(BOLD)building test fixtures from $(PWD)$(RESET)" + @for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \ + if [ -f "$$dir/Makefile" ]; then \ + echo "$(ORANGE)• running make '$@' in $$dir$(RESET)"; \ + $(MAKE) -C $$dir $@; \ + fi; \ + done diff --git a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile index 98083904234..c80c69044fe 100644 --- a/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile +++ b/syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile @@ -1,5 +1,10 @@ CACHE_DIR = cache CACHE_PATH = $(shell pwd)/cache +FINGERPRINT_FILE=$(CACHE_DIR).fingerprint + +ifndef CACHE_DIR + $(error CACHE_DIR is not set) +endif JACKSON_CORE = jackson-core-2.15.2 SBT_JACKSON_CORE = com.fasterxml.jackson.core.jackson-core-2.15.2 @@ -8,28 +13,53 @@ API_ALL_SOURCES = api-all-2.0.0-sources SPRING_INSTRUMENTATION = spring-instrumentation-4.3.0-1.0 MULTIPLE_MATCHING = multiple-matching-2.11.5 -$(CACHE_DIR): - mkdir -p $(CACHE_DIR) -$(CACHE_DIR)/$(JACKSON_CORE).jar: $(CACHE_DIR) +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: $(CACHE_DIR) + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) + +$(CACHE_DIR): $(CACHE_DIR)/$(JACKSON_CORE).jar $(CACHE_DIR)/$(SBT_JACKSON_CORE).jar $(CACHE_DIR)/$(OPENSAML_CORE).jar $(CACHE_DIR)/$(API_ALL_SOURCES).jar $(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar $(CACHE_DIR)/$(MULTIPLE_MATCHING).jar + +$(CACHE_DIR)/$(JACKSON_CORE).jar: + mkdir -p $(CACHE_DIR) cd $(JACKSON_CORE) && zip -r $(CACHE_PATH)/$(JACKSON_CORE).jar . -$(CACHE_DIR)/$(SBT_JACKSON_CORE).jar: $(CACHE_DIR) +$(CACHE_DIR)/$(SBT_JACKSON_CORE).jar: + mkdir -p $(CACHE_DIR) cd $(SBT_JACKSON_CORE) && zip -r $(CACHE_PATH)/$(SBT_JACKSON_CORE).jar . -$(CACHE_DIR)/$(OPENSAML_CORE).jar: $(CACHE_DIR) +$(CACHE_DIR)/$(OPENSAML_CORE).jar: + mkdir -p $(CACHE_DIR) cd $(OPENSAML_CORE) && zip -r $(CACHE_PATH)/$(OPENSAML_CORE).jar . -$(CACHE_DIR)/$(API_ALL_SOURCES).jar: $(CACHE_DIR) +$(CACHE_DIR)/$(API_ALL_SOURCES).jar: + mkdir -p $(CACHE_DIR) cd $(API_ALL_SOURCES) && zip -r $(CACHE_PATH)/$(API_ALL_SOURCES).jar . -$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar: $(CACHE_DIR) +$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar: + mkdir -p $(CACHE_DIR) cd $(SPRING_INSTRUMENTATION) && zip -r $(CACHE_PATH)/$(SPRING_INSTRUMENTATION).jar . -$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar: $(CACHE_DIR) +$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar: + mkdir -p $(CACHE_DIR) cd $(MULTIPLE_MATCHING) && zip -r $(CACHE_PATH)/$(MULTIPLE_MATCHING).jar . # Jenkins plugins typically do not have the version included in the archive name, # so it is important to not include it in the generated test fixture -$(CACHE_DIR)/gradle.hpi: $(CACHE_DIR) - cd jenkins-plugins/gradle/2.11 && zip -r $(CACHE_PATH)/gradle.hpi . \ No newline at end of file +$(CACHE_DIR)/gradle.hpi: + mkdir -p $(CACHE_DIR) + cd jenkins-plugins/gradle/2.11 && zip -r $(CACHE_PATH)/gradle.hpi . + +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find . ! -path '*/cache*' -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum + +# requirement 4: 'clean' goal to remove all generated test fixtures +clean: + rm -rf $(CACHE_DIR)/* $(FINGERPRINT_FILE) diff --git a/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile b/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile index 1970b42f805..66b4fbf3db6 100644 --- a/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile +++ b/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile @@ -1,17 +1,18 @@ PKGSDIR=packages +FINGERPRINT_FILE=$(PKGSDIR).fingerprint ifndef PKGSDIR $(error PKGSDIR is not set) endif -all: jars archives native-image -clean: clean-examples - rm -f $(PKGSDIR)/* +.DEFAULT_GOAL := fixtures -clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: jars archives native-image -.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar jars archives +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) jars: $(PKGSDIR)/example-java-app-maven-0.1.0.jar $(PKGSDIR)/example-java-app-gradle-0.1.0.jar $(PKGSDIR)/example-jenkins-plugin.hpi $(PKGSDIR)/spring-boot-0.0.1-SNAPSHOT.jar @@ -71,8 +72,16 @@ $(PKGSDIR)/example-java-app: $(PKGSDIR)/example-java-app-maven-0.1.0.jar $(PKGSDIR)/gcc-amd64-darwin-exec-debug: ./build-example-macho-binary.sh $(PKGSDIR) -# we need a way to determine if CI should bust the test cache based on the source material -.PHONY: cache.fingerprint -cache.fingerprint: - find example* build* gradle* Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee cache.fingerprint - sha256sum cache.fingerprint +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find example* build* gradle* Makefile -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum + +# requirement 4: 'clean' goal to remove all generated test fixtures +clean: clean-examples + rm -rf $(PKGSDIR) $(FINGERPRINT_FILE) + +clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar + +.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar jars archives diff --git a/syft/pkg/cataloger/kernel/test-fixtures/Makefile b/syft/pkg/cataloger/kernel/test-fixtures/Makefile index 4a2849919c3..af9204980c7 100644 --- a/syft/pkg/cataloger/kernel/test-fixtures/Makefile +++ b/syft/pkg/cataloger/kernel/test-fixtures/Makefile @@ -1,7 +1,21 @@ -all: +FINGERPRINT_FILE=cache.fingerprint -# we need a way to determine if CI should bust the test cache based on the source material -.PHONY: cache.fingerprint -cache.fingerprint: - find Makefile **/Dockerfile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee cache.fingerprint - sha256sum cache.fingerprint + +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: + @echo "nothing to do" + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) + +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find Makefile **/Dockerfile -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum + +# requirement 4: 'clean' goal to remove all generated test fixtures +clean: + rm -f $(FINGERPRINT_FILE) diff --git a/syft/pkg/cataloger/redhat/test-fixtures/Makefile b/syft/pkg/cataloger/redhat/test-fixtures/Makefile index e280d5e60e7..f55bd6976cb 100644 --- a/syft/pkg/cataloger/redhat/test-fixtures/Makefile +++ b/syft/pkg/cataloger/redhat/test-fixtures/Makefile @@ -1,21 +1,38 @@ RPMSDIR=rpms +FINGERPRINT_FILE=$(RPMSDIR).fingerprint ifndef RPMSDIR $(error RPMSDIR is not set) endif -all: rpms -clean: - rm -rf $(RPMSDIR) +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: rpms + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) rpms: mkdir -p $(RPMSDIR) - cd $(RPMSDIR) && curl https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/a/abc-1.01-9.hg20160905.el7.x86_64.rpm -O - cd $(RPMSDIR) && curl https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/z/zork-1.0.3-1.el7.x86_64.rpm -O - -# we need a way to determine if CI should bust the test cache based on the source material -.PHONY: $(RPMSDIR).fingerprint -$(RPMSDIR).fingerprint: - find Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(RPMSDIR).fingerprint - sha256sum $(RPMSDIR).fingerprint + @# see note from https://dl.fedoraproject.org/pub/epel/7/README + @# ATTENTION + @# ====================================== + @# The contents of this directory have been moved to our archives available at: + @# + @# http://archives.fedoraproject.org/pub/archive/epel/ + + cd $(RPMSDIR) && curl -LO https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/a/abc-1.01-9.hg20160905.el7.x86_64.rpm + cd $(RPMSDIR) && curl -LO https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/z/zork-1.0.3-1.el7.x86_64.rpm + +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find Makefile -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum + +# requirement 4: 'clean' goal to remove all generated test fixtures +.PHONY: clean +clean: + rm -rf $(RPMSDIR) $(FINGERPRINT_FILE) diff --git a/test/cli/test-fixtures/Makefile b/test/cli/test-fixtures/Makefile index 5042a5aad65..6d7c135d5ea 100644 --- a/test/cli/test-fixtures/Makefile +++ b/test/cli/test-fixtures/Makefile @@ -1,6 +1,26 @@ # change these if you want CI to not use previous stored cache -CLI_CACHE_BUSTER := "e5cdfd8" +CACHE_BUSTER := "fd8e5cd" + +FINGERPRINT_FILE=cache.fingerprint + +.DEFAULT_GOAL := fixtures + +# requirement 1: 'fixtures' goal to generate any and all test fixtures +fixtures: + @echo "nothing to do" + +# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted +fingerprint: $(FINGERPRINT_FILE) + +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find image-* -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @echo "$(CACHE_BUSTER)" >> $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum + +# requirement 4: 'clean' goal to remove all generated test fixtures +.PHONY: clean +clean: + rm -f $(FINGERPRINT_FILE) -.PHONY: cache.fingerprint -cache.fingerprint: - find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> cache.fingerprint diff --git a/test/install/Makefile b/test/install/Makefile index 2a632cd2825..3cf1c2034b3 100644 --- a/test/install/Makefile +++ b/test/install/Makefile @@ -1,5 +1,9 @@ NAME=syft +# CI cache busting values; change these if you want CI to not use previous stored cache +CACHE_BUSTER=da88c94 +FINGERPRINT_FILE := cache.fingerprint + # for local testing (not testing within containers) use the binny-managed version of cosign. # this also means that the user does not need to install cosign on their system to run tests. COSIGN_BINARY=../../.tool/cosign @@ -21,8 +25,6 @@ ACCEPTANCE_CMD=sh -c '../../install.sh -v -b /usr/local/bin && syft version && r PREVIOUS_RELEASE=v0.33.0 ACCEPTANCE_PREVIOUS_RELEASE_CMD=sh -c "../../install.sh -b /usr/local/bin $(PREVIOUS_RELEASE) && syft version" -# CI cache busting values; change these if you want CI to not use previous stored cache -INSTALL_TEST_CACHE_BUSTER=894d8ca define title @printf '\n≡≡≡[ $(1) ]≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n' @@ -130,7 +132,9 @@ busybox-1.36: ## For CI ######################################################## -.PHONY: cache.fingerprint -cache.fingerprint: - $(call title,Install test fixture fingerprint) - @find ./environments/* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INSTALL_TEST_CACHE_BUSTER)" >> cache.fingerprint +# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint +.PHONY: $(FINGERPRINT_FILE) +$(FINGERPRINT_FILE): + @find ./environments/* -type f -exec sha256sum {} \; | sort -k2 | tee /dev/stderr > $(FINGERPRINT_FILE) + @echo "$(CACHE_BUSTER)" >> $(FINGERPRINT_FILE) + @cat $(FINGERPRINT_FILE) | sha256sum