From 608dbded061016c2645cc7306473a6272ec40052 Mon Sep 17 00:00:00 2001 From: "anchore-actions-token-generator[bot]" <102182147+anchore-actions-token-generator[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:10:30 -0400 Subject: [PATCH 01/17] chore(deps): update bootstrap tools to latest versions (#1905) Signed-off-by: GitHub Co-authored-by: spiffcs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 21c4ad6cd77..fd1ee97590f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ GOLANGCILINT_VERSION := v1.53.3 GOSIMPORTS_VERSION := v0.3.8 BOUNCER_VERSION := v0.4.0 CHRONICLE_VERSION := v0.6.0 -GORELEASER_VERSION := v1.18.2 +GORELEASER_VERSION := v1.19.0 YAJSV_VERSION := v1.4.1 COSIGN_VERSION := v2.1.1 QUILL_VERSION := v0.2.0 From 4da3be864fab86dec963c231366fdad11eaef21c Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 30 Jun 2023 10:19:16 -0400 Subject: [PATCH 02/17] Refactor source API (#1846) * refactor source API and syft json source block Signed-off-by: Alex Goodman * update source detection and format test utils Signed-off-by: Alex Goodman * generate list of all source metadata types Signed-off-by: Alex Goodman * extract base and root normalization into helper functions Signed-off-by: Alex Goodman * preserve syftjson model package name import ref Signed-off-by: Alex Goodman * alias should not be a pointer Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman Signed-off-by: Alex Goodman --- .gitignore | 1 + Makefile | 2 +- cmd/syft/cli/attest/attest.go | 67 +- cmd/syft/cli/eventloop/tasks.go | 14 +- cmd/syft/cli/packages/packages.go | 60 +- cmd/syft/cli/poweruser/poweruser.go | 55 +- internal/constants.go | 2 +- schema/json/README.md | 12 +- schema/json/generate/main.go | 50 - schema/json/internal/generated.go | 39 - schema/json/main_test.go | 39 - schema/json/schema-9.0.0.json | 1881 +++++++++++++++++ .../cataloger/filedigest/cataloger_test.go | 4 +- .../cataloger/filemetadata/cataloger_test.go | 2 +- .../internal/all_regular_files_test.go | 4 +- syft/file/test-fixtures/req-resp/.gitignore | 2 + .../req-resp/path/to/rel-inside.txt | 1 + .../req-resp/path/to/the/file.txt | 1 + .../req-resp/path/to/the/rel-outside.txt | 1 + syft/file/test-fixtures/req-resp/root-link | 1 + .../req-resp/somewhere/outside.txt | 1 + .../common/cyclonedxhelpers/decoder.go | 34 +- .../formats/common/cyclonedxhelpers/format.go | 49 +- .../common/spdxhelpers/document_name.go | 14 +- .../common/spdxhelpers/document_name_test.go | 31 +- .../common/spdxhelpers/document_namespace.go | 16 +- .../spdxhelpers/document_namespace_test.go | 42 +- .../common/spdxhelpers/to_syft_model.go | 23 +- .../common/spdxhelpers/to_syft_model_test.go | 27 +- syft/formats/cyclonedxjson/encoder_test.go | 69 +- .../TestCycloneDxDirectoryEncoder.golden | 12 +- .../snapshot/TestCycloneDxImageEncoder.golden | 16 +- syft/formats/cyclonedxxml/encoder_test.go | 65 +- .../TestCycloneDxDirectoryEncoder.golden | 10 +- .../snapshot/TestCycloneDxImageEncoder.golden | 14 +- syft/formats/github/encoder.go | 107 +- syft/formats/github/encoder_test.go | 182 +- .../internal/testutils/directory_input.go | 204 ++ .../internal/testutils/file_relationships.go | 32 + .../formats/internal/testutils/image_input.go | 113 + syft/formats/internal/testutils/redactor.go | 142 ++ syft/formats/internal/testutils/snapshot.go | 88 + syft/formats/internal/testutils/utils.go | 396 ---- syft/formats/spdxjson/encoder_test.go | 76 +- .../TestSPDXJSONDirectoryEncoder.golden | 8 +- .../snapshot/TestSPDXJSONImageEncoder.golden | 6 +- .../snapshot/TestSPDXRelationshipOrder.golden | 6 +- syft/formats/spdxtagvalue/encoder_test.go | 121 +- .../snapshot/TestSPDXJSONSPDXIDs.golden | 6 +- .../snapshot/TestSPDXRelationshipOrder.golden | 6 +- .../TestSPDXTagValueDirectoryEncoder.golden | 8 +- .../TestSPDXTagValueImageEncoder.golden | 6 +- syft/formats/syftjson/encoder_test.go | 75 +- syft/formats/syftjson/model/package.go | 4 +- syft/formats/syftjson/model/package_test.go | 2 +- syft/formats/syftjson/model/source.go | 101 +- syft/formats/syftjson/model/source_test.go | 203 +- .../snapshot/TestDirectoryEncoder.golden | 12 +- .../TestEncodeFullJSONDocument.golden | 8 +- .../snapshot/TestImageEncoder.golden | 8 +- syft/formats/syftjson/to_format_model.go | 44 +- syft/formats/syftjson/to_format_model_test.go | 176 +- syft/formats/syftjson/to_syft_model.go | 52 +- syft/formats/syftjson/to_syft_model_test.go | 154 +- syft/formats/table/encoder_test.go | 13 +- syft/formats/template/encoder_test.go | 27 +- syft/formats/text/encoder.go | 14 +- syft/formats/text/encoder_test.go | 38 +- .../snapshot/TestTextDirectoryEncoder.golden | 2 +- syft/internal/fileresolver/chroot_context.go | 165 ++ .../fileresolver/chroot_context_test.go | 481 +++++ .../container_image_squash_test.go | 21 + syft/internal/fileresolver/directory.go | 135 +- .../fileresolver/directory_indexer.go | 6 +- syft/internal/fileresolver/excluding_file.go | 4 +- .../fileresolver/excluding_file_test.go | 2 +- .../test-fixtures/req-resp/.gitignore | 2 + .../fileresolver/unindexed_directory_test.go | 21 - syft/internal/generate.go | 4 + syft/internal/jsonschema/README.md | 1 + .../json => syft/internal/jsonschema}/main.go | 71 +- .../packagemetadata/discover_type_names.go | 8 +- .../internal/packagemetadata/generate/main.go | 55 + syft/internal/packagemetadata/generated.go | 10 + syft/internal/packagemetadata/names.go | 13 + syft/internal/packagemetadata/names_test.go | 25 + .../sourcemetadata/completion_tester.go | 69 + .../sourcemetadata/discover_type_names.go | 148 ++ syft/internal/sourcemetadata/generate/main.go | 55 + syft/internal/sourcemetadata/generated.go | 10 + syft/internal/sourcemetadata/names.go | 41 + syft/internal/sourcemetadata/names_test.go | 29 + syft/internal/windows/path.go | 41 + syft/lib.go | 17 +- syft/linux/identify_release_test.go | 2 +- syft/pkg/cataloger/binary/cataloger_test.go | 6 +- .../internal/pkgtest/test_generic_parser.go | 4 +- syft/pkg/cataloger/search_config.go | 4 +- syft/sbom/sbom.go | 2 +- syft/source/alias.go | 13 + syft/source/description.go | 9 + syft/source/detection.go | 200 ++ .../{scheme_test.go => detection_test.go} | 50 +- syft/source/digest_utils.go | 11 + syft/source/directory_source.go | 215 ++ syft/source/directory_source_test.go | 560 +++++ syft/source/directory_source_win_test.go | 65 + syft/source/exclude.go | 5 + syft/source/file_source.go | 280 +++ syft/source/file_source_test.go | 278 +++ syft/source/image_metadata.go | 62 - syft/source/metadata.go | 12 - syft/source/scheme.go | 74 - syft/source/scope.go | 2 +- syft/source/source.go | 626 +----- syft/source/source_test.go | 920 -------- syft/source/source_win_test.go | 54 - syft/source/stereoscope_image_metadata.go | 62 + syft/source/stereoscope_image_source.go | 245 +++ syft/source/stereoscope_image_source_test.go | 243 +++ .../file-index-filter/.1/something | 1 + .../source/test-fixtures/file-index-filter/.2 | 1 + .../test-fixtures/file-index-filter/.vimrc | 1 + .../test-fixtures/file-index-filter/empty | 0 test/integration/catalog_packages_test.go | 8 +- test/integration/utils_test.go | 24 +- 126 files changed, 7384 insertions(+), 3190 deletions(-) delete mode 100644 schema/json/generate/main.go delete mode 100644 schema/json/internal/generated.go delete mode 100644 schema/json/main_test.go create mode 100644 schema/json/schema-9.0.0.json create mode 100644 syft/file/test-fixtures/req-resp/.gitignore create mode 120000 syft/file/test-fixtures/req-resp/path/to/rel-inside.txt create mode 100644 syft/file/test-fixtures/req-resp/path/to/the/file.txt create mode 120000 syft/file/test-fixtures/req-resp/path/to/the/rel-outside.txt create mode 120000 syft/file/test-fixtures/req-resp/root-link create mode 100644 syft/file/test-fixtures/req-resp/somewhere/outside.txt create mode 100644 syft/formats/internal/testutils/directory_input.go create mode 100644 syft/formats/internal/testutils/file_relationships.go create mode 100644 syft/formats/internal/testutils/image_input.go create mode 100644 syft/formats/internal/testutils/redactor.go create mode 100644 syft/formats/internal/testutils/snapshot.go delete mode 100644 syft/formats/internal/testutils/utils.go create mode 100644 syft/internal/fileresolver/chroot_context.go create mode 100644 syft/internal/fileresolver/chroot_context_test.go create mode 100644 syft/internal/fileresolver/test-fixtures/req-resp/.gitignore create mode 100644 syft/internal/generate.go create mode 100644 syft/internal/jsonschema/README.md rename {schema/json => syft/internal/jsonschema}/main.go (56%) rename schema/json/internal/metadata_types.go => syft/internal/packagemetadata/discover_type_names.go (96%) create mode 100644 syft/internal/packagemetadata/generate/main.go create mode 100644 syft/internal/packagemetadata/generated.go create mode 100644 syft/internal/packagemetadata/names.go create mode 100644 syft/internal/packagemetadata/names_test.go create mode 100644 syft/internal/sourcemetadata/completion_tester.go create mode 100644 syft/internal/sourcemetadata/discover_type_names.go create mode 100644 syft/internal/sourcemetadata/generate/main.go create mode 100644 syft/internal/sourcemetadata/generated.go create mode 100644 syft/internal/sourcemetadata/names.go create mode 100644 syft/internal/sourcemetadata/names_test.go create mode 100644 syft/internal/windows/path.go create mode 100644 syft/source/alias.go create mode 100644 syft/source/description.go create mode 100644 syft/source/detection.go rename syft/source/{scheme_test.go => detection_test.go} (88%) create mode 100644 syft/source/digest_utils.go create mode 100644 syft/source/directory_source.go create mode 100644 syft/source/directory_source_test.go create mode 100644 syft/source/directory_source_win_test.go create mode 100644 syft/source/exclude.go create mode 100644 syft/source/file_source.go create mode 100644 syft/source/file_source_test.go delete mode 100644 syft/source/image_metadata.go delete mode 100644 syft/source/metadata.go delete mode 100644 syft/source/scheme.go delete mode 100644 syft/source/source_test.go delete mode 100644 syft/source/source_win_test.go create mode 100644 syft/source/stereoscope_image_metadata.go create mode 100644 syft/source/stereoscope_image_source.go create mode 100644 syft/source/stereoscope_image_source_test.go create mode 100644 syft/source/test-fixtures/file-index-filter/.1/something create mode 100644 syft/source/test-fixtures/file-index-filter/.2 create mode 100644 syft/source/test-fixtures/file-index-filter/.vimrc create mode 100644 syft/source/test-fixtures/file-index-filter/empty diff --git a/.gitignore b/.gitignore index 423be3f9051..ab7501da491 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/.bin CHANGELOG.md VERSION /test/results diff --git a/Makefile b/Makefile index fd1ee97590f..f04590d8004 100644 --- a/Makefile +++ b/Makefile @@ -302,7 +302,7 @@ compare-test-rpm-package-install: $(TEMP_DIR) $(SNAPSHOT_DIR) .PHONY: generate-json-schema generate-json-schema: ## Generate a new json schema - cd schema/json && go generate . && go run . + cd syft/internal && go generate . && cd jsonschema && go run . .PHONY: generate-license-list generate-license-list: ## Generate an updated spdx license list diff --git a/cmd/syft/cli/attest/attest.go b/cmd/syft/cli/attest/attest.go index 9264c8e8b95..9ed452b359c 100644 --- a/cmd/syft/cli/attest/attest.go +++ b/cmd/syft/cli/attest/attest.go @@ -12,6 +12,7 @@ import ( "golang.org/x/exp/slices" "github.com/anchore/stereoscope" + "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" "github.com/anchore/syft/cmd/syft/cli/packages" @@ -34,17 +35,8 @@ func Run(_ context.Context, app *config.Application, args []string) error { return err } - // could be an image or a directory, with or without a scheme - // TODO: validate that source is image + // note: must be a container image userInput := args[0] - si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource) - if err != nil { - return fmt.Errorf("could not generate source input for packages command: %w", err) - } - - if si.Scheme != source.ImageScheme { - return fmt.Errorf("attestations are only supported for oci images at this time") - } eventBus := partybus.NewBus() stereoscope.SetBus(eventBus) @@ -52,7 +44,7 @@ func Run(_ context.Context, app *config.Application, args []string) error { subscription := eventBus.Subscribe() return eventloop.EventLoop( - execWorker(app, *si), + execWorker(app, userInput), eventloop.SetupSignals(), subscription, stereoscope.Cleanup, @@ -60,13 +52,48 @@ func Run(_ context.Context, app *config.Application, args []string) error { ) } -func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom.SBOM, error) { - src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions) - if cleanup != nil { - defer cleanup() +func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbom.SBOM, error) { + cfg := source.DetectConfig{ + DefaultImageSource: app.DefaultImagePullSource, + } + detection, err := source.Detect(userInput, cfg) + if err != nil { + return nil, fmt.Errorf("could not deteremine source: %w", err) + } + + if detection.IsContainerImage() { + return nil, fmt.Errorf("attestations are only supported for oci images at this time") + } + + var platform *image.Platform + + if app.Platform != "" { + platform, err = image.NewPlatform(app.Platform) + if err != nil { + return nil, fmt.Errorf("invalid platform: %w", err) + } + } + + src, err := detection.NewSource( + source.DetectionSourceConfig{ + Alias: source.Alias{ + Name: app.SourceName, + Version: app.SourceVersion, + }, + RegistryOptions: app.Registry.ToOptions(), + Platform: platform, + Exclude: source.ExcludeConfig{ + Paths: app.Exclusions, + }, + DigestAlgorithms: nil, + }, + ) + + if src != nil { + defer src.Close() } if err != nil { - return nil, fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err) + return nil, fmt.Errorf("failed to construct source from user input %q: %w", userInput, err) } s, err := packages.GenerateSBOM(src, errs, app) @@ -75,20 +102,20 @@ func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom } if s == nil { - return nil, fmt.Errorf("no SBOM produced for %q", si.UserInput) + return nil, fmt.Errorf("no SBOM produced for %q", userInput) } return s, nil } //nolint:funlen -func execWorker(app *config.Application, si source.Input) <-chan error { +func execWorker(app *config.Application, userInput string) <-chan error { errs := make(chan error) go func() { defer close(errs) defer bus.Publish(partybus.Event{Type: event.Exit}) - s, err := buildSBOM(app, si, errs) + s, err := buildSBOM(app, userInput, errs) if err != nil { errs <- fmt.Errorf("unable to build SBOM: %w", err) return @@ -136,7 +163,7 @@ func execWorker(app *config.Application, si source.Input) <-chan error { predicateType = "custom" } - args := []string{"attest", si.UserInput, "--predicate", f.Name(), "--type", predicateType} + args := []string{"attest", userInput, "--predicate", f.Name(), "--type", predicateType} if app.Attest.Key != "" { args = append(args, "--key", app.Attest.Key) } diff --git a/cmd/syft/cli/eventloop/tasks.go b/cmd/syft/cli/eventloop/tasks.go index 536a39ee6f1..4c04565427c 100644 --- a/cmd/syft/cli/eventloop/tasks.go +++ b/cmd/syft/cli/eventloop/tasks.go @@ -16,7 +16,7 @@ import ( "github.com/anchore/syft/syft/source" ) -type Task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error) +type Task func(*sbom.Artifacts, source.Source) ([]artifact.Relationship, error) func Tasks(app *config.Application) ([]Task, error) { var tasks []Task @@ -48,7 +48,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) { return nil, nil } - task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) { packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig()) results.Packages = packageCatalog @@ -67,7 +67,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) { metadataCataloger := filemetadata.NewCataloger() - task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt) if err != nil { return nil, err @@ -110,7 +110,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) { digestsCataloger := filedigest.NewCataloger(hashes) - task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt) if err != nil { return nil, err @@ -142,7 +142,7 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(app.Secrets.Cataloger.ScopeOpt) if err != nil { return nil, err @@ -169,7 +169,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(app.FileContents.Cataloger.ScopeOpt) if err != nil { return nil, err @@ -186,7 +186,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) { return task, nil } -func RunTask(t Task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) { +func RunTask(t Task, a *sbom.Artifacts, src source.Source, c chan<- artifact.Relationship, errs chan<- error) { defer close(c) relationships, err := t(a, src) diff --git a/cmd/syft/cli/packages/packages.go b/cmd/syft/cli/packages/packages.go index 544a1b502b6..da52581935b 100644 --- a/cmd/syft/cli/packages/packages.go +++ b/cmd/syft/cli/packages/packages.go @@ -7,6 +7,7 @@ import ( "github.com/wagoodman/go-partybus" "github.com/anchore/stereoscope" + "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" "github.com/anchore/syft/internal" @@ -35,10 +36,6 @@ func Run(_ context.Context, app *config.Application, args []string) error { // could be an image or a directory, with or without a scheme userInput := args[0] - si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource) - if err != nil { - return fmt.Errorf("could not generate source input for packages command: %w", err) - } eventBus := partybus.NewBus() stereoscope.SetBus(eventBus) @@ -46,7 +43,7 @@ func Run(_ context.Context, app *config.Application, args []string) error { subscription := eventBus.Subscribe() return eventloop.EventLoop( - execWorker(app, *si, writer), + execWorker(app, userInput, writer), eventloop.SetupSignals(), subscription, stereoscope.Cleanup, @@ -54,17 +51,52 @@ func Run(_ context.Context, app *config.Application, args []string) error { ) } -func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error { +func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error { errs := make(chan error) go func() { defer close(errs) - src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions) - if cleanup != nil { - defer cleanup() + detection, err := source.Detect( + userInput, + source.DetectConfig{ + DefaultImageSource: app.DefaultImagePullSource, + }, + ) + if err != nil { + errs <- fmt.Errorf("could not deteremine source: %w", err) + return + } + + var platform *image.Platform + + if app.Platform != "" { + platform, err = image.NewPlatform(app.Platform) + if err != nil { + errs <- fmt.Errorf("invalid platform: %w", err) + return + } + } + + src, err := detection.NewSource( + source.DetectionSourceConfig{ + Alias: source.Alias{ + Name: app.SourceName, + Version: app.SourceVersion, + }, + RegistryOptions: app.Registry.ToOptions(), + Platform: platform, + Exclude: source.ExcludeConfig{ + Paths: app.Exclusions, + }, + DigestAlgorithms: nil, + }, + ) + + if src != nil { + defer src.Close() } if err != nil { - errs <- fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err) + errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err) return } @@ -75,7 +107,7 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <- } if s == nil { - errs <- fmt.Errorf("no SBOM produced for %q", si.UserInput) + errs <- fmt.Errorf("no SBOM produced for %q", userInput) } bus.Publish(partybus.Event{ @@ -86,14 +118,14 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <- return errs } -func GenerateSBOM(src *source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) { +func GenerateSBOM(src source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) { tasks, err := eventloop.Tasks(app) if err != nil { return nil, err } s := sbom.SBOM{ - Source: src.Metadata, + Source: src.Describe(), Descriptor: sbom.Descriptor{ Name: internal.ApplicationName, Version: version.FromBuild().Version, @@ -106,7 +138,7 @@ func GenerateSBOM(src *source.Source, errs chan error, app *config.Application) return &s, nil } -func buildRelationships(s *sbom.SBOM, src *source.Source, tasks []eventloop.Task, errs chan error) { +func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task, errs chan error) { var relationships []<-chan artifact.Relationship for _, task := range tasks { c := make(chan artifact.Relationship) diff --git a/cmd/syft/cli/poweruser/poweruser.go b/cmd/syft/cli/poweruser/poweruser.go index 724f9a81fa2..dd1b758fe0f 100644 --- a/cmd/syft/cli/poweruser/poweruser.go +++ b/cmd/syft/cli/poweruser/poweruser.go @@ -9,6 +9,7 @@ import ( "github.com/wagoodman/go-partybus" "github.com/anchore/stereoscope" + "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" "github.com/anchore/syft/cmd/syft/cli/packages" @@ -38,10 +39,6 @@ func Run(_ context.Context, app *config.Application, args []string) error { }() userInput := args[0] - si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource) - if err != nil { - return fmt.Errorf("could not generate source input for packages command: %w", err) - } eventBus := partybus.NewBus() stereoscope.SetBus(eventBus) @@ -49,7 +46,7 @@ func Run(_ context.Context, app *config.Application, args []string) error { subscription := eventBus.Subscribe() return eventloop.EventLoop( - execWorker(app, *si, writer), + execWorker(app, userInput, writer), eventloop.SetupSignals(), subscription, stereoscope.Cleanup, @@ -57,7 +54,8 @@ func Run(_ context.Context, app *config.Application, args []string) error { ) } -func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error { +//nolint:funlen +func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error { errs := make(chan error) go func() { defer close(errs) @@ -72,17 +70,52 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <- return } - src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions) + detection, err := source.Detect( + userInput, + source.DetectConfig{ + DefaultImageSource: app.DefaultImagePullSource, + }, + ) if err != nil { - errs <- err + errs <- fmt.Errorf("could not deteremine source: %w", err) return } - if cleanup != nil { - defer cleanup() + + var platform *image.Platform + + if app.Platform != "" { + platform, err = image.NewPlatform(app.Platform) + if err != nil { + errs <- fmt.Errorf("invalid platform: %w", err) + return + } + } + + src, err := detection.NewSource( + source.DetectionSourceConfig{ + Alias: source.Alias{ + Name: app.SourceName, + Version: app.SourceVersion, + }, + RegistryOptions: app.Registry.ToOptions(), + Platform: platform, + Exclude: source.ExcludeConfig{ + Paths: app.Exclusions, + }, + DigestAlgorithms: nil, + }, + ) + + if src != nil { + defer src.Close() + } + if err != nil { + errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err) + return } s := sbom.SBOM{ - Source: src.Metadata, + Source: src.Describe(), Descriptor: sbom.Descriptor{ Name: internal.ApplicationName, Version: version.FromBuild().Version, diff --git a/internal/constants.go b/internal/constants.go index 73bd40a41b8..bf520ab9068 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -6,5 +6,5 @@ const ( // JSONSchemaVersion is the current schema version output by the JSON encoder // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "8.0.1" + JSONSchemaVersion = "9.0.0" ) diff --git a/schema/json/README.md b/schema/json/README.md index 40af8b4cd44..7de19b64d88 100644 --- a/schema/json/README.md +++ b/schema/json/README.md @@ -3,8 +3,8 @@ This is the JSON schema for output from the JSON presenters (`syft packages -o json` and `syft power-user `). The required inputs for defining the JSON schema are as follows: - the value of `internal.JSONSchemaVersion` that governs the schema filename -- the `Document` struct definition within `internal/presenters/poweruser/json_document.go` that governs the overall document shape -- the `artifactMetadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata` +- the `Document` struct definition within `github.com/anchore/syft/syft/formats/syftjson/model/document.go` that governs the overall document shape +- generated `AllTypes()` helper function within the `syft/internal/sourcemetadata` and `syft/internal/packagemetadata` packages With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file. @@ -22,15 +22,13 @@ Given a version number format `MODEL.REVISION.ADDITION`: ## Adding a New `pkg.*Metadata` Type -When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things -are done: +When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field you must add a test case to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata. -- a new integration test case is added to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata -- the new metadata struct is added to the `artifactMetadataContainer` struct within `schema/json/generate.go` +Additionally it is important to generate a new JSON schema since the `pkg.Package.Metadata` field is covered by the schema. ## Generating a New Schema -Create the new schema by running `cd schema/json && go run generate.go` (note you must be in the `schema/json` dir while running this): +Create the new schema by running `make generate-json-schema` from the root of the repo: - If there is **not** an existing schema for the given version, then the new schema file will be written to `schema/json/schema-$VERSION.json` - If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken diff --git a/schema/json/generate/main.go b/schema/json/generate/main.go deleted file mode 100644 index fc8dc120a21..00000000000 --- a/schema/json/generate/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/dave/jennifer/jen" - - "github.com/anchore/syft/schema/json/internal" -) - -// This program generates internal/generated.go. - -const ( - pkgImport = "github.com/anchore/syft/syft/pkg" - path = "internal/generated.go" -) - -func main() { - typeNames, err := internal.AllSyftMetadataTypeNames() - if err != nil { - panic(fmt.Errorf("unable to get all metadata type names: %w", err)) - } - - fmt.Printf("updating metadata container object with %+v types\n", len(typeNames)) - - f := jen.NewFile("internal") - f.HeaderComment("DO NOT EDIT: generated by schema/json/generate/main.go") - f.ImportName(pkgImport, "pkg") - f.Comment("ArtifactMetadataContainer is a struct that contains all the metadata types for a package, as represented in the pkg.Package.Metadata field.") - f.Type().Id("ArtifactMetadataContainer").StructFunc(func(g *jen.Group) { - for _, typeName := range typeNames { - g.Id(typeName).Qual(pkgImport, typeName) - } - }) - - rendered := fmt.Sprintf("%#v", f) - - fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - panic(fmt.Errorf("unable to open file: %w", err)) - } - _, err = fh.WriteString(rendered) - if err != nil { - panic(fmt.Errorf("unable to write file: %w", err)) - } - if err := fh.Close(); err != nil { - panic(fmt.Errorf("unable to close file: %w", err)) - } -} diff --git a/schema/json/internal/generated.go b/schema/json/internal/generated.go deleted file mode 100644 index 3341818deb7..00000000000 --- a/schema/json/internal/generated.go +++ /dev/null @@ -1,39 +0,0 @@ -// DO NOT EDIT: generated by schema/json/generate/main.go - -package internal - -import "github.com/anchore/syft/syft/pkg" - -// ArtifactMetadataContainer is a struct that contains all the metadata types for a package, as represented in the pkg.Package.Metadata field. -type ArtifactMetadataContainer struct { - AlpmMetadata pkg.AlpmMetadata - ApkMetadata pkg.ApkMetadata - BinaryMetadata pkg.BinaryMetadata - CargoPackageMetadata pkg.CargoPackageMetadata - CocoapodsMetadata pkg.CocoapodsMetadata - ConanLockMetadata pkg.ConanLockMetadata - ConanMetadata pkg.ConanMetadata - DartPubMetadata pkg.DartPubMetadata - DotnetDepsMetadata pkg.DotnetDepsMetadata - DpkgMetadata pkg.DpkgMetadata - GemMetadata pkg.GemMetadata - GolangBinMetadata pkg.GolangBinMetadata - GolangModMetadata pkg.GolangModMetadata - HackageMetadata pkg.HackageMetadata - JavaMetadata pkg.JavaMetadata - KbPackageMetadata pkg.KbPackageMetadata - LinuxKernelMetadata pkg.LinuxKernelMetadata - LinuxKernelModuleMetadata pkg.LinuxKernelModuleMetadata - MixLockMetadata pkg.MixLockMetadata - NixStoreMetadata pkg.NixStoreMetadata - NpmPackageJSONMetadata pkg.NpmPackageJSONMetadata - NpmPackageLockJSONMetadata pkg.NpmPackageLockJSONMetadata - PhpComposerJSONMetadata pkg.PhpComposerJSONMetadata - PortageMetadata pkg.PortageMetadata - PythonPackageMetadata pkg.PythonPackageMetadata - PythonPipfileLockMetadata pkg.PythonPipfileLockMetadata - PythonRequirementsMetadata pkg.PythonRequirementsMetadata - RDescriptionFileMetadata pkg.RDescriptionFileMetadata - RebarLockMetadata pkg.RebarLockMetadata - RpmMetadata pkg.RpmMetadata -} diff --git a/schema/json/main_test.go b/schema/json/main_test.go deleted file mode 100644 index 0903b4dde39..00000000000 --- a/schema/json/main_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "reflect" - "sort" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/schema/json/internal" -) - -func TestAllMetadataRepresented(t *testing.T) { - // this test checks that all the metadata types are represented in the currently generated ArtifactMetadataContainer struct - // such that PRs will reflect when there is drift from the implemented set of metadata types and the generated struct - // which controls the JSON schema content. - expected, err := internal.AllSyftMetadataTypeNames() - require.NoError(t, err) - actual := allTypeNamesFromStruct(internal.ArtifactMetadataContainer{}) - if !assert.ElementsMatch(t, expected, actual) { - t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual)) - t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?") - t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)") - } -} - -func allTypeNamesFromStruct(instance any) []string { - // get all the type names from the struct (not recursively) - var typeNames []string - tt := reflect.TypeOf(instance) - for i := 0; i < tt.NumField(); i++ { - field := tt.Field(i) - typeNames = append(typeNames, field.Type.Name()) - } - sort.Strings(typeNames) - return typeNames -} diff --git a/schema/json/schema-9.0.0.json b/schema/json/schema-9.0.0.json new file mode 100644 index 00000000000..0f24da576de --- /dev/null +++ b/schema/json/schema-9.0.0.json @@ -0,0 +1,1881 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "anchore.io/schema/syft/json/9.0.0/document", + "$ref": "#/$defs/Document", + "$defs": { + "AlpmFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "type": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "gid": { + "type": "string" + }, + "time": { + "type": "string", + "format": "date-time" + }, + "size": { + "type": "string" + }, + "link": { + "type": "string" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object" + }, + "AlpmMetadata": { + "properties": { + "basepackage": { + "type": "string" + }, + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "description": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "packager": { + "type": "string" + }, + "url": { + "type": "string" + }, + "validation": { + "type": "string" + }, + "reason": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + }, + "backup": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "basepackage", + "package", + "version", + "description", + "architecture", + "size", + "packager", + "url", + "validation", + "reason", + "files", + "backup" + ] + }, + "ApkFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "ownerGid": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "ApkMetadata": { + "properties": { + "package": { + "type": "string" + }, + "originPackage": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "url": { + "type": "string" + }, + "description": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "installedSize": { + "type": "integer" + }, + "pullDependencies": { + "items": { + "type": "string" + }, + "type": "array" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array" + }, + "pullChecksum": { + "type": "string" + }, + "gitCommitOfApkPort": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/ApkFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "provides", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ] + }, + "BinaryMetadata": { + "properties": { + "matches": { + "items": { + "$ref": "#/$defs/ClassifierMatch" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "matches" + ] + }, + "CargoPackageMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + }, + "checksum": { + "type": "string" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "checksum", + "dependencies" + ] + }, + "ClassifierMatch": { + "properties": { + "classifier": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Location" + } + }, + "type": "object", + "required": [ + "classifier", + "location" + ] + }, + "CocoapodsMetadata": { + "properties": { + "checksum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "checksum" + ] + }, + "ConanLockMetadata": { + "properties": { + "ref": { + "type": "string" + }, + "package_id": { + "type": "string" + }, + "prev": { + "type": "string" + }, + "requires": { + "type": "string" + }, + "build_requires": { + "type": "string" + }, + "py_requires": { + "type": "string" + }, + "options": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "path": { + "type": "string" + }, + "context": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "ConanMetadata": { + "properties": { + "ref": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "Coordinates": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "DartPubMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hosted_url": { + "type": "string" + }, + "vcs_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Descriptor": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "configuration": true + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Digest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "Document": { + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/Package" + }, + "type": "array" + }, + "artifactRelationships": { + "items": { + "$ref": "#/$defs/Relationship" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + "secrets": { + "items": { + "$ref": "#/$defs/Secrets" + }, + "type": "array" + }, + "source": { + "$ref": "#/$defs/Source" + }, + "distro": { + "$ref": "#/$defs/LinuxRelease" + }, + "descriptor": { + "$ref": "#/$defs/Descriptor" + }, + "schema": { + "$ref": "#/$defs/Schema" + } + }, + "type": "object", + "required": [ + "artifacts", + "artifactRelationships", + "source", + "distro", + "descriptor", + "schema" + ] + }, + "DotnetDepsMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "path": { + "type": "string" + }, + "sha512": { + "type": "string" + }, + "hashPath": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "path", + "sha512", + "hashPath" + ] + }, + "DpkgFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "isConfigFile": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "path", + "isConfigFile" + ] + }, + "DpkgMetadata": { + "properties": { + "package": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/DpkgFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ] + }, + "File": { + "properties": { + "id": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Coordinates" + }, + "metadata": { + "$ref": "#/$defs/FileMetadataEntry" + }, + "contents": { + "type": "string" + }, + "digests": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "id", + "location" + ] + }, + "FileMetadataEntry": { + "properties": { + "mode": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "linkDestination": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "groupID": { + "type": "integer" + }, + "mimeType": { + "type": "string" + }, + "size": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "mode", + "type", + "userID", + "groupID", + "mimeType", + "size" + ] + }, + "GemMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "GolangBinMetadata": { + "properties": { + "goBuildSettings": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "goCompiledVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "h1Digest": { + "type": "string" + }, + "mainModule": { + "type": "string" + } + }, + "type": "object", + "required": [ + "goCompiledVersion", + "architecture" + ] + }, + "GolangModMetadata": { + "properties": { + "h1Digest": { + "type": "string" + } + }, + "type": "object" + }, + "HackageMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "snapshotURL": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "IDLikes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "JavaManifest": { + "properties": { + "main": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "namedSections": { + "patternProperties": { + ".*": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "JavaMetadata": { + "properties": { + "virtualPath": { + "type": "string" + }, + "manifest": { + "$ref": "#/$defs/JavaManifest" + }, + "pomProperties": { + "$ref": "#/$defs/PomProperties" + }, + "pomProject": { + "$ref": "#/$defs/PomProject" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "virtualPath" + ] + }, + "KbPackageMetadata": { + "properties": { + "product_id": { + "type": "string" + }, + "kb": { + "type": "string" + } + }, + "type": "object", + "required": [ + "product_id", + "kb" + ] + }, + "License": { + "properties": { + "value": { + "type": "string" + }, + "spdxExpression": { + "type": "string" + }, + "type": { + "type": "string" + }, + "urls": { + "items": { + "type": "string" + }, + "type": "array" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type", + "urls", + "locations" + ] + }, + "LinuxKernelMetadata": { + "properties": { + "name": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extendedVersion": { + "type": "string" + }, + "buildTime": { + "type": "string" + }, + "author": { + "type": "string" + }, + "format": { + "type": "string" + }, + "rwRootFS": { + "type": "boolean" + }, + "swapDevice": { + "type": "integer" + }, + "rootDevice": { + "type": "integer" + }, + "videoMode": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "architecture", + "version" + ] + }, + "LinuxKernelModuleMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "path": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "license": { + "type": "string" + }, + "kernelVersion": { + "type": "string" + }, + "versionMagic": { + "type": "string" + }, + "parameters": { + "patternProperties": { + ".*": { + "$ref": "#/$defs/LinuxKernelModuleParameter" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "LinuxKernelModuleParameter": { + "properties": { + "type": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "type": "object" + }, + "LinuxRelease": { + "properties": { + "prettyName": { + "type": "string" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "idLike": { + "$ref": "#/$defs/IDLikes" + }, + "version": { + "type": "string" + }, + "versionID": { + "type": "string" + }, + "versionCodename": { + "type": "string" + }, + "buildID": { + "type": "string" + }, + "imageID": { + "type": "string" + }, + "imageVersion": { + "type": "string" + }, + "variant": { + "type": "string" + }, + "variantID": { + "type": "string" + }, + "homeURL": { + "type": "string" + }, + "supportURL": { + "type": "string" + }, + "bugReportURL": { + "type": "string" + }, + "privacyPolicyURL": { + "type": "string" + }, + "cpeName": { + "type": "string" + }, + "supportEnd": { + "type": "string" + } + }, + "type": "object" + }, + "Location": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + }, + "annotations": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "MixLockMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "NixStoreMetadata": { + "properties": { + "outputHash": { + "type": "string" + }, + "output": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "outputHash", + "files" + ] + }, + "NpmPackageJSONMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + }, + "private": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "homepage", + "description", + "url", + "private" + ] + }, + "NpmPackageLockJSONMetadata": { + "properties": { + "resolved": { + "type": "string" + }, + "integrity": { + "type": "string" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity" + ] + }, + "Package": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + }, + "licenses": { + "$ref": "#/$defs/licenses" + }, + "language": { + "type": "string" + }, + "cpes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/AlpmMetadata" + }, + { + "$ref": "#/$defs/ApkMetadata" + }, + { + "$ref": "#/$defs/BinaryMetadata" + }, + { + "$ref": "#/$defs/CargoPackageMetadata" + }, + { + "$ref": "#/$defs/CocoapodsMetadata" + }, + { + "$ref": "#/$defs/ConanLockMetadata" + }, + { + "$ref": "#/$defs/ConanMetadata" + }, + { + "$ref": "#/$defs/DartPubMetadata" + }, + { + "$ref": "#/$defs/DotnetDepsMetadata" + }, + { + "$ref": "#/$defs/DpkgMetadata" + }, + { + "$ref": "#/$defs/GemMetadata" + }, + { + "$ref": "#/$defs/GolangBinMetadata" + }, + { + "$ref": "#/$defs/GolangModMetadata" + }, + { + "$ref": "#/$defs/HackageMetadata" + }, + { + "$ref": "#/$defs/JavaMetadata" + }, + { + "$ref": "#/$defs/KbPackageMetadata" + }, + { + "$ref": "#/$defs/LinuxKernelMetadata" + }, + { + "$ref": "#/$defs/LinuxKernelModuleMetadata" + }, + { + "$ref": "#/$defs/MixLockMetadata" + }, + { + "$ref": "#/$defs/NixStoreMetadata" + }, + { + "$ref": "#/$defs/NpmPackageJSONMetadata" + }, + { + "$ref": "#/$defs/NpmPackageLockJSONMetadata" + }, + { + "$ref": "#/$defs/PhpComposerJSONMetadata" + }, + { + "$ref": "#/$defs/PortageMetadata" + }, + { + "$ref": "#/$defs/PythonPackageMetadata" + }, + { + "$ref": "#/$defs/PythonPipfileLockMetadata" + }, + { + "$ref": "#/$defs/PythonRequirementsMetadata" + }, + { + "$ref": "#/$defs/RDescriptionFileMetadata" + }, + { + "$ref": "#/$defs/RebarLockMetadata" + }, + { + "$ref": "#/$defs/RpmMetadata" + } + ] + } + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl" + ] + }, + "PhpComposerAuthors": { + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "PhpComposerExternalReference": { + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": "string" + }, + "reference": { + "type": "string" + }, + "shasum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "type", + "url", + "reference" + ] + }, + "PhpComposerJSONMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "require": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "provide": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "require-dev": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "suggest": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": { + "type": "string" + }, + "notification-url": { + "type": "string" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array" + }, + "time": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ] + }, + "PomParent": { + "properties": { + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "type": "object", + "required": [ + "groupId", + "artifactId", + "version" + ] + }, + "PomProject": { + "properties": { + "path": { + "type": "string" + }, + "parent": { + "$ref": "#/$defs/PomParent" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "groupId", + "artifactId", + "version", + "name" + ] + }, + "PomProperties": { + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "extraFields": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version" + ] + }, + "PortageFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PortageMetadata": { + "properties": { + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/PortageFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "installedSize", + "files" + ] + }, + "PythonDirectURLOriginInfo": { + "properties": { + "url": { + "type": "string" + }, + "commitId": { + "type": "string" + }, + "vcs": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "PythonFileDigest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "PythonFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/PythonFileDigest" + }, + "size": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PythonPackageMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorEmail": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/PythonFileRecord" + }, + "type": "array" + }, + "sitePackagesRootPath": { + "type": "string" + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array" + }, + "directUrlOrigin": { + "$ref": "#/$defs/PythonDirectURLOriginInfo" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ] + }, + "PythonPipfileLockMetadata": { + "properties": { + "hashes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "index": { + "type": "string" + } + }, + "type": "object", + "required": [ + "hashes", + "index" + ] + }, + "PythonRequirementsMetadata": { + "properties": { + "name": { + "type": "string" + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array" + }, + "versionConstraint": { + "type": "string" + }, + "url": { + "type": "string" + }, + "markers": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "name", + "extras", + "versionConstraint", + "url", + "markers" + ] + }, + "RDescriptionFileMetadata": { + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "url": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repository": { + "type": "string" + }, + "built": { + "type": "string" + }, + "needsCompilation": { + "type": "boolean" + }, + "imports": { + "items": { + "type": "string" + }, + "type": "array" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "suggests": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "RebarLockMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "Relationship": { + "properties": { + "parent": { + "type": "string" + }, + "child": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "parent", + "child", + "type" + ] + }, + "RpmMetadata": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "vendor": { + "type": "string" + }, + "modularityLabel": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/RpmdbFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "modularityLabel", + "files" + ] + }, + "RpmdbFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "mode": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "userName": { + "type": "string" + }, + "groupName": { + "type": "string" + }, + "flags": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "mode", + "size", + "digest", + "userName", + "groupName", + "flags" + ] + }, + "Schema": { + "properties": { + "version": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "version", + "url" + ] + }, + "SearchResult": { + "properties": { + "classification": { + "type": "string" + }, + "lineNumber": { + "type": "integer" + }, + "lineOffset": { + "type": "integer" + }, + "seekPosition": { + "type": "integer" + }, + "length": { + "type": "integer" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "classification", + "lineNumber", + "lineOffset", + "seekPosition", + "length" + ] + }, + "Secrets": { + "properties": { + "location": { + "$ref": "#/$defs/Coordinates" + }, + "secrets": { + "items": { + "$ref": "#/$defs/SearchResult" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "location", + "secrets" + ] + }, + "Source": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "metadata" + ] + }, + "licenses": { + "items": { + "$ref": "#/$defs/License" + }, + "type": "array" + } + } +} diff --git a/syft/file/cataloger/filedigest/cataloger_test.go b/syft/file/cataloger/filedigest/cataloger_test.go index ed8562cbd38..1ed1af58878 100644 --- a/syft/file/cataloger/filedigest/cataloger_test.go +++ b/syft/file/cataloger/filedigest/cataloger_test.go @@ -75,7 +75,7 @@ func TestDigestsCataloger(t *testing.T) { t.Run(test.name, func(t *testing.T) { c := NewCataloger(test.digests) - src, err := source.NewFromDirectory("test-fixtures/last/") + src, err := source.NewFromDirectoryPath("test-fixtures/last/") require.NoError(t, err) resolver, err := src.FileResolver(source.SquashedScope) @@ -94,7 +94,7 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) { img := imagetest.GetFixtureImage(t, "docker-archive", testImage) - src, err := source.NewFromImage(img, "---") + src, err := source.NewFromStereoscopeImageObject(img, testImage, nil) if err != nil { t.Fatalf("could not create source: %+v", err) } diff --git a/syft/file/cataloger/filemetadata/cataloger_test.go b/syft/file/cataloger/filemetadata/cataloger_test.go index 99dfa908a9f..9c84c9f76d8 100644 --- a/syft/file/cataloger/filemetadata/cataloger_test.go +++ b/syft/file/cataloger/filemetadata/cataloger_test.go @@ -20,7 +20,7 @@ func TestFileMetadataCataloger(t *testing.T) { c := NewCataloger() - src, err := source.NewFromImage(img, "---") + src, err := source.NewFromStereoscopeImageObject(img, testImage, nil) if err != nil { t.Fatalf("could not create source: %+v", err) } diff --git a/syft/file/cataloger/internal/all_regular_files_test.go b/syft/file/cataloger/internal/all_regular_files_test.go index 714e733e689..ced33384083 100644 --- a/syft/file/cataloger/internal/all_regular_files_test.go +++ b/syft/file/cataloger/internal/all_regular_files_test.go @@ -27,7 +27,7 @@ func Test_allRegularFiles(t *testing.T) { img := imagetest.GetFixtureImage(t, "docker-archive", testImage) - s, err := source.NewFromImage(img, "---") + s, err := source.NewFromStereoscopeImageObject(img, testImage, nil) require.NoError(t, err) r, err := s.FileResolver(source.SquashedScope) @@ -41,7 +41,7 @@ func Test_allRegularFiles(t *testing.T) { { name: "directory", setup: func() file.Resolver { - s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root") + s, err := source.NewFromDirectoryPath("test-fixtures/symlinked-root/nested/link-root") require.NoError(t, err) r, err := s.FileResolver(source.SquashedScope) require.NoError(t, err) diff --git a/syft/file/test-fixtures/req-resp/.gitignore b/syft/file/test-fixtures/req-resp/.gitignore new file mode 100644 index 00000000000..c944599212e --- /dev/null +++ b/syft/file/test-fixtures/req-resp/.gitignore @@ -0,0 +1,2 @@ +path/to/abs-inside.txt +path/to/the/abs-outside.txt \ No newline at end of file diff --git a/syft/file/test-fixtures/req-resp/path/to/rel-inside.txt b/syft/file/test-fixtures/req-resp/path/to/rel-inside.txt new file mode 120000 index 00000000000..f2bc06e87c4 --- /dev/null +++ b/syft/file/test-fixtures/req-resp/path/to/rel-inside.txt @@ -0,0 +1 @@ +./the/file.txt \ No newline at end of file diff --git a/syft/file/test-fixtures/req-resp/path/to/the/file.txt b/syft/file/test-fixtures/req-resp/path/to/the/file.txt new file mode 100644 index 00000000000..fbfd79f5e48 --- /dev/null +++ b/syft/file/test-fixtures/req-resp/path/to/the/file.txt @@ -0,0 +1 @@ +file-1 diff --git a/syft/file/test-fixtures/req-resp/path/to/the/rel-outside.txt b/syft/file/test-fixtures/req-resp/path/to/the/rel-outside.txt new file mode 120000 index 00000000000..6ad08d35758 --- /dev/null +++ b/syft/file/test-fixtures/req-resp/path/to/the/rel-outside.txt @@ -0,0 +1 @@ +../../../somewhere/outside.txt \ No newline at end of file diff --git a/syft/file/test-fixtures/req-resp/root-link b/syft/file/test-fixtures/req-resp/root-link new file mode 120000 index 00000000000..6a043149e81 --- /dev/null +++ b/syft/file/test-fixtures/req-resp/root-link @@ -0,0 +1 @@ +./ \ No newline at end of file diff --git a/syft/file/test-fixtures/req-resp/somewhere/outside.txt b/syft/file/test-fixtures/req-resp/somewhere/outside.txt new file mode 100644 index 00000000000..37ad5611998 --- /dev/null +++ b/syft/file/test-fixtures/req-resp/somewhere/outside.txt @@ -0,0 +1 @@ +file-2 diff --git a/syft/formats/common/cyclonedxhelpers/decoder.go b/syft/formats/common/cyclonedxhelpers/decoder.go index ef81bf9997e..cb4c9974e28 100644 --- a/syft/formats/common/cyclonedxhelpers/decoder.go +++ b/syft/formats/common/cyclonedxhelpers/decoder.go @@ -229,32 +229,34 @@ func collectRelationships(bom *cyclonedx.BOM, s *sbom.SBOM, idMap map[string]int } } -func extractComponents(meta *cyclonedx.Metadata) source.Metadata { +func extractComponents(meta *cyclonedx.Metadata) source.Description { if meta == nil || meta.Component == nil { - return source.Metadata{} + return source.Description{} } c := meta.Component - image := source.ImageMetadata{ - UserInput: c.Name, - ID: c.BOMRef, - ManifestDigest: c.Version, - } - switch c.Type { case cyclonedx.ComponentTypeContainer: - return source.Metadata{ - Scheme: source.ImageScheme, - ImageMetadata: image, + return source.Description{ + ID: "", + // TODO: can we decode alias name-version somehow? (it isn't be encoded in the first place yet) + + Metadata: source.StereoscopeImageSourceMetadata{ + UserInput: c.Name, + ID: c.BOMRef, + ManifestDigest: c.Version, + }, } case cyclonedx.ComponentTypeFile: - return source.Metadata{ - Scheme: source.FileScheme, // or source.DirectoryScheme - Path: c.Name, - ImageMetadata: image, + // TODO: can we decode alias name-version somehow? (it isn't be encoded in the first place yet) + + // TODO: this is lossy... we can't know if this is a file or a directory + return source.Description{ + ID: "", + Metadata: source.FileSourceMetadata{Path: c.Name}, } } - return source.Metadata{} + return source.Description{} } // if there is more than one tool in meta.Tools' list the last item will be used diff --git a/syft/formats/common/cyclonedxhelpers/format.go b/syft/formats/common/cyclonedxhelpers/format.go index 2facf558d92..34ca3509416 100644 --- a/syft/formats/common/cyclonedxhelpers/format.go +++ b/syft/formats/common/cyclonedxhelpers/format.go @@ -110,7 +110,7 @@ func formatCPE(cpeString string) string { } // NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details. -func toBomDescriptor(name, version string, srcMetadata source.Metadata) *cyclonedx.Metadata { +func toBomDescriptor(name, version string, srcMetadata source.Description) *cyclonedx.Metadata { return &cyclonedx.Metadata{ Timestamp: time.Now().Format(time.RFC3339), Tools: &[]cyclonedx.Tool{ @@ -170,35 +170,56 @@ func toDependencies(relationships []artifact.Relationship) []cyclonedx.Dependenc return result } -func toBomDescriptorComponent(srcMetadata source.Metadata) *cyclonedx.Component { +func toBomDescriptorComponent(srcMetadata source.Description) *cyclonedx.Component { name := srcMetadata.Name - switch srcMetadata.Scheme { - case source.ImageScheme: + version := srcMetadata.Version + switch metadata := srcMetadata.Metadata.(type) { + case source.StereoscopeImageSourceMetadata: if name == "" { - name = srcMetadata.ImageMetadata.UserInput + name = metadata.UserInput } - bomRef, err := artifact.IDByHash(srcMetadata.ImageMetadata.ID) + if version == "" { + version = metadata.ManifestDigest + } + bomRef, err := artifact.IDByHash(metadata.ID) if err != nil { - log.Warnf("unable to get fingerprint of image metadata=%s: %+v", srcMetadata.ImageMetadata.ID, err) + log.Warnf("unable to get fingerprint of source image metadata=%s: %+v", metadata.ID, err) } return &cyclonedx.Component{ BOMRef: string(bomRef), Type: cyclonedx.ComponentTypeContainer, Name: name, - Version: srcMetadata.ImageMetadata.ManifestDigest, + Version: version, + } + case source.DirectorySourceMetadata: + if name == "" { + name = metadata.Path + } + bomRef, err := artifact.IDByHash(metadata.Path) + if err != nil { + log.Warnf("unable to get fingerprint of source directory metadata path=%s: %+v", metadata.Path, err) + } + return &cyclonedx.Component{ + BOMRef: string(bomRef), + // TODO: this is lossy... we can't know if this is a file or a directory + Type: cyclonedx.ComponentTypeFile, + Name: name, + Version: version, } - case source.DirectoryScheme, source.FileScheme: + case source.FileSourceMetadata: if name == "" { - name = srcMetadata.Path + name = metadata.Path } - bomRef, err := artifact.IDByHash(srcMetadata.Path) + bomRef, err := artifact.IDByHash(metadata.Path) if err != nil { - log.Warnf("unable to get fingerprint of source metadata path=%s: %+v", srcMetadata.Path, err) + log.Warnf("unable to get fingerprint of source file metadata path=%s: %+v", metadata.Path, err) } return &cyclonedx.Component{ BOMRef: string(bomRef), - Type: cyclonedx.ComponentTypeFile, - Name: name, + // TODO: this is lossy... we can't know if this is a file or a directory + Type: cyclonedx.ComponentTypeFile, + Name: name, + Version: version, } } diff --git a/syft/formats/common/spdxhelpers/document_name.go b/syft/formats/common/spdxhelpers/document_name.go index 8967117e919..6932f2b4e81 100644 --- a/syft/formats/common/spdxhelpers/document_name.go +++ b/syft/formats/common/spdxhelpers/document_name.go @@ -4,16 +4,18 @@ import ( "github.com/anchore/syft/syft/source" ) -func DocumentName(srcMetadata source.Metadata) string { +func DocumentName(srcMetadata source.Description) string { if srcMetadata.Name != "" { return srcMetadata.Name } - switch srcMetadata.Scheme { - case source.ImageScheme: - return srcMetadata.ImageMetadata.UserInput - case source.DirectoryScheme, source.FileScheme: - return srcMetadata.Path + switch metadata := srcMetadata.Metadata.(type) { + case source.StereoscopeImageSourceMetadata: + return metadata.UserInput + case source.DirectorySourceMetadata: + return metadata.Path + case source.FileSourceMetadata: + return metadata.Path default: return "unknown" } diff --git a/syft/formats/common/spdxhelpers/document_name_test.go b/syft/formats/common/spdxhelpers/document_name_test.go index cc1944247c2..f993f41a602 100644 --- a/syft/formats/common/spdxhelpers/document_name_test.go +++ b/syft/formats/common/spdxhelpers/document_name_test.go @@ -5,31 +5,27 @@ import ( "strings" "testing" - "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/source" ) func Test_DocumentName(t *testing.T) { - allSchemes := strset.New() - for _, s := range source.AllSchemes { - allSchemes.Add(string(s)) - } - testedSchemes := strset.New() + + tracker := sourcemetadata.NewCompletionTester(t) tests := []struct { name string inputName string - srcMetadata source.Metadata + srcMetadata source.Description expected string }{ { name: "image", inputName: "my-name", - srcMetadata: source.Metadata{ - Scheme: source.ImageScheme, - ImageMetadata: source.ImageMetadata{ + srcMetadata: source.Description{ + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "image-repo/name:tag", ID: "id", ManifestDigest: "digest", @@ -40,18 +36,16 @@ func Test_DocumentName(t *testing.T) { { name: "directory", inputName: "my-name", - srcMetadata: source.Metadata{ - Scheme: source.DirectoryScheme, - Path: "some/path/to/place", + srcMetadata: source.Description{ + Metadata: source.DirectorySourceMetadata{Path: "some/path/to/place"}, }, expected: "some/path/to/place", }, { name: "file", inputName: "my-name", - srcMetadata: source.Metadata{ - Scheme: source.FileScheme, - Path: "some/path/to/place", + srcMetadata: source.Description{ + Metadata: source.FileSourceMetadata{Path: "some/path/to/place"}, }, expected: "some/path/to/place", }, @@ -62,10 +56,7 @@ func Test_DocumentName(t *testing.T) { assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual)) // track each scheme tested (passed or not) - testedSchemes.Add(string(test.srcMetadata.Scheme)) + tracker.Tested(t, test.srcMetadata.Metadata) }) } - - // assert all possible schemes were under test - assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test") } diff --git a/syft/formats/common/spdxhelpers/document_namespace.go b/syft/formats/common/spdxhelpers/document_namespace.go index c2a2bd1296c..3b6d30b69eb 100644 --- a/syft/formats/common/spdxhelpers/document_namespace.go +++ b/syft/formats/common/spdxhelpers/document_namespace.go @@ -18,20 +18,20 @@ const ( inputFile = "file" ) -func DocumentNameAndNamespace(srcMetadata source.Metadata) (string, string) { - name := DocumentName(srcMetadata) - return name, DocumentNamespace(name, srcMetadata) +func DocumentNameAndNamespace(src source.Description) (string, string) { + name := DocumentName(src) + return name, DocumentNamespace(name, src) } -func DocumentNamespace(name string, srcMetadata source.Metadata) string { +func DocumentNamespace(name string, src source.Description) string { name = cleanName(name) input := "unknown-source-type" - switch srcMetadata.Scheme { - case source.ImageScheme: + switch src.Metadata.(type) { + case source.StereoscopeImageSourceMetadata: input = inputImage - case source.DirectoryScheme: + case source.DirectorySourceMetadata: input = inputDirectory - case source.FileScheme: + case source.FileSourceMetadata: input = inputFile } diff --git a/syft/formats/common/spdxhelpers/document_namespace_test.go b/syft/formats/common/spdxhelpers/document_namespace_test.go index 5452276644f..00bed35362e 100644 --- a/syft/formats/common/spdxhelpers/document_namespace_test.go +++ b/syft/formats/common/spdxhelpers/document_namespace_test.go @@ -5,31 +5,26 @@ import ( "strings" "testing" - "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/source" ) func Test_documentNamespace(t *testing.T) { - allSchemes := strset.New() - for _, s := range source.AllSchemes { - allSchemes.Add(string(s)) - } - testedSchemes := strset.New() + tracker := sourcemetadata.NewCompletionTester(t) tests := []struct { - name string - inputName string - srcMetadata source.Metadata - expected string + name string + inputName string + src source.Description + expected string }{ { name: "image", inputName: "my-name", - srcMetadata: source.Metadata{ - Scheme: source.ImageScheme, - ImageMetadata: source.ImageMetadata{ + src: source.Description{ + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "image-repo/name:tag", ID: "id", ManifestDigest: "digest", @@ -40,33 +35,32 @@ func Test_documentNamespace(t *testing.T) { { name: "directory", inputName: "my-name", - srcMetadata: source.Metadata{ - Scheme: source.DirectoryScheme, - Path: "some/path/to/place", + src: source.Description{ + Metadata: source.DirectorySourceMetadata{ + Path: "some/path/to/place", + }, }, expected: "https://anchore.com/syft/dir/my-name-", }, { name: "file", inputName: "my-name", - srcMetadata: source.Metadata{ - Scheme: source.FileScheme, - Path: "some/path/to/place", + src: source.Description{ + Metadata: source.FileSourceMetadata{ + Path: "some/path/to/place", + }, }, expected: "https://anchore.com/syft/file/my-name-", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - actual := DocumentNamespace(test.inputName, test.srcMetadata) + actual := DocumentNamespace(test.inputName, test.src) // note: since the namespace ends with a UUID we check the prefix assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual namespace %q", actual)) // track each scheme tested (passed or not) - testedSchemes.Add(string(test.srcMetadata.Scheme)) + tracker.Tested(t, test.src.Metadata) }) } - - // assert all possible schemes were under test - assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test") } diff --git a/syft/formats/common/spdxhelpers/to_syft_model.go b/syft/formats/common/spdxhelpers/to_syft_model.go index fd34541df99..54ecd145537 100644 --- a/syft/formats/common/spdxhelpers/to_syft_model.go +++ b/syft/formats/common/spdxhelpers/to_syft_model.go @@ -28,8 +28,7 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { spdxIDMap := make(map[string]interface{}) - src := source.Metadata{Scheme: source.UnknownScheme} - src.Scheme = extractSchemeFromNamespace(doc.DocumentNamespace) + src := extractSourceFromNamespace(doc.DocumentNamespace) s := &sbom.SBOM{ Source: src, @@ -54,24 +53,32 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { // image, directory, for example. This is our best effort to determine // the scheme. Syft-generated SBOMs have in the namespace // field a type encoded, which we try to identify here. -func extractSchemeFromNamespace(ns string) source.Scheme { +func extractSourceFromNamespace(ns string) source.Description { u, err := url.Parse(ns) if err != nil { - return source.UnknownScheme + return source.Description{ + Metadata: nil, + } } parts := strings.Split(u.Path, "/") for _, p := range parts { switch p { case inputFile: - return source.FileScheme + return source.Description{ + Metadata: source.FileSourceMetadata{}, + } case inputImage: - return source.ImageScheme + return source.Description{ + Metadata: source.StereoscopeImageSourceMetadata{}, + } case inputDirectory: - return source.DirectoryScheme + return source.Description{ + Metadata: source.DirectorySourceMetadata{}, + } } } - return source.UnknownScheme + return source.Description{} } func findLinuxReleaseByPURL(doc *spdx.Document) *linux.Release { diff --git a/syft/formats/common/spdxhelpers/to_syft_model_test.go b/syft/formats/common/spdxhelpers/to_syft_model_test.go index e4a98f5abd2..6c9cf7c8f66 100644 --- a/syft/formats/common/spdxhelpers/to_syft_model_test.go +++ b/syft/formats/common/spdxhelpers/to_syft_model_test.go @@ -1,6 +1,7 @@ package spdxhelpers import ( + "reflect" "testing" "github.com/spdx/tools-golang/spdx" @@ -197,36 +198,46 @@ func Test_extractMetadata(t *testing.T) { func TestExtractSourceFromNamespaces(t *testing.T) { tests := []struct { namespace string - expected source.Scheme + expected any }{ { namespace: "https://anchore.com/syft/file/d42b01d0-7325-409b-b03f-74082935c4d3", - expected: source.FileScheme, + expected: source.FileSourceMetadata{}, }, { namespace: "https://anchore.com/syft/image/d42b01d0-7325-409b-b03f-74082935c4d3", - expected: source.ImageScheme, + expected: source.StereoscopeImageSourceMetadata{}, }, { namespace: "https://anchore.com/syft/dir/d42b01d0-7325-409b-b03f-74082935c4d3", - expected: source.DirectoryScheme, + expected: source.DirectorySourceMetadata{}, }, { namespace: "https://another-host/blob/123", - expected: source.UnknownScheme, + expected: nil, }, { namespace: "bla bla", - expected: source.UnknownScheme, + expected: nil, }, { namespace: "", - expected: source.UnknownScheme, + expected: nil, }, } for _, tt := range tests { - require.Equal(t, tt.expected, extractSchemeFromNamespace(tt.namespace)) + desc := extractSourceFromNamespace(tt.namespace) + if tt.expected == nil && desc.Metadata == nil { + return + } + if tt.expected != nil && desc.Metadata == nil { + t.Fatal("expected metadata but got nil") + } + if tt.expected == nil && desc.Metadata != nil { + t.Fatal("expected nil metadata but got something") + } + require.Equal(t, reflect.TypeOf(tt.expected), reflect.TypeOf(desc.Metadata)) } } diff --git a/syft/formats/cyclonedxjson/encoder_test.go b/syft/formats/cyclonedxjson/encoder_test.go index 2159fca5ee4..dec29164e06 100644 --- a/syft/formats/cyclonedxjson/encoder_test.go +++ b/syft/formats/cyclonedxjson/encoder_test.go @@ -2,49 +2,62 @@ package cyclonedxjson import ( "flag" - "regexp" "testing" "github.com/anchore/syft/syft/formats/internal/testutils" ) -var updateCycloneDx = flag.Bool("update-cyclonedx", false, "update the *.golden files for cyclone-dx encoders") +var updateSnapshot = flag.Bool("update-cyclonedx-json", false, "update the *.golden files for cyclone-dx JSON encoders") +var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing") func TestCycloneDxDirectoryEncoder(t *testing.T) { + dir := t.TempDir() testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateCycloneDx, - true, - cycloneDxRedactor, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, dir), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(dir), + }, ) } func TestCycloneDxImageEncoder(t *testing.T) { testImage := "image-simple" testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - testutils.ImageInput(t, testImage), - testImage, - *updateCycloneDx, - true, - cycloneDxRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.ImageInput(t, testImage), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(), + }, ) } -func cycloneDxRedactor(s []byte) []byte { - replacements := map[string]string{ - // UUIDs - `urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`: `urn:uuid:redacted`, - // timestamps - `([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `timestamp:redacted`, - // image hashes - `sha256:[A-Fa-f0-9]{64}`: `sha256:redacted`, - // bom-refs - `"bom-ref":\s*"[^"]+"`: `"bom-ref": "redacted"`, - } - for pattern, replacement := range replacements { - s = regexp.MustCompile(pattern).ReplaceAll(s, []byte(replacement)) - } - return s +func redactor(values ...string) testutils.Redactor { + return testutils.NewRedactions(). + WithValuesRedacted(values...). + WithPatternRedactors( + map[string]string{ + // UUIDs + `urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`: `urn:uuid:redacted`, + + // timestamps + `([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `timestamp:redacted`, + + // image hashes + `sha256:[A-Fa-f0-9]{64}`: `sha256:redacted`, + + // BOM refs + `"bom-ref":\s*"[^"]+"`: `"bom-ref":"redacted"`, + }, + ) } diff --git a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden index 3cab5be0e47..77ff36dbaf7 100644 --- a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden +++ b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden @@ -2,10 +2,10 @@ "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.4", - "serialNumber": "urn:uuid:1b71a5b4-4bc5-4548-a51a-212e631976cd", + "serialNumber": "urn:uuid:redacted", "version": 1, "metadata": { - "timestamp": "2023-05-08T14:40:32-04:00", + "timestamp": "timestamp:redacted", "tools": [ { "vendor": "anchore", @@ -14,14 +14,14 @@ } ], "component": { - "bom-ref": "163686ac6e30c752", + "bom-ref":"redacted", "type": "file", - "name": "/some/path" + "name": "some/path" } }, "components": [ { - "bom-ref": "8c7e1242588c971a", + "bom-ref":"redacted", "type": "library", "name": "package-1", "version": "1.0.1", @@ -58,7 +58,7 @@ ] }, { - "bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=db4abfe497c180d3", + "bom-ref":"redacted", "type": "library", "name": "package-2", "version": "2.0.1", diff --git a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden index 4d4765f5468..ac4799123d1 100644 --- a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden +++ b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden @@ -2,10 +2,10 @@ "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.4", - "serialNumber": "urn:uuid:1695d6ae-0ddf-4e77-9c9d-74df1bdd8d5b", + "serialNumber": "urn:uuid:redacted", "version": 1, "metadata": { - "timestamp": "2023-05-08T14:40:32-04:00", + "timestamp": "timestamp:redacted", "tools": [ { "vendor": "anchore", @@ -14,15 +14,15 @@ } ], "component": { - "bom-ref": "38160ebc2a6876e8", + "bom-ref":"redacted", "type": "container", "name": "user-image-input", - "version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" + "version": "sha256:redacted" } }, "components": [ { - "bom-ref": "ec2e0c93617507ef", + "bom-ref":"redacted", "type": "library", "name": "package-1", "version": "1.0.1", @@ -54,7 +54,7 @@ }, { "name": "syft:location:0:layerID", - "value": "sha256:ab62016f9bec7286af65604081564cadeeb364a48faca2346c3f5a5a1f5ef777" + "value": "sha256:redacted" }, { "name": "syft:location:0:path", @@ -63,7 +63,7 @@ ] }, { - "bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=958443e2d9304af4", + "bom-ref":"redacted", "type": "library", "name": "package-2", "version": "2.0.1", @@ -84,7 +84,7 @@ }, { "name": "syft:location:0:layerID", - "value": "sha256:f1803845b6747d94d6e4ecce2331457e5f1c4fb97de5216f392a76f4582f63b2" + "value": "sha256:redacted" }, { "name": "syft:location:0:path", diff --git a/syft/formats/cyclonedxxml/encoder_test.go b/syft/formats/cyclonedxxml/encoder_test.go index 1070f44cad5..5b8781affe5 100644 --- a/syft/formats/cyclonedxxml/encoder_test.go +++ b/syft/formats/cyclonedxxml/encoder_test.go @@ -2,51 +2,62 @@ package cyclonedxxml import ( "flag" - "regexp" "testing" "github.com/anchore/syft/syft/formats/internal/testutils" ) -var updateCycloneDx = flag.Bool("update-cyclonedx", false, "update the *.golden files for cyclone-dx encoders") +var updateSnapshot = flag.Bool("update-cyclonedx-xml", false, "update the *.golden files for cyclone-dx XML encoders") +var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing") func TestCycloneDxDirectoryEncoder(t *testing.T) { + dir := t.TempDir() testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateCycloneDx, - false, - cycloneDxRedactor, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, dir), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(dir), + }, ) } func TestCycloneDxImageEncoder(t *testing.T) { testImage := "image-simple" testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - testutils.ImageInput(t, testImage), - testImage, - *updateCycloneDx, - false, - cycloneDxRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.ImageInput(t, testImage), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(), + }, ) } -func cycloneDxRedactor(s []byte) []byte { - serialPattern := regexp.MustCompile(`serialNumber="[a-zA-Z0-9\-:]+"`) - rfc3339Pattern := regexp.MustCompile(`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`) - sha256Pattern := regexp.MustCompile(`sha256:[A-Fa-f0-9]{64}`) +func redactor(values ...string) testutils.Redactor { + return testutils.NewRedactions(). + WithValuesRedacted(values...). + WithPatternRedactors( + map[string]string{ + // serial numbers + `serialNumber="[a-zA-Z0-9\-:]+`: `serialNumber="redacted`, - for _, pattern := range []*regexp.Regexp{serialPattern, rfc3339Pattern, sha256Pattern} { - s = pattern.ReplaceAll(s, []byte("redacted")) - } + // dates + `([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `redacted`, - // the bom-ref will be autogenerated every time, the value here should not be directly tested in snapshot tests - bomRefPattern := regexp.MustCompile(` bom-ref="[a-zA-Z0-9\-:]+"`) - bomRef3339Pattern := regexp.MustCompile(`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`) - for _, pattern := range []*regexp.Regexp{bomRefPattern, bomRef3339Pattern} { - s = pattern.ReplaceAll(s, []byte("")) - } + // image hashes + `sha256:[A-Za-z0-9]{64}`: `sha256:redacted`, - return s + // BOM refs + `bom-ref="[a-zA-Z0-9\-:]+"`: `bom-ref:redacted`, + }, + ) } diff --git a/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden b/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden index 32eaf274708..592072d2005 100644 --- a/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden +++ b/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden @@ -1,7 +1,7 @@ - + - 2023-05-08T14:40:52-04:00 + redacted anchore @@ -9,12 +9,12 @@ v0.42.0-bogus - - /some/path + + some/path - + package-1 1.0.1 diff --git a/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden b/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden index 67ad5f052a5..95701d1005d 100644 --- a/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden +++ b/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden @@ -1,7 +1,7 @@ - + - 2023-05-08T14:40:52-04:00 + redacted anchore @@ -9,13 +9,13 @@ v0.42.0-bogus - + user-image-input - sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368 + sha256:redacted - + package-1 1.0.1 @@ -30,7 +30,7 @@ python PythonPackageMetadata python - sha256:ab62016f9bec7286af65604081564cadeeb364a48faca2346c3f5a5a1f5ef777 + sha256:redacted /somefile-1.txt @@ -43,7 +43,7 @@ the-cataloger-2 DpkgMetadata deb - sha256:f1803845b6747d94d6e4ecce2331457e5f1c4fb97de5216f392a76f4582f63b2 + sha256:redacted /somefile-2.txt 0 diff --git a/syft/formats/github/encoder.go b/syft/formats/github/encoder.go index e03c7f504de..261ff6b184c 100644 --- a/syft/formats/github/encoder.go +++ b/syft/formats/github/encoder.go @@ -64,45 +64,6 @@ func filesystem(p pkg.Package) string { return "" } -// isArchive returns true if the path appears to be an archive -func isArchive(path string) bool { - _, err := archiver.ByExtension(path) - return err == nil -} - -// toPath Generates a string representation of the package location, optionally including the layer hash -func toPath(s source.Metadata, p pkg.Package) string { - inputPath := strings.TrimPrefix(s.Path, "./") - if inputPath == "." { - inputPath = "" - } - locations := p.Locations.ToSlice() - if len(locations) > 0 { - location := locations[0] - packagePath := location.RealPath - if location.VirtualPath != "" { - packagePath = location.VirtualPath - } - packagePath = strings.TrimPrefix(packagePath, "/") - switch s.Scheme { - case source.ImageScheme: - image := strings.ReplaceAll(s.ImageMetadata.UserInput, ":/", "//") - return fmt.Sprintf("%s:/%s", image, packagePath) - case source.FileScheme: - if isArchive(inputPath) { - return fmt.Sprintf("%s:/%s", inputPath, packagePath) - } - return inputPath - case source.DirectoryScheme: - if inputPath != "" { - return fmt.Sprintf("%s/%s", inputPath, packagePath) - } - return packagePath - } - } - return fmt.Sprintf("%s%s", inputPath, s.ImageMetadata.UserInput) -} - // toGithubManifests manifests, each of which represents a specific location that has dependencies func toGithubManifests(s *sbom.SBOM) Manifests { manifests := map[string]*Manifest{} @@ -144,6 +105,63 @@ func toGithubManifests(s *sbom.SBOM) Manifests { return out } +// toPath Generates a string representation of the package location, optionally including the layer hash +func toPath(s source.Description, p pkg.Package) string { + inputPath := trimRelative(s.Name) + locations := p.Locations.ToSlice() + if len(locations) > 0 { + location := locations[0] + packagePath := location.RealPath + if location.VirtualPath != "" { + packagePath = location.VirtualPath + } + packagePath = strings.TrimPrefix(packagePath, "/") + switch metadata := s.Metadata.(type) { + case source.StereoscopeImageSourceMetadata: + image := strings.ReplaceAll(metadata.UserInput, ":/", "//") + return fmt.Sprintf("%s:/%s", image, packagePath) + case source.FileSourceMetadata: + path := trimRelative(metadata.Path) + if isArchive(metadata.Path) { + return fmt.Sprintf("%s:/%s", path, packagePath) + } + return path + case source.DirectorySourceMetadata: + path := trimRelative(metadata.Path) + if path != "" { + return fmt.Sprintf("%s/%s", path, packagePath) + } + return packagePath + } + } + return inputPath +} + +func trimRelative(s string) string { + s = strings.TrimPrefix(s, "./") + if s == "." { + s = "" + } + return s +} + +// isArchive returns true if the path appears to be an archive +func isArchive(path string) bool { + _, err := archiver.ByExtension(path) + return err == nil +} + +func toDependencies(s *sbom.SBOM, p pkg.Package) (out []string) { + for _, r := range s.Relationships { + if r.From.ID() == p.ID() { + if p, ok := r.To.(pkg.Package); ok { + out = append(out, dependencyName(p)) + } + } + } + return +} + // dependencyName to make things a little nicer to read; this might end up being lossy func dependencyName(p pkg.Package) string { purl, err := packageurl.FromString(p.PURL) @@ -171,14 +189,3 @@ func toDependencyMetadata(_ pkg.Package) Metadata { // so we don't need anything here yet return Metadata{} } - -func toDependencies(s *sbom.SBOM, p pkg.Package) (out []string) { - for _, r := range s.Relationships { - if r.From.ID() == p.ID() { - if p, ok := r.To.(pkg.Package); ok { - out = append(out, dependencyName(p)) - } - } - } - return -} diff --git a/syft/formats/github/encoder_test.go b/syft/formats/github/encoder_test.go index a0770f2520e..3325509ce5f 100644 --- a/syft/formats/github/encoder_test.go +++ b/syft/formats/github/encoder_test.go @@ -1,24 +1,25 @@ package github import ( - "encoding/json" "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/anchore/packageurl-go" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" ) -func Test_toGithubModel(t *testing.T) { +func sbomFixture() sbom.SBOM { s := sbom.SBOM{ - Source: source.Metadata{ - Scheme: source.ImageScheme, - ImageMetadata: source.ImageMetadata{ + Source: source.Description{ + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "ubuntu:18.04", Architecture: "amd64", }, @@ -75,88 +76,121 @@ func Test_toGithubModel(t *testing.T) { s.Artifacts.Packages.Add(p) } - actual := toGithubModel(&s) + return s +} - expected := DependencySnapshot{ - Version: 0, - Detector: DetectorMetadata{ - Name: "syft", - Version: "0.0.0-dev", - URL: "https://github.com/anchore/syft", - }, - Metadata: Metadata{ - "syft:distro": "pkg:generic/ubuntu@18.04?like=debian", - }, - Scanned: actual.Scanned, - Manifests: Manifests{ - "ubuntu:18.04:/usr/lib": Manifest{ - Name: "ubuntu:18.04:/usr/lib", - File: FileInfo{ - SourceLocation: "ubuntu:18.04:/usr/lib", +func Test_toGithubModel(t *testing.T) { + tracker := sourcemetadata.NewCompletionTester(t) + + tests := []struct { + name string + metadata any + testPath string + expected *DependencySnapshot + }{ + { + name: "image", + expected: &DependencySnapshot{ + Version: 0, + Detector: DetectorMetadata{ + Name: "syft", + Version: "0.0.0-dev", + URL: "https://github.com/anchore/syft", }, Metadata: Metadata{ - "syft:filesystem": "fsid-1", + "syft:distro": "pkg:generic/ubuntu@18.04?like=debian", }, - Resolved: DependencyGraph{ - "pkg:generic/pkg-1@1.0.1": DependencyNode{ - PackageURL: "pkg:generic/pkg-1@1.0.1", - Scope: DependencyScopeRuntime, - Relationship: DependencyRelationshipDirect, + //Scanned: actual.Scanned, + Manifests: Manifests{ + "ubuntu:18.04:/usr/lib": Manifest{ + Name: "ubuntu:18.04:/usr/lib", + File: FileInfo{ + SourceLocation: "ubuntu:18.04:/usr/lib", + }, + Metadata: Metadata{ + "syft:filesystem": "fsid-1", + }, + Resolved: DependencyGraph{ + "pkg:generic/pkg-1@1.0.1": DependencyNode{ + PackageURL: "pkg:generic/pkg-1@1.0.1", + Scope: DependencyScopeRuntime, + Relationship: DependencyRelationshipDirect, + Metadata: Metadata{}, + }, + "pkg:generic/pkg-2@2.0.2": DependencyNode{ + PackageURL: "pkg:generic/pkg-2@2.0.2", + Scope: DependencyScopeRuntime, + Relationship: DependencyRelationshipDirect, + Metadata: Metadata{}, + }, + }, }, - "pkg:generic/pkg-2@2.0.2": DependencyNode{ - PackageURL: "pkg:generic/pkg-2@2.0.2", - Scope: DependencyScopeRuntime, - Relationship: DependencyRelationshipDirect, - }, - }, - }, - "ubuntu:18.04:/etc": Manifest{ - Name: "ubuntu:18.04:/etc", - File: FileInfo{ - SourceLocation: "ubuntu:18.04:/etc", - }, - Metadata: Metadata{ - "syft:filesystem": "fsid-1", - }, - Resolved: DependencyGraph{ - "pkg:generic/pkg-3@3.0.3": DependencyNode{ - PackageURL: "pkg:generic/pkg-3@3.0.3", - Scope: DependencyScopeRuntime, - Relationship: DependencyRelationshipDirect, + "ubuntu:18.04:/etc": Manifest{ + Name: "ubuntu:18.04:/etc", + File: FileInfo{ + SourceLocation: "ubuntu:18.04:/etc", + }, + Metadata: Metadata{ + "syft:filesystem": "fsid-1", + }, + Resolved: DependencyGraph{ + "pkg:generic/pkg-3@3.0.3": DependencyNode{ + PackageURL: "pkg:generic/pkg-3@3.0.3", + Scope: DependencyScopeRuntime, + Relationship: DependencyRelationshipDirect, + Metadata: Metadata{}, + }, + }, }, }, }, }, + { + name: "current directory", + metadata: source.DirectorySourceMetadata{Path: "."}, + testPath: "etc", + }, + { + name: "relative directory", + metadata: source.DirectorySourceMetadata{Path: "./artifacts"}, + testPath: "artifacts/etc", + }, + { + name: "absolute directory", + metadata: source.DirectorySourceMetadata{Path: "/artifacts"}, + testPath: "/artifacts/etc", + }, + { + name: "file", + metadata: source.FileSourceMetadata{Path: "./executable"}, + testPath: "executable", + }, + { + name: "archive", + metadata: source.FileSourceMetadata{Path: "./archive.tar.gz"}, + testPath: "archive.tar.gz:/etc", + }, } - // just using JSONEq because it gives a comprehensible diff - s1, _ := json.Marshal(expected) - s2, _ := json.Marshal(actual) - assert.JSONEq(t, string(s1), string(s2)) - - // Just test the other schemes: - s.Source.Path = "." - s.Source.Scheme = source.DirectoryScheme - actual = toGithubModel(&s) - assert.Equal(t, "etc", actual.Manifests["etc"].Name) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := sbomFixture() - s.Source.Path = "./artifacts" - s.Source.Scheme = source.DirectoryScheme - actual = toGithubModel(&s) - assert.Equal(t, "artifacts/etc", actual.Manifests["artifacts/etc"].Name) + if test.metadata != nil { + s.Source.Metadata = test.metadata + } + actual := toGithubModel(&s) - s.Source.Path = "/artifacts" - s.Source.Scheme = source.DirectoryScheme - actual = toGithubModel(&s) - assert.Equal(t, "/artifacts/etc", actual.Manifests["/artifacts/etc"].Name) + if test.expected != nil { + if d := cmp.Diff(*test.expected, actual, cmpopts.IgnoreFields(DependencySnapshot{}, "Scanned")); d != "" { + t.Errorf("unexpected result (-want +got):\n%s", d) + } + } - s.Source.Path = "./executable" - s.Source.Scheme = source.FileScheme - actual = toGithubModel(&s) - assert.Equal(t, "executable", actual.Manifests["executable"].Name) + assert.Equal(t, test.testPath, actual.Manifests[test.testPath].Name) - s.Source.Path = "./archive.tar.gz" - s.Source.Scheme = source.FileScheme - actual = toGithubModel(&s) - assert.Equal(t, "archive.tar.gz:/etc", actual.Manifests["archive.tar.gz:/etc"].Name) + // track each scheme tested (passed or not) + tracker.Tested(t, s.Source.Metadata) + }) + } } diff --git a/syft/formats/internal/testutils/directory_input.go b/syft/formats/internal/testutils/directory_input.go new file mode 100644 index 00000000000..232ac9c1275 --- /dev/null +++ b/syft/formats/internal/testutils/directory_input.go @@ -0,0 +1,204 @@ +package testutils + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cpe" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/linux" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" +) + +func DirectoryInput(t testing.TB, dir string) sbom.SBOM { + catalog := newDirectoryCatalog() + + path := filepath.Join(dir, "some", "path") + + require.NoError(t, os.MkdirAll(path, 0755)) + + src, err := source.NewFromDirectory( + source.DirectoryConfig{ + Path: path, + Base: dir, + }, + ) + require.NoError(t, err) + + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: catalog, + LinuxDistribution: &linux.Release{ + PrettyName: "debian", + Name: "debian", + ID: "debian", + IDLike: []string{"like!"}, + Version: "1.2.3", + VersionID: "1.2.3", + }, + }, + Source: src.Describe(), + Descriptor: sbom.Descriptor{ + Name: "syft", + Version: "v0.42.0-bogus", + // the application configuration should be persisted here, however, we do not want to import + // the application configuration in this package (it's reserved only for ingestion by the cmd package) + Configuration: map[string]string{ + "config-key": "config-value", + }, + }, + } +} + +func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM { + catalog := newDirectoryCatalogWithAuthorField() + + dir := t.TempDir() + path := filepath.Join(dir, "some", "path") + + require.NoError(t, os.MkdirAll(path, 0755)) + + src, err := source.NewFromDirectory( + source.DirectoryConfig{ + Path: path, + Base: dir, + }, + ) + require.NoError(t, err) + + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: catalog, + LinuxDistribution: &linux.Release{ + PrettyName: "debian", + Name: "debian", + ID: "debian", + IDLike: []string{"like!"}, + Version: "1.2.3", + VersionID: "1.2.3", + }, + }, + Source: src.Describe(), + Descriptor: sbom.Descriptor{ + Name: "syft", + Version: "v0.42.0-bogus", + // the application configuration should be persisted here, however, we do not want to import + // the application configuration in this package (it's reserved only for ingestion by the cmd package) + Configuration: map[string]string{ + "config-key": "config-value", + }, + }, + } +} + +func newDirectoryCatalog() *pkg.Collection { + catalog := pkg.NewCollection() + + // populate catalog with test data + catalog.Add(pkg.Package{ + Name: "package-1", + Version: "1.0.1", + Type: pkg.PythonPkg, + FoundBy: "the-cataloger-1", + Locations: file.NewLocationSet( + file.NewLocation("/some/path/pkg1"), + ), + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: pkg.NewLicenseSet( + pkg.NewLicense("MIT"), + ), + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + Files: []pkg.PythonFileRecord{ + { + Path: "/some/path/pkg1/dependencies/foo", + }, + }, + }, + PURL: "a-purl-2", // intentionally a bad pURL for test fixtures + CPEs: []cpe.CPE{ + cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), + }, + }) + catalog.Add(pkg.Package{ + Name: "package-2", + Version: "2.0.1", + Type: pkg.DebPkg, + FoundBy: "the-cataloger-2", + Locations: file.NewLocationSet( + file.NewLocation("/some/path/pkg1"), + ), + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, + PURL: "pkg:deb/debian/package-2@2.0.1", + CPEs: []cpe.CPE{ + cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), + }, + }) + + return catalog +} + +func newDirectoryCatalogWithAuthorField() *pkg.Collection { + catalog := pkg.NewCollection() + + // populate catalog with test data + catalog.Add(pkg.Package{ + Name: "package-1", + Version: "1.0.1", + Type: pkg.PythonPkg, + FoundBy: "the-cataloger-1", + Locations: file.NewLocationSet( + file.NewLocation("/some/path/pkg1"), + ), + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: pkg.NewLicenseSet( + pkg.NewLicense("MIT"), + ), + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + Author: "test-author", + Files: []pkg.PythonFileRecord{ + { + Path: "/some/path/pkg1/dependencies/foo", + }, + }, + }, + PURL: "a-purl-2", // intentionally a bad pURL for test fixtures + CPEs: []cpe.CPE{ + cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), + }, + }) + catalog.Add(pkg.Package{ + Name: "package-2", + Version: "2.0.1", + Type: pkg.DebPkg, + FoundBy: "the-cataloger-2", + Locations: file.NewLocationSet( + file.NewLocation("/some/path/pkg1"), + ), + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, + PURL: "pkg:deb/debian/package-2@2.0.1", + CPEs: []cpe.CPE{ + cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), + }, + }) + + return catalog +} diff --git a/syft/formats/internal/testutils/file_relationships.go b/syft/formats/internal/testutils/file_relationships.go new file mode 100644 index 00000000000..39ab8211fec --- /dev/null +++ b/syft/formats/internal/testutils/file_relationships.go @@ -0,0 +1,32 @@ +package testutils + +import ( + "math/rand" + "time" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/sbom" +) + +//nolint:gosec +func AddSampleFileRelationships(s *sbom.SBOM) { + catalog := s.Artifacts.Packages.Sorted() + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} + + files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"} + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) + + for _, f := range files { + meta := file.Metadata{} + coords := file.Coordinates{RealPath: f} + s.Artifacts.FileMetadata[coords] = meta + + s.Relationships = append(s.Relationships, artifact.Relationship{ + From: catalog[0], + To: coords, + Type: artifact.ContainsRelationship, + }) + } +} diff --git a/syft/formats/internal/testutils/image_input.go b/syft/formats/internal/testutils/image_input.go new file mode 100644 index 00000000000..e53bc5f7d10 --- /dev/null +++ b/syft/formats/internal/testutils/image_input.go @@ -0,0 +1,113 @@ +package testutils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/cpe" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/linux" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" +) + +func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM { + t.Helper() + catalog := pkg.NewCollection() + var cfg imageCfg + var img *image.Image + for _, opt := range options { + opt(&cfg) + } + + switch cfg.fromSnapshot { + case true: + img = imagetest.GetGoldenFixtureImage(t, testImage) + default: + img = imagetest.GetFixtureImage(t, "docker-archive", testImage) + } + + populateImageCatalog(catalog, img) + + // this is a hard coded value that is not given by the fixture helper and must be provided manually + img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" + + src, err := source.NewFromStereoscopeImageObject(img, "user-image-input", nil) + assert.NoError(t, err) + + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: catalog, + LinuxDistribution: &linux.Release{ + PrettyName: "debian", + Name: "debian", + ID: "debian", + IDLike: []string{"like!"}, + Version: "1.2.3", + VersionID: "1.2.3", + }, + }, + Source: src.Describe(), + Descriptor: sbom.Descriptor{ + Name: "syft", + Version: "v0.42.0-bogus", + // the application configuration should be persisted here, however, we do not want to import + // the application configuration in this package (it's reserved only for ingestion by the cmd package) + Configuration: map[string]string{ + "config-key": "config-value", + }, + }, + } +} + +func populateImageCatalog(catalog *pkg.Collection, img *image.Image) { + _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) + _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) + + // populate catalog with test data + catalog.Add(pkg.Package{ + Name: "package-1", + Version: "1.0.1", + Locations: file.NewLocationSet( + file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img), + ), + Type: pkg.PythonPkg, + FoundBy: "the-cataloger-1", + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: pkg.NewLicenseSet( + pkg.NewLicense("MIT"), + ), + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + }, + PURL: "a-purl-1", // intentionally a bad pURL for test fixtures + CPEs: []cpe.CPE{ + cpe.Must("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"), + }, + }) + catalog.Add(pkg.Package{ + Name: "package-2", + Version: "2.0.1", + Locations: file.NewLocationSet( + file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img), + ), + Type: pkg.DebPkg, + FoundBy: "the-cataloger-2", + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, + PURL: "pkg:deb/debian/package-2@2.0.1", + CPEs: []cpe.CPE{ + cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), + }, + }) +} diff --git a/syft/formats/internal/testutils/redactor.go b/syft/formats/internal/testutils/redactor.go new file mode 100644 index 00000000000..0c5505d1b00 --- /dev/null +++ b/syft/formats/internal/testutils/redactor.go @@ -0,0 +1,142 @@ +package testutils + +import ( + "bytes" + "regexp" +) + +var ( + _ Redactor = (*RedactorFn)(nil) + _ Redactor = (*PatternReplacement)(nil) + _ Redactor = (*ValueReplacement)(nil) + _ Redactor = (*Redactions)(nil) +) + +type Redactor interface { + Redact([]byte) []byte +} + +// Replace by function ////////////////////////////// + +type RedactorFn func([]byte) []byte + +func (r RedactorFn) Redact(b []byte) []byte { + return r(b) +} + +// Replace by regex ////////////////////////////// + +type PatternReplacement struct { + Search *regexp.Regexp + Replace string +} + +func NewPatternReplacement(r *regexp.Regexp) PatternReplacement { + return PatternReplacement{ + Search: r, + Replace: "redacted", + } +} + +func (p PatternReplacement) Redact(b []byte) []byte { + return p.Search.ReplaceAll(b, []byte(p.Replace)) +} + +// Replace by value ////////////////////////////// + +type ValueReplacement struct { + Search string + Replace string +} + +func NewValueReplacement(v string) ValueReplacement { + return ValueReplacement{ + Search: v, + Replace: "redacted", + } +} + +func (v ValueReplacement) Redact(b []byte) []byte { + return bytes.ReplaceAll(b, []byte(v.Search), []byte(v.Replace)) +} + +// Handle a collection of redactors ////////////////////////////// + +type Redactions struct { + redactors []Redactor +} + +func NewRedactions(redactors ...Redactor) *Redactions { + r := &Redactions{ + redactors: redactors, + } + + return r.WithFunctions(carriageRedactor) +} + +func (r *Redactions) WithPatternRedactors(values map[string]string) *Redactions { + for k, v := range values { + r.redactors = append(r.redactors, + PatternReplacement{ + Search: regexp.MustCompile(k), + Replace: v, + }, + ) + } + return r +} + +func (r *Redactions) WithValueRedactors(values map[string]string) *Redactions { + for k, v := range values { + r.redactors = append(r.redactors, + ValueReplacement{ + Search: k, + Replace: v, + }, + ) + } + return r +} + +func (r *Redactions) WithPatternsRedacted(values ...string) *Redactions { + for _, pattern := range values { + r.redactors = append(r.redactors, + NewPatternReplacement(regexp.MustCompile(pattern)), + ) + } + return r +} + +func (r *Redactions) WithValuesRedacted(values ...string) *Redactions { + for _, v := range values { + r.redactors = append(r.redactors, + NewValueReplacement(v), + ) + } + return r +} + +func (r *Redactions) WithFunctions(values ...func([]byte) []byte) *Redactions { + for _, fn := range values { + r.redactors = append(r.redactors, + RedactorFn(fn), + ) + } + return r +} + +func (r *Redactions) WithRedactors(rs ...Redactor) *Redactions { + r.redactors = append(r.redactors, rs...) + return r +} + +func (r Redactions) Redact(b []byte) []byte { + for _, redactor := range r.redactors { + b = redactor.Redact(b) + } + return b +} + +func carriageRedactor(s []byte) []byte { + return bytes.ReplaceAll(s, []byte("\r\n"), []byte("\n")) +} diff --git a/syft/formats/internal/testutils/snapshot.go b/syft/formats/internal/testutils/snapshot.go new file mode 100644 index 00000000000..7eae36594ed --- /dev/null +++ b/syft/formats/internal/testutils/snapshot.go @@ -0,0 +1,88 @@ +package testutils + +import ( + "bytes" + "testing" + + "github.com/sergi/go-diff/diffmatchpatch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/go-testutils" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/sbom" +) + +type imageCfg struct { + fromSnapshot bool +} + +type ImageOption func(cfg *imageCfg) + +func FromSnapshot() ImageOption { + return func(cfg *imageCfg) { + cfg.fromSnapshot = true + } +} + +type EncoderSnapshotTestConfig struct { + Subject sbom.SBOM + Format sbom.Format + UpdateSnapshot bool + PersistRedactionsInSnapshot bool + IsJSON bool + Redactor Redactor +} + +func AssertEncoderAgainstGoldenSnapshot(t *testing.T, cfg EncoderSnapshotTestConfig) { + t.Helper() + var buffer bytes.Buffer + + err := cfg.Format.Encode(&buffer, cfg.Subject) + assert.NoError(t, err) + actual := buffer.Bytes() + + if cfg.UpdateSnapshot && !cfg.PersistRedactionsInSnapshot { + // replace the expected snapshot contents with the current (unredacted) encoder contents + testutils.UpdateGoldenFileContents(t, actual) + return + } + + var expected []byte + if cfg.Redactor != nil { + actual = cfg.Redactor.Redact(actual) + expected = cfg.Redactor.Redact(testutils.GetGoldenFileContents(t)) + } else { + expected = testutils.GetGoldenFileContents(t) + } + + if cfg.UpdateSnapshot && cfg.PersistRedactionsInSnapshot { + // replace the expected snapshot contents with the current (redacted) encoder contents + testutils.UpdateGoldenFileContents(t, actual) + return + } + + if cfg.IsJSON { + require.JSONEq(t, string(expected), string(actual)) + } else if !bytes.Equal(expected, actual) { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(string(expected), string(actual), true) + t.Logf("len: %d\nexpected: %s", len(expected), expected) + t.Logf("len: %d\nactual: %s", len(actual), actual) + t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) + } +} + +type ImageSnapshotTestConfig struct { + Image string + UpdateImageSnapshot bool +} + +func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, imgCfg ImageSnapshotTestConfig, cfg EncoderSnapshotTestConfig) { + if imgCfg.UpdateImageSnapshot { + // grab the latest image contents and persist + imagetest.UpdateGoldenFixtureImage(t, imgCfg.Image) + } + + AssertEncoderAgainstGoldenSnapshot(t, cfg) +} diff --git a/syft/formats/internal/testutils/utils.go b/syft/formats/internal/testutils/utils.go deleted file mode 100644 index f9f4941d4e0..00000000000 --- a/syft/formats/internal/testutils/utils.go +++ /dev/null @@ -1,396 +0,0 @@ -package testutils - -import ( - "bytes" - "math/rand" - "strings" - "testing" - "time" - - "github.com/sergi/go-diff/diffmatchpatch" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/go-testutils" - "github.com/anchore/stereoscope/pkg/filetree" - "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/cpe" - "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/linux" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/sbom" - "github.com/anchore/syft/syft/source" -) - -type redactor func(s []byte) []byte - -type imageCfg struct { - fromSnapshot bool -} - -type ImageOption func(cfg *imageCfg) - -func FromSnapshot() ImageOption { - return func(cfg *imageCfg) { - cfg.fromSnapshot = true - } -} - -func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, format sbom.Format, sbom sbom.SBOM, testImage string, updateSnapshot bool, json bool, redactors ...redactor) { - var buffer bytes.Buffer - - // grab the latest image contents and persist - if updateSnapshot { - imagetest.UpdateGoldenFixtureImage(t, testImage) - } - - err := format.Encode(&buffer, sbom) - assert.NoError(t, err) - actual := buffer.Bytes() - - // replace the expected snapshot contents with the current encoder contents - if updateSnapshot { - testutils.UpdateGoldenFileContents(t, actual) - } - - actual = redact(actual, redactors...) - expected := redact(testutils.GetGoldenFileContents(t), redactors...) - - if json { - require.JSONEq(t, string(expected), string(actual)) - } else if !bytes.Equal(expected, actual) { - // assert that the golden file snapshot matches the actual contents - dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(expected), string(actual), true) - t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) - } -} - -func AssertEncoderAgainstGoldenSnapshot(t *testing.T, format sbom.Format, sbom sbom.SBOM, updateSnapshot bool, json bool, redactors ...redactor) { - var buffer bytes.Buffer - - err := format.Encode(&buffer, sbom) - assert.NoError(t, err) - actual := buffer.Bytes() - - // replace the expected snapshot contents with the current encoder contents - if updateSnapshot { - testutils.UpdateGoldenFileContents(t, actual) - } - - actual = redact(actual, redactors...) - expected := redact(testutils.GetGoldenFileContents(t), redactors...) - - if json { - require.JSONEq(t, string(expected), string(actual)) - } else if !bytes.Equal(expected, actual) { - dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(string(expected), string(actual), true) - t.Logf("len: %d\nexpected: %s", len(expected), expected) - t.Logf("len: %d\nactual: %s", len(actual), actual) - t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) - } -} - -func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM { - t.Helper() - catalog := pkg.NewCollection() - var cfg imageCfg - var img *image.Image - for _, opt := range options { - opt(&cfg) - } - - switch cfg.fromSnapshot { - case true: - img = imagetest.GetGoldenFixtureImage(t, testImage) - default: - img = imagetest.GetFixtureImage(t, "docker-archive", testImage) - } - - populateImageCatalog(catalog, img) - - // this is a hard coded value that is not given by the fixture helper and must be provided manually - img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" - - src, err := source.NewFromImage(img, "user-image-input") - assert.NoError(t, err) - - return sbom.SBOM{ - Artifacts: sbom.Artifacts{ - Packages: catalog, - LinuxDistribution: &linux.Release{ - PrettyName: "debian", - Name: "debian", - ID: "debian", - IDLike: []string{"like!"}, - Version: "1.2.3", - VersionID: "1.2.3", - }, - }, - Source: src.Metadata, - Descriptor: sbom.Descriptor{ - Name: "syft", - Version: "v0.42.0-bogus", - // the application configuration should be persisted here, however, we do not want to import - // the application configuration in this package (it's reserved only for ingestion by the cmd package) - Configuration: map[string]string{ - "config-key": "config-value", - }, - }, - } -} - -func carriageRedactor(s []byte) []byte { - msg := strings.ReplaceAll(string(s), "\r\n", "\n") - return []byte(msg) -} - -func populateImageCatalog(catalog *pkg.Collection, img *image.Image) { - _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) - _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) - - // populate catalog with test data - catalog.Add(pkg.Package{ - Name: "package-1", - Version: "1.0.1", - Locations: file.NewLocationSet( - file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img), - ), - Type: pkg.PythonPkg, - FoundBy: "the-cataloger-1", - Language: pkg.Python, - MetadataType: pkg.PythonPackageMetadataType, - Licenses: pkg.NewLicenseSet( - pkg.NewLicense("MIT"), - ), - Metadata: pkg.PythonPackageMetadata{ - Name: "package-1", - Version: "1.0.1", - }, - PURL: "a-purl-1", // intentionally a bad pURL for test fixtures - CPEs: []cpe.CPE{ - cpe.Must("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"), - }, - }) - catalog.Add(pkg.Package{ - Name: "package-2", - Version: "2.0.1", - Locations: file.NewLocationSet( - file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img), - ), - Type: pkg.DebPkg, - FoundBy: "the-cataloger-2", - MetadataType: pkg.DpkgMetadataType, - Metadata: pkg.DpkgMetadata{ - Package: "package-2", - Version: "2.0.1", - }, - PURL: "pkg:deb/debian/package-2@2.0.1", - CPEs: []cpe.CPE{ - cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), - }, - }) -} - -func DirectoryInput(t testing.TB) sbom.SBOM { - catalog := newDirectoryCatalog() - - src, err := source.NewFromDirectory("/some/path") - assert.NoError(t, err) - - return sbom.SBOM{ - Artifacts: sbom.Artifacts{ - Packages: catalog, - LinuxDistribution: &linux.Release{ - PrettyName: "debian", - Name: "debian", - ID: "debian", - IDLike: []string{"like!"}, - Version: "1.2.3", - VersionID: "1.2.3", - }, - }, - Source: src.Metadata, - Descriptor: sbom.Descriptor{ - Name: "syft", - Version: "v0.42.0-bogus", - // the application configuration should be persisted here, however, we do not want to import - // the application configuration in this package (it's reserved only for ingestion by the cmd package) - Configuration: map[string]string{ - "config-key": "config-value", - }, - }, - } -} - -func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM { - catalog := newDirectoryCatalogWithAuthorField() - - src, err := source.NewFromDirectory("/some/path") - assert.NoError(t, err) - - return sbom.SBOM{ - Artifacts: sbom.Artifacts{ - Packages: catalog, - LinuxDistribution: &linux.Release{ - PrettyName: "debian", - Name: "debian", - ID: "debian", - IDLike: []string{"like!"}, - Version: "1.2.3", - VersionID: "1.2.3", - }, - }, - Source: src.Metadata, - Descriptor: sbom.Descriptor{ - Name: "syft", - Version: "v0.42.0-bogus", - // the application configuration should be persisted here, however, we do not want to import - // the application configuration in this package (it's reserved only for ingestion by the cmd package) - Configuration: map[string]string{ - "config-key": "config-value", - }, - }, - } -} - -func newDirectoryCatalog() *pkg.Collection { - catalog := pkg.NewCollection() - - // populate catalog with test data - catalog.Add(pkg.Package{ - Name: "package-1", - Version: "1.0.1", - Type: pkg.PythonPkg, - FoundBy: "the-cataloger-1", - Locations: file.NewLocationSet( - file.NewLocation("/some/path/pkg1"), - ), - Language: pkg.Python, - MetadataType: pkg.PythonPackageMetadataType, - Licenses: pkg.NewLicenseSet( - pkg.NewLicense("MIT"), - ), - Metadata: pkg.PythonPackageMetadata{ - Name: "package-1", - Version: "1.0.1", - Files: []pkg.PythonFileRecord{ - { - Path: "/some/path/pkg1/dependencies/foo", - }, - }, - }, - PURL: "a-purl-2", // intentionally a bad pURL for test fixtures - CPEs: []cpe.CPE{ - cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), - }, - }) - catalog.Add(pkg.Package{ - Name: "package-2", - Version: "2.0.1", - Type: pkg.DebPkg, - FoundBy: "the-cataloger-2", - Locations: file.NewLocationSet( - file.NewLocation("/some/path/pkg1"), - ), - MetadataType: pkg.DpkgMetadataType, - Metadata: pkg.DpkgMetadata{ - Package: "package-2", - Version: "2.0.1", - }, - PURL: "pkg:deb/debian/package-2@2.0.1", - CPEs: []cpe.CPE{ - cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), - }, - }) - - return catalog -} - -func newDirectoryCatalogWithAuthorField() *pkg.Collection { - catalog := pkg.NewCollection() - - // populate catalog with test data - catalog.Add(pkg.Package{ - Name: "package-1", - Version: "1.0.1", - Type: pkg.PythonPkg, - FoundBy: "the-cataloger-1", - Locations: file.NewLocationSet( - file.NewLocation("/some/path/pkg1"), - ), - Language: pkg.Python, - MetadataType: pkg.PythonPackageMetadataType, - Licenses: pkg.NewLicenseSet( - pkg.NewLicense("MIT"), - ), - Metadata: pkg.PythonPackageMetadata{ - Name: "package-1", - Version: "1.0.1", - Author: "test-author", - Files: []pkg.PythonFileRecord{ - { - Path: "/some/path/pkg1/dependencies/foo", - }, - }, - }, - PURL: "a-purl-2", // intentionally a bad pURL for test fixtures - CPEs: []cpe.CPE{ - cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), - }, - }) - catalog.Add(pkg.Package{ - Name: "package-2", - Version: "2.0.1", - Type: pkg.DebPkg, - FoundBy: "the-cataloger-2", - Locations: file.NewLocationSet( - file.NewLocation("/some/path/pkg1"), - ), - MetadataType: pkg.DpkgMetadataType, - Metadata: pkg.DpkgMetadata{ - Package: "package-2", - Version: "2.0.1", - }, - PURL: "pkg:deb/debian/package-2@2.0.1", - CPEs: []cpe.CPE{ - cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"), - }, - }) - - return catalog -} - -//nolint:gosec -func AddSampleFileRelationships(s *sbom.SBOM) { - catalog := s.Artifacts.Packages.Sorted() - s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} - - files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"} - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) - - for _, f := range files { - meta := file.Metadata{} - coords := file.Coordinates{RealPath: f} - s.Artifacts.FileMetadata[coords] = meta - - s.Relationships = append(s.Relationships, artifact.Relationship{ - From: catalog[0], - To: coords, - Type: artifact.ContainsRelationship, - }) - } -} - -// remove dynamic values, which should be tested independently -func redact(b []byte, redactors ...redactor) []byte { - redactors = append(redactors, carriageRedactor) - for _, r := range redactors { - b = r(b) - } - return b -} diff --git a/syft/formats/spdxjson/encoder_test.go b/syft/formats/spdxjson/encoder_test.go index f33a87708b3..b684777aa35 100644 --- a/syft/formats/spdxjson/encoder_test.go +++ b/syft/formats/spdxjson/encoder_test.go @@ -2,57 +2,81 @@ package spdxjson import ( "flag" - "regexp" "testing" "github.com/anchore/syft/syft/formats/internal/testutils" ) -var updateSpdxJson = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json encoders") +var updateSnapshot = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json encoders") +var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing") func TestSPDXJSONDirectoryEncoder(t *testing.T) { + dir := t.TempDir() testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateSpdxJson, - true, - spdxJsonRedactor, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, dir), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(dir), + }, ) } func TestSPDXJSONImageEncoder(t *testing.T) { testImage := "image-simple" testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - testutils.ImageInput(t, testImage, testutils.FromSnapshot()), - testImage, - *updateSpdxJson, - true, - spdxJsonRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(), + }, ) } func TestSPDXRelationshipOrder(t *testing.T) { testImage := "image-simple" + s := testutils.ImageInput(t, testImage, testutils.FromSnapshot()) testutils.AddSampleFileRelationships(&s) + testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - s, - testImage, - *updateSpdxJson, - true, - spdxJsonRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: s, + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(), + }, ) } -func spdxJsonRedactor(s []byte) []byte { - // each SBOM reports the time it was generated, which is not useful during snapshot testing - s = regexp.MustCompile(`"created":\s+"[^"]*"`).ReplaceAll(s, []byte(`"created":""`)) +func redactor(values ...string) testutils.Redactor { + return testutils.NewRedactions(). + WithValuesRedacted(values...). + WithPatternRedactors( + map[string]string{ + // each SBOM reports the time it was generated, which is not useful during snapshot testing + `"created":\s+"[^"]*"`: `"created":"redacted"`, - // each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing - s = regexp.MustCompile(`"documentNamespace":\s+"[^"]*"`).ReplaceAll(s, []byte(`"documentNamespace":""`)) + // each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing + `"documentNamespace":\s+"[^"]*"`: `"documentNamespace":"redacted"`, - // the license list will be updated periodically, the value here should not be directly tested in snapshot tests - return regexp.MustCompile(`"licenseListVersion":\s+"[^"]*"`).ReplaceAll(s, []byte(`"licenseListVersion":""`)) + // the license list will be updated periodically, the value here should not be directly tested in snapshot tests + `"licenseListVersion":\s+"[^"]*"`: `"licenseListVersion":"redacted"`, + }, + ) } diff --git a/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONDirectoryEncoder.golden b/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONDirectoryEncoder.golden index e4ba2ccd9cd..bfcc72b9681 100644 --- a/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONDirectoryEncoder.golden +++ b/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONDirectoryEncoder.golden @@ -2,15 +2,15 @@ "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", - "name": "/some/path", - "documentNamespace": "https://anchore.com/syft/dir/some/path-303fccb4-22d1-4039-9061-553bc875f086", + "name": "some/path", + "documentNamespace":"redacted", "creationInfo": { - "licenseListVersion": "3.20", + "licenseListVersion":"redacted", "creators": [ "Organization: Anchore, Inc", "Tool: syft-v0.42.0-bogus" ], - "created": "2023-06-05T18:49:13Z" + "created":"redacted" }, "packages": [ { diff --git a/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONImageEncoder.golden b/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONImageEncoder.golden index 3c49c336271..3caf98e8548 100644 --- a/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONImageEncoder.golden +++ b/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONImageEncoder.golden @@ -3,14 +3,14 @@ "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": "user-image-input", - "documentNamespace": "https://anchore.com/syft/image/user-image-input-5b9aac79-334c-4d6a-b2e6-95a819c1d45a", + "documentNamespace":"redacted", "creationInfo": { - "licenseListVersion": "3.20", + "licenseListVersion":"redacted", "creators": [ "Organization: Anchore, Inc", "Tool: syft-v0.42.0-bogus" ], - "created": "2023-06-05T18:49:14Z" + "created":"redacted" }, "packages": [ { diff --git a/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden b/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden index 0dba256d9b7..acdb202ee3a 100644 --- a/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden +++ b/syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden @@ -3,14 +3,14 @@ "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": "user-image-input", - "documentNamespace": "https://anchore.com/syft/image/user-image-input-2a1392ab-7eb5-4f2a-86f6-777aef3232e1", + "documentNamespace":"redacted", "creationInfo": { - "licenseListVersion": "3.20", + "licenseListVersion":"redacted", "creators": [ "Organization: Anchore, Inc", "Tool: syft-v0.42.0-bogus" ], - "created": "2023-06-05T18:49:14Z" + "created":"redacted" }, "packages": [ { diff --git a/syft/formats/spdxtagvalue/encoder_test.go b/syft/formats/spdxtagvalue/encoder_test.go index 5d95f639799..f414d88faf8 100644 --- a/syft/formats/spdxtagvalue/encoder_test.go +++ b/syft/formats/spdxtagvalue/encoder_test.go @@ -2,7 +2,6 @@ package spdxtagvalue import ( "flag" - "regexp" "testing" "github.com/anchore/syft/syft/formats/internal/testutils" @@ -11,28 +10,38 @@ import ( "github.com/anchore/syft/syft/source" ) -var updateSpdxTagValue = flag.Bool("update-spdx-tv", false, "update the *.golden files for spdx-tv encoders") +var updateSnapshot = flag.Bool("update-spdx-tv", false, "update the *.golden files for spdx-tv encoders") +var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing") func TestSPDXTagValueDirectoryEncoder(t *testing.T) { - + dir := t.TempDir() testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateSpdxTagValue, - false, - spdxTagValueRedactor, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, dir), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(dir), + }, ) } func TestSPDXTagValueImageEncoder(t *testing.T) { testImage := "image-simple" testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - testutils.ImageInput(t, testImage, testutils.FromSnapshot()), - testImage, - *updateSpdxTagValue, - false, - spdxTagValueRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(), + }, ) } @@ -45,28 +54,33 @@ func TestSPDXJSONSPDXIDs(t *testing.T) { p.SetID() pkgs = append(pkgs, p) } - testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - sbom.SBOM{ - Artifacts: sbom.Artifacts{ - Packages: pkg.NewCollection(pkgs...), - }, - Relationships: nil, - Source: source.Metadata{ - Scheme: source.DirectoryScheme, - Path: "foobar/baz", // in this case, foobar is used as the spdx docment name - }, - Descriptor: sbom.Descriptor{ - Name: "syft", - Version: "v0.42.0-bogus", - Configuration: map[string]string{ - "config-key": "config-value", - }, + + s := sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(pkgs...), + }, + Relationships: nil, + Source: source.Description{ + Metadata: source.DirectorySourceMetadata{Path: "foobar/baz"}, // in this case, foobar is used as the spdx docment name + }, + Descriptor: sbom.Descriptor{ + Name: "syft", + Version: "v0.42.0-bogus", + Configuration: map[string]string{ + "config-key": "config-value", }, }, - *updateSpdxTagValue, - false, - spdxTagValueRedactor, + } + + testutils.AssertEncoderAgainstGoldenSnapshot(t, + testutils.EncoderSnapshotTestConfig{ + Subject: s, + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(), + }, ) } @@ -74,23 +88,36 @@ func TestSPDXRelationshipOrder(t *testing.T) { testImage := "image-simple" s := testutils.ImageInput(t, testImage, testutils.FromSnapshot()) testutils.AddSampleFileRelationships(&s) + testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - s, - testImage, - *updateSpdxTagValue, - false, - spdxTagValueRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: s, + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(), + }, ) } -func spdxTagValueRedactor(s []byte) []byte { - // each SBOM reports the time it was generated, which is not useful during snapshot testing - s = regexp.MustCompile(`Created: .*`).ReplaceAll(s, []byte("redacted")) +func redactor(values ...string) testutils.Redactor { + return testutils.NewRedactions(). + WithValuesRedacted(values...). + WithPatternRedactors( + map[string]string{ + // each SBOM reports the time it was generated, which is not useful during snapshot testing + `Created: .*`: "Created: redacted", - // each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing - s = regexp.MustCompile(`DocumentNamespace: https://anchore.com/syft/.*`).ReplaceAll(s, []byte("redacted")) + // each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing + `DocumentNamespace: https://anchore.com/syft/.*`: "DocumentNamespace: redacted", - // the license list will be updated periodically, the value here should not be directly tested in snapshot tests - return regexp.MustCompile(`LicenseListVersion: .*`).ReplaceAll(s, []byte("redacted")) + // the license list will be updated periodically, the value here should not be directly tested in snapshot tests + `LicenseListVersion: .*`: "LicenseListVersion: redacted", + }, + ) } diff --git a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXJSONSPDXIDs.golden b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXJSONSPDXIDs.golden index ca1775a5de1..7f6320ebad9 100644 --- a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXJSONSPDXIDs.golden +++ b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXJSONSPDXIDs.golden @@ -2,11 +2,11 @@ SPDXVersion: SPDX-2.3 DataLicense: CC0-1.0 SPDXID: SPDXRef-DOCUMENT DocumentName: foobar/baz -DocumentNamespace: https://anchore.com/syft/dir/foobar/baz-1813dede-1ac5-4c44-a640-4c56e213d575 -LicenseListVersion: 3.20 +DocumentNamespace: redacted +LicenseListVersion: redacted Creator: Organization: Anchore, Inc Creator: Tool: syft-v0.42.0-bogus -Created: 2023-05-09T17:11:49Z +Created: redacted ##### Package: @at-sign diff --git a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden index 339b17c2c1c..26e6e9a67ab 100644 --- a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden +++ b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden @@ -2,11 +2,11 @@ SPDXVersion: SPDX-2.3 DataLicense: CC0-1.0 SPDXID: SPDXRef-DOCUMENT DocumentName: user-image-input -DocumentNamespace: https://anchore.com/syft/image/user-image-input-96ea886a-3297-4847-b211-6da405ff1f8f -LicenseListVersion: 3.20 +DocumentNamespace: redacted +LicenseListVersion: redacted Creator: Organization: Anchore, Inc Creator: Tool: syft-v0.42.0-bogus -Created: 2023-05-09T17:11:49Z +Created: redacted ##### Unpackaged files diff --git a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden index 818d62e7d79..52b63c984ae 100644 --- a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden +++ b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden @@ -1,12 +1,12 @@ SPDXVersion: SPDX-2.3 DataLicense: CC0-1.0 SPDXID: SPDXRef-DOCUMENT -DocumentName: /some/path -DocumentNamespace: https://anchore.com/syft/dir/some/path-f7bdb1ee-7fef-48e7-a386-6ee3836d4a28 -LicenseListVersion: 3.20 +DocumentName: some/path +DocumentNamespace: redacted +LicenseListVersion: redacted Creator: Organization: Anchore, Inc Creator: Tool: syft-v0.42.0-bogus -Created: 2023-05-09T17:11:49Z +Created: redacted ##### Package: package-2 diff --git a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden index 867e8e0396f..65b11110712 100644 --- a/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden +++ b/syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden @@ -2,11 +2,11 @@ SPDXVersion: SPDX-2.3 DataLicense: CC0-1.0 SPDXID: SPDXRef-DOCUMENT DocumentName: user-image-input -DocumentNamespace: https://anchore.com/syft/image/user-image-input-44d44a85-2207-4b51-bd73-d0c7b080f6d3 -LicenseListVersion: 3.20 +DocumentNamespace: redacted +LicenseListVersion: redacted Creator: Organization: Anchore, Inc Creator: Tool: syft-v0.42.0-bogus -Created: 2023-05-09T17:11:49Z +Created: redacted ##### Package: package-2 diff --git a/syft/formats/syftjson/encoder_test.go b/syft/formats/syftjson/encoder_test.go index 231333bb869..8ad7ababda2 100644 --- a/syft/formats/syftjson/encoder_test.go +++ b/syft/formats/syftjson/encoder_test.go @@ -2,7 +2,6 @@ package syftjson import ( "flag" - "regexp" "testing" stereoFile "github.com/anchore/stereoscope/pkg/file" @@ -16,36 +15,41 @@ import ( "github.com/anchore/syft/syft/source" ) -var updateJson = flag.Bool("update-json", false, "update the *.golden files for json encoders") +var updateSnapshot = flag.Bool("update-json", false, "update the *.golden files for json encoders") +var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing") func TestDirectoryEncoder(t *testing.T) { + dir := t.TempDir() testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateJson, - true, - schemaVersionRedactor, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, dir), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(dir), + }, ) } func TestImageEncoder(t *testing.T) { testImage := "image-simple" testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - testutils.ImageInput(t, testImage, testutils.FromSnapshot()), - testImage, - *updateJson, - true, - schemaVersionRedactor, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(), + }, ) } -func schemaVersionRedactor(s []byte) []byte { - pattern := regexp.MustCompile(`,?\s*"schema":\s*\{[^}]*}`) - out := pattern.ReplaceAll(s, []byte("")) - return out -} - func TestEncodeFullJSONDocument(t *testing.T) { catalog := pkg.NewCollection() @@ -176,10 +180,9 @@ func TestEncodeFullJSONDocument(t *testing.T) { }, }, }, - Source: source.Metadata{ - ID: "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", - Scheme: source.ImageScheme, - ImageMetadata: source.ImageMetadata{ + Source: source.Description{ + ID: "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "user-image-input", ID: "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", ManifestDigest: "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", @@ -188,7 +191,7 @@ func TestEncodeFullJSONDocument(t *testing.T) { "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b", }, Size: 38, - Layers: []source.LayerMetadata{ + Layers: []source.StereoscopeLayerMetadata{ { MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", Digest: "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b", @@ -217,10 +220,24 @@ func TestEncodeFullJSONDocument(t *testing.T) { } testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - s, - *updateJson, - true, - schemaVersionRedactor, + testutils.EncoderSnapshotTestConfig{ + Subject: s, + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: true, + Redactor: redactor(), + }, ) } + +func redactor(values ...string) testutils.Redactor { + return testutils.NewRedactions(). + WithValuesRedacted(values...). + WithPatternRedactors( + map[string]string{ + // remove schema version (don't even show the key or value) + `,?\s*"schema":\s*\{[^}]*}`: "", + }, + ) +} diff --git a/syft/formats/syftjson/model/package.go b/syft/formats/syftjson/model/package.go index fccf04c0bda..d4a819f2ac1 100644 --- a/syft/formats/syftjson/model/package.go +++ b/syft/formats/syftjson/model/package.go @@ -102,7 +102,7 @@ func (p *Package) UnmarshalJSON(b []byte) error { return err } - err := unpackMetadata(p, unpacker) + err := unpackPkgMetadata(p, unpacker) if errors.Is(err, errUnknownMetadataType) { log.Warnf("unknown package metadata type=%q for packageID=%q", p.MetadataType, p.ID) return nil @@ -111,7 +111,7 @@ func (p *Package) UnmarshalJSON(b []byte) error { return err } -func unpackMetadata(p *Package, unpacker packageMetadataUnpacker) error { +func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error { p.MetadataType = pkg.CleanMetadataType(unpacker.MetadataType) typ, ok := pkg.MetadataTypeByName[p.MetadataType] diff --git a/syft/formats/syftjson/model/package_test.go b/syft/formats/syftjson/model/package_test.go index 8027764d85e..aec5d1ec8e3 100644 --- a/syft/formats/syftjson/model/package_test.go +++ b/syft/formats/syftjson/model/package_test.go @@ -362,7 +362,7 @@ func Test_unpackMetadata(t *testing.T) { var unpacker packageMetadataUnpacker require.NoError(t, json.Unmarshal(test.packageData, &unpacker)) - err := unpackMetadata(p, unpacker) + err := unpackPkgMetadata(p, unpacker) assert.Equal(t, test.metadataType, p.MetadataType) test.wantErr(t, err) diff --git a/syft/formats/syftjson/model/source.go b/syft/formats/syftjson/model/source.go index d546cf11c24..b0e843ef20d 100644 --- a/syft/formats/syftjson/model/source.go +++ b/syft/formats/syftjson/model/source.go @@ -3,53 +3,114 @@ package model import ( "encoding/json" "fmt" + "reflect" "strconv" + "strings" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/source" ) // Source object represents the thing that was cataloged type Source struct { - ID string `json:"id"` - Type string `json:"type"` - Target interface{} `json:"target"` + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Metadata interface{} `json:"metadata"` } // sourceUnpacker is used to unmarshal Source objects type sourceUnpacker struct { - ID string `json:"id,omitempty"` - Type string `json:"type"` - Target json.RawMessage `json:"target"` + ID string `json:"id,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Metadata json.RawMessage `json:"metadata"` + Target json.RawMessage `json:"target"` // pre-v9 schema support } // UnmarshalJSON populates a source object from JSON bytes. func (s *Source) UnmarshalJSON(b []byte) error { var unpacker sourceUnpacker - if err := json.Unmarshal(b, &unpacker); err != nil { + err := json.Unmarshal(b, &unpacker) + if err != nil { return err } + s.Name = unpacker.Name + s.Version = unpacker.Version s.Type = unpacker.Type s.ID = unpacker.ID - switch s.Type { - case "directory", "file": - if target, err := strconv.Unquote(string(unpacker.Target)); err == nil { - s.Target = target - } else { - s.Target = string(unpacker.Target[:]) + if len(unpacker.Target) > 0 { + s.Type = cleanPreSchemaV9MetadataType(s.Type) + s.Metadata, err = extractPreSchemaV9Metadata(s.Type, unpacker.Target) + if err != nil { + return fmt.Errorf("unable to extract pre-schema-v9 source metadata: %w", err) } + return nil + } - case "image": - var payload source.ImageMetadata - if err := json.Unmarshal(unpacker.Target, &payload); err != nil { + return unpackSrcMetadata(s, unpacker) +} + +func unpackSrcMetadata(s *Source, unpacker sourceUnpacker) error { + rt := sourcemetadata.ReflectTypeFromJSONName(s.Type) + if rt == nil { + return fmt.Errorf("unable to find source metadata type=%q", s.Type) + } + + val := reflect.New(rt).Interface() + if len(unpacker.Metadata) > 0 { + if err := json.Unmarshal(unpacker.Metadata, val); err != nil { return err } - s.Target = payload - - default: - return fmt.Errorf("unsupported package metadata type: %+v", s.Type) } + s.Metadata = reflect.ValueOf(val).Elem().Interface() + return nil } + +func cleanPreSchemaV9MetadataType(t string) string { + t = strings.ToLower(t) + if t == "dir" { + return "directory" + } + return t +} + +func extractPreSchemaV9Metadata(t string, target []byte) (interface{}, error) { + switch t { + case "directory", "dir": + cleanTarget, err := strconv.Unquote(string(target)) + if err != nil { + cleanTarget = string(target) + } + + return source.DirectorySourceMetadata{ + Path: cleanTarget, + }, nil + + case "file": + cleanTarget, err := strconv.Unquote(string(target)) + if err != nil { + cleanTarget = string(target) + } + + return source.FileSourceMetadata{ + Path: cleanTarget, + }, nil + + case "image": + var payload source.StereoscopeImageSourceMetadata + if err := json.Unmarshal(target, &payload); err != nil { + return nil, err + } + return payload, nil + + default: + return nil, fmt.Errorf("unsupported package metadata type: %+v", t) + } +} diff --git a/syft/formats/syftjson/model/source_test.go b/syft/formats/syftjson/model/source_test.go index ee7969d7490..e9118f09099 100644 --- a/syft/formats/syftjson/model/source_test.go +++ b/syft/formats/syftjson/model/source_test.go @@ -6,17 +6,194 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/source" ) func TestSource_UnmarshalJSON(t *testing.T) { + tracker := sourcemetadata.NewCompletionTester(t) + + cases := []struct { + name string + input []byte + expected *Source + wantErr require.ErrorAssertionFunc + }{ + { + name: "directory", + input: []byte(`{ + "id": "foobar", + "type": "directory", + "metadata": {"path": "/var/lib/foo", "base":"/nope"} + }`), + expected: &Source{ + ID: "foobar", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "/var/lib/foo", + //Base: "/nope", // note: should be ignored entirely + }, + }, + }, + { + name: "image", + input: []byte(`{ + "id": "foobar", + "type": "image", + "metadata": { + "userInput": "alpine:3.10", + "imageID": "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a", + "manifestDigest": "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [], + "imageSize": 5576169, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635", + "size": 5576169 + } + ], + "manifest": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMTQ3MiwKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZTdiMzAwYWVlOWY5YmYzNDMzZDMyYmM5MzA1YmZkZDIyMTgzYmViNTlkOTMzYjQ4ZDc3YWI1NmJhNTNhMTk3YSIKICAgfSwKICAgImxheWVycyI6IFsKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDI3OTgzMzgsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjM5NmMzMTgzNzExNmFjMjkwNDU4YWZjYjkyOGY2OGI2Y2MxYzdiZGQ2OTYzZmM3MmY1MmYzNjVhMmE4OWMxYjUiCiAgICAgIH0KICAgXQp9", + "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9iaW4vc2giXSwiSW1hZ2UiOiJzaGEyNTY6ZWIyMDgwYzQ1NWU5NGMyMmFlMzViM2FlZjllMDc4YzQ5MmEwMDc5NTQxMmUwMjZlNGQ2YjQxZWY2NGJjN2RkOCIsIlZvbHVtZXMiOm51bGwsIldvcmtpbmdEaXIiOiIiLCJFbnRyeXBvaW50IjpudWxsLCJPbkJ1aWxkIjpudWxsLCJMYWJlbHMiOm51bGx9LCJjb250YWluZXIiOiJmZGI3ZTgwZTMzMzllOGQwNTk5MjgyZTYwNmM5MDdhYTU4ODFlZTRjNjY4YTY4MTM2MTE5ZTZkZmFjNmNlM2E0IiwiY29udGFpbmVyX2NvbmZpZyI6eyJIb3N0bmFtZSI6ImZkYjdlODBlMzMzOSIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9iaW4vc2giLCItYyIsIiMobm9wKSAiLCJDTUQgW1wiL2Jpbi9zaFwiXSJdLCJJbWFnZSI6InNoYTI1NjplYjIwODBjNDU1ZTk0YzIyYWUzNWIzYWVmOWUwNzhjNDkyYTAwNzk1NDEyZTAyNmU0ZDZiNDFlZjY0YmM3ZGQ4IiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6e319LCJjcmVhdGVkIjoiMjAyMS0wNC0xNFQxOToyMDowNS4zMzgzOTc3NjFaIiwiZG9ja2VyX3ZlcnNpb24iOiIxOS4wMy4xMiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIxLTA0LTE0VDE5OjIwOjA0Ljk4NzIxOTEyNFoiLCJjcmVhdGVkX2J5IjoiL2Jpbi9zaCAtYyAjKG5vcCkgQUREIGZpbGU6YzUzNzdlYWE5MjZiZjQxMmRkOGQ0YTA4YjBhMWYyMzk5Y2ZkNzA4NzQzNTMzYjBhYTAzYjUzZDE0Y2I0YmI0ZSBpbiAvICJ9LHsiY3JlYXRlZCI6IjIwMjEtMDQtMTRUMTk6MjA6MDUuMzM4Mzk3NzYxWiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSAgQ01EIFtcIi9iaW4vc2hcIl0iLCJlbXB0eV9sYXllciI6dHJ1ZX1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6OWZiM2FhMmY4YjgwMjNhNGJlYmJmOTJhYTU2N2NhZjg4ZTM4ZTk2OWFkYTlmMGFjMTI2NDNiMjg0NzM5MTYzNSJdfX0=", + "repoDigests": [ + "index.docker.io/library/alpine@sha256:451eee8bedcb2f029756dc3e9d73bab0e7943c1ac55cff3a4861c52a0fdd3e98" + ] + } + }`), + expected: &Source{ + ID: "foobar", + Type: "image", + Metadata: source.StereoscopeImageSourceMetadata{ + UserInput: "alpine:3.10", + ID: "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a", + ManifestDigest: "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c", + MediaType: "application/vnd.docker.distribution.manifest.v2+json", + Tags: []string{}, + Size: 5576169, + Layers: []source.StereoscopeLayerMetadata{ + { + MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + Digest: "sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635", + Size: 5576169, + }, + }, + RawManifest: []byte(`{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1472, + "digest": "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 2798338, + "digest": "sha256:396c31837116ac290458afcb928f68b6cc1c7bdd6963fc72f52f365a2a89c1b5" + } + ] +}`), + RawConfig: []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"Image":"sha256:eb2080c455e94c22ae35b3aef9e078c492a00795412e026e4d6b41ef64bc7dd8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fdb7e80e3339e8d0599282e606c907aa5881ee4c668a68136119e6dfac6ce3a4","container_config":{"Hostname":"fdb7e80e3339","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/bin/sh\"]"],"Image":"sha256:eb2080c455e94c22ae35b3aef9e078c492a00795412e026e4d6b41ef64bc7dd8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":{}},"created":"2021-04-14T19:20:05.338397761Z","docker_version":"19.03.12","history":[{"created":"2021-04-14T19:20:04.987219124Z","created_by":"/bin/sh -c #(nop) ADD file:c5377eaa926bf412dd8d4a08b0a1f2399cfd708743533b0aa03b53d14cb4bb4e in / "},{"created":"2021-04-14T19:20:05.338397761Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635"]}}`), + RepoDigests: []string{ + "index.docker." + + "io/library/alpine@sha256:451eee8bedcb2f029756dc3e9d73bab0e7943c1ac55cff3a4861c52a0fdd3e98", + }, + }, + }, + }, + { + name: "file", + input: []byte(`{ + "id": "foobar", + "type": "file", + "metadata": { + "path": "/var/lib/foo/go.mod", + "mimeType": "text/plain", + "digests": [ + { + "algorithm": "sha256", + "value": "e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a" + } + ] + } + }`), + expected: &Source{ + ID: "foobar", + Type: "file", + Metadata: source.FileSourceMetadata{ + Path: "/var/lib/foo/go.mod", + Digests: []file.Digest{ + { + Algorithm: "sha256", + Value: "e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a", + }, + }, + MIMEType: "text/plain", + }, + }, + }, + { + name: "unknown source type", + input: []byte(`{ + "id": "foobar", + "type": "unknown-thing", + "target":"/var/lib/foo" + }`), + expected: &Source{ + ID: "foobar", + Type: "unknown-thing", + }, + wantErr: require.Error, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + var src Source + err := json.Unmarshal(tt.input, &src) + + tt.wantErr(t, err) + + if diff := cmp.Diff(tt.expected, &src); diff != "" { + t.Errorf("unexpected result from Source unmarshaling (-want +got)\n%s", diff) + } + + tracker.Tested(t, tt.expected.Metadata) + }) + } +} + +func TestSource_UnmarshalJSON_PreSchemaV9(t *testing.T) { cases := []struct { name string input []byte expectedSource *Source errAssertion assert.ErrorAssertionFunc }{ + { + name: "abbreviated directory", + input: []byte(`{ + "id": "foobar", + "type": "dir", + "target":"/var/lib/foo" + }`), + expectedSource: &Source{ + ID: "foobar", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "/var/lib/foo", + }, + }, + errAssertion: assert.NoError, + }, { name: "directory", input: []byte(`{ @@ -25,9 +202,11 @@ func TestSource_UnmarshalJSON(t *testing.T) { "target":"/var/lib/foo" }`), expectedSource: &Source{ - ID: "foobar", - Type: "directory", - Target: "/var/lib/foo", + ID: "foobar", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "/var/lib/foo", + }, }, errAssertion: assert.NoError, }, @@ -60,14 +239,14 @@ func TestSource_UnmarshalJSON(t *testing.T) { expectedSource: &Source{ ID: "foobar", Type: "image", - Target: source.ImageMetadata{ + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "alpine:3.10", ID: "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a", ManifestDigest: "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c", MediaType: "application/vnd.docker.distribution.manifest.v2+json", Tags: []string{}, Size: 5576169, - Layers: []source.LayerMetadata{ + Layers: []source.StereoscopeLayerMetadata{ { MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", Digest: "sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635", @@ -107,9 +286,11 @@ func TestSource_UnmarshalJSON(t *testing.T) { "target":"/var/lib/foo/go.mod" }`), expectedSource: &Source{ - ID: "foobar", - Type: "file", - Target: "/var/lib/foo/go.mod", + ID: "foobar", + Type: "file", + Metadata: source.FileSourceMetadata{ + Path: "/var/lib/foo/go.mod", + }, }, errAssertion: assert.NoError, }, @@ -130,12 +311,12 @@ func TestSource_UnmarshalJSON(t *testing.T) { for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { - source := new(Source) + src := new(Source) - err := json.Unmarshal(testCase.input, source) + err := json.Unmarshal(testCase.input, src) testCase.errAssertion(t, err) - if diff := cmp.Diff(testCase.expectedSource, source); diff != "" { + if diff := cmp.Diff(testCase.expectedSource, src); diff != "" { t.Errorf("unexpected result from Source unmarshaling (-want +got)\n%s", diff) } }) diff --git a/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden b/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden index a162e983559..61bb2efe42d 100644 --- a/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden +++ b/syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden @@ -72,9 +72,13 @@ ], "artifactRelationships": [], "source": { - "id": "eda6cf0b63f1a1d2eaf7792a2a98c832c21a18e6992bcebffe6381781cc85cbc", + "id": "d1563248892cd59af469f406eee907c76fa4f9041f5410d45b93aef903bc4216", + "name": "some/path", + "version": "", "type": "directory", - "target": "/some/path" + "metadata": { + "path": "redacted/some/path" + } }, "distro": { "prettyName": "debian", @@ -92,9 +96,5 @@ "configuration": { "config-key": "config-value" } - }, - "schema": { - "version": "8.0.0", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-8.0.0.json" } } diff --git a/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden b/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden index daca7eb5ed6..f83789be5ec 100644 --- a/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden +++ b/syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden @@ -149,8 +149,10 @@ ], "source": { "id": "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", + "name": "", + "version": "", "type": "image", - "target": { + "metadata": { "userInput": "user-image-input", "imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", @@ -192,9 +194,5 @@ "configuration": { "config-key": "config-value" } - }, - "schema": { - "version": "8.0.0", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-8.0.0.json" } } diff --git a/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden b/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden index c0a2f758d36..518a90ab50d 100644 --- a/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden +++ b/syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden @@ -70,8 +70,10 @@ "artifactRelationships": [], "source": { "id": "c8ac88bbaf3d1c036f6a1d601c3d52bafbf05571c97d68322e7cb3a7ecaa304f", + "name": "user-image-input", + "version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", "type": "image", - "target": { + "metadata": { "userInput": "user-image-input", "imageID": "sha256:a3c61dc134d2f31b415c50324e75842d7f91622f39a89468e51938330b3fd3af", "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", @@ -115,9 +117,5 @@ "configuration": { "config-key": "config-value" } - }, - "schema": { - "version": "8.0.0", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-8.0.0.json" } } diff --git a/syft/formats/syftjson/to_format_model.go b/syft/formats/syftjson/to_format_model.go index 7b3688ced69..2cafddf1408 100644 --- a/syft/formats/syftjson/to_format_model.go +++ b/syft/formats/syftjson/to_format_model.go @@ -12,6 +12,7 @@ import ( "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/formats/syftjson/model" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" @@ -20,17 +21,12 @@ import ( // ToFormatModel transforms the sbom import a format-specific model. func ToFormatModel(s sbom.SBOM) model.Document { - src, err := toSourceModel(s.Source) - if err != nil { - log.Warnf("unable to create syft-json source object: %+v", err) - } - return model.Document{ Artifacts: toPackageModels(s.Artifacts.Packages), ArtifactRelationships: toRelationshipModel(s.Relationships), Files: toFile(s), Secrets: toSecrets(s.Artifacts.Secrets), - Source: src, + Source: toSourceModel(s.Source), Distro: toLinuxReleaser(s.Artifacts.LinuxDistribution), Descriptor: toDescriptor(s.Descriptor), Schema: model.Schema{ @@ -267,10 +263,16 @@ func toRelationshipModel(relationships []artifact.Relationship) []model.Relation } // toSourceModel creates a new source object to be represented into JSON. -func toSourceModel(src source.Metadata) (model.Source, error) { - switch src.Scheme { - case source.ImageScheme: - metadata := src.ImageMetadata +func toSourceModel(src source.Description) model.Source { + m := model.Source{ + ID: src.ID, + Name: src.Name, + Version: src.Version, + Type: sourcemetadata.JSONName(src.Metadata), + Metadata: src.Metadata, + } + + if metadata, ok := src.Metadata.(source.StereoscopeImageSourceMetadata); ok { // ensure that empty collections are not shown as null if metadata.RepoDigests == nil { metadata.RepoDigests = []string{} @@ -278,24 +280,8 @@ func toSourceModel(src source.Metadata) (model.Source, error) { if metadata.Tags == nil { metadata.Tags = []string{} } - return model.Source{ - ID: src.ID, - Type: "image", - Target: metadata, - }, nil - case source.DirectoryScheme: - return model.Source{ - ID: src.ID, - Type: "directory", - Target: src.Path, - }, nil - case source.FileScheme: - return model.Source{ - ID: src.ID, - Type: "file", - Target: src.Path, - }, nil - default: - return model.Source{}, fmt.Errorf("unsupported source: %q", src.Scheme) + m.Metadata = metadata } + + return m } diff --git a/syft/formats/syftjson/to_format_model_test.go b/syft/formats/syftjson/to_format_model_test.go index 98f03c7b08f..8dd3475d8dd 100644 --- a/syft/formats/syftjson/to_format_model_test.go +++ b/syft/formats/syftjson/to_format_model_test.go @@ -1,62 +1,174 @@ package syftjson import ( + "encoding/json" "testing" - "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/formats/syftjson/model" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/source" ) -func Test_toSourceModel(t *testing.T) { - allSchemes := strset.New() - for _, s := range source.AllSchemes { - allSchemes.Add(string(s)) +func Test_toSourceModel_IgnoreBase(t *testing.T) { + tests := []struct { + name string + src source.Description + }{ + { + name: "directory", + src: source.Description{ + ID: "test-id", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, + }, + }, } - testedSchemes := strset.New() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // assert the model transformation is correct + actual := toSourceModel(test.src) + + by, err := json.Marshal(actual) + require.NoError(t, err) + assert.NotContains(t, string(by), "some/base") + }) + } +} + +func Test_toSourceModel(t *testing.T) { + tracker := sourcemetadata.NewCompletionTester(t) tests := []struct { name string - src source.Metadata + src source.Description expected model.Source }{ { name: "directory", - src: source.Metadata{ - ID: "test-id", - Scheme: source.DirectoryScheme, - Path: "some/path", + src: source.Description{ + ID: "test-id", + Name: "some-name", + Version: "some-version", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, }, expected: model.Source{ - ID: "test-id", - Type: "directory", - Target: "some/path", + ID: "test-id", + Name: "some-name", + Version: "some-version", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, }, }, { name: "file", - src: source.Metadata{ - ID: "test-id", - Scheme: source.FileScheme, - Path: "some/path", + src: source.Description{ + ID: "test-id", + Name: "some-name", + Version: "some-version", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, }, expected: model.Source{ - ID: "test-id", - Type: "file", - Target: "some/path", + ID: "test-id", + Name: "some-name", + Version: "some-version", + Type: "file", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, }, }, { name: "image", - src: source.Metadata{ - ID: "test-id", - Scheme: source.ImageScheme, - ImageMetadata: source.ImageMetadata{ + src: source.Description{ + ID: "test-id", + Name: "some-name", + Version: "some-version", + Metadata: source.StereoscopeImageSourceMetadata{ + UserInput: "user-input", + ID: "id...", + ManifestDigest: "digest...", + MediaType: "type...", + }, + }, + expected: model.Source{ + ID: "test-id", + Name: "some-name", + Version: "some-version", + Type: "image", + Metadata: source.StereoscopeImageSourceMetadata{ + UserInput: "user-input", + ID: "id...", + ManifestDigest: "digest...", + MediaType: "type...", + RepoDigests: []string{}, + Tags: []string{}, + }, + }, + }, + // below are regression tests for when the name/version are not provided + // historically we've hoisted up the name/version from the metadata, now it is a simple pass-through + { + name: "directory - no name/version", + src: source.Description{ + ID: "test-id", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, + }, + expected: model.Source{ + ID: "test-id", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, + }, + }, + { + name: "file - no name/version", + src: source.Description{ + ID: "test-id", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, + }, + expected: model.Source{ + ID: "test-id", + Type: "file", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, + }, + }, + { + name: "image - no name/version", + src: source.Description{ + ID: "test-id", + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "user-input", ID: "id...", ManifestDigest: "digest...", @@ -66,7 +178,7 @@ func Test_toSourceModel(t *testing.T) { expected: model.Source{ ID: "test-id", Type: "image", - Target: source.ImageMetadata{ + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "user-input", ID: "id...", ManifestDigest: "digest...", @@ -79,18 +191,14 @@ func Test_toSourceModel(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // track each scheme tested (passed or not) - testedSchemes.Add(string(test.src.Scheme)) - // assert the model transformation is correct - actual, err := toSourceModel(test.src) - require.NoError(t, err) + actual := toSourceModel(test.src) assert.Equal(t, test.expected, actual) + + // track each scheme tested (passed or not) + tracker.Tested(t, test.expected.Metadata) }) } - - // assert all possible schemes were under test - assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test") } func Test_toFileType(t *testing.T) { diff --git a/syft/formats/syftjson/to_syft_model.go b/syft/formats/syftjson/to_syft_model.go index aeb0c24f165..419cf3ed4ba 100644 --- a/syft/formats/syftjson/to_syft_model.go +++ b/syft/formats/syftjson/to_syft_model.go @@ -202,12 +202,12 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations return out, conversionErrors } -func toSyftSource(s model.Source) *source.Source { - newSrc := &source.Source{ - Metadata: *toSyftSourceData(s), +func toSyftSource(s model.Source) source.Source { + description := toSyftSourceData(s) + if description == nil { + return nil } - newSrc.SetID() - return newSrc + return source.FromDescription(*description) } func toSyftRelationship(idMap map[string]interface{}, relationship model.Relationship, idAliases map[string]string) (*artifact.Relationship, error) { @@ -257,43 +257,13 @@ func toSyftDescriptor(d model.Descriptor) sbom.Descriptor { } } -func toSyftSourceData(s model.Source) *source.Metadata { - switch s.Type { - case "directory": - path, ok := s.Target.(string) - if !ok { - log.Warnf("unable to parse source target as string: %+v", s.Target) - return nil - } - return &source.Metadata{ - ID: s.ID, - Scheme: source.DirectoryScheme, - Path: path, - } - case "file": - path, ok := s.Target.(string) - if !ok { - log.Warnf("unable to parse source target as string: %+v", s.Target) - return nil - } - return &source.Metadata{ - ID: s.ID, - Scheme: source.FileScheme, - Path: path, - } - case "image": - metadata, ok := s.Target.(source.ImageMetadata) - if !ok { - log.Warnf("unable to parse source target as image metadata: %+v", s.Target) - return nil - } - return &source.Metadata{ - ID: s.ID, - Scheme: source.ImageScheme, - ImageMetadata: metadata, - } +func toSyftSourceData(s model.Source) *source.Description { + return &source.Description{ + ID: s.ID, + Name: s.Name, + Version: s.Version, + Metadata: s.Metadata, } - return nil } func toSyftCatalog(pkgs []model.Package, idAliases map[string]string) *pkg.Collection { diff --git a/syft/formats/syftjson/to_syft_model_test.go b/syft/formats/syftjson/to_syft_model_test.go index dabc33f3841..5600ec155f1 100644 --- a/syft/formats/syftjson/to_syft_model_test.go +++ b/syft/formats/syftjson/to_syft_model_test.go @@ -4,66 +4,154 @@ import ( "errors" "testing" - "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" stereoFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/formats/syftjson/model" + "github.com/anchore/syft/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" ) func Test_toSyftSourceData(t *testing.T) { - allSchemes := strset.New() - for _, s := range source.AllSchemes { - allSchemes.Add(string(s)) - } - testedSchemes := strset.New() + tracker := sourcemetadata.NewCompletionTester(t) tests := []struct { name string src model.Source - expected source.Metadata + expected *source.Description }{ { name: "directory", - expected: source.Metadata{ - Scheme: source.DirectoryScheme, - Path: "some/path", - }, src: model.Source{ - Type: "directory", - Target: "some/path", + ID: "the-id", + Name: "some-name", + Version: "some-version", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, + }, + expected: &source.Description{ + ID: "the-id", + Name: "some-name", + Version: "some-version", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, }, }, { name: "file", - expected: source.Metadata{ - Scheme: source.FileScheme, - Path: "some/path", - }, src: model.Source{ - Type: "file", - Target: "some/path", + ID: "the-id", + Name: "some-name", + Version: "some-version", + Type: "file", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, + }, + expected: &source.Description{ + ID: "the-id", + Name: "some-name", + Version: "some-version", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, }, }, { name: "image", - expected: source.Metadata{ - Scheme: source.ImageScheme, - ImageMetadata: source.ImageMetadata{ + src: model.Source{ + ID: "the-id", + Name: "some-name", + Version: "some-version", + Type: "image", + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "user-input", ID: "id...", ManifestDigest: "digest...", MediaType: "type...", }, }, + expected: &source.Description{ + ID: "the-id", + Name: "some-name", + Version: "some-version", + Metadata: source.StereoscopeImageSourceMetadata{ + UserInput: "user-input", + ID: "id...", + ManifestDigest: "digest...", + MediaType: "type...", + }, + }, + }, + // below are regression tests for when the name/version are not provided + // historically we've hoisted up the name/version from the metadata, now it is a simple pass-through + { + name: "directory - no name/version", src: model.Source{ + ID: "the-id", + Type: "directory", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, + }, + expected: &source.Description{ + ID: "the-id", + Metadata: source.DirectorySourceMetadata{ + Path: "some/path", + Base: "some/base", + }, + }, + }, + { + name: "file - no name/version", + src: model.Source{ + ID: "the-id", + Type: "file", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, + }, + expected: &source.Description{ + ID: "the-id", + Metadata: source.FileSourceMetadata{ + Path: "some/path", + Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}}, + MIMEType: "text/plain", + }, + }, + }, + { + name: "image - no name/version", + src: model.Source{ + ID: "the-id", Type: "image", - Target: source.ImageMetadata{ + Metadata: source.StereoscopeImageSourceMetadata{ + UserInput: "user-input", + ID: "id...", + ManifestDigest: "digest...", + MediaType: "type...", + }, + }, + expected: &source.Description{ + ID: "the-id", + Metadata: source.StereoscopeImageSourceMetadata{ UserInput: "user-input", ID: "id...", ManifestDigest: "digest...", @@ -76,22 +164,18 @@ func Test_toSyftSourceData(t *testing.T) { t.Run(test.name, func(t *testing.T) { // assert the model transformation is correct actual := toSyftSourceData(test.src) - assert.Equal(t, test.expected, *actual) + assert.Equal(t, test.expected, actual) - // track each scheme tested (passed or not) - testedSchemes.Add(string(test.expected.Scheme)) + tracker.Tested(t, test.expected.Metadata) }) } - - // assert all possible schemes were under test - assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test") } func Test_idsHaveChanged(t *testing.T) { s, err := toSyftModel(model.Document{ Source: model.Source{ - Type: "file", - Target: "some/path", + Type: "file", + Metadata: source.FileSourceMetadata{Path: "some/path"}, }, Artifacts: []model.Package{ { @@ -116,17 +200,17 @@ func Test_idsHaveChanged(t *testing.T) { }, }) - assert.NoError(t, err) - assert.Len(t, s.Relationships, 1) + require.NoError(t, err) + require.Len(t, s.Relationships, 1) r := s.Relationships[0] from := s.Artifacts.Packages.Package(r.From.ID()) - assert.NotNil(t, from) + require.NotNil(t, from) assert.Equal(t, "pkg-1", from.Name) to := s.Artifacts.Packages.Package(r.To.ID()) - assert.NotNil(t, to) + require.NotNil(t, to) assert.Equal(t, "pkg-2", to.Name) } diff --git a/syft/formats/table/encoder_test.go b/syft/formats/table/encoder_test.go index 44e9f473069..d0c672237d8 100644 --- a/syft/formats/table/encoder_test.go +++ b/syft/formats/table/encoder_test.go @@ -9,14 +9,17 @@ import ( "github.com/anchore/syft/syft/formats/internal/testutils" ) -var updateTableGoldenFiles = flag.Bool("update-table", false, "update the *.golden files for table format") +var updateSnapshot = flag.Bool("update-table", false, "update the *.golden files for table format") func TestTableEncoder(t *testing.T) { testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateTableGoldenFiles, - false, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, t.TempDir()), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + }, ) } diff --git a/syft/formats/template/encoder_test.go b/syft/formats/template/encoder_test.go index aed6dca526a..82a54498a07 100644 --- a/syft/formats/template/encoder_test.go +++ b/syft/formats/template/encoder_test.go @@ -9,19 +9,21 @@ import ( "github.com/anchore/syft/syft/formats/internal/testutils" ) -var updateTmpl = flag.Bool("update-tmpl", false, "update the *.golden files for json encoders") +var updateSnapshot = flag.Bool("update-template", false, "update the *.golden files for json encoders") func TestFormatWithOption(t *testing.T) { f := OutputFormat{} f.SetTemplatePath("test-fixtures/csv.template") testutils.AssertEncoderAgainstGoldenSnapshot(t, - f, - testutils.DirectoryInput(t), - *updateTmpl, - false, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, t.TempDir()), + Format: f, + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + }, ) - } func TestFormatWithOptionAndHasField(t *testing.T) { @@ -29,16 +31,19 @@ func TestFormatWithOptionAndHasField(t *testing.T) { f.SetTemplatePath("test-fixtures/csv-hasField.template") testutils.AssertEncoderAgainstGoldenSnapshot(t, - f, - testutils.DirectoryInputWithAuthorField(t), - *updateTmpl, - false, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInputWithAuthorField(t), + Format: f, + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + }, ) } func TestFormatWithoutOptions(t *testing.T) { f := Format() - err := f.Encode(nil, testutils.DirectoryInput(t)) + err := f.Encode(nil, testutils.DirectoryInput(t, t.TempDir())) assert.ErrorContains(t, err, "no template file: please provide a template path") } diff --git a/syft/formats/text/encoder.go b/syft/formats/text/encoder.go index d16ef17989a..1c19084d5b3 100644 --- a/syft/formats/text/encoder.go +++ b/syft/formats/text/encoder.go @@ -14,13 +14,15 @@ func encoder(output io.Writer, s sbom.SBOM) error { w := new(tabwriter.Writer) w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight) - switch s.Source.Scheme { - case source.DirectoryScheme, source.FileScheme: - fmt.Fprintf(w, "[Path: %s]\n", s.Source.Path) - case source.ImageScheme: + switch metadata := s.Source.Metadata.(type) { + case source.DirectorySourceMetadata: + fmt.Fprintf(w, "[Path: %s]\n", metadata.Path) + case source.FileSourceMetadata: + fmt.Fprintf(w, "[Path: %s]\n", metadata.Path) + case source.StereoscopeImageSourceMetadata: fmt.Fprintln(w, "[Image]") - for idx, l := range s.Source.ImageMetadata.Layers { + for idx, l := range metadata.Layers { fmt.Fprintln(w, " Layer:\t", idx) fmt.Fprintln(w, " Digest:\t", l.Digest) fmt.Fprintln(w, " Size:\t", l.Size) @@ -29,7 +31,7 @@ func encoder(output io.Writer, s sbom.SBOM) error { w.Flush() } default: - return fmt.Errorf("unsupported source: %T", s.Source.Scheme) + return fmt.Errorf("unsupported source: %T", s.Source.Metadata) } // populate artifacts... diff --git a/syft/formats/text/encoder_test.go b/syft/formats/text/encoder_test.go index 7b5e9f4727f..4a52d28344f 100644 --- a/syft/formats/text/encoder_test.go +++ b/syft/formats/text/encoder_test.go @@ -7,24 +7,42 @@ import ( "github.com/anchore/syft/syft/formats/internal/testutils" ) -var updateTextEncoderGoldenFiles = flag.Bool("update-text", false, "update the *.golden files for text encoder") +var updateSnapshot = flag.Bool("update-text", false, "update the *.golden files for text encoder") +var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing") func TestTextDirectoryEncoder(t *testing.T) { + dir := t.TempDir() testutils.AssertEncoderAgainstGoldenSnapshot(t, - Format(), - testutils.DirectoryInput(t), - *updateTextEncoderGoldenFiles, - false, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.DirectoryInput(t, dir), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(dir), + }, ) } func TestTextImageEncoder(t *testing.T) { testImage := "image-simple" testutils.AssertEncoderAgainstGoldenImageSnapshot(t, - Format(), - testutils.ImageInput(t, testImage, testutils.FromSnapshot()), - testImage, - *updateTextEncoderGoldenFiles, - false, + testutils.ImageSnapshotTestConfig{ + Image: testImage, + UpdateImageSnapshot: *updateImage, + }, + testutils.EncoderSnapshotTestConfig{ + Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()), + Format: Format(), + UpdateSnapshot: *updateSnapshot, + PersistRedactionsInSnapshot: true, + IsJSON: false, + Redactor: redactor(), + }, ) } + +func redactor(values ...string) testutils.Redactor { + return testutils.NewRedactions(). + WithValuesRedacted(values...) +} diff --git a/syft/formats/text/test-fixtures/snapshot/TestTextDirectoryEncoder.golden b/syft/formats/text/test-fixtures/snapshot/TestTextDirectoryEncoder.golden index 25881f2d952..9efab380486 100644 --- a/syft/formats/text/test-fixtures/snapshot/TestTextDirectoryEncoder.golden +++ b/syft/formats/text/test-fixtures/snapshot/TestTextDirectoryEncoder.golden @@ -1,4 +1,4 @@ -[Path: /some/path] +[Path: redacted/some/path] [package-1] Version: 1.0.1 Type: python diff --git a/syft/internal/fileresolver/chroot_context.go b/syft/internal/fileresolver/chroot_context.go new file mode 100644 index 00000000000..a5245952b85 --- /dev/null +++ b/syft/internal/fileresolver/chroot_context.go @@ -0,0 +1,165 @@ +package fileresolver + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/anchore/syft/syft/internal/windows" +) + +// ChrootContext helps to modify path from a real filesystem to a chroot-like filesystem, taking into account +// the user given root, the base path (if any) to consider as the root, and the current working directory. +// Note: this only works on a real filesystem, not on a virtual filesystem (such as a stereoscope filetree). +type ChrootContext struct { + root string + base string + cwd string + cwdRelativeToRoot string +} + +func NewChrootContextFromCWD(root, base string) (*ChrootContext, error) { + currentWD, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("could not get current working directory: %w", err) + } + + return NewChrootContext(root, base, currentWD) +} + +func NewChrootContext(root, base, cwd string) (*ChrootContext, error) { + cleanRoot, err := NormalizeRootDirectory(root) + if err != nil { + return nil, err + } + + cleanBase, err := NormalizeBaseDirectory(base) + if err != nil { + return nil, err + } + + chroot := &ChrootContext{ + root: cleanRoot, + base: cleanBase, + cwd: cwd, + } + + return chroot, chroot.ChangeDirectory(cwd) +} + +func NormalizeRootDirectory(root string) (string, error) { + cleanRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return "", fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err) + } + return cleanRoot, nil +} + +func NormalizeBaseDirectory(base string) (string, error) { + if base == "" { + return "", nil + } + + cleanBase, err := filepath.EvalSymlinks(base) + if err != nil { + return "", fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err) + } + + return filepath.Abs(cleanBase) +} + +// Root returns the root path with all symlinks evaluated. +func (r ChrootContext) Root() string { + return r.root +} + +// Base returns the absolute base path with all symlinks evaluated. +func (r ChrootContext) Base() string { + return r.base +} + +// ChangeRoot swaps the path for the chroot. +func (r *ChrootContext) ChangeRoot(dir string) error { + newR, err := NewChrootContext(dir, r.base, r.cwd) + if err != nil { + return fmt.Errorf("could not change root: %w", err) + } + + *r = *newR + + return nil +} + +// ChangeDirectory changes the current working directory so that any relative paths passed +// into ToNativePath() and ToChrootPath() honor the new CWD. If the process changes the CWD in-flight, this should be +// called again to ensure correct functionality of ToNativePath() and ToChrootPath(). +func (r *ChrootContext) ChangeDirectory(dir string) error { + var ( + cwdRelativeToRoot string + err error + ) + + dir, err = filepath.Abs(dir) + if err != nil { + return fmt.Errorf("could not determine absolute path to CWD: %w", err) + } + + if path.IsAbs(r.root) { + cwdRelativeToRoot, err = filepath.Rel(dir, r.root) + if err != nil { + return fmt.Errorf("could not determine given root path to CWD: %w", err) + } + } else { + cwdRelativeToRoot = filepath.Clean(r.root) + } + + r.cwd = dir + r.cwdRelativeToRoot = cwdRelativeToRoot + return nil +} + +// ToNativePath takes a path in the context of the chroot-like filesystem and converts it to a path in the underlying fs domain. +func (r ChrootContext) ToNativePath(chrootPath string) (string, error) { + responsePath := chrootPath + + if filepath.IsAbs(responsePath) { + // don't allow input to potentially hop above root path + responsePath = path.Join(r.root, responsePath) + } else { + // ensure we take into account any relative difference between the root path and the CWD for relative requests + responsePath = path.Join(r.cwdRelativeToRoot, responsePath) + } + + var err error + responsePath, err = filepath.Abs(responsePath) + if err != nil { + return "", err + } + return responsePath, nil +} + +// ToChrootPath takes a path from the underlying fs domain and converts it to a path that is relative to the current root context. +func (r ChrootContext) ToChrootPath(nativePath string) string { + responsePath := nativePath + // check to see if we need to encode back to Windows from posix + if windows.HostRunningOnWindows() { + responsePath = windows.FromPosix(responsePath) + } + + // clean references to the request path (either the root, or the base if set) + if filepath.IsAbs(responsePath) { + var prefix string + if r.base != "" { + prefix = r.base + } else { + // we need to account for the cwd relative to the running process and the given root for the directory resolver + prefix = filepath.Clean(filepath.Join(r.cwd, r.cwdRelativeToRoot)) + prefix += string(filepath.Separator) + } + responsePath = strings.TrimPrefix(responsePath, prefix) + } + + return responsePath +} diff --git a/syft/internal/fileresolver/chroot_context_test.go b/syft/internal/fileresolver/chroot_context_test.go new file mode 100644 index 00000000000..2cd8befe136 --- /dev/null +++ b/syft/internal/fileresolver/chroot_context_test.go @@ -0,0 +1,481 @@ +package fileresolver + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_ChrootContext_RequestResponse(t *testing.T) { + // / + // somewhere/ + // outside.txt + // root-link -> ./ + // path/ + // to/ + // abs-inside.txt -> /path/to/the/file.txt # absolute link to somewhere inside of the root + // rel-inside.txt -> ./the/file.txt # relative link to somewhere inside of the root + // the/ + // file.txt + // abs-outside.txt -> /somewhere/outside.txt # absolute link to outside of the root + // rel-outside -> ../../../somewhere/outside.txt # relative link to outside of the root + // + + testDir, err := os.Getwd() + require.NoError(t, err) + relative := filepath.Join("test-fixtures", "req-resp") + absolute := filepath.Join(testDir, relative) + + absPathToTheFile := filepath.Join(absolute, "path", "to", "the", "file.txt") + + absAbsInsidePath := filepath.Join(absolute, "path", "to", "abs-inside.txt") + absAbsOutsidePath := filepath.Join(absolute, "path", "to", "the", "abs-outside.txt") + + absRelOutsidePath := filepath.Join(absolute, "path", "to", "the", "rel-outside.txt") + + relViaLink := filepath.Join(relative, "root-link") + absViaLink := filepath.Join(absolute, "root-link") + + absViaLinkPathToTheFile := filepath.Join(absViaLink, "path", "to", "the", "file.txt") + absViaLinkAbsOutsidePath := filepath.Join(absViaLink, "path", "to", "the", "abs-outside.txt") + absViaLinkRelOutsidePath := filepath.Join(absViaLink, "path", "to", "the", "rel-outside.txt") + + relViaDoubleLink := filepath.Join(relative, "root-link", "root-link") + absViaDoubleLink := filepath.Join(absolute, "root-link", "root-link") + + absViaDoubleLinkPathToTheFile := filepath.Join(absViaDoubleLink, "path", "to", "the", "file.txt") + absViaDoubleLinkRelOutsidePath := filepath.Join(absViaDoubleLink, "path", "to", "the", "rel-outside.txt") + + cleanup := func() { + _ = os.Remove(absAbsInsidePath) + _ = os.Remove(absAbsOutsidePath) + } + + // ensure the absolute symlinks are cleaned up from any previous runs + cleanup() + + require.NoError(t, os.Symlink(filepath.Join(absolute, "path", "to", "the", "file.txt"), absAbsInsidePath)) + require.NoError(t, os.Symlink(filepath.Join(absolute, "somewhere", "outside.txt"), absAbsOutsidePath)) + + t.Cleanup(cleanup) + + cases := []struct { + name string + cwd string + root string + base string + input string + expectedNativePath string + expectedChrootPath string + }{ + { + name: "relative root, relative request, direct", + root: relative, + input: "path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, relative request, direct", + root: absolute, + input: "path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "relative root, abs request, direct", + root: relative, + input: "/path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, abs request, direct", + root: absolute, + input: "/path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + // cwd within root... + { + name: "relative root, relative request, direct, cwd within root", + cwd: filepath.Join(relative, "path/to"), + root: "../../", + input: "path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, relative request, direct, cwd within root", + cwd: filepath.Join(relative, "path/to"), + root: absolute, + input: "path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "relative root, abs request, direct, cwd within root", + cwd: filepath.Join(relative, "path/to"), + root: "../../", + input: "/path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, abs request, direct, cwd within root", + cwd: filepath.Join(relative, "path/to"), + + root: absolute, + input: "/path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + // cwd within symlink root... + { + name: "relative root, relative request, direct, cwd within symlink root", + cwd: relViaLink, + root: "./", + input: "path/to/the/file.txt", + expectedNativePath: absViaLinkPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, relative request, direct, cwd within symlink root", + cwd: relViaLink, + root: absViaLink, + input: "path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "relative root, abs request, direct, cwd within symlink root", + cwd: relViaLink, + root: "./", + input: "/path/to/the/file.txt", + expectedNativePath: absViaLinkPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, abs request, direct, cwd within symlink root", + cwd: relViaLink, + root: absViaLink, + input: "/path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + // cwd within symlink root, request nested within... + { + name: "relative root, relative nested request, direct, cwd within symlink root", + cwd: relViaLink, + root: "./path", + input: "to/the/file.txt", + expectedNativePath: absViaLinkPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "abs root, relative nested request, direct, cwd within symlink root", + cwd: relViaLink, + root: filepath.Join(absViaLink, "path"), + input: "to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "relative root, abs nested request, direct, cwd within symlink root", + cwd: relViaLink, + root: "./path", + input: "/to/the/file.txt", + expectedNativePath: absViaLinkPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "abs root, abs nested request, direct, cwd within symlink root", + cwd: relViaLink, + root: filepath.Join(absViaLink, "path"), + input: "/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + // cwd within DOUBLE symlink root... + { + name: "relative root, relative request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: "./", + input: "path/to/the/file.txt", + expectedNativePath: absViaDoubleLinkPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, relative request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: absViaDoubleLink, + input: "path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "relative root, abs request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: "./", + input: "/path/to/the/file.txt", + expectedNativePath: absViaDoubleLinkPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + { + name: "abs root, abs request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: absViaDoubleLink, + input: "/path/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "path/to/the/file.txt", + }, + // cwd within DOUBLE symlink root, request nested within... + { + name: "relative root, relative nested request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: "./path", + input: "to/the/file.txt", + expectedNativePath: absViaDoubleLinkPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "abs root, relative nested request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: filepath.Join(absViaDoubleLink, "path"), + input: "to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "relative root, abs nested request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: "./path", + input: "/to/the/file.txt", + expectedNativePath: absViaDoubleLinkPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "abs root, abs nested request, direct, cwd within (double) symlink root", + cwd: relViaDoubleLink, + root: filepath.Join(absViaDoubleLink, "path"), + input: "/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + // cwd within DOUBLE symlink root, request nested DEEP within... + { + name: "relative root, relative nested request, direct, cwd deep within (double) symlink root", + cwd: filepath.Join(relViaDoubleLink, "path", "to"), + root: "../", + input: "to/the/file.txt", + expectedNativePath: absViaDoubleLinkPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "abs root, relative nested request, direct, cwd deep within (double) symlink root", + cwd: filepath.Join(relViaDoubleLink, "path", "to"), + root: filepath.Join(absViaDoubleLink, "path"), + input: "to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "relative root, abs nested request, direct, cwd deep within (double) symlink root", + cwd: filepath.Join(relViaDoubleLink, "path", "to"), + root: "../", + input: "/to/the/file.txt", + expectedNativePath: absViaDoubleLinkPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + { + name: "abs root, abs nested request, direct, cwd deep within (double) symlink root", + cwd: filepath.Join(relViaDoubleLink, "path", "to"), + root: filepath.Join(absViaDoubleLink, "path"), + input: "/to/the/file.txt", + expectedNativePath: absPathToTheFile, + expectedChrootPath: "to/the/file.txt", + }, + // link to outside of root cases... + { + name: "relative root, relative request, abs indirect (outside of root)", + root: filepath.Join(relative, "path"), + input: "to/the/abs-outside.txt", + expectedNativePath: absAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "abs root, relative request, abs indirect (outside of root)", + root: filepath.Join(absolute, "path"), + input: "to/the/abs-outside.txt", + expectedNativePath: absAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "relative root, abs request, abs indirect (outside of root)", + root: filepath.Join(relative, "path"), + input: "/to/the/abs-outside.txt", + expectedNativePath: absAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "abs root, abs request, abs indirect (outside of root)", + root: filepath.Join(absolute, "path"), + input: "/to/the/abs-outside.txt", + expectedNativePath: absAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "relative root, relative request, relative indirect (outside of root)", + root: filepath.Join(relative, "path"), + input: "to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "abs root, relative request, relative indirect (outside of root)", + root: filepath.Join(absolute, "path"), + input: "to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "relative root, abs request, relative indirect (outside of root)", + root: filepath.Join(relative, "path"), + input: "/to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "abs root, abs request, relative indirect (outside of root)", + root: filepath.Join(absolute, "path"), + input: "/to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + // link to outside of root cases... cwd within symlink root + { + name: "relative root, relative request, abs indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: "path", + input: "to/the/abs-outside.txt", + expectedNativePath: absViaLinkAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "abs root, relative request, abs indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: filepath.Join(absolute, "path"), + input: "to/the/abs-outside.txt", + expectedNativePath: absAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "relative root, abs request, abs indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: "path", + input: "/to/the/abs-outside.txt", + expectedNativePath: absViaLinkAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "abs root, abs request, abs indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: filepath.Join(absolute, "path"), + input: "/to/the/abs-outside.txt", + expectedNativePath: absAbsOutsidePath, + expectedChrootPath: "to/the/abs-outside.txt", + }, + { + name: "relative root, relative request, relative indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: "path", + input: "to/the/rel-outside.txt", + expectedNativePath: absViaLinkRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "abs root, relative request, relative indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: filepath.Join(absolute, "path"), + input: "to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "relative root, abs request, relative indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: "path", + input: "/to/the/rel-outside.txt", + expectedNativePath: absViaLinkRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "abs root, abs request, relative indirect (outside of root), cwd within symlink root", + cwd: relViaLink, + root: filepath.Join(absolute, "path"), + input: "/to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "relative root, relative request, relative indirect (outside of root), cwd within DOUBLE symlink root", + cwd: relViaDoubleLink, + root: "path", + input: "to/the/rel-outside.txt", + expectedNativePath: absViaDoubleLinkRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "abs root, relative request, relative indirect (outside of root), cwd within DOUBLE symlink root", + cwd: relViaDoubleLink, + root: filepath.Join(absolute, "path"), + input: "to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "relative root, abs request, relative indirect (outside of root), cwd within DOUBLE symlink root", + cwd: relViaDoubleLink, + root: "path", + input: "/to/the/rel-outside.txt", + expectedNativePath: absViaDoubleLinkRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + { + name: "abs root, abs request, relative indirect (outside of root), cwd within DOUBLE symlink root", + cwd: relViaDoubleLink, + root: filepath.Join(absolute, "path"), + input: "/to/the/rel-outside.txt", + expectedNativePath: absRelOutsidePath, + expectedChrootPath: "to/the/rel-outside.txt", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + // we need to mimic a shell, otherwise we won't get a path within a symlink + targetPath := filepath.Join(testDir, c.cwd) + t.Setenv("PWD", filepath.Clean(targetPath)) + + require.NoError(t, err) + require.NoError(t, os.Chdir(targetPath)) + t.Cleanup(func() { + require.NoError(t, os.Chdir(testDir)) + }) + + chroot, err := NewChrootContextFromCWD(c.root, c.base) + require.NoError(t, err) + require.NotNil(t, chroot) + + req, err := chroot.ToNativePath(c.input) + require.NoError(t, err) + assert.Equal(t, c.expectedNativePath, req, "native path different") + + resp := chroot.ToChrootPath(req) + assert.Equal(t, c.expectedChrootPath, resp, "chroot path different") + }) + } +} diff --git a/syft/internal/fileresolver/container_image_squash_test.go b/syft/internal/fileresolver/container_image_squash_test.go index d65d0bccc88..642f6b52053 100644 --- a/syft/internal/fileresolver/container_image_squash_test.go +++ b/syft/internal/fileresolver/container_image_squash_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -513,6 +514,26 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) { } +func compareLocations(t *testing.T, expected, actual []file.Location) { + t.Helper() + ignoreUnexported := cmpopts.IgnoreUnexported(file.LocationData{}) + ignoreMetadata := cmpopts.IgnoreFields(file.LocationMetadata{}, "Annotations") + ignoreFS := cmpopts.IgnoreFields(file.Coordinates{}, "FileSystemID") + + sort.Sort(file.Locations(expected)) + sort.Sort(file.Locations(actual)) + + if d := cmp.Diff(expected, actual, + ignoreUnexported, + ignoreFS, + ignoreMetadata, + ); d != "" { + + t.Errorf("unexpected locations (-want +got):\n%s", d) + } + +} + func TestSquashResolver_AllLocations(t *testing.T) { img := imagetest.GetFixtureImage(t, "docker-archive", "image-files-deleted") diff --git a/syft/internal/fileresolver/directory.go b/syft/internal/fileresolver/directory.go index 2d634cf1eed..766d53c8fba 100644 --- a/syft/internal/fileresolver/directory.go +++ b/syft/internal/fileresolver/directory.go @@ -5,19 +5,14 @@ import ( "fmt" "io" "os" - "path" - "path/filepath" - "runtime" - "strings" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/windows" ) -const WindowsOS = "windows" - var unixSystemRuntimePrefixes = []string{ "/proc", "/dev", @@ -30,14 +25,12 @@ var _ file.Resolver = (*Directory)(nil) // Directory implements path and content access for the directory data source. type Directory struct { - path string - base string - currentWdRelativeToRoot string - currentWd string - tree filetree.Reader - index filetree.IndexReader - searchContext filetree.Searcher - indexer *directoryIndexer + path string + chroot ChrootContext + tree filetree.Reader + index filetree.IndexReader + searchContext filetree.Searcher + indexer *directoryIndexer } func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) { @@ -50,46 +43,20 @@ func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) } func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) { - currentWD, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("could not get CWD: %w", err) - } - - cleanRoot, err := filepath.EvalSymlinks(root) + chroot, err := NewChrootContextFromCWD(root, base) if err != nil { - return nil, fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err) - } - - cleanBase := "" - if base != "" { - cleanBase, err = filepath.EvalSymlinks(base) - if err != nil { - return nil, fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err) - } - cleanBase, err = filepath.Abs(cleanBase) - if err != nil { - return nil, err - } + return nil, fmt.Errorf("unable to interpret chroot context: %w", err) } - var currentWdRelRoot string - if path.IsAbs(cleanRoot) { - currentWdRelRoot, err = filepath.Rel(currentWD, cleanRoot) - if err != nil { - return nil, fmt.Errorf("could not determine given root path to CWD: %w", err) - } - } else { - currentWdRelRoot = filepath.Clean(cleanRoot) - } + cleanRoot := chroot.Root() + cleanBase := chroot.Base() return &Directory{ - path: cleanRoot, - base: cleanBase, - currentWd: currentWD, - currentWdRelativeToRoot: currentWdRelRoot, - tree: filetree.New(), - index: filetree.NewIndex(), - indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...), + path: cleanRoot, + chroot: *chroot, + tree: filetree.New(), + index: filetree.NewIndex(), + indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...), }, nil } @@ -110,43 +77,12 @@ func (r *Directory) buildIndex() error { } func (r Directory) requestPath(userPath string) (string, error) { - if filepath.IsAbs(userPath) { - // don't allow input to potentially hop above root path - userPath = path.Join(r.path, userPath) - } else { - // ensure we take into account any relative difference between the root path and the CWD for relative requests - userPath = path.Join(r.currentWdRelativeToRoot, userPath) - } - - var err error - userPath, err = filepath.Abs(userPath) - if err != nil { - return "", err - } - return userPath, nil + return r.chroot.ToNativePath(userPath) } // responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the directory resolver. func (r Directory) responsePath(path string) string { - // check to see if we need to encode back to Windows from posix - if runtime.GOOS == WindowsOS { - path = posixToWindows(path) - } - - // clean references to the request path (either the root, or the base if set) - if filepath.IsAbs(path) { - var prefix string - if r.base != "" { - prefix = r.base - } else { - // we need to account for the cwd relative to the running process and the given root for the directory resolver - prefix = filepath.Clean(filepath.Join(r.currentWd, r.currentWdRelativeToRoot)) - prefix += string(filepath.Separator) - } - path = strings.TrimPrefix(path, prefix) - } - - return path + return r.chroot.ToChrootPath(path) } // HasPath indicates if the given path exists in the underlying source. @@ -196,8 +132,8 @@ func (r Directory) FilesByPath(userPaths ...string) ([]file.Location, error) { continue } - if runtime.GOOS == WindowsOS { - userStrPath = windowsToPosix(userStrPath) + if windows.HostRunningOnWindows() { + userStrPath = windows.ToPosix(userStrPath) } if ref.HasReference() { @@ -286,8 +222,8 @@ func (r Directory) FileContentsByLocation(location file.Location) (io.ReadCloser // RealPath is posix so for windows directory resolver we need to translate // to its true on disk path. filePath := string(location.Reference().RealPath) - if runtime.GOOS == WindowsOS { - filePath = posixToWindows(filePath) + if windows.HostRunningOnWindows() { + filePath = windows.FromPosix(filePath) } return stereoscopeFile.NewLazyReadCloser(filePath), nil @@ -338,30 +274,3 @@ func (r *Directory) FilesByMIMEType(types ...string) ([]file.Location, error) { return uniqueLocations, nil } - -func windowsToPosix(windowsPath string) (posixPath string) { - // volume should be encoded at the start (e.g /c/) where c is the volume - volumeName := filepath.VolumeName(windowsPath) - pathWithoutVolume := strings.TrimPrefix(windowsPath, volumeName) - volumeLetter := strings.ToLower(strings.TrimSuffix(volumeName, ":")) - - // translate non-escaped backslash to forwardslash - translatedPath := strings.ReplaceAll(pathWithoutVolume, "\\", "/") - - // always have `/` as the root... join all components, e.g.: - // convert: C:\\some\windows\Place - // into: /c/some/windows/Place - return path.Clean("/" + strings.Join([]string{volumeLetter, translatedPath}, "/")) -} - -func posixToWindows(posixPath string) (windowsPath string) { - // decode the volume (e.g. /c/ --> C:\\) - There should always be a volume name. - pathFields := strings.Split(posixPath, "/") - volumeName := strings.ToUpper(pathFields[1]) + `:\\` - - // translate non-escaped forward slashes into backslashes - remainingTranslatedPath := strings.Join(pathFields[2:], "\\") - - // combine volume name and backslash components - return filepath.Clean(volumeName + remainingTranslatedPath) -} diff --git a/syft/internal/fileresolver/directory_indexer.go b/syft/internal/fileresolver/directory_indexer.go index 6bdbae0c7f0..47349a4456b 100644 --- a/syft/internal/fileresolver/directory_indexer.go +++ b/syft/internal/fileresolver/directory_indexer.go @@ -7,7 +7,6 @@ import ( "os" "path" "path/filepath" - "runtime" "strings" "github.com/wagoodman/go-partybus" @@ -19,6 +18,7 @@ import ( "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/internal/windows" ) type PathIndexVisitor func(string, os.FileInfo, error) error @@ -263,8 +263,8 @@ func (r *directoryIndexer) indexPath(path string, info os.FileInfo, err error) ( } // here we check to see if we need to normalize paths to posix on the way in coming from windows - if runtime.GOOS == WindowsOS { - path = windowsToPosix(path) + if windows.HostRunningOnWindows() { + path = windows.ToPosix(path) } newRoot, err := r.addPathToIndex(path, info) diff --git a/syft/internal/fileresolver/excluding_file.go b/syft/internal/fileresolver/excluding_file.go index 81caa49c765..34c4948a232 100644 --- a/syft/internal/fileresolver/excluding_file.go +++ b/syft/internal/fileresolver/excluding_file.go @@ -16,9 +16,9 @@ type excluding struct { excludeFn excludeFn } -// NewExcluding create a new resolver which wraps the provided delegate and excludes +// NewExcludingDecorator create a new resolver which wraps the provided delegate and excludes // entries based on a provided path exclusion function -func NewExcluding(delegate file.Resolver, excludeFn excludeFn) file.Resolver { +func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Resolver { return &excluding{ delegate, excludeFn, diff --git a/syft/internal/fileresolver/excluding_file_test.go b/syft/internal/fileresolver/excluding_file_test.go index 2ba51473682..bb4e3ce1e4d 100644 --- a/syft/internal/fileresolver/excluding_file_test.go +++ b/syft/internal/fileresolver/excluding_file_test.go @@ -56,7 +56,7 @@ func TestExcludingResolver(t *testing.T) { resolver := &mockResolver{ locations: test.locations, } - er := NewExcluding(resolver, test.excludeFn) + er := NewExcludingDecorator(resolver, test.excludeFn) locations, _ := er.FilesByPath() assert.ElementsMatch(t, locationPaths(locations), test.expected) diff --git a/syft/internal/fileresolver/test-fixtures/req-resp/.gitignore b/syft/internal/fileresolver/test-fixtures/req-resp/.gitignore new file mode 100644 index 00000000000..c944599212e --- /dev/null +++ b/syft/internal/fileresolver/test-fixtures/req-resp/.gitignore @@ -0,0 +1,2 @@ +path/to/abs-inside.txt +path/to/the/abs-outside.txt \ No newline at end of file diff --git a/syft/internal/fileresolver/unindexed_directory_test.go b/syft/internal/fileresolver/unindexed_directory_test.go index 3714d8d55eb..44ec69bf74b 100644 --- a/syft/internal/fileresolver/unindexed_directory_test.go +++ b/syft/internal/fileresolver/unindexed_directory_test.go @@ -14,7 +14,6 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1263,23 +1262,3 @@ func testWithTimeout(t *testing.T, timeout time.Duration, test func(*testing.T)) case <-done: } } - -func compareLocations(t *testing.T, expected, actual []file.Location) { - t.Helper() - ignoreUnexported := cmpopts.IgnoreFields(file.LocationData{}, "ref") - ignoreMetadata := cmpopts.IgnoreFields(file.LocationMetadata{}, "Annotations") - ignoreFS := cmpopts.IgnoreFields(file.Coordinates{}, "FileSystemID") - - sort.Sort(file.Locations(expected)) - sort.Sort(file.Locations(actual)) - - if d := cmp.Diff(expected, actual, - ignoreUnexported, - ignoreFS, - ignoreMetadata, - ); d != "" { - - t.Errorf("unexpected locations (-want +got):\n%s", d) - } - -} diff --git a/syft/internal/generate.go b/syft/internal/generate.go new file mode 100644 index 00000000000..6780080d3a1 --- /dev/null +++ b/syft/internal/generate.go @@ -0,0 +1,4 @@ +package internal + +//go:generate go run ./sourcemetadata/generate/main.go +//go:generate go run ./packagemetadata/generate/main.go diff --git a/syft/internal/jsonschema/README.md b/syft/internal/jsonschema/README.md new file mode 100644 index 00000000000..dc7b76c91ff --- /dev/null +++ b/syft/internal/jsonschema/README.md @@ -0,0 +1 @@ +Please see [schema/json/README.md](../../../schema/json/README.md) for more information on the JSON schema files in this directory. \ No newline at end of file diff --git a/schema/json/main.go b/syft/internal/jsonschema/main.go similarity index 56% rename from schema/json/main.go rename to syft/internal/jsonschema/main.go index 246abc532a5..26148eb23d7 100644 --- a/schema/json/main.go +++ b/syft/internal/jsonschema/main.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "reflect" "sort" "strings" @@ -13,8 +14,8 @@ import ( "github.com/invopop/jsonschema" "github.com/anchore/syft/internal" - genInt "github.com/anchore/syft/schema/json/internal" - syftjsonModel "github.com/anchore/syft/syft/formats/syftjson/model" + syftJsonModel "github.com/anchore/syft/syft/formats/syftjson/model" + "github.com/anchore/syft/syft/internal/packagemetadata" ) /* @@ -24,30 +25,59 @@ are not captured (empty interfaces). This means that pkg.Package.Metadata is not can be extended to include specific package metadata struct shapes in the future. */ -//go:generate go run ./generate/main.go - -const schemaVersion = internal.JSONSchemaVersion - func main() { write(encode(build())) } +func schemaID() jsonschema.ID { + // Today we do not host the schemas at this address, but per the JSON schema spec we should be referencing + // the schema by a URL in a domain we control. This is a placeholder for now. + return jsonschema.ID(fmt.Sprintf("anchore.io/schema/syft/json/%s", internal.JSONSchemaVersion)) +} + +func assembleTypeContainer(items []any) any { + structFields := make([]reflect.StructField, len(items)) + + for i, item := range items { + itemType := reflect.TypeOf(item) + fieldName := itemType.Name() + + structFields[i] = reflect.StructField{ + Name: fieldName, + Type: itemType, + } + } + + structType := reflect.StructOf(structFields) + return reflect.New(structType).Elem().Interface() +} + func build() *jsonschema.Schema { reflector := &jsonschema.Reflector{ + BaseSchemaID: schemaID(), AllowAdditionalProperties: true, Namer: func(r reflect.Type) string { return strings.TrimPrefix(r.Name(), "JSON") }, } - documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftjsonModel.Document{})) - metadataSchema := reflector.ReflectFromType(reflect.TypeOf(&genInt.ArtifactMetadataContainer{})) - // TODO: inject source definitions - // inject the definitions of all metadatas into the schema definitions + pkgMetadataContainer := assembleTypeContainer(packagemetadata.AllTypes()) + pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer) + + // srcMetadataContainer := assembleTypeContainer(sourcemetadata.AllTypes()) + // srcMetadataContainerType := reflect.TypeOf(srcMetadataContainer) + + documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftJsonModel.Document{})) + pkgMetadataSchema := reflector.ReflectFromType(reflect.TypeOf(pkgMetadataContainer)) + // srcMetadataSchema := reflector.ReflectFromType(reflect.TypeOf(srcMetadataContainer)) + + // TODO: add source metadata types + + // inject the definitions of all packages metadatas into the schema definitions var metadataNames []string - for name, definition := range metadataSchema.Definitions { - if name == reflect.TypeOf(genInt.ArtifactMetadataContainer{}).Name() { + for name, definition := range pkgMetadataSchema.Definitions { + if name == pkgMetadataContainerType.Name() { // ignore the definition for the fake container continue } @@ -93,11 +123,16 @@ func encode(schema *jsonschema.Schema) []byte { } func write(schema []byte) { - filename := fmt.Sprintf("schema-%s.json", schemaVersion) + repoRoot, err := packagemetadata.RepoRoot() + if err != nil { + fmt.Println("unable to determine repo root") + os.Exit(1) + } + schemaPath := filepath.Join(repoRoot, "schema", "json", fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)) - if _, err := os.Stat(filename); !os.IsNotExist(err) { + if _, err := os.Stat(schemaPath); !os.IsNotExist(err) { // check if the schema is the same... - existingFh, err := os.Open(filename) + existingFh, err := os.Open(schemaPath) if err != nil { panic(err) } @@ -114,11 +149,11 @@ func write(schema []byte) { } // the generated schema is different, bail with error :( - fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the schema/json/README.md for how to increment\n", filename) + fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the schema/json/README.md for how to increment\n", schemaPath) os.Exit(1) } - fh, err := os.Create(filename) + fh, err := os.Create(schemaPath) if err != nil { panic(err) } @@ -130,5 +165,5 @@ func write(schema []byte) { defer fh.Close() - fmt.Printf("Wrote new schema to %q\n", filename) + fmt.Printf("Wrote new schema to %q\n", schemaPath) } diff --git a/schema/json/internal/metadata_types.go b/syft/internal/packagemetadata/discover_type_names.go similarity index 96% rename from schema/json/internal/metadata_types.go rename to syft/internal/packagemetadata/discover_type_names.go index 4d515a18890..467b0ffc5de 100644 --- a/schema/json/internal/metadata_types.go +++ b/syft/internal/packagemetadata/discover_type_names.go @@ -1,4 +1,4 @@ -package internal +package packagemetadata import ( "fmt" @@ -18,8 +18,8 @@ var metadataExceptions = strset.New( "FileMetadata", ) -func AllSyftMetadataTypeNames() ([]string, error) { - root, err := repoRoot() +func DiscoverTypeNames() ([]string, error) { + root, err := RepoRoot() if err != nil { return nil, err } @@ -30,7 +30,7 @@ func AllSyftMetadataTypeNames() ([]string, error) { return findMetadataDefinitionNames(files...) } -func repoRoot() (string, error) { +func RepoRoot() (string, error) { root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() if err != nil { return "", fmt.Errorf("unable to find repo root dir: %+v", err) diff --git a/syft/internal/packagemetadata/generate/main.go b/syft/internal/packagemetadata/generate/main.go new file mode 100644 index 00000000000..55c7de41bc5 --- /dev/null +++ b/syft/internal/packagemetadata/generate/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "os" + + "github.com/dave/jennifer/jen" + + "github.com/anchore/syft/syft/internal/packagemetadata" +) + +// This program is invoked from syft/internal and generates packagemetadata/generated.go + +const ( + pkgImport = "github.com/anchore/syft/syft/pkg" + path = "packagemetadata/generated.go" +) + +func main() { + typeNames, err := packagemetadata.DiscoverTypeNames() + if err != nil { + panic(fmt.Errorf("unable to get all metadata type names: %w", err)) + } + + fmt.Printf("updating package metadata type list with %+v types\n", len(typeNames)) + + f := jen.NewFile("packagemetadata") + f.HeaderComment("DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go") + f.ImportName(pkgImport, "pkg") + f.Comment("AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).") + + f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) { + g.ReturnFunc(func(g *jen.Group) { + g.Index().Any().ValuesFunc(func(g *jen.Group) { + for _, typeName := range typeNames { + g.Qual(pkgImport, typeName).Values() + } + }) + }) + }) + + rendered := fmt.Sprintf("%#v", f) + + fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + panic(fmt.Errorf("unable to open file: %w", err)) + } + _, err = fh.WriteString(rendered) + if err != nil { + panic(fmt.Errorf("unable to write file: %w", err)) + } + if err := fh.Close(); err != nil { + panic(fmt.Errorf("unable to close file: %w", err)) + } +} diff --git a/syft/internal/packagemetadata/generated.go b/syft/internal/packagemetadata/generated.go new file mode 100644 index 00000000000..42bca884f07 --- /dev/null +++ b/syft/internal/packagemetadata/generated.go @@ -0,0 +1,10 @@ +// DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go + +package packagemetadata + +import "github.com/anchore/syft/syft/pkg" + +// AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field). +func AllTypes() []any { + return []any{pkg.AlpmMetadata{}, pkg.ApkMetadata{}, pkg.BinaryMetadata{}, pkg.CargoPackageMetadata{}, pkg.CocoapodsMetadata{}, pkg.ConanLockMetadata{}, pkg.ConanMetadata{}, pkg.DartPubMetadata{}, pkg.DotnetDepsMetadata{}, pkg.DpkgMetadata{}, pkg.GemMetadata{}, pkg.GolangBinMetadata{}, pkg.GolangModMetadata{}, pkg.HackageMetadata{}, pkg.JavaMetadata{}, pkg.KbPackageMetadata{}, pkg.LinuxKernelMetadata{}, pkg.LinuxKernelModuleMetadata{}, pkg.MixLockMetadata{}, pkg.NixStoreMetadata{}, pkg.NpmPackageJSONMetadata{}, pkg.NpmPackageLockJSONMetadata{}, pkg.PhpComposerJSONMetadata{}, pkg.PortageMetadata{}, pkg.PythonPackageMetadata{}, pkg.PythonPipfileLockMetadata{}, pkg.PythonRequirementsMetadata{}, pkg.RDescriptionFileMetadata{}, pkg.RebarLockMetadata{}, pkg.RpmMetadata{}} +} diff --git a/syft/internal/packagemetadata/names.go b/syft/internal/packagemetadata/names.go new file mode 100644 index 00000000000..f3dec93478c --- /dev/null +++ b/syft/internal/packagemetadata/names.go @@ -0,0 +1,13 @@ +package packagemetadata + +import ( + "reflect" +) + +func AllNames() []string { + names := make([]string, 0) + for _, t := range AllTypes() { + names = append(names, reflect.TypeOf(t).Name()) + } + return names +} diff --git a/syft/internal/packagemetadata/names_test.go b/syft/internal/packagemetadata/names_test.go new file mode 100644 index 00000000000..60c0abaf32e --- /dev/null +++ b/syft/internal/packagemetadata/names_test.go @@ -0,0 +1,25 @@ +package packagemetadata + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAllNames(t *testing.T) { + // note: this is a form of completion testing relative to the current code base. + + expected, err := DiscoverTypeNames() + require.NoError(t, err) + + actual := AllNames() + + // ensure that the codebase (from ast analysis) reflects the latest code generated state + if !assert.ElementsMatch(t, expected, actual) { + t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual)) + t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?") + t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)") + } +} diff --git a/syft/internal/sourcemetadata/completion_tester.go b/syft/internal/sourcemetadata/completion_tester.go new file mode 100644 index 00000000000..8dc9ce0c444 --- /dev/null +++ b/syft/internal/sourcemetadata/completion_tester.go @@ -0,0 +1,69 @@ +package sourcemetadata + +import ( + "reflect" + "testing" +) + +type CompletionTester struct { + saw []any + valid []any + ignore []any +} + +func NewCompletionTester(t testing.TB, ignore ...any) *CompletionTester { + tester := &CompletionTester{ + valid: AllTypes(), + ignore: ignore, + } + t.Cleanup(func() { + t.Helper() + tester.validate(t) + }) + return tester +} + +func (tr *CompletionTester) Tested(t testing.TB, m any) { + t.Helper() + + if m == nil { + return + } + if len(tr.valid) == 0 { + t.Fatal("no valid metadata types to test against") + } + ty := reflect.TypeOf(m) + for _, v := range tr.valid { + if reflect.TypeOf(v) == ty { + tr.saw = append(tr.saw, m) + return + } + } + + t.Fatalf("tested metadata type is not valid: %s", ty.Name()) +} + +func (tr *CompletionTester) validate(t testing.TB) { + t.Helper() + + count := make(map[reflect.Type]int) + for _, m := range tr.saw { + count[reflect.TypeOf(m)]++ + } + +validations: + for _, v := range tr.valid { + ty := reflect.TypeOf(v) + + for _, ignore := range tr.ignore { + if ty == reflect.TypeOf(ignore) { + // skip ignored types + continue validations + } + } + + if c, exists := count[ty]; c == 0 || !exists { + t.Errorf("metadata type %s is not covered by a test", ty.Name()) + } + } +} diff --git a/syft/internal/sourcemetadata/discover_type_names.go b/syft/internal/sourcemetadata/discover_type_names.go new file mode 100644 index 00000000000..9b1ac2f5843 --- /dev/null +++ b/syft/internal/sourcemetadata/discover_type_names.go @@ -0,0 +1,148 @@ +package sourcemetadata + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os/exec" + "path/filepath" + "sort" + "strings" + "unicode" + + "github.com/scylladb/go-set/strset" +) + +var metadataExceptions = strset.New() + +func DiscoverTypeNames() ([]string, error) { + root, err := repoRoot() + if err != nil { + return nil, err + } + files, err := filepath.Glob(filepath.Join(root, "syft/source/*.go")) + if err != nil { + return nil, err + } + return findMetadataDefinitionNames(files...) +} + +func repoRoot() (string, error) { + root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + return "", fmt.Errorf("unable to find repo root dir: %+v", err) + } + absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root))) + if err != nil { + return "", fmt.Errorf("unable to get abs path to repo root: %w", err) + } + return absRepoRoot, nil +} + +func findMetadataDefinitionNames(paths ...string) ([]string, error) { + names := strset.New() + usedNames := strset.New() + for _, path := range paths { + metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path) + if err != nil { + return nil, err + } + + // useful for debugging... + // fmt.Println(path) + // fmt.Println("Defs:", metadataDefinitions) + // fmt.Println("Used Types:", usedTypeNames) + // fmt.Println() + + names.Add(metadataDefinitions...) + usedNames.Add(usedTypeNames...) + } + + // any definition that is used within another struct should not be considered a top-level metadata definition + names.Remove(usedNames.List()...) + + strNames := names.List() + sort.Strings(strNames) + + // note: 3 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required. + // it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions. + if len(strNames) < 3 { + return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")") + } + + return strNames, nil +} + +func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) { + // set up the parser + fs := token.NewFileSet() + f, err := parser.ParseFile(fs, path, nil, parser.ParseComments) + if err != nil { + return nil, nil, err + } + + var metadataDefinitions []string + var usedTypeNames []string + for _, decl := range f.Decls { + // check if the declaration is a type declaration + spec, ok := decl.(*ast.GenDecl) + if !ok || spec.Tok != token.TYPE { + continue + } + + // loop over all types declared in the type declaration + for _, typ := range spec.Specs { + // check if the type is a struct type + spec, ok := typ.(*ast.TypeSpec) + if !ok || spec.Type == nil { + continue + } + + structType, ok := spec.Type.(*ast.StructType) + if !ok { + continue + } + + // check if the struct type ends with "Metadata" + name := spec.Name.String() + + // only look for exported types that end with "Metadata" + if isMetadataTypeCandidate(name) { + // print the full declaration of the struct type + metadataDefinitions = append(metadataDefinitions, name) + usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...) + } + } + } + return metadataDefinitions, usedTypeNames, nil +} + +func typeNamesUsedInStruct(structType *ast.StructType) []string { + // recursively find all type names used in the struct type + var names []string + for i := range structType.Fields.List { + // capture names of all of the types (not field names) + ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool { + ident, ok := n.(*ast.Ident) + if !ok { + return true + } + + // add the type name to the list + names = append(names, ident.Name) + + // continue inspecting + return true + }) + } + + return names +} + +func isMetadataTypeCandidate(name string) bool { + return len(name) > 0 && + strings.HasSuffix(name, "Metadata") && + unicode.IsUpper(rune(name[0])) && // must be exported + !metadataExceptions.Has(name) +} diff --git a/syft/internal/sourcemetadata/generate/main.go b/syft/internal/sourcemetadata/generate/main.go new file mode 100644 index 00000000000..ea40960a073 --- /dev/null +++ b/syft/internal/sourcemetadata/generate/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "os" + + "github.com/dave/jennifer/jen" + + "github.com/anchore/syft/syft/internal/sourcemetadata" +) + +// This program is invoked from syft/internal and generates sourcemetadata/generated.go + +const ( + srcImport = "github.com/anchore/syft/syft/source" + path = "sourcemetadata/generated.go" +) + +func main() { + typeNames, err := sourcemetadata.DiscoverTypeNames() + if err != nil { + panic(fmt.Errorf("unable to get all metadata type names: %w", err)) + } + + fmt.Printf("updating source metadata type list with %+v types\n", len(typeNames)) + + f := jen.NewFile("sourcemetadata") + f.HeaderComment("DO NOT EDIT: generated by syft/internal/sourcemetadata/generate/main.go") + f.ImportName(srcImport, "source") + f.Comment("AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).") + + f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) { + g.ReturnFunc(func(g *jen.Group) { + g.Index().Any().ValuesFunc(func(g *jen.Group) { + for _, typeName := range typeNames { + g.Qual(srcImport, typeName).Values() + } + }) + }) + }) + + rendered := fmt.Sprintf("%#v", f) + + fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + panic(fmt.Errorf("unable to open file: %w", err)) + } + _, err = fh.WriteString(rendered) + if err != nil { + panic(fmt.Errorf("unable to write file: %w", err)) + } + if err := fh.Close(); err != nil { + panic(fmt.Errorf("unable to close file: %w", err)) + } +} diff --git a/syft/internal/sourcemetadata/generated.go b/syft/internal/sourcemetadata/generated.go new file mode 100644 index 00000000000..c829f7eac37 --- /dev/null +++ b/syft/internal/sourcemetadata/generated.go @@ -0,0 +1,10 @@ +// DO NOT EDIT: generated by syft/internal/sourcemetadata/generate/main.go + +package sourcemetadata + +import "github.com/anchore/syft/syft/source" + +// AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field). +func AllTypes() []any { + return []any{source.DirectorySourceMetadata{}, source.FileSourceMetadata{}, source.StereoscopeImageSourceMetadata{}} +} diff --git a/syft/internal/sourcemetadata/names.go b/syft/internal/sourcemetadata/names.go new file mode 100644 index 00000000000..b33e7f9422c --- /dev/null +++ b/syft/internal/sourcemetadata/names.go @@ -0,0 +1,41 @@ +package sourcemetadata + +import ( + "reflect" + "strings" + + "github.com/anchore/syft/syft/source" +) + +var jsonNameFromType = map[reflect.Type][]string{ + reflect.TypeOf(source.DirectorySourceMetadata{}): {"directory", "dir"}, + reflect.TypeOf(source.FileSourceMetadata{}): {"file"}, + reflect.TypeOf(source.StereoscopeImageSourceMetadata{}): {"image"}, +} + +func AllNames() []string { + names := make([]string, 0) + for _, t := range AllTypes() { + names = append(names, reflect.TypeOf(t).Name()) + } + return names +} + +func JSONName(metadata any) string { + if vs, exists := jsonNameFromType[reflect.TypeOf(metadata)]; exists { + return vs[0] + } + return "" +} + +func ReflectTypeFromJSONName(name string) reflect.Type { + name = strings.ToLower(name) + for t, vs := range jsonNameFromType { + for _, v := range vs { + if v == name { + return t + } + } + } + return nil +} diff --git a/syft/internal/sourcemetadata/names_test.go b/syft/internal/sourcemetadata/names_test.go new file mode 100644 index 00000000000..d3ff762117d --- /dev/null +++ b/syft/internal/sourcemetadata/names_test.go @@ -0,0 +1,29 @@ +package sourcemetadata + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAllNames(t *testing.T) { + // note: this is a form of completion testing relative to the current code base. + + expected, err := DiscoverTypeNames() + require.NoError(t, err) + + actual := AllNames() + + // ensure that the codebase (from ast analysis) reflects the latest code generated state + if !assert.ElementsMatch(t, expected, actual) { + t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual)) + t.Log("did you add a new source.*Metadata type without updating the JSON schema?") + t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)") + } + + for _, ty := range AllTypes() { + assert.NotEmpty(t, JSONName(ty), "metadata type %q does not have a JSON name", ty) + } +} diff --git a/syft/internal/windows/path.go b/syft/internal/windows/path.go new file mode 100644 index 00000000000..7aa59d1ca9d --- /dev/null +++ b/syft/internal/windows/path.go @@ -0,0 +1,41 @@ +package windows + +import ( + "path" + "path/filepath" + "runtime" + "strings" +) + +const windowsGoOS = "windows" + +func HostRunningOnWindows() bool { + return runtime.GOOS == windowsGoOS +} + +func ToPosix(windowsPath string) (posixPath string) { + // volume should be encoded at the start (e.g /c/) where c is the volume + volumeName := filepath.VolumeName(windowsPath) + pathWithoutVolume := strings.TrimPrefix(windowsPath, volumeName) + volumeLetter := strings.ToLower(strings.TrimSuffix(volumeName, ":")) + + // translate non-escaped backslash to forwardslash + translatedPath := strings.ReplaceAll(pathWithoutVolume, "\\", "/") + + // always have `/` as the root... join all components, e.g.: + // convert: C:\\some\windows\Place + // into: /c/some/windows/Place + return path.Clean("/" + strings.Join([]string{volumeLetter, translatedPath}, "/")) +} + +func FromPosix(posixPath string) (windowsPath string) { + // decode the volume (e.g. /c/ --> C:\\) - There should always be a volume name. + pathFields := strings.Split(posixPath, "/") + volumeName := strings.ToUpper(pathFields[1]) + `:\\` + + // translate non-escaped forward slashes into backslashes + remainingTranslatedPath := strings.Join(pathFields[2:], "\\") + + // combine volume name and backslash components + return filepath.Clean(volumeName + remainingTranslatedPath) +} diff --git a/syft/lib.go b/syft/lib.go index ea28690066a..849584ab728 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -34,7 +34,7 @@ import ( // CatalogPackages takes an inventory of packages from the given image from a particular perspective // (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux // distribution, and the source object used to wrap the data source. -func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) { +func CatalogPackages(src source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) { resolver, err := src.FileResolver(cfg.Search.Scope) if err != nil { return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err) @@ -54,18 +54,21 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection, catalogers = cataloger.AllCatalogers(cfg) } else { // otherwise conditionally use the correct set of loggers based on the input type (container image or directory) - switch src.Metadata.Scheme { - case source.ImageScheme: + + // TODO: this is bad, we should not be using the concrete type to determine the cataloger set + // instead this should be a caller concern (pass the catalogers you want to use). The SBOM build PR will do this. + switch src.(type) { + case *source.StereoscopeImageSource: log.Info("cataloging an image") catalogers = cataloger.ImageCatalogers(cfg) - case source.FileScheme: + case *source.FileSource: log.Info("cataloging a file") catalogers = cataloger.AllCatalogers(cfg) - case source.DirectoryScheme: + case *source.DirectorySource: log.Info("cataloging a directory") catalogers = cataloger.DirectoryCatalogers(cfg) default: - return nil, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme) + return nil, nil, nil, fmt.Errorf("unsupported source type: %T", src) } } @@ -76,7 +79,7 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection, return catalog, relationships, release, err } -func newSourceRelationshipsFromCatalog(src *source.Source, c *pkg.Collection) []artifact.Relationship { +func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []artifact.Relationship { relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method? for p := range c.Enumerate() { relationships = append(relationships, artifact.Relationship{ diff --git a/syft/linux/identify_release_test.go b/syft/linux/identify_release_test.go index 00d04ee71fc..9af79fcdf4f 100644 --- a/syft/linux/identify_release_test.go +++ b/syft/linux/identify_release_test.go @@ -336,7 +336,7 @@ func TestIdentifyRelease(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - s, err := source.NewFromDirectory(test.fixture) + s, err := source.NewFromDirectoryPath(test.fixture) require.NoError(t, err) resolver, err := s.FileResolver(source.SquashedScope) diff --git a/syft/pkg/cataloger/binary/cataloger_test.go b/syft/pkg/cataloger/binary/cataloger_test.go index d6622423bb5..4a36a5e1dea 100644 --- a/syft/pkg/cataloger/binary/cataloger_test.go +++ b/syft/pkg/cataloger/binary/cataloger_test.go @@ -649,7 +649,7 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) { t.Run(test.name, func(t *testing.T) { c := NewCataloger() - src, err := source.NewFromDirectory(test.fixtureDir) + src, err := source.NewFromDirectoryPath(test.fixtureDir) require.NoError(t, err) resolver, err := src.FileResolver(source.SquashedScope) @@ -688,7 +688,7 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) { c := NewCataloger() img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage) - src, err := source.NewFromImage(img, "test-img") + src, err := source.NewFromStereoscopeImageObject(img, test.fixtureImage, nil) require.NoError(t, err) resolver, err := src.FileResolver(source.SquashedScope) @@ -718,7 +718,7 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) { func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) { c := NewCataloger() - src, err := source.NewFromDirectory("test-fixtures/classifiers/negative") + src, err := source.NewFromDirectoryPath("test-fixtures/classifiers/negative") assert.NoError(t, err) resolver, err := src.FileResolver(source.SquashedScope) diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index 9545c66b6fd..573cc5bee76 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -85,7 +85,7 @@ func DefaultLicenseComparer(x, y pkg.License) bool { func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester { t.Helper() - s, err := source.NewFromDirectory(path) + s, err := source.NewFromDirectoryPath(path) require.NoError(t, err) resolver, err := s.FileResolver(source.AllLayersScope) @@ -149,7 +149,7 @@ func (p *CatalogTester) WithImageResolver(t *testing.T, fixtureName string) *Cat t.Helper() img := imagetest.GetFixtureImage(t, "docker-archive", fixtureName) - s, err := source.NewFromImage(img, fixtureName) + s, err := source.NewFromStereoscopeImageObject(img, fixtureName, nil) require.NoError(t, err) r, err := s.FileResolver(source.SquashedScope) diff --git a/syft/pkg/cataloger/search_config.go b/syft/pkg/cataloger/search_config.go index f92dc9928a9..17a6a3019cc 100644 --- a/syft/pkg/cataloger/search_config.go +++ b/syft/pkg/cataloger/search_config.go @@ -1,6 +1,8 @@ package cataloger -import "github.com/anchore/syft/syft/source" +import ( + "github.com/anchore/syft/syft/source" +) type SearchConfig struct { IncludeIndexedArchives bool diff --git a/syft/sbom/sbom.go b/syft/sbom/sbom.go index 0bc8feb0cfa..8592d8440c8 100644 --- a/syft/sbom/sbom.go +++ b/syft/sbom/sbom.go @@ -15,7 +15,7 @@ import ( type SBOM struct { Artifacts Artifacts Relationships []artifact.Relationship - Source source.Metadata + Source source.Description Descriptor Descriptor } diff --git a/syft/source/alias.go b/syft/source/alias.go new file mode 100644 index 00000000000..e1c5c670173 --- /dev/null +++ b/syft/source/alias.go @@ -0,0 +1,13 @@ +package source + +type Alias struct { + Name string `json:"name" yaml:"name" mapstructure:"name"` + Version string `json:"version" yaml:"version" mapstructure:"version"` +} + +func (a *Alias) IsEmpty() bool { + if a == nil { + return true + } + return a.Name == "" && a.Version == "" +} diff --git a/syft/source/description.go b/syft/source/description.go new file mode 100644 index 00000000000..0aae582573f --- /dev/null +++ b/syft/source/description.go @@ -0,0 +1,9 @@ +package source + +// Description represents any static source data that helps describe "what" was cataloged. +type Description struct { + ID string `hash:"ignore"` // the id generated from the parent source struct + Name string `hash:"ignore"` + Version string `hash:"ignore"` + Metadata interface{} +} diff --git a/syft/source/detection.go b/syft/source/detection.go new file mode 100644 index 00000000000..3d301f14da5 --- /dev/null +++ b/syft/source/detection.go @@ -0,0 +1,200 @@ +package source + +import ( + "crypto" + "fmt" + "strings" + + "github.com/mitchellh/go-homedir" + "github.com/spf13/afero" + + "github.com/anchore/stereoscope/pkg/image" +) + +type detectedType string + +const ( + // unknownType is the default scheme + unknownType detectedType = "unknown-type" + + // directoryType indicates the source being cataloged is a directory on the root filesystem + directoryType detectedType = "directory-type" + + // containerImageType indicates the source being cataloged is a container image + containerImageType detectedType = "container-image-type" + + // fileType indicates the source being cataloged is a single file + fileType detectedType = "file-type" +) + +type sourceResolver func(string) (image.Source, string, error) + +// Detection is an object that captures the detected user input regarding source location, scheme, and provider type. +// It acts as a struct input for some source constructors. +type Detection struct { + detectedType detectedType + imageSource image.Source + location string +} + +func (d Detection) IsContainerImage() bool { + return d.detectedType == containerImageType +} + +type DetectConfig struct { + DefaultImageSource string +} + +func DefaultDetectConfig() DetectConfig { + return DetectConfig{} +} + +// Detect generates a source Detection that can be used as an argument to generate a new source +// from specific providers including a registry, with an explicit name. +func Detect(userInput string, cfg DetectConfig) (*Detection, error) { + fs := afero.NewOsFs() + ty, src, location, err := detect(fs, image.DetectSource, userInput) + if err != nil { + return nil, err + } + + if src == image.UnknownSource { + // only run for these two schemes + // only check on packages command, attest we automatically try to pull from userInput + switch ty { + case containerImageType, unknownType: + ty = containerImageType + location = userInput + if cfg.DefaultImageSource != "" { + src = parseDefaultImageSource(cfg.DefaultImageSource) + } else { + src = image.DetermineDefaultImagePullSource(userInput) + } + } + } + + // collect user input for downstream consumption + return &Detection{ + detectedType: ty, + imageSource: src, + location: location, + }, nil +} + +type DetectionSourceConfig struct { + Alias Alias + RegistryOptions *image.RegistryOptions + Platform *image.Platform + Exclude ExcludeConfig + DigestAlgorithms []crypto.Hash +} + +func DefaultDetectionSourceConfig() DetectionSourceConfig { + return DetectionSourceConfig{ + DigestAlgorithms: []crypto.Hash{ + crypto.SHA256, + }, + } +} + +// NewSource produces a Source based on userInput like dir: or image:tag +func (d Detection) NewSource(cfg DetectionSourceConfig) (Source, error) { + var err error + var src Source + + if d.detectedType != containerImageType && cfg.Platform != nil { + return nil, fmt.Errorf("cannot specify a platform for a non-image source") + } + + switch d.detectedType { + case fileType: + src, err = NewFromFile( + FileConfig{ + Path: d.location, + Exclude: cfg.Exclude, + DigestAlgorithms: cfg.DigestAlgorithms, + Alias: cfg.Alias, + }, + ) + case directoryType: + src, err = NewFromDirectory( + DirectoryConfig{ + Path: d.location, + Base: d.location, + Exclude: cfg.Exclude, + Alias: cfg.Alias, + }, + ) + case containerImageType: + src, err = NewFromStereoscopeImage( + StereoscopeImageConfig{ + Reference: d.location, + From: d.imageSource, + Platform: cfg.Platform, + RegistryOptions: cfg.RegistryOptions, + Exclude: cfg.Exclude, + Alias: cfg.Alias, + }, + ) + default: + err = fmt.Errorf("unable to process input for scanning") + } + + return src, err +} + +func detect(fs afero.Fs, imageSourceResolver sourceResolver, userInput string) (detectedType, image.Source, string, error) { + switch { + case strings.HasPrefix(userInput, "dir:"): + dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:")) + if err != nil { + return unknownType, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err) + } + return directoryType, image.UnknownSource, dirLocation, nil + + case strings.HasPrefix(userInput, "file:"): + fileLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "file:")) + if err != nil { + return unknownType, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err) + } + return fileType, image.UnknownSource, fileLocation, nil + } + + // try the most specific sources first and move out towards more generic sources. + + // first: let's try the image detector, which has more scheme parsing internal to stereoscope + src, imageSpec, err := imageSourceResolver(userInput) + if err == nil && src != image.UnknownSource { + return containerImageType, src, imageSpec, nil + } + + // next: let's try more generic sources (dir, file, etc.) + location, err := homedir.Expand(userInput) + if err != nil { + return unknownType, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err) + } + + fileMeta, err := fs.Stat(location) + if err != nil { + return unknownType, src, "", nil + } + + if fileMeta.IsDir() { + return directoryType, src, location, nil + } + + return fileType, src, location, nil +} + +func parseDefaultImageSource(defaultImageSource string) image.Source { + switch defaultImageSource { + case "registry": + return image.OciRegistrySource + case "docker": + return image.DockerDaemonSource + case "podman": + return image.PodmanDaemonSource + default: + return image.UnknownSource + } +} diff --git a/syft/source/scheme_test.go b/syft/source/detection_test.go similarity index 88% rename from syft/source/scheme_test.go rename to syft/source/detection_test.go index 0523f977ec1..380ca8e658b 100644 --- a/syft/source/scheme_test.go +++ b/syft/source/detection_test.go @@ -11,7 +11,7 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -func TestDetectScheme(t *testing.T) { +func Test_Detect(t *testing.T) { type detectorResult struct { src image.Source ref string @@ -24,7 +24,7 @@ func TestDetectScheme(t *testing.T) { dirs []string files []string detection detectorResult - expectedScheme Scheme + expectedScheme detectedType expectedLocation string }{ { @@ -34,7 +34,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "wagoodman/dive:latest", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "wagoodman/dive:latest", }, { @@ -44,7 +44,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "wagoodman/dive", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "wagoodman/dive", }, { @@ -54,7 +54,7 @@ func TestDetectScheme(t *testing.T) { src: image.OciRegistrySource, ref: "wagoodman/dive:latest", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "wagoodman/dive:latest", }, { @@ -64,7 +64,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "wagoodman/dive:latest", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "wagoodman/dive:latest", }, { @@ -74,7 +74,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "wagoodman/dive", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "wagoodman/dive", }, { @@ -84,7 +84,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "latest", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, // we expected to be able to handle this case better, however, I don't see a way to do this // the user will need to provide more explicit input (docker:docker:latest) expectedLocation: "latest", @@ -96,7 +96,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "docker:latest", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, // we expected to be able to handle this case better, however, I don't see a way to do this // the user will need to provide more explicit input (docker:docker:latest) expectedLocation: "docker:latest", @@ -108,7 +108,7 @@ func TestDetectScheme(t *testing.T) { src: image.OciTarballSource, ref: "some/path-to-file", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "some/path-to-file", }, { @@ -119,7 +119,7 @@ func TestDetectScheme(t *testing.T) { ref: "some/path-to-dir", }, dirs: []string{"some/path-to-dir"}, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "some/path-to-dir", }, { @@ -130,7 +130,7 @@ func TestDetectScheme(t *testing.T) { ref: "", }, dirs: []string{"some/path-to-dir"}, - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: "some/path-to-dir", }, { @@ -140,7 +140,7 @@ func TestDetectScheme(t *testing.T) { src: image.DockerDaemonSource, ref: "some/path-to-dir", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "some/path-to-dir", }, { @@ -150,7 +150,7 @@ func TestDetectScheme(t *testing.T) { src: image.PodmanDaemonSource, ref: "something:latest", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "something:latest", }, { @@ -161,7 +161,7 @@ func TestDetectScheme(t *testing.T) { ref: "", }, dirs: []string{"some/path-to-dir"}, - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: "some/path-to-dir", }, { @@ -172,7 +172,7 @@ func TestDetectScheme(t *testing.T) { ref: "", }, files: []string{"some/path-to-file"}, - expectedScheme: FileScheme, + expectedScheme: fileType, expectedLocation: "some/path-to-file", }, { @@ -183,7 +183,7 @@ func TestDetectScheme(t *testing.T) { ref: "", }, files: []string{"some/path-to-file"}, - expectedScheme: FileScheme, + expectedScheme: fileType, expectedLocation: "some/path-to-file", }, { @@ -193,7 +193,7 @@ func TestDetectScheme(t *testing.T) { src: image.UnknownSource, ref: "", }, - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: ".", }, { @@ -203,7 +203,7 @@ func TestDetectScheme(t *testing.T) { src: image.UnknownSource, ref: "", }, - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: ".", }, // we should support tilde expansion @@ -214,7 +214,7 @@ func TestDetectScheme(t *testing.T) { src: image.OciDirectorySource, ref: "~/some-path", }, - expectedScheme: ImageScheme, + expectedScheme: containerImageType, expectedLocation: "~/some-path", }, { @@ -225,26 +225,26 @@ func TestDetectScheme(t *testing.T) { ref: "", }, dirs: []string{"~/some-path"}, - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: "~/some-path", }, { name: "tilde-expansion-dir-explicit-exists", userInput: "dir:~/some-path", dirs: []string{"~/some-path"}, - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: "~/some-path", }, { name: "tilde-expansion-dir-explicit-dne", userInput: "dir:~/some-path", - expectedScheme: DirectoryScheme, + expectedScheme: directoryType, expectedLocation: "~/some-path", }, { name: "tilde-expansion-dir-implicit-dne", userInput: "~/some-path", - expectedScheme: UnknownScheme, + expectedScheme: unknownType, expectedLocation: "", }, } @@ -288,7 +288,7 @@ func TestDetectScheme(t *testing.T) { } } - actualScheme, actualSource, actualLocation, err := DetectScheme(fs, imageDetector, test.userInput) + actualScheme, actualSource, actualLocation, err := detect(fs, imageDetector, test.userInput) if err != nil { t.Fatalf("unexpected err : %+v", err) } diff --git a/syft/source/digest_utils.go b/syft/source/digest_utils.go new file mode 100644 index 00000000000..6c7f2feebd9 --- /dev/null +++ b/syft/source/digest_utils.go @@ -0,0 +1,11 @@ +package source + +import ( + "strings" + + "github.com/anchore/syft/syft/artifact" +) + +func artifactIDFromDigest(input string) artifact.ID { + return artifact.ID(strings.TrimPrefix(input, "sha256:")) +} diff --git a/syft/source/directory_source.go b/syft/source/directory_source.go new file mode 100644 index 00000000000..ab7f3d46231 --- /dev/null +++ b/syft/source/directory_source.go @@ -0,0 +1,215 @@ +package source + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/bmatcuk/doublestar/v4" + "github.com/opencontainers/go-digest" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" +) + +var _ Source = (*DirectorySource)(nil) + +type DirectoryConfig struct { + Path string + Base string + Exclude ExcludeConfig + Alias Alias +} + +type DirectorySourceMetadata struct { + Path string `json:"path" yaml:"path"` + Base string `json:"-" yaml:"-"` // though this is important, for display purposes it leaks too much information (abs paths) +} + +type DirectorySource struct { + id artifact.ID + config DirectoryConfig + resolver *fileresolver.Directory + mutex *sync.Mutex +} + +func NewFromDirectoryPath(path string) (*DirectorySource, error) { + cfg := DirectoryConfig{ + Path: path, + } + return NewFromDirectory(cfg) +} + +func NewFromDirectory(cfg DirectoryConfig) (*DirectorySource, error) { + fi, err := os.Stat(cfg.Path) + if err != nil { + return nil, fmt.Errorf("unable to stat path=%q: %w", cfg.Path, err) + } + + if !fi.IsDir() { + return nil, fmt.Errorf("given path is not a directory: %q", cfg.Path) + } + + return &DirectorySource{ + id: deriveIDFromDirectory(cfg), + config: cfg, + mutex: &sync.Mutex{}, + }, nil +} + +// deriveIDFromDirectory generates an artifact ID from the given directory config. If an alias is provided, then +// the artifact ID is derived exclusively from the alias name and version. Otherwise, the artifact ID is derived +// from the path provided with an attempt to prune a prefix if a base is given. Since the contents of the directory +// are not considered, there is no semantic meaning to the artifact ID -- this is why the alias is preferred without +// consideration for the path. +func deriveIDFromDirectory(cfg DirectoryConfig) artifact.ID { + var info string + if !cfg.Alias.IsEmpty() { + // don't use any of the path information -- instead use the alias name and version as the artifact ID. + // why? this allows the user to set a dependable stable value for the artifact ID in case the + // scanning root changes (e.g. a user scans a directory, then moves it to a new location and scans again). + info = fmt.Sprintf("%s@%s", cfg.Alias.Name, cfg.Alias.Version) + } else { + log.Warn("no explicit name and version provided for directory source, deriving artifact ID from the given path (which is not ideal)") + info = cleanDirPath(cfg.Path, cfg.Base) + } + + return artifactIDFromDigest(digest.SHA256.FromString(filepath.Clean(info)).String()) +} + +func cleanDirPath(path, base string) string { + if path == base { + return path + } + + if base != "" { + cleanRoot, rootErr := fileresolver.NormalizeRootDirectory(path) + cleanBase, baseErr := fileresolver.NormalizeBaseDirectory(base) + + if rootErr == nil && baseErr == nil { + // allows for normalizing inputs: + // cleanRoot: /var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/TestDirectoryEncoder1121632790/001/some/path + // cleanBase: /var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/TestDirectoryEncoder1121632790/001 + // normalized: some/path + + relPath, err := filepath.Rel(cleanBase, cleanRoot) + if err == nil { + path = relPath + } + // this is odd, but this means we can't use base + } + // if the base is not a valid chroot, then just use the path as-is + } + + return path +} + +func (s DirectorySource) ID() artifact.ID { + return s.id +} + +func (s DirectorySource) Describe() Description { + name := cleanDirPath(s.config.Path, s.config.Base) + version := "" + if !s.config.Alias.IsEmpty() { + a := s.config.Alias + if a.Name != "" { + name = a.Name + } + } + return Description{ + ID: string(s.id), + Name: name, + Version: version, + Metadata: DirectorySourceMetadata{ + Path: s.config.Path, + Base: s.config.Base, + }, + } +} + +func (s *DirectorySource) FileResolver(_ Scope) (file.Resolver, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + if s.resolver == nil { + exclusionFunctions, err := getDirectoryExclusionFunctions(s.config.Path, s.config.Exclude.Paths) + if err != nil { + return nil, err + } + + res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...) + if err != nil { + return nil, fmt.Errorf("unable to create directory resolver: %w", err) + } + + s.resolver = res + } + + return s.resolver, nil +} + +func (s *DirectorySource) Close() error { + s.mutex.Lock() + defer s.mutex.Unlock() + s.resolver = nil + return nil +} + +func getDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) { + if len(exclusions) == 0 { + return nil, nil + } + + // this is what directoryResolver.indexTree is doing to get the absolute path: + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + + // this handles Windows file paths by converting them to C:/something/else format + root = filepath.ToSlash(root) + + if !strings.HasSuffix(root, "/") { + root += "/" + } + + var errors []string + for idx, exclusion := range exclusions { + // check exclusions for supported paths, these are all relative to the "scan root" + if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") { + exclusion = strings.TrimPrefix(exclusion, "./") + exclusions[idx] = root + exclusion + } else { + errors = append(errors, exclusion) + } + } + + if errors != nil { + return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '")) + } + + return []fileresolver.PathIndexVisitor{ + func(path string, info os.FileInfo, _ error) error { + for _, exclusion := range exclusions { + // this is required to handle Windows filepaths + path = filepath.ToSlash(path) + matches, err := doublestar.Match(exclusion, path) + if err != nil { + return nil + } + if matches { + if info != nil && info.IsDir() { + return filepath.SkipDir + } + return fileresolver.ErrSkipPath + } + } + return nil + }, + }, nil +} diff --git a/syft/source/directory_source_test.go b/syft/source/directory_source_test.go new file mode 100644 index 00000000000..c324fb91b9c --- /dev/null +++ b/syft/source/directory_source_test.go @@ -0,0 +1,560 @@ +package source + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/internal/fileresolver" +) + +func TestNewFromDirectory(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + inputPaths []string + expectedRefs int + cxErr require.ErrorAssertionFunc + }{ + { + desc: "no paths exist", + input: "foobar/", + inputPaths: []string{"/opt/", "/other"}, + cxErr: require.Error, + }, + { + desc: "path detected", + input: "test-fixtures", + inputPaths: []string{"path-detected/.vimrc"}, + expectedRefs: 1, + }, + { + desc: "directory ignored", + input: "test-fixtures", + inputPaths: []string{"path-detected"}, + expectedRefs: 0, + }, + { + desc: "no files-by-path detected", + input: "test-fixtures", + inputPaths: []string{"no-path-detected"}, + expectedRefs: 0, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + if test.cxErr == nil { + test.cxErr = require.NoError + } + src, err := NewFromDirectory(DirectoryConfig{ + Path: test.input, + }) + test.cxErr(t, err) + if err != nil { + return + } + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + assert.Equal(t, test.input, src.Describe().Metadata.(DirectorySourceMetadata).Path) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := res.FilesByPath(test.inputPaths...) + require.NoError(t, err) + + if len(refs) != test.expectedRefs { + t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expectedRefs) + } + + }) + } +} + +func Test_DirectorySource_FilesByGlob(t *testing.T) { + testCases := []struct { + desc string + input string + glob string + expected int + }{ + { + input: "test-fixtures", + desc: "no matches", + glob: "bar/foo", + expected: 0, + }, + { + input: "test-fixtures/path-detected", + desc: "a single match", + glob: "**/*vimrc", + expected: 1, + }, + { + input: "test-fixtures/path-detected", + desc: "multiple matches", + glob: "**", + expected: 2, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := NewFromDirectory(DirectoryConfig{Path: test.input}) + require.NoError(t, err) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + contents, err := res.FilesByGlob(test.glob) + require.NoError(t, err) + if len(contents) != test.expected { + t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected) + } + + }) + } +} + +func Test_DirectorySource_Exclusions(t *testing.T) { + testCases := []struct { + desc string + input string + glob string + expected []string + exclusions []string + err bool + }{ + { + input: "test-fixtures/system_paths", + desc: "exclude everything", + glob: "**", + expected: nil, + exclusions: []string{"**/*"}, + }, + { + input: "test-fixtures/image-simple", + desc: "a single path excluded", + glob: "**", + expected: []string{ + "Dockerfile", + "file-1.txt", + "file-2.txt", + }, + exclusions: []string{"**/target/**"}, + }, + { + input: "test-fixtures/image-simple", + desc: "exclude explicit directory relative to the root", + glob: "**", + expected: []string{ + "Dockerfile", + "file-1.txt", + "file-2.txt", + //"target/really/nested/file-3.txt", // explicitly skipped + }, + exclusions: []string{"./target"}, + }, + { + input: "test-fixtures/image-simple", + desc: "exclude explicit file relative to the root", + glob: "**", + expected: []string{ + "Dockerfile", + //"file-1.txt", // explicitly skipped + "file-2.txt", + "target/really/nested/file-3.txt", + }, + exclusions: []string{"./file-1.txt"}, + }, + { + input: "test-fixtures/image-simple", + desc: "exclude wildcard relative to the root", + glob: "**", + expected: []string{ + "Dockerfile", + //"file-1.txt", // explicitly skipped + //"file-2.txt", // explicitly skipped + "target/really/nested/file-3.txt", + }, + exclusions: []string{"./*.txt"}, + }, + { + input: "test-fixtures/image-simple", + desc: "exclude files deeper", + glob: "**", + expected: []string{ + "Dockerfile", + "file-1.txt", + "file-2.txt", + //"target/really/nested/file-3.txt", // explicitly skipped + }, + exclusions: []string{"**/really/**"}, + }, + { + input: "test-fixtures/image-simple", + desc: "files excluded with extension", + glob: "**", + expected: []string{ + "Dockerfile", + //"file-1.txt", // explicitly skipped + //"file-2.txt", // explicitly skipped + //"target/really/nested/file-3.txt", // explicitly skipped + }, + exclusions: []string{"**/*.txt"}, + }, + { + input: "test-fixtures/image-simple", + desc: "keep files with different extensions", + glob: "**", + expected: []string{ + "Dockerfile", + "file-1.txt", + "file-2.txt", + "target/really/nested/file-3.txt", + }, + exclusions: []string{"**/target/**/*.jar"}, + }, + { + input: "test-fixtures/path-detected", + desc: "file directly excluded", + glob: "**", + expected: []string{ + ".vimrc", + }, + exclusions: []string{"**/empty"}, + }, + { + input: "test-fixtures/path-detected", + desc: "pattern error containing **/", + glob: "**", + expected: []string{ + ".vimrc", + }, + exclusions: []string{"/**/empty"}, + err: true, + }, + { + input: "test-fixtures/path-detected", + desc: "pattern error incorrect start", + glob: "**", + expected: []string{ + ".vimrc", + }, + exclusions: []string{"empty"}, + err: true, + }, + { + input: "test-fixtures/path-detected", + desc: "pattern error starting with /", + glob: "**", + expected: []string{ + ".vimrc", + }, + exclusions: []string{"/empty"}, + err: true, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := NewFromDirectory(DirectoryConfig{ + Path: test.input, + Exclude: ExcludeConfig{ + Paths: test.exclusions, + }, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + if test.err { + _, err = src.FileResolver(SquashedScope) + require.Error(t, err) + return + } + require.NoError(t, err) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + locations, err := res.FilesByGlob(test.glob) + require.NoError(t, err) + + var actual []string + for _, l := range locations { + actual = append(actual, l.RealPath) + } + + assert.ElementsMatchf(t, test.expected, actual, "diff \n"+cmp.Diff(test.expected, actual)) + }) + } +} + +func Test_getDirectoryExclusionFunctions_crossPlatform(t *testing.T) { + testCases := []struct { + desc string + root string + path string + finfo os.FileInfo + exclude string + walkHint error + }{ + { + desc: "directory exclusion", + root: "/", + path: "/usr/var/lib", + exclude: "**/var/lib", + finfo: file.ManualInfo{ModeValue: os.ModeDir}, + walkHint: fs.SkipDir, + }, + { + desc: "no file info", + root: "/", + path: "/usr/var/lib", + exclude: "**/var/lib", + walkHint: fileresolver.ErrSkipPath, + }, + // linux specific tests... + { + desc: "linux doublestar", + root: "/usr", + path: "/usr/var/lib/etc.txt", + exclude: "**/*.txt", + finfo: file.ManualInfo{}, + walkHint: fileresolver.ErrSkipPath, + }, + { + desc: "linux relative", + root: "/usr/var/lib", + path: "/usr/var/lib/etc.txt", + exclude: "./*.txt", + finfo: file.ManualInfo{}, + + walkHint: fileresolver.ErrSkipPath, + }, + { + desc: "linux one level", + root: "/usr", + path: "/usr/var/lib/etc.txt", + exclude: "*/*.txt", + finfo: file.ManualInfo{}, + walkHint: nil, + }, + // NOTE: since these tests will run in linux and macOS, the windows paths will be + // considered relative if they do not start with a forward slash and paths with backslashes + // won't be modified by the filepath.ToSlash call, so these are emulating the result of + // filepath.ToSlash usage + + // windows specific tests... + { + desc: "windows doublestar", + root: "/C:/User/stuff", + path: "/C:/User/stuff/thing.txt", + exclude: "**/*.txt", + finfo: file.ManualInfo{}, + walkHint: fileresolver.ErrSkipPath, + }, + { + desc: "windows relative", + root: "/C:/User/stuff", + path: "/C:/User/stuff/thing.txt", + exclude: "./*.txt", + finfo: file.ManualInfo{}, + walkHint: fileresolver.ErrSkipPath, + }, + { + desc: "windows one level", + root: "/C:/User/stuff", + path: "/C:/User/stuff/thing.txt", + exclude: "*/*.txt", + finfo: file.ManualInfo{}, + walkHint: nil, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude}) + require.NoError(t, err) + + for _, f := range fns { + result := f(test.path, test.finfo, nil) + require.Equal(t, test.walkHint, result) + } + }) + } +} + +func Test_DirectorySource_FilesByPathDoesNotExist(t *testing.T) { + testCases := []struct { + desc string + input string + path string + expected string + }{ + { + input: "test-fixtures/path-detected", + desc: "path does not exist", + path: "foo", + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := NewFromDirectory(DirectoryConfig{Path: test.input}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := res.FilesByPath(test.path) + require.NoError(t, err) + + assert.Len(t, refs, 0) + }) + } +} + +func Test_DirectorySource_ID(t *testing.T) { + tests := []struct { + name string + cfg DirectoryConfig + want artifact.ID + wantErr require.ErrorAssertionFunc + }{ + { + name: "empty", + cfg: DirectoryConfig{}, + wantErr: require.Error, + }, + { + name: "to a non-existent directory", + cfg: DirectoryConfig{ + Path: "./test-fixtures/does-not-exist", + }, + wantErr: require.Error, + }, + { + name: "with odd unclean path through non-existent directory", + cfg: DirectoryConfig{Path: "test-fixtures/does-not-exist/../"}, + wantErr: require.Error, + }, + { + name: "to a file (not a directory)", + cfg: DirectoryConfig{ + Path: "./test-fixtures/image-simple/Dockerfile", + }, + wantErr: require.Error, + }, + { + name: "to dir with name and version", + cfg: DirectoryConfig{ + Path: "./test-fixtures", + Alias: Alias{ + Name: "name-me-that!", + Version: "version-me-this!", + }, + }, + want: artifact.ID("51a5f2a1536cf4b5220d4247814b07eec5862ab0547050f90e9ae216548ded7e"), + }, + { + name: "to different dir with name and version", + cfg: DirectoryConfig{ + Path: "./test-fixtures/image-simple", + Alias: Alias{ + Name: "name-me-that!", + Version: "version-me-this!", + }, + }, + // note: this must match the previous value because the alias should trump the path info + want: artifact.ID("51a5f2a1536cf4b5220d4247814b07eec5862ab0547050f90e9ae216548ded7e"), + }, + { + name: "with path", + cfg: DirectoryConfig{Path: "./test-fixtures"}, + want: artifact.ID("c2f936b0054dc6114fc02a3446bf8916bde8fdf87166a23aee22ea011b443522"), + }, + { + name: "with unclean path", + cfg: DirectoryConfig{Path: "test-fixtures/image-simple/../"}, + want: artifact.ID("c2f936b0054dc6114fc02a3446bf8916bde8fdf87166a23aee22ea011b443522"), + }, + { + name: "other fields do not affect ID", + cfg: DirectoryConfig{ + Path: "test-fixtures", + Base: "a-base!", + Exclude: ExcludeConfig{ + Paths: []string{"a", "b"}, + }, + }, + want: artifact.ID("c2f936b0054dc6114fc02a3446bf8916bde8fdf87166a23aee22ea011b443522"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + s, err := NewFromDirectory(tt.cfg) + tt.wantErr(t, err) + if err != nil { + return + } + assert.Equalf(t, tt.want, s.ID(), "ID()") + }) + } +} + +func Test_cleanDirPath(t *testing.T) { + + abs, err := filepath.Abs("test-fixtures") + require.NoError(t, err) + + tests := []struct { + name string + path string + base string + want string + }{ + { + name: "abs path, abs base, base contained in path", + path: filepath.Join(abs, "system_paths/outside_root"), + base: abs, + want: "system_paths/outside_root", + }, + { + name: "abs path, abs base, base not contained in path", + path: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path", + base: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/002", + want: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path", + }, + { + name: "path and base match", + path: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path", + base: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path", + want: "/var/folders/8x/gw98pp6535s4r8drc374tb1r0000gn/T/001/some/path", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, cleanDirPath(tt.path, tt.base)) + }) + } +} diff --git a/syft/source/directory_source_win_test.go b/syft/source/directory_source_win_test.go new file mode 100644 index 00000000000..aa0d0a27703 --- /dev/null +++ b/syft/source/directory_source_win_test.go @@ -0,0 +1,65 @@ +//go:build windows +// +build windows + +// why the build tags? there is behavior from filepath.ToSlash() that must be tested, but can't be tested on non-windows +// since the stdlib keeps this functionality behind a build tag (specifically filepath.Separator): +// - https://github.com/golang/go/blob/3aea422e2cb8b1ec2e0c2774be97fe96c7299838/src/path/filepath/path.go#L224-L227 +// - https://github.com/golang/go/blob/3aea422e2cb8b1ec2e0c2774be97fe96c7299838/src/path/filepath/path.go#L63 +// - https://github.com/golang/go/blob/master/src/os/path_windows.go#L8 +// +// It would be nice to extract this to simplify testing, however, we also need filepath.Abs(), which in windows +// requires a specific syscall: +// - https://github.com/golang/go/blob/3aea422e2cb8b1ec2e0c2774be97fe96c7299838/src/path/filepath/path_windows.go#L216 +// ... which means we can't extract this functionality without build tags. + +package source + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_DirectorySource_crossPlatformExclusions(t *testing.T) { + testCases := []struct { + desc string + root string + path string + exclude string + match bool + }{ + { + desc: "windows doublestar", + root: "C:\\User\\stuff", + path: "C:\\User\\stuff\\thing.txt", + exclude: "**/*.txt", + match: true, + }, + { + desc: "windows relative", + root: "C:\\User\\stuff", + path: "C:\\User\\stuff\\thing.txt", + exclude: "./*.txt", + match: true, + }, + { + desc: "windows one level", + root: "C:\\User\\stuff", + path: "C:\\User\\stuff\\thing.txt", + exclude: "*/*.txt", + match: false, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude}) + require.NoError(t, err) + + for _, f := range fns { + result := f(test.path, nil, nil) + require.Equal(t, test.match, result) + } + }) + } +} diff --git a/syft/source/exclude.go b/syft/source/exclude.go new file mode 100644 index 00000000000..f41dc0e31d6 --- /dev/null +++ b/syft/source/exclude.go @@ -0,0 +1,5 @@ +package source + +type ExcludeConfig struct { + Paths []string +} diff --git a/syft/source/file_source.go b/syft/source/file_source.go new file mode 100644 index 00000000000..2025d085676 --- /dev/null +++ b/syft/source/file_source.go @@ -0,0 +1,280 @@ +package source + +import ( + "crypto" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sync" + + "github.com/mholt/archiver/v3" + "github.com/opencontainers/go-digest" + + stereoFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" +) + +var _ Source = (*FileSource)(nil) + +type FileConfig struct { + Path string + Exclude ExcludeConfig + DigestAlgorithms []crypto.Hash + Alias Alias +} + +type FileSourceMetadata struct { + Path string `json:"path" yaml:"path"` + Digests []file.Digest `json:"digests,omitempty" yaml:"digests,omitempty"` + MIMEType string `json:"mimeType" yaml:"mimeType"` +} + +type FileSource struct { + id artifact.ID + digestForVersion string + config FileConfig + resolver *fileresolver.Directory + mutex *sync.Mutex + closer func() error + digests []file.Digest + mimeType string + analysisPath string +} + +func NewFromFile(cfg FileConfig) (*FileSource, error) { + fileMeta, err := os.Stat(cfg.Path) + if err != nil { + return nil, fmt.Errorf("unable to stat path=%q: %w", cfg.Path, err) + } + + if fileMeta.IsDir() { + return nil, fmt.Errorf("given path is a directory: %q", cfg.Path) + } + + analysisPath, cleanupFn := fileAnalysisPath(cfg.Path) + + var digests []file.Digest + if len(cfg.DigestAlgorithms) > 0 { + fh, err := os.Open(cfg.Path) + if err != nil { + return nil, fmt.Errorf("unable to open file=%q: %w", cfg.Path, err) + } + + defer fh.Close() + + digests, err = file.NewDigestsFromFile(fh, cfg.DigestAlgorithms) + if err != nil { + return nil, fmt.Errorf("unable to calculate digests for file=%q: %w", cfg.Path, err) + } + } + + fh, err := os.Open(cfg.Path) + if err != nil { + return nil, fmt.Errorf("unable to open file=%q: %w", cfg.Path, err) + } + + defer fh.Close() + + id, versionDigest := deriveIDFromFile(cfg) + + return &FileSource{ + id: id, + config: cfg, + mutex: &sync.Mutex{}, + closer: cleanupFn, + analysisPath: analysisPath, + digestForVersion: versionDigest, + digests: digests, + mimeType: stereoFile.MIMEType(fh), + }, nil +} + +// deriveIDFromFile derives an artifact ID from the contents of a file. If an alias is provided, it will be included +// in the ID derivation (along with contents). This way if the user scans the same item but is considered to be +// logically different, then ID will express that. +func deriveIDFromFile(cfg FileConfig) (artifact.ID, string) { + d := digestOfFileContents(cfg.Path) + info := d + + if !cfg.Alias.IsEmpty() { + // if the user provided an alias, we want to consider that in the artifact ID. This way if the user + // scans the same item but is considered to be logically different, then ID will express that. + info += fmt.Sprintf(":%s@%s", cfg.Alias.Name, cfg.Alias.Version) + } + + if d != "" { + d = fmt.Sprintf("sha256:%s", d) + } + + return artifactIDFromDigest(digest.SHA256.FromString(info).String()), d +} + +func (s FileSource) ID() artifact.ID { + return s.id +} + +func (s FileSource) Describe() Description { + name := path.Base(s.config.Path) + version := s.digestForVersion + if !s.config.Alias.IsEmpty() { + a := s.config.Alias + if a.Name != "" { + name = a.Name + } + + if a.Version != "" { + version = a.Version + } + } + return Description{ + ID: string(s.id), + Name: name, + Version: version, + Metadata: FileSourceMetadata{ + Path: s.config.Path, + Digests: s.digests, + MIMEType: s.mimeType, + }, + } +} + +func (s FileSource) FileResolver(_ Scope) (file.Resolver, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + if s.resolver != nil { + return s.resolver, nil + } + + exclusionFunctions, err := getDirectoryExclusionFunctions(s.analysisPath, s.config.Exclude.Paths) + if err != nil { + return nil, err + } + + fi, err := os.Stat(s.analysisPath) + if err != nil { + return nil, fmt.Errorf("unable to stat path=%q: %w", s.analysisPath, err) + } + isArchiveAnalysis := fi.IsDir() + + absAnalysisPath, err := filepath.Abs(s.analysisPath) + if err != nil { + return nil, fmt.Errorf("unable to get absolute path for analysis path=%q: %w", s.analysisPath, err) + } + absParentDir := filepath.Dir(absAnalysisPath) + + var res *fileresolver.Directory + if isArchiveAnalysis { + // this is an analysis of an archive file... we should scan the directory where the archive contents + res, err = fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...) + if err != nil { + return nil, fmt.Errorf("unable to create directory resolver: %w", err) + } + } else { + // this is an analysis of a single file. We want to ultimately scan the directory that the file is in, but we + // don't want to include any other files except this the given file. + exclusionFunctions = append([]fileresolver.PathIndexVisitor{ + + // note: we should exclude these kinds of paths first before considering any other user-provided exclusions + func(p string, info os.FileInfo, err error) error { + if p == absParentDir { + // this is the root directory... always include it + return nil + } + + if filepath.Dir(p) != absParentDir { + // we are no longer in the root directory containing the single file we want to scan... + // we should skip the directory this path resides in entirely! + return fs.SkipDir + } + + if path.Base(p) != path.Base(s.config.Path) { + // we're in the root directory, but this is not the file we want to scan... + // we should selectively skip this file (not the directory we're in). + return fileresolver.ErrSkipPath + } + return nil + }, + }, exclusionFunctions...) + + res, err = fileresolver.NewFromDirectory(absParentDir, absParentDir, exclusionFunctions...) + if err != nil { + return nil, fmt.Errorf("unable to create directory resolver: %w", err) + } + } + + s.resolver = res + + return s.resolver, nil +} + +func (s *FileSource) Close() error { + if s.closer == nil { + return nil + } + s.resolver = nil + return s.closer() +} + +// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive +// contents have been made available. A cleanup function is provided for any temp files created (if any). +func fileAnalysisPath(path string) (string, func() error) { + var analysisPath = path + var cleanupFn = func() error { return nil } + + // if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and + // use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is + // unarchived. + envelopedUnarchiver, err := archiver.ByExtension(path) + if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok { + if tar, ok := unarchiver.(*archiver.Tar); ok { + // when tar files are extracted, if there are multiple entries at the same + // location, the last entry wins + // NOTE: this currently does not display any messages if an overwrite happens + tar.OverwriteExisting = true + } + unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver) + if err != nil { + log.Warnf("file could not be unarchived: %+v", err) + } else { + log.Debugf("source path is an archive") + analysisPath = unarchivedPath + } + if tmpCleanup != nil { + cleanupFn = tmpCleanup + } + } + + return analysisPath, cleanupFn +} + +func digestOfFileContents(path string) string { + file, err := os.Open(path) + if err != nil { + return digest.SHA256.FromString(path).String() + } + defer file.Close() + di, err := digest.SHA256.FromReader(file) + if err != nil { + return digest.SHA256.FromString(path).String() + } + return di.String() +} + +func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func() error, error) { + tempDir, err := os.MkdirTemp("", "syft-archive-contents-") + if err != nil { + return "", func() error { return nil }, fmt.Errorf("unable to create tempdir for archive processing: %w", err) + } + + cleanupFn := func() error { + return os.RemoveAll(tempDir) + } + + return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir) +} diff --git a/syft/source/file_source_test.go b/syft/source/file_source_test.go new file mode 100644 index 00000000000..11fcc3428cf --- /dev/null +++ b/syft/source/file_source_test.go @@ -0,0 +1,278 @@ +package source + +import ( + "io" + "os" + "os/exec" + "path" + "path/filepath" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" +) + +func TestNewFromFile(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + testPathFn func(file.Resolver) ([]file.Location, error) + expRefs int + }{ + { + desc: "path detected by glob", + input: "test-fixtures/file-index-filter/.vimrc", + testPathFn: func(resolver file.Resolver) ([]file.Location, error) { + return resolver.FilesByGlob("**/.vimrc", "**/.2", "**/.1/*", "**/empty") + }, + expRefs: 1, + }, + { + desc: "path detected by abs path", + input: "test-fixtures/file-index-filter/.vimrc", + testPathFn: func(resolver file.Resolver) ([]file.Location, error) { + return resolver.FilesByPath("/.vimrc", "/.2", "/.1/something", "/empty") + }, + expRefs: 1, + }, + { + desc: "path detected by relative path", + input: "test-fixtures/file-index-filter/.vimrc", + testPathFn: func(resolver file.Resolver) ([]file.Location, error) { + return resolver.FilesByPath(".vimrc", "/.2", "/.1/something", "empty") + }, + expRefs: 1, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := NewFromFile(FileConfig{ + Path: test.input, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + assert.Equal(t, test.input, src.Describe().Metadata.(FileSourceMetadata).Path) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := test.testPathFn(res) + require.NoError(t, err) + require.Len(t, refs, test.expRefs) + if test.expRefs == 1 { + assert.Equal(t, path.Base(test.input), path.Base(refs[0].RealPath)) + } + + }) + } +} + +func TestNewFromFile_WithArchive(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + inputPaths []string + expRefs int + layer2 bool + contents string + }{ + { + desc: "path detected", + input: "test-fixtures/path-detected", + inputPaths: []string{"/.vimrc"}, + expRefs: 1, + }, + { + desc: "use first entry for duplicate paths", + input: "test-fixtures/path-detected", + inputPaths: []string{"/.vimrc"}, + expRefs: 1, + layer2: true, + contents: "Another .vimrc file", + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + archivePath := setupArchiveTest(t, test.input, test.layer2) + + src, err := NewFromFile(FileConfig{ + Path: archivePath, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + assert.Equal(t, archivePath, src.Describe().Metadata.(FileSourceMetadata).Path) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := res.FilesByPath(test.inputPaths...) + require.NoError(t, err) + assert.Len(t, refs, test.expRefs) + + if test.contents != "" { + reader, err := res.FileContentsByLocation(refs[0]) + require.NoError(t, err) + + data, err := io.ReadAll(reader) + require.NoError(t, err) + + assert.Equal(t, test.contents, string(data)) + } + + }) + } +} + +// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function, +// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error, +// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil +// (even if there's an error), and it should always be called. +func setupArchiveTest(t testing.TB, sourceDirPath string, layer2 bool) string { + t.Helper() + + archivePrefix, err := os.CreateTemp("", "syft-archive-TEST-") + require.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, os.Remove(archivePrefix.Name())) + }) + + destinationArchiveFilePath := archivePrefix.Name() + ".tar" + t.Logf("archive path: %s", destinationArchiveFilePath) + createArchive(t, sourceDirPath, destinationArchiveFilePath, layer2) + + t.Cleanup(func() { + assert.NoError(t, os.Remove(destinationArchiveFilePath)) + }) + + cwd, err := os.Getwd() + require.NoError(t, err) + + t.Logf("running from: %s", cwd) + + return destinationArchiveFilePath +} + +// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath. +func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string, layer2 bool) { + t.Helper() + + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("unable to get cwd: %+v", err) + } + + cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath)) + cmd.Dir = filepath.Join(cwd, "test-fixtures") + + if err := cmd.Start(); err != nil { + t.Fatalf("unable to start generate zip fixture script: %+v", err) + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + + // This works on both Unix and Windows. Although package + // syscall is generally platform dependent, WaitStatus is + // defined for both Unix and Windows and in both cases has + // an ExitStatus() method with the same signature. + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() != 0 { + t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus()) + } + } + } else { + t.Fatalf("unable to get generate fixture script result: %+v", err) + } + } + + if layer2 { + cmd = exec.Command("tar", "-rvf", destinationArchivePath, ".") + cmd.Dir = filepath.Join(cwd, "test-fixtures", path.Base(sourceDirPath+"-2")) + if err := cmd.Start(); err != nil { + t.Fatalf("unable to start tar appending fixture script: %+v", err) + } + _ = cmd.Wait() + } +} + +func Test_FileSource_ID(t *testing.T) { + tests := []struct { + name string + cfg FileConfig + want artifact.ID + wantErr require.ErrorAssertionFunc + }{ + { + name: "empty", + cfg: FileConfig{}, + wantErr: require.Error, + }, + { + name: "does not exist", + cfg: FileConfig{ + Path: "./test-fixtures/does-not-exist", + }, + wantErr: require.Error, + }, + { + name: "to dir", + cfg: FileConfig{ + Path: "./test-fixtures/image-simple", + }, + wantErr: require.Error, + }, + { + name: "with path", + cfg: FileConfig{Path: "./test-fixtures/image-simple/Dockerfile"}, + want: artifact.ID("db7146472cf6d49b3ac01b42812fb60020b0b4898b97491b21bb690c808d5159"), + }, + { + name: "with path and alias", + cfg: FileConfig{ + Path: "./test-fixtures/image-simple/Dockerfile", + Alias: Alias{ + Name: "name-me-that!", + Version: "version-me-this!", + }, + }, + want: artifact.ID("3c713003305ac6605255cec8bf4ea649aa44b2b9a9f3a07bd683869d1363438a"), + }, + { + name: "other fields do not affect ID", + cfg: FileConfig{ + Path: "test-fixtures/image-simple/Dockerfile", + Exclude: ExcludeConfig{ + Paths: []string{"a", "b"}, + }, + }, + want: artifact.ID("db7146472cf6d49b3ac01b42812fb60020b0b4898b97491b21bb690c808d5159"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + s, err := NewFromFile(tt.cfg) + tt.wantErr(t, err) + if err != nil { + return + } + assert.Equalf(t, tt.want, s.ID(), "ID()") + }) + } +} diff --git a/syft/source/image_metadata.go b/syft/source/image_metadata.go deleted file mode 100644 index 0d70ed775ab..00000000000 --- a/syft/source/image_metadata.go +++ /dev/null @@ -1,62 +0,0 @@ -package source - -import "github.com/anchore/stereoscope/pkg/image" - -// ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe -// "what" was cataloged without needing the more complicated stereoscope Image objects or FileResolver objects. -type ImageMetadata struct { - UserInput string `json:"userInput"` - ID string `json:"imageID"` - ManifestDigest string `json:"manifestDigest"` - MediaType string `json:"mediaType"` - Tags []string `json:"tags"` - Size int64 `json:"imageSize"` - Layers []LayerMetadata `json:"layers"` - RawManifest []byte `json:"manifest"` - RawConfig []byte `json:"config"` - RepoDigests []string `json:"repoDigests"` - Architecture string `json:"architecture"` - Variant string `json:"architectureVariant,omitempty"` - OS string `json:"os"` -} - -// LayerMetadata represents all static metadata that defines what a container image layer is. -type LayerMetadata struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -// NewImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration. -func NewImageMetadata(img *image.Image, userInput string) ImageMetadata { - // populate artifacts... - tags := make([]string, len(img.Metadata.Tags)) - for idx, tag := range img.Metadata.Tags { - tags[idx] = tag.String() - } - theImg := ImageMetadata{ - ID: img.Metadata.ID, - UserInput: userInput, - ManifestDigest: img.Metadata.ManifestDigest, - Size: img.Metadata.Size, - MediaType: string(img.Metadata.MediaType), - Tags: tags, - Layers: make([]LayerMetadata, len(img.Layers)), - RawConfig: img.Metadata.RawConfig, - RawManifest: img.Metadata.RawManifest, - RepoDigests: img.Metadata.RepoDigests, - Architecture: img.Metadata.Architecture, - Variant: img.Metadata.Variant, - OS: img.Metadata.OS, - } - - // populate image metadata - for idx, l := range img.Layers { - theImg.Layers[idx] = LayerMetadata{ - MediaType: string(l.Metadata.MediaType), - Digest: l.Metadata.Digest, - Size: l.Metadata.Size, - } - } - return theImg -} diff --git a/syft/source/metadata.go b/syft/source/metadata.go deleted file mode 100644 index ecbad4f1dd8..00000000000 --- a/syft/source/metadata.go +++ /dev/null @@ -1,12 +0,0 @@ -package source - -// Metadata represents any static source data that helps describe "what" was cataloged. -type Metadata struct { - ID string `hash:"ignore"` // the id generated from the parent source struct - Scheme Scheme // the source data scheme type (directory or image) - ImageMetadata ImageMetadata // all image info (image only) - Path string // the root path to be cataloged (directory only) - Base string // the base path to be cataloged (directory only) - Name string - Version string -} diff --git a/syft/source/scheme.go b/syft/source/scheme.go deleted file mode 100644 index 46a6214782c..00000000000 --- a/syft/source/scheme.go +++ /dev/null @@ -1,74 +0,0 @@ -package source - -import ( - "fmt" - "strings" - - "github.com/mitchellh/go-homedir" - "github.com/spf13/afero" - - "github.com/anchore/stereoscope/pkg/image" -) - -// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:"). -type Scheme string - -const ( - // UnknownScheme is the default scheme - UnknownScheme Scheme = "UnknownScheme" - // DirectoryScheme indicates the source being cataloged is a directory on the root filesystem - DirectoryScheme Scheme = "DirectoryScheme" - // ImageScheme indicates the source being cataloged is a container image - ImageScheme Scheme = "ImageScheme" - // FileScheme indicates the source being cataloged is a single file - FileScheme Scheme = "FileScheme" -) - -var AllSchemes = []Scheme{ - DirectoryScheme, - ImageScheme, - FileScheme, -} - -func DetectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, image.Source, string, error) { - switch { - case strings.HasPrefix(userInput, "dir:"): - dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:")) - if err != nil { - return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err) - } - return DirectoryScheme, image.UnknownSource, dirLocation, nil - - case strings.HasPrefix(userInput, "file:"): - fileLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "file:")) - if err != nil { - return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err) - } - return FileScheme, image.UnknownSource, fileLocation, nil - } - - // try the most specific sources first and move out towards more generic sources. - - // first: let's try the image detector, which has more scheme parsing internal to stereoscope - source, imageSpec, err := imageDetector(userInput) - if err == nil && source != image.UnknownSource { - return ImageScheme, source, imageSpec, nil - } - - // next: let's try more generic sources (dir, file, etc.) - location, err := homedir.Expand(userInput) - if err != nil { - return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err) - } - - fileMeta, err := fs.Stat(location) - if err != nil { - return UnknownScheme, source, "", nil - } - - if fileMeta.IsDir() { - return DirectoryScheme, source, location, nil - } - - return FileScheme, source, location, nil -} diff --git a/syft/source/scope.go b/syft/source/scope.go index e959d1a420f..05f14644a49 100644 --- a/syft/source/scope.go +++ b/syft/source/scope.go @@ -10,7 +10,7 @@ const ( UnknownScope Scope = "UnknownScope" // SquashedScope indicates to only catalog content visible from the squashed filesystem representation (what can be seen only within the container at runtime) SquashedScope Scope = "Squashed" - // AllLayersScope indicates to catalog content on all layers, irregardless if it is visible from the container at runtime. + // AllLayersScope indicates to catalog content on all layers, regardless if it is visible from the container at runtime. AllLayersScope Scope = "AllLayers" ) diff --git a/syft/source/source.go b/syft/source/source.go index 4ff747ae297..6b77b16f040 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -6,628 +6,42 @@ within this package. package source import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "sync" + "errors" + "io" - "github.com/bmatcuk/doublestar/v4" - "github.com/mholt/archiver/v3" - digest "github.com/opencontainers/go-digest" - "github.com/spf13/afero" - - "github.com/anchore/stereoscope" - "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/internal/fileresolver" ) -// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used -// in cataloging (based on the data source and configuration) -type Source struct { - id artifact.ID `hash:"ignore"` - Image *image.Image `hash:"ignore"` // the image object to be cataloged (image only) - Metadata Metadata - directoryResolver *fileresolver.Directory `hash:"ignore"` - path string - base string - mutex *sync.Mutex - Exclusions []string `hash:"ignore"` -} - -// Input is an object that captures the detected user input regarding source location, scheme, and provider type. -// It acts as a struct input for some source constructors. -type Input struct { - UserInput string - Scheme Scheme - ImageSource image.Source - Location string - Platform string - Name string - Version string -} - -// ParseInput generates a source Input that can be used as an argument to generate a new source -// from specific providers including a registry. -func ParseInput(userInput string, platform string) (*Input, error) { - return ParseInputWithName(userInput, platform, "", "") -} - -// ParseInputWithName generates a source Input that can be used as an argument to generate a new source -// from specific providers including a registry, with an explicit name. -func ParseInputWithName(userInput string, platform, name, defaultImageSource string) (*Input, error) { - return ParseInputWithNameVersion(userInput, platform, name, "", defaultImageSource) -} - -// ParseInputWithNameVersion generates a source Input that can be used as an argument to generate a new source -// from specific providers including a registry, with an explicit name and version. -func ParseInputWithNameVersion(userInput, platform, name, version, defaultImageSource string) (*Input, error) { - fs := afero.NewOsFs() - scheme, source, location, err := DetectScheme(fs, image.DetectSource, userInput) - if err != nil { - return nil, err - } - - if source == image.UnknownSource { - // only run for these two scheme - // only check on packages command, attest we automatically try to pull from userInput - switch scheme { - case ImageScheme, UnknownScheme: - scheme = ImageScheme - location = userInput - if defaultImageSource != "" { - source = parseDefaultImageSource(defaultImageSource) - } else { - imagePullSource := image.DetermineDefaultImagePullSource(userInput) - source = imagePullSource - } - if location == "" { - location = userInput - } - default: - } - } - - if scheme != ImageScheme && platform != "" { - return nil, fmt.Errorf("cannot specify a platform for a non-image source") - } - - // collect user input for downstream consumption - return &Input{ - UserInput: userInput, - Scheme: scheme, - ImageSource: source, - Location: location, - Platform: platform, - Name: name, - Version: version, - }, nil -} - -func parseDefaultImageSource(defaultImageSource string) image.Source { - switch defaultImageSource { - case "registry": - return image.OciRegistrySource - case "docker": - return image.DockerDaemonSource - case "podman": - return image.PodmanDaemonSource - default: - return image.UnknownSource - } -} - -type sourceDetector func(string) (image.Source, string, error) - -func NewFromRegistry(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) { - source, cleanupFn, err := generateImageSource(in, registryOptions) - if source != nil { - source.Exclusions = exclusions - } - return source, cleanupFn, err -} - -// New produces a Source based on userInput like dir: or image:tag -func New(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) { - var err error - fs := afero.NewOsFs() - var source *Source - cleanupFn := func() {} - - switch in.Scheme { - case FileScheme: - source, cleanupFn, err = generateFileSource(fs, in) - case DirectoryScheme: - source, cleanupFn, err = generateDirectorySource(fs, in) - case ImageScheme: - source, cleanupFn, err = generateImageSource(in, registryOptions) - default: - err = fmt.Errorf("unable to process input for scanning: %q", in.UserInput) - } - - if err == nil { - source.Exclusions = exclusions - } - - return source, cleanupFn, err -} - -func generateImageSource(in Input, registryOptions *image.RegistryOptions) (*Source, func(), error) { - img, cleanup, err := getImageWithRetryStrategy(in, registryOptions) - if err != nil || img == nil { - return nil, cleanup, fmt.Errorf("could not fetch image %q: %w", in.Location, err) - } - - s, err := NewFromImageWithNameVersion(img, in.Location, in.Name, in.Version) - if err != nil { - return nil, cleanup, fmt.Errorf("could not populate source with image: %w", err) - } - - return &s, cleanup, nil +type Source interface { + artifact.Identifiable + FileResolver(Scope) (file.Resolver, error) + Describe() Description + io.Closer } -func parseScheme(userInput string) string { - parts := strings.SplitN(userInput, ":", 2) - if len(parts) < 2 { - return "" - } - - return parts[0] +type emptySource struct { + description Description } -func getImageWithRetryStrategy(in Input, registryOptions *image.RegistryOptions) (*image.Image, func(), error) { - ctx := context.TODO() - - var opts []stereoscope.Option - if registryOptions != nil { - opts = append(opts, stereoscope.WithRegistryOptions(*registryOptions)) - } - - if in.Platform != "" { - opts = append(opts, stereoscope.WithPlatform(in.Platform)) - } - - img, err := stereoscope.GetImageFromSource(ctx, in.Location, in.ImageSource, opts...) - cleanup := func() { - if err := img.Cleanup(); err != nil { - log.Warnf("unable to cleanup image=%q: %w", in.UserInput, err) - } - } - if err == nil { - // Success on the first try! - return img, cleanup, nil - } - - scheme := parseScheme(in.UserInput) - if !(scheme == "docker" || scheme == "registry") { - // Image retrieval failed, and we shouldn't retry it. It's most likely that the - // user _did_ intend the parsed scheme, but there was a legitimate failure with - // using the scheme to load the image. Alert the user to this failure, so they - // can fix the problem. - return nil, nil, err - } - - // Maybe the user wanted "docker" or "registry" to refer to an _image name_ - // (e.g. "docker:latest"), not a scheme. We'll retry image retrieval with this - // alternative interpretation, in an attempt to avoid unnecessary user friction. - - log.Warnf( - "scheme %q specified, but it coincides with a common image name; re-examining user input %q"+ - " without scheme parsing because image retrieval using scheme parsing was unsuccessful: %v", - scheme, - in.UserInput, - err, - ) - - // We need to determine the image source again, such that this determination - // doesn't take scheme parsing into account. - in.ImageSource = image.DetermineDefaultImagePullSource(in.UserInput) - img, userInputErr := stereoscope.GetImageFromSource(ctx, in.UserInput, in.ImageSource, opts...) - cleanup = func() { - if err := img.Cleanup(); err != nil { - log.Warnf("unable to cleanup image=%q: %w", in.UserInput, err) - } +func FromDescription(d Description) Source { + return &emptySource{ + description: d, } - if userInputErr != nil { - // Image retrieval failed on both tries, we will want to return both errors. - return nil, nil, fmt.Errorf( - "scheme %q specified; "+ - "image retrieval using scheme parsing (%s) was unsuccessful: %v; "+ - "image retrieval without scheme parsing (%s) was unsuccessful: %v", - scheme, - in.Location, - err, - in.UserInput, - userInputErr, - ) - } - - return img, cleanup, nil } -func generateDirectorySource(fs afero.Fs, in Input) (*Source, func(), error) { - fileMeta, err := fs.Stat(in.Location) - if err != nil { - return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err) - } - - if !fileMeta.IsDir() { - return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err) - } - - s, err := NewFromDirectoryWithNameVersion(in.Location, in.Name, in.Version) - if err != nil { - return nil, func() {}, fmt.Errorf("could not populate source from path=%q: %w", in.Location, err) - } - - return &s, func() {}, nil +func (e emptySource) ID() artifact.ID { + return artifact.ID(e.description.ID) } -func generateFileSource(fs afero.Fs, in Input) (*Source, func(), error) { - fileMeta, err := fs.Stat(in.Location) - if err != nil { - return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err) - } - - if fileMeta.IsDir() { - return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err) - } - - s, cleanupFn := NewFromFileWithNameVersion(in.Location, in.Name, in.Version) - - return &s, cleanupFn, nil +func (e emptySource) FileResolver(_ Scope) (file.Resolver, error) { + return nil, errors.New("no file resolver available for description-only source") } -// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. -func NewFromDirectory(path string) (Source, error) { - return NewFromDirectoryWithName(path, "") +func (e emptySource) Describe() Description { + return e.description } -// NewFromDirectoryWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name. -func NewFromDirectoryWithName(path string, name string) (Source, error) { - return NewFromDirectoryWithNameVersion(path, name, "") -} - -// NewFromDirectoryWithNameVersion creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name. -func NewFromDirectoryWithNameVersion(path string, name string, version string) (Source, error) { - s := Source{ - mutex: &sync.Mutex{}, - Metadata: Metadata{ - Name: name, - Version: version, - Scheme: DirectoryScheme, - Path: path, - }, - path: path, - } - s.SetID() - return s, nil -} - -// NewFromDirectoryRoot creates a new source object tailored to catalog a given filesystem directory recursively. -func NewFromDirectoryRoot(path string) (Source, error) { - return NewFromDirectoryRootWithName(path, "") -} - -// NewFromDirectoryRootWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name. -func NewFromDirectoryRootWithName(path string, name string) (Source, error) { - return NewFromDirectoryRootWithNameVersion(path, name, "") -} - -// NewFromDirectoryRootWithNameVersion creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name. -func NewFromDirectoryRootWithNameVersion(path string, name string, version string) (Source, error) { - s := Source{ - mutex: &sync.Mutex{}, - Metadata: Metadata{ - Name: name, - Version: version, - Scheme: DirectoryScheme, - Path: path, - Base: path, - }, - path: path, - base: path, - } - s.SetID() - return s, nil -} - -// NewFromFile creates a new source object tailored to catalog a file. -func NewFromFile(path string) (Source, func()) { - return NewFromFileWithName(path, "") -} - -// NewFromFileWithName creates a new source object tailored to catalog a file, with an explicitly provided name. -func NewFromFileWithName(path string, name string) (Source, func()) { - return NewFromFileWithNameVersion(path, name, "") -} - -// NewFromFileWithNameVersion creates a new source object tailored to catalog a file, with an explicitly provided name and version. -func NewFromFileWithNameVersion(path string, name string, version string) (Source, func()) { - analysisPath, cleanupFn := fileAnalysisPath(path) - - s := Source{ - mutex: &sync.Mutex{}, - Metadata: Metadata{ - Name: name, - Version: version, - Scheme: FileScheme, - Path: path, - }, - path: analysisPath, - } - - s.SetID() - return s, cleanupFn -} - -// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive -// contents have been made available. A cleanup function is provided for any temp files created (if any). -func fileAnalysisPath(path string) (string, func()) { - var analysisPath = path - var cleanupFn = func() {} - - // if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and - // use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is - // unarchived. - envelopedUnarchiver, err := archiver.ByExtension(path) - if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok { - if tar, ok := unarchiver.(*archiver.Tar); ok { - // when tar files are extracted, if there are multiple entries at the same - // location, the last entry wins - // NOTE: this currently does not display any messages if an overwrite happens - tar.OverwriteExisting = true - } - unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver) - if err != nil { - log.Warnf("file could not be unarchived: %+v", err) - } else { - log.Debugf("source path is an archive") - analysisPath = unarchivedPath - } - if tmpCleanup != nil { - cleanupFn = tmpCleanup - } - } - - return analysisPath, cleanupFn -} - -// NewFromImage creates a new source object tailored to catalog a given container image, relative to the -// option given (e.g. all-layers, squashed, etc) -func NewFromImage(img *image.Image, userImageStr string) (Source, error) { - return NewFromImageWithName(img, userImageStr, "") -} - -// NewFromImageWithName creates a new source object tailored to catalog a given container image, relative to the -// option given (e.g. all-layers, squashed, etc), with an explicit name. -func NewFromImageWithName(img *image.Image, userImageStr string, name string) (Source, error) { - return NewFromImageWithNameVersion(img, userImageStr, name, "") -} - -// NewFromImageWithNameVersion creates a new source object tailored to catalog a given container image, relative to the -// option given (e.g. all-layers, squashed, etc), with an explicit name and version. -func NewFromImageWithNameVersion(img *image.Image, userImageStr string, name string, version string) (Source, error) { - if img == nil { - return Source{}, fmt.Errorf("no image given") - } - - s := Source{ - Image: img, - Metadata: Metadata{ - Name: name, - Version: version, - Scheme: ImageScheme, - ImageMetadata: NewImageMetadata(img, userImageStr), - }, - } - s.SetID() - return s, nil -} - -func (s *Source) ID() artifact.ID { - if s.id == "" { - s.SetID() - } - return s.id -} - -func (s *Source) SetID() { - var d string - switch s.Metadata.Scheme { - case DirectoryScheme: - d = digest.FromString(s.Metadata.Path).String() - case FileScheme: - // attempt to use the digest of the contents of the file as the ID - file, err := os.Open(s.Metadata.Path) - if err != nil { - d = digest.FromString(s.Metadata.Path).String() - break - } - defer file.Close() - di, err := digest.FromReader(file) - if err != nil { - d = digest.FromString(s.Metadata.Path).String() - break - } - d = di.String() - case ImageScheme: - manifestDigest := digest.FromBytes(s.Metadata.ImageMetadata.RawManifest).String() - if manifestDigest != "" { - d = manifestDigest - break - } - - // calcuate chain ID for image sources where manifestDigest is not available - // https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid - d = calculateChainID(s.Metadata.ImageMetadata.Layers) - if d == "" { - // TODO what happens here if image has no layers? - // Is this case possible - d = digest.FromString(s.Metadata.ImageMetadata.UserInput).String() - } - default: // for UnknownScheme we hash the struct - id, _ := artifact.IDByHash(s) - d = string(id) - } - - s.id = artifact.ID(strings.TrimPrefix(d, "sha256:")) - s.Metadata.ID = strings.TrimPrefix(d, "sha256:") -} - -func calculateChainID(lm []LayerMetadata) string { - if len(lm) < 1 { - return "" - } - - // DiffID(L0) = digest of layer 0 - // https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32 - chainID := lm[0].Digest - id := chain(chainID, lm[1:]) - - return id -} - -func chain(chainID string, layers []LayerMetadata) string { - if len(layers) < 1 { - return chainID - } - - chainID = digest.FromString(layers[0].Digest + " " + chainID).String() - return chain(chainID, layers[1:]) -} - -func (s *Source) FileResolver(scope Scope) (file.Resolver, error) { - switch s.Metadata.Scheme { - case DirectoryScheme, FileScheme: - s.mutex.Lock() - defer s.mutex.Unlock() - if s.directoryResolver == nil { - exclusionFunctions, err := getDirectoryExclusionFunctions(s.path, s.Exclusions) - if err != nil { - return nil, err - } - res, err := fileresolver.NewFromDirectory(s.path, s.base, exclusionFunctions...) - if err != nil { - return nil, fmt.Errorf("unable to create directory resolver: %w", err) - } - s.directoryResolver = res - } - return s.directoryResolver, nil - case ImageScheme: - var res file.Resolver - var err error - switch scope { - case SquashedScope: - res, err = fileresolver.NewFromContainerImageSquash(s.Image) - case AllLayersScope: - res, err = fileresolver.NewFromContainerImageAllLayers(s.Image) - default: - return nil, fmt.Errorf("bad image scope provided: %+v", scope) - } - if err != nil { - return nil, err - } - // image tree contains all paths, so we filter out the excluded entries afterwards - if len(s.Exclusions) > 0 { - res = fileresolver.NewExcluding(res, getImageExclusionFunction(s.Exclusions)) - } - return res, nil - } - return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme) -} - -func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func(), error) { - tempDir, err := os.MkdirTemp("", "syft-archive-contents-") - if err != nil { - return "", func() {}, fmt.Errorf("unable to create tempdir for archive processing: %w", err) - } - - cleanupFn := func() { - if err := os.RemoveAll(tempDir); err != nil { - log.Warnf("unable to cleanup archive tempdir: %+v", err) - } - } - - return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir) -} - -func getImageExclusionFunction(exclusions []string) func(string) bool { - if len(exclusions) == 0 { - return nil - } - // add subpath exclusions - for _, exclusion := range exclusions { - exclusions = append(exclusions, exclusion+"/**") - } - return func(path string) bool { - for _, exclusion := range exclusions { - matches, err := doublestar.Match(exclusion, path) - if err != nil { - return false - } - if matches { - return true - } - } - return false - } -} - -func getDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) { - if len(exclusions) == 0 { - return nil, nil - } - - // this is what Directory.indexTree is doing to get the absolute path: - root, err := filepath.Abs(root) - if err != nil { - return nil, err - } - - // this handles Windows file paths by converting them to C:/something/else format - root = filepath.ToSlash(root) - - if !strings.HasSuffix(root, "/") { - root += "/" - } - - var errors []string - for idx, exclusion := range exclusions { - // check exclusions for supported paths, these are all relative to the "scan root" - if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") { - exclusion = strings.TrimPrefix(exclusion, "./") - exclusions[idx] = root + exclusion - } else { - errors = append(errors, exclusion) - } - } - - if errors != nil { - return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '")) - } - - return []fileresolver.PathIndexVisitor{ - func(path string, info os.FileInfo, _ error) error { - for _, exclusion := range exclusions { - // this is required to handle Windows filepaths - path = filepath.ToSlash(path) - matches, err := doublestar.Match(exclusion, path) - if err != nil { - return nil - } - if matches { - if info != nil && info.IsDir() { - return filepath.SkipDir - } - return fileresolver.ErrSkipPath - } - } - return nil - }, - }, nil +func (e emptySource) Close() error { + return nil // no-op } diff --git a/syft/source/source_test.go b/syft/source/source_test.go deleted file mode 100644 index bfa085d09a6..00000000000 --- a/syft/source/source_test.go +++ /dev/null @@ -1,920 +0,0 @@ -//go:build !windows -// +build !windows - -package source - -import ( - "io" - "io/fs" - "os" - "os/exec" - "path" - "path/filepath" - "sort" - "strings" - "syscall" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/internal/fileresolver" -) - -func TestParseInput(t *testing.T) { - tests := []struct { - name string - input string - platform string - expected Scheme - errFn require.ErrorAssertionFunc - }{ - { - name: "ParseInput parses a file input", - input: "test-fixtures/image-simple/file-1.txt", - expected: FileScheme, - }, - { - name: "errors out when using platform for non-image scheme", - input: "test-fixtures/image-simple/file-1.txt", - platform: "arm64", - errFn: require.Error, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if test.errFn == nil { - test.errFn = require.NoError - } - sourceInput, err := ParseInput(test.input, test.platform) - test.errFn(t, err) - if test.expected != "" { - require.NotNil(t, sourceInput) - assert.Equal(t, sourceInput.Scheme, test.expected) - } - }) - } -} - -func TestNewFromImageFails(t *testing.T) { - t.Run("no image given", func(t *testing.T) { - _, err := NewFromImage(nil, "") - if err == nil { - t.Errorf("expected an error condition but none was given") - } - }) -} - -func TestSetID(t *testing.T) { - layer := image.NewLayer(nil) - layer.Metadata = image.LayerMetadata{ - Digest: "sha256:6f4fb385d4e698647bf2a450749dfbb7bc2831ec9a730ef4046c78c08d468e89", - } - img := image.Image{ - Layers: []*image.Layer{layer}, - } - - tests := []struct { - name string - input *Source - expected artifact.ID - }{ - { - name: "source.SetID sets the ID for FileScheme", - input: &Source{ - Metadata: Metadata{ - Scheme: FileScheme, - Path: "test-fixtures/image-simple/file-1.txt", - }, - }, - expected: artifact.ID("55096713247489add592ce977637be868497132b36d1e294a3831925ec64319a"), - }, - { - name: "source.SetID sets the ID for ImageScheme", - input: &Source{ - Image: &img, - Metadata: Metadata{ - Scheme: ImageScheme, - }, - }, - expected: artifact.ID("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - }, - { - name: "source.SetID sets the ID for DirectoryScheme", - input: &Source{ - Image: &img, - Metadata: Metadata{ - Scheme: DirectoryScheme, - Path: "test-fixtures/image-simple", - }, - }, - expected: artifact.ID("91db61e5e0ae097ef764796ce85e442a93f2a03e5313d4c7307e9b413f62e8c4"), - }, - { - name: "source.SetID sets the ID for UnknownScheme", - input: &Source{ - Image: &img, - Metadata: Metadata{ - Scheme: UnknownScheme, - Path: "test-fixtures/image-simple", - }, - }, - expected: artifact.ID("9ee9e786412d6ae5"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - test.input.SetID() - assert.Equal(t, test.expected, test.input.ID()) - }) - } -} - -func TestNewFromImage(t *testing.T) { - layer := image.NewLayer(nil) - img := image.Image{ - Layers: []*image.Layer{layer}, - } - - t.Run("create a new source object from image", func(t *testing.T) { - _, err := NewFromImage(&img, "") - if err != nil { - t.Errorf("unexpected error when creating a new Locations from img: %+v", err) - } - }) -} - -func TestNewFromDirectory(t *testing.T) { - testCases := []struct { - desc string - input string - expString string - inputPaths []string - expectedRefs int - expectedErr bool - }{ - { - desc: "no paths exist", - input: "foobar/", - inputPaths: []string{"/opt/", "/other"}, - expectedErr: true, - }, - { - desc: "path detected", - input: "test-fixtures", - inputPaths: []string{"path-detected/.vimrc"}, - expectedRefs: 1, - }, - { - desc: "directory ignored", - input: "test-fixtures", - inputPaths: []string{"path-detected"}, - expectedRefs: 0, - }, - { - desc: "no files-by-path detected", - input: "test-fixtures", - inputPaths: []string{"no-path-detected"}, - expectedRefs: 0, - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - src, err := NewFromDirectory(test.input) - require.NoError(t, err) - assert.Equal(t, test.input, src.Metadata.Path) - - res, err := src.FileResolver(SquashedScope) - if test.expectedErr { - if err == nil { - t.Fatal("expected an error when making the resolver but got none") - } - return - } else { - require.NoError(t, err) - } - - refs, err := res.FilesByPath(test.inputPaths...) - if err != nil { - t.Errorf("FilesByPath call produced an error: %+v", err) - } - if len(refs) != test.expectedRefs { - t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expectedRefs) - - } - - }) - } -} - -func TestNewFromFile(t *testing.T) { - testCases := []struct { - desc string - input string - expString string - inputPaths []string - expRefs int - }{ - { - desc: "path detected", - input: "test-fixtures/path-detected", - inputPaths: []string{"/.vimrc"}, - expRefs: 1, - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - src, cleanup := NewFromFile(test.input) - if cleanup != nil { - t.Cleanup(cleanup) - } - - assert.Equal(t, test.input, src.Metadata.Path) - assert.Equal(t, src.Metadata.Path, src.path) - - res, err := src.FileResolver(SquashedScope) - require.NoError(t, err) - - refs, err := res.FilesByPath(test.inputPaths...) - require.NoError(t, err) - assert.Len(t, refs, test.expRefs) - - }) - } -} - -func TestNewFromFile_WithArchive(t *testing.T) { - testCases := []struct { - desc string - input string - expString string - inputPaths []string - expRefs int - layer2 bool - contents string - }{ - { - desc: "path detected", - input: "test-fixtures/path-detected", - inputPaths: []string{"/.vimrc"}, - expRefs: 1, - }, - { - desc: "lest entry for duplicate paths", - input: "test-fixtures/path-detected", - inputPaths: []string{"/.vimrc"}, - expRefs: 1, - layer2: true, - contents: "Another .vimrc file", - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - archivePath := setupArchiveTest(t, test.input, test.layer2) - - src, cleanup := NewFromFile(archivePath) - if cleanup != nil { - t.Cleanup(cleanup) - } - - assert.Equal(t, archivePath, src.Metadata.Path) - assert.NotEqual(t, src.Metadata.Path, src.path) - - res, err := src.FileResolver(SquashedScope) - require.NoError(t, err) - - refs, err := res.FilesByPath(test.inputPaths...) - require.NoError(t, err) - assert.Len(t, refs, test.expRefs) - - if test.contents != "" { - reader, err := res.FileContentsByLocation(refs[0]) - require.NoError(t, err) - - data, err := io.ReadAll(reader) - require.NoError(t, err) - - assert.Equal(t, test.contents, string(data)) - } - - }) - } -} - -func TestNewFromDirectoryShared(t *testing.T) { - testCases := []struct { - desc string - input string - expString string - notExist string - inputPaths []string - expRefs int - }{ - { - desc: "path detected", - input: "test-fixtures", - notExist: "foobar/", - inputPaths: []string{"path-detected/.vimrc"}, - expRefs: 1, - }, - { - desc: "directory ignored", - input: "test-fixtures", - notExist: "foobar/", - inputPaths: []string{"path-detected"}, - expRefs: 0, - }, - { - desc: "no files-by-path detected", - input: "test-fixtures", - notExist: "foobar/", - inputPaths: []string{"no-path-detected"}, - expRefs: 0, - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - src, err := NewFromDirectory(test.input) - - if err != nil { - t.Errorf("could not create NewDirScope: %+v", err) - } - if src.Metadata.Path != test.input { - t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input) - } - - _, err = src.FileResolver(SquashedScope) - assert.NoError(t, err) - - src.Metadata.Path = test.notExist - resolver, err := src.FileResolver(SquashedScope) - assert.NoError(t, err) - - refs, err := resolver.FilesByPath(test.inputPaths...) - if err != nil { - t.Errorf("FilesByPath call produced an error: %+v", err) - } - if len(refs) != test.expRefs { - t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs) - - } - - }) - } -} - -func TestFilesByPathDoesNotExist(t *testing.T) { - testCases := []struct { - desc string - input string - path string - expected string - }{ - { - input: "test-fixtures/path-detected", - desc: "path does not exist", - path: "foo", - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - src, err := NewFromDirectory(test.input) - if err != nil { - t.Errorf("could not create NewDirScope: %+v", err) - } - res, err := src.FileResolver(SquashedScope) - if err != nil { - t.Errorf("could not get resolver error: %+v", err) - } - refs, err := res.FilesByPath(test.path) - if err != nil { - t.Errorf("could not get file references from path: %s, %v", test.path, err) - } - - if len(refs) != 0 { - t.Errorf("didnt' expect a ref, but got: %d", len(refs)) - } - - }) - } -} - -func TestFilesByGlob(t *testing.T) { - testCases := []struct { - desc string - input string - glob string - expected int - }{ - { - input: "test-fixtures", - desc: "no matches", - glob: "bar/foo", - expected: 0, - }, - { - input: "test-fixtures/path-detected", - desc: "a single match", - glob: "**/*vimrc", - expected: 1, - }, - { - input: "test-fixtures/path-detected", - desc: "multiple matches", - glob: "**", - expected: 2, - }, - } - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - src, err := NewFromDirectory(test.input) - if err != nil { - t.Errorf("could not create NewDirScope: %+v", err) - } - res, err := src.FileResolver(SquashedScope) - if err != nil { - t.Errorf("could not get resolver error: %+v", err) - } - contents, err := res.FilesByGlob(test.glob) - if err != nil { - t.Errorf("could not get files by glob: %s+v", err) - } - if len(contents) != test.expected { - t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected) - } - - }) - } -} - -func TestDirectoryExclusions(t *testing.T) { - testCases := []struct { - desc string - input string - glob string - expected []string - exclusions []string - err bool - }{ - { - input: "test-fixtures/system_paths", - desc: "exclude everything", - glob: "**", - expected: nil, - exclusions: []string{"**/*"}, - }, - { - input: "test-fixtures/image-simple", - desc: "a single path excluded", - glob: "**", - expected: []string{ - "Dockerfile", - "file-1.txt", - "file-2.txt", - }, - exclusions: []string{"**/target/**"}, - }, - { - input: "test-fixtures/image-simple", - desc: "exclude explicit directory relative to the root", - glob: "**", - expected: []string{ - "Dockerfile", - "file-1.txt", - "file-2.txt", - //"target/really/nested/file-3.txt", // explicitly skipped - }, - exclusions: []string{"./target"}, - }, - { - input: "test-fixtures/image-simple", - desc: "exclude explicit file relative to the root", - glob: "**", - expected: []string{ - "Dockerfile", - //"file-1.txt", // explicitly skipped - "file-2.txt", - "target/really/nested/file-3.txt", - }, - exclusions: []string{"./file-1.txt"}, - }, - { - input: "test-fixtures/image-simple", - desc: "exclude wildcard relative to the root", - glob: "**", - expected: []string{ - "Dockerfile", - //"file-1.txt", // explicitly skipped - //"file-2.txt", // explicitly skipped - "target/really/nested/file-3.txt", - }, - exclusions: []string{"./*.txt"}, - }, - { - input: "test-fixtures/image-simple", - desc: "exclude files deeper", - glob: "**", - expected: []string{ - "Dockerfile", - "file-1.txt", - "file-2.txt", - //"target/really/nested/file-3.txt", // explicitly skipped - }, - exclusions: []string{"**/really/**"}, - }, - { - input: "test-fixtures/image-simple", - desc: "files excluded with extension", - glob: "**", - expected: []string{ - "Dockerfile", - //"file-1.txt", // explicitly skipped - //"file-2.txt", // explicitly skipped - //"target/really/nested/file-3.txt", // explicitly skipped - }, - exclusions: []string{"**/*.txt"}, - }, - { - input: "test-fixtures/image-simple", - desc: "keep files with different extensions", - glob: "**", - expected: []string{ - "Dockerfile", - "file-1.txt", - "file-2.txt", - "target/really/nested/file-3.txt", - }, - exclusions: []string{"**/target/**/*.jar"}, - }, - { - input: "test-fixtures/path-detected", - desc: "file directly excluded", - glob: "**", - expected: []string{ - ".vimrc", - }, - exclusions: []string{"**/empty"}, - }, - { - input: "test-fixtures/path-detected", - desc: "pattern error containing **/", - glob: "**", - expected: []string{ - ".vimrc", - }, - exclusions: []string{"/**/empty"}, - err: true, - }, - { - input: "test-fixtures/path-detected", - desc: "pattern error incorrect start", - glob: "**", - expected: []string{ - ".vimrc", - }, - exclusions: []string{"empty"}, - err: true, - }, - { - input: "test-fixtures/path-detected", - desc: "pattern error starting with /", - glob: "**", - expected: []string{ - ".vimrc", - }, - exclusions: []string{"/empty"}, - err: true, - }, - } - registryOpts := &image.RegistryOptions{} - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - sourceInput, err := ParseInput("dir:"+test.input, "") - require.NoError(t, err) - src, fn, err := New(*sourceInput, registryOpts, test.exclusions) - defer fn() - - if test.err { - _, err = src.FileResolver(SquashedScope) - if err == nil { - t.Errorf("expected an error for patterns: %s", strings.Join(test.exclusions, " or ")) - } - return - } - - if err != nil { - t.Errorf("could not create NewDirScope: %+v", err) - } - res, err := src.FileResolver(SquashedScope) - if err != nil { - t.Errorf("could not get resolver error: %+v", err) - } - locations, err := res.FilesByGlob(test.glob) - if err != nil { - t.Errorf("could not get files by glob: %s+v", err) - } - var actual []string - for _, l := range locations { - actual = append(actual, l.RealPath) - } - - sort.Strings(test.expected) - sort.Strings(actual) - - assert.Equal(t, test.expected, actual, "diff \n"+cmp.Diff(test.expected, actual)) - }) - } -} - -func TestImageExclusions(t *testing.T) { - testCases := []struct { - desc string - input string - glob string - expected int - exclusions []string - }{ - // NOTE: in the Dockerfile, /target is moved to /, which makes /really a top-level dir - { - input: "image-simple", - desc: "a single path excluded", - glob: "**", - expected: 2, - exclusions: []string{"/really/**"}, - }, - { - input: "image-simple", - desc: "a directly referenced directory is excluded", - glob: "**", - expected: 2, - exclusions: []string{"/really"}, - }, - { - input: "image-simple", - desc: "a partial directory is not excluded", - glob: "**", - expected: 3, - exclusions: []string{"/reall"}, - }, - { - input: "image-simple", - desc: "exclude files deeper", - glob: "**", - expected: 2, - exclusions: []string{"**/nested/**"}, - }, - { - input: "image-simple", - desc: "files excluded with extension", - glob: "**", - expected: 2, - exclusions: []string{"**/*1.txt"}, - }, - { - input: "image-simple", - desc: "keep files with different extensions", - glob: "**", - expected: 3, - exclusions: []string{"**/target/**/*.jar"}, - }, - { - input: "image-simple", - desc: "file directly excluded", - glob: "**", - expected: 2, - exclusions: []string{"**/somefile-1.txt"}, // file-1 renamed to somefile-1 in Dockerfile - }, - } - registryOpts := &image.RegistryOptions{} - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - archiveLocation := imagetest.PrepareFixtureImage(t, "docker-archive", test.input) - sourceInput, err := ParseInput(archiveLocation, "") - require.NoError(t, err) - src, fn, err := New(*sourceInput, registryOpts, test.exclusions) - defer fn() - - if err != nil { - t.Errorf("could not create NewDirScope: %+v", err) - } - res, err := src.FileResolver(SquashedScope) - if err != nil { - t.Errorf("could not get resolver error: %+v", err) - } - contents, err := res.FilesByGlob(test.glob) - if err != nil { - t.Errorf("could not get files by glob: %s+v", err) - } - if len(contents) != test.expected { - t.Errorf("wrong number of files after exclusions (%s): %d != %d", test.glob, len(contents), test.expected) - } - }) - } -} - -type dummyInfo struct { - isDir bool -} - -func (d dummyInfo) Name() string { - //TODO implement me - panic("implement me") -} - -func (d dummyInfo) Size() int64 { - //TODO implement me - panic("implement me") -} - -func (d dummyInfo) Mode() fs.FileMode { - //TODO implement me - panic("implement me") -} - -func (d dummyInfo) ModTime() time.Time { - //TODO implement me - panic("implement me") -} - -func (d dummyInfo) IsDir() bool { - return d.isDir -} - -func (d dummyInfo) Sys() any { - //TODO implement me - panic("implement me") -} - -func Test_crossPlatformExclusions(t *testing.T) { - testCases := []struct { - desc string - root string - path string - finfo os.FileInfo - exclude string - walkHint error - }{ - { - desc: "directory exclusion", - root: "/", - path: "/usr/var/lib", - exclude: "**/var/lib", - finfo: dummyInfo{isDir: true}, - walkHint: fs.SkipDir, - }, - { - desc: "no file info", - root: "/", - path: "/usr/var/lib", - exclude: "**/var/lib", - walkHint: fileresolver.ErrSkipPath, - }, - // linux specific tests... - { - desc: "linux doublestar", - root: "/usr", - path: "/usr/var/lib/etc.txt", - exclude: "**/*.txt", - finfo: dummyInfo{isDir: false}, - walkHint: fileresolver.ErrSkipPath, - }, - { - desc: "linux relative", - root: "/usr/var/lib", - path: "/usr/var/lib/etc.txt", - exclude: "./*.txt", - finfo: dummyInfo{isDir: false}, - - walkHint: fileresolver.ErrSkipPath, - }, - { - desc: "linux one level", - root: "/usr", - path: "/usr/var/lib/etc.txt", - exclude: "*/*.txt", - finfo: dummyInfo{isDir: false}, - walkHint: nil, - }, - // NOTE: since these tests will run in linux and macOS, the windows paths will be - // considered relative if they do not start with a forward slash and paths with backslashes - // won't be modified by the filepath.ToSlash call, so these are emulating the result of - // filepath.ToSlash usage - - // windows specific tests... - { - desc: "windows doublestar", - root: "/C:/User/stuff", - path: "/C:/User/stuff/thing.txt", - exclude: "**/*.txt", - finfo: dummyInfo{isDir: false}, - walkHint: fileresolver.ErrSkipPath, - }, - { - desc: "windows relative", - root: "/C:/User/stuff", - path: "/C:/User/stuff/thing.txt", - exclude: "./*.txt", - finfo: dummyInfo{isDir: false}, - walkHint: fileresolver.ErrSkipPath, - }, - { - desc: "windows one level", - root: "/C:/User/stuff", - path: "/C:/User/stuff/thing.txt", - exclude: "*/*.txt", - finfo: dummyInfo{isDir: false}, - walkHint: nil, - }, - } - - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude}) - require.NoError(t, err) - - for _, f := range fns { - result := f(test.path, test.finfo, nil) - require.Equal(t, test.walkHint, result) - } - }) - } -} - -// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath. -func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string, layer2 bool) { - t.Helper() - - cwd, err := os.Getwd() - if err != nil { - t.Fatalf("unable to get cwd: %+v", err) - } - - cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath)) - cmd.Dir = filepath.Join(cwd, "test-fixtures") - - if err := cmd.Start(); err != nil { - t.Fatalf("unable to start generate zip fixture script: %+v", err) - } - - if err := cmd.Wait(); err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - - // This works on both Unix and Windows. Although package - // syscall is generally platform dependent, WaitStatus is - // defined for both Unix and Windows and in both cases has - // an ExitStatus() method with the same signature. - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - if status.ExitStatus() != 0 { - t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus()) - } - } - } else { - t.Fatalf("unable to get generate fixture script result: %+v", err) - } - } - - if layer2 { - cmd = exec.Command("tar", "-rvf", destinationArchivePath, ".") - cmd.Dir = filepath.Join(cwd, "test-fixtures", path.Base(sourceDirPath+"-2")) - if err := cmd.Start(); err != nil { - t.Fatalf("unable to start tar appending fixture script: %+v", err) - } - _ = cmd.Wait() - } -} - -// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function, -// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error, -// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil -// (even if there's an error), and it should always be called. -func setupArchiveTest(t testing.TB, sourceDirPath string, layer2 bool) string { - t.Helper() - - archivePrefix, err := os.CreateTemp(t.TempDir(), "syft-archive-TEST-") - require.NoError(t, err) - - destinationArchiveFilePath := archivePrefix.Name() + ".tar" - t.Logf("archive path: %s", destinationArchiveFilePath) - createArchive(t, sourceDirPath, destinationArchiveFilePath, layer2) - - cwd, err := os.Getwd() - require.NoError(t, err) - - t.Logf("running from: %s", cwd) - - return destinationArchiveFilePath -} - -func assertNoError(t testing.TB, fn func() error) func() { - return func() { - assert.NoError(t, fn()) - } -} diff --git a/syft/source/source_win_test.go b/syft/source/source_win_test.go deleted file mode 100644 index 8fd5eb4b700..00000000000 --- a/syft/source/source_win_test.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build windows -// +build windows - -package source - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_crossPlatformExclusions(t *testing.T) { - testCases := []struct { - desc string - root string - path string - exclude string - match bool - }{ - { - desc: "windows doublestar", - root: "C:\\User\\stuff", - path: "C:\\User\\stuff\\thing.txt", - exclude: "**/*.txt", - match: true, - }, - { - desc: "windows relative", - root: "C:\\User\\stuff", - path: "C:\\User\\stuff\\thing.txt", - exclude: "./*.txt", - match: true, - }, - { - desc: "windows one level", - root: "C:\\User\\stuff", - path: "C:\\User\\stuff\\thing.txt", - exclude: "*/*.txt", - match: false, - }, - } - - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - fns, err := getDirectoryExclusionFunctions(test.root, []string{test.exclude}) - require.NoError(t, err) - - for _, f := range fns { - result := f(test.path, nil) - require.Equal(t, test.match, result) - } - }) - } -} diff --git a/syft/source/stereoscope_image_metadata.go b/syft/source/stereoscope_image_metadata.go new file mode 100644 index 00000000000..ade4f592347 --- /dev/null +++ b/syft/source/stereoscope_image_metadata.go @@ -0,0 +1,62 @@ +package source + +import "github.com/anchore/stereoscope/pkg/image" + +// StereoscopeImageSourceMetadata represents all static metadata that defines what a container image is. This is useful to later describe +// "what" was cataloged without needing the more complicated stereoscope Image objects or FileResolver objects. +type StereoscopeImageSourceMetadata struct { + UserInput string `json:"userInput"` + ID string `json:"imageID"` + ManifestDigest string `json:"manifestDigest"` + MediaType string `json:"mediaType"` + Tags []string `json:"tags"` + Size int64 `json:"imageSize"` + Layers []StereoscopeLayerMetadata `json:"layers"` + RawManifest []byte `json:"manifest"` + RawConfig []byte `json:"config"` + RepoDigests []string `json:"repoDigests"` + Architecture string `json:"architecture"` + Variant string `json:"architectureVariant,omitempty"` + OS string `json:"os"` +} + +// StereoscopeLayerMetadata represents all static metadata that defines what a container image layer is. +type StereoscopeLayerMetadata struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// NewStereoscopeImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration. +func NewStereoscopeImageMetadata(img *image.Image, userInput string) StereoscopeImageSourceMetadata { + // populate artifacts... + tags := make([]string, len(img.Metadata.Tags)) + for idx, tag := range img.Metadata.Tags { + tags[idx] = tag.String() + } + theImg := StereoscopeImageSourceMetadata{ + ID: img.Metadata.ID, + UserInput: userInput, + ManifestDigest: img.Metadata.ManifestDigest, + Size: img.Metadata.Size, + MediaType: string(img.Metadata.MediaType), + Tags: tags, + Layers: make([]StereoscopeLayerMetadata, len(img.Layers)), + RawConfig: img.Metadata.RawConfig, + RawManifest: img.Metadata.RawManifest, + RepoDigests: img.Metadata.RepoDigests, + Architecture: img.Metadata.Architecture, + Variant: img.Metadata.Variant, + OS: img.Metadata.OS, + } + + // populate image metadata + for idx, l := range img.Layers { + theImg.Layers[idx] = StereoscopeLayerMetadata{ + MediaType: string(l.Metadata.MediaType), + Digest: l.Metadata.Digest, + Size: l.Metadata.Size, + } + } + return theImg +} diff --git a/syft/source/stereoscope_image_source.go b/syft/source/stereoscope_image_source.go new file mode 100644 index 00000000000..e9c39d17f34 --- /dev/null +++ b/syft/source/stereoscope_image_source.go @@ -0,0 +1,245 @@ +package source + +import ( + "context" + "fmt" + + "github.com/bmatcuk/doublestar/v4" + "github.com/opencontainers/go-digest" + + "github.com/anchore/stereoscope" + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" +) + +var _ Source = (*StereoscopeImageSource)(nil) + +type StereoscopeImageConfig struct { + Reference string + From image.Source + Platform *image.Platform + RegistryOptions *image.RegistryOptions + Exclude ExcludeConfig + Alias Alias +} + +type StereoscopeImageSource struct { + id artifact.ID + config StereoscopeImageConfig + image *image.Image + metadata StereoscopeImageSourceMetadata +} + +func NewFromStereoscopeImageObject(img *image.Image, reference string, alias *Alias) (*StereoscopeImageSource, error) { + var aliasVal Alias + if !alias.IsEmpty() { + aliasVal = *alias + } + cfg := StereoscopeImageConfig{ + Reference: reference, + Alias: aliasVal, + } + metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference) + + return &StereoscopeImageSource{ + id: deriveIDFromStereoscopeImage(cfg.Alias, metadata), + config: cfg, + image: img, + metadata: metadata, + }, nil +} + +func NewFromStereoscopeImage(cfg StereoscopeImageConfig) (*StereoscopeImageSource, error) { + ctx := context.TODO() + + var opts []stereoscope.Option + if cfg.RegistryOptions != nil { + opts = append(opts, stereoscope.WithRegistryOptions(*cfg.RegistryOptions)) + } + + if cfg.Platform != nil { + opts = append(opts, stereoscope.WithPlatform(cfg.Platform.String())) + } + + img, err := stereoscope.GetImageFromSource(ctx, cfg.Reference, cfg.From, opts...) + if err != nil { + return nil, fmt.Errorf("unable to load image: %w", err) + } + + metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference) + + return &StereoscopeImageSource{ + id: deriveIDFromStereoscopeImage(cfg.Alias, metadata), + config: cfg, + image: img, + metadata: metadata, + }, nil +} + +func (s StereoscopeImageSource) ID() artifact.ID { + return s.id +} + +func (s StereoscopeImageSource) Describe() Description { + name := s.metadata.UserInput + version := s.metadata.ManifestDigest + + a := s.config.Alias + if a.Name != "" { + name = a.Name + } + + if a.Version != "" { + version = a.Version + } + + return Description{ + ID: string(s.id), + Name: name, + Version: version, + Metadata: s.metadata, + } +} + +func (s StereoscopeImageSource) FileResolver(scope Scope) (file.Resolver, error) { + var res file.Resolver + var err error + + switch scope { + case SquashedScope: + res, err = fileresolver.NewFromContainerImageSquash(s.image) + case AllLayersScope: + res, err = fileresolver.NewFromContainerImageAllLayers(s.image) + default: + return nil, fmt.Errorf("bad image scope provided: %+v", scope) + } + + if err != nil { + return nil, err + } + + // image tree contains all paths, so we filter out the excluded entries afterward + if len(s.config.Exclude.Paths) > 0 { + res = fileresolver.NewExcludingDecorator(res, getImageExclusionFunction(s.config.Exclude.Paths)) + } + + return res, nil +} + +func (s StereoscopeImageSource) Close() error { + if s.image == nil { + return nil + } + return s.image.Cleanup() +} + +func imageMetadataFromStereoscopeImage(img *image.Image, reference string) StereoscopeImageSourceMetadata { + tags := make([]string, len(img.Metadata.Tags)) + for idx, tag := range img.Metadata.Tags { + tags[idx] = tag.String() + } + + layers := make([]StereoscopeLayerMetadata, len(img.Layers)) + for idx, l := range img.Layers { + layers[idx] = StereoscopeLayerMetadata{ + MediaType: string(l.Metadata.MediaType), + Digest: l.Metadata.Digest, + Size: l.Metadata.Size, + } + } + + return StereoscopeImageSourceMetadata{ + ID: img.Metadata.ID, + UserInput: reference, + ManifestDigest: img.Metadata.ManifestDigest, + Size: img.Metadata.Size, + MediaType: string(img.Metadata.MediaType), + Tags: tags, + Layers: layers, + RawConfig: img.Metadata.RawConfig, + RawManifest: img.Metadata.RawManifest, + RepoDigests: img.Metadata.RepoDigests, + Architecture: img.Metadata.Architecture, + Variant: img.Metadata.Variant, + OS: img.Metadata.OS, + } +} + +// deriveIDFromStereoscopeImage derives an artifact ID from the given image metadata. The order of data precedence is: +// 1. prefer a digest of the raw container image manifest +// 2. if no manifest digest is available, calculate a chain ID from the image layer metadata +// 3. if no layer metadata is available, use the user input string +// +// in all cases, if an alias is provided, it is additionally considered in the ID calculation. This allows for the +// same image to be scanned multiple times with different aliases and be considered logically different. +func deriveIDFromStereoscopeImage(alias Alias, metadata StereoscopeImageSourceMetadata) artifact.ID { + var input string + + if len(metadata.RawManifest) > 0 { + input = digest.Canonical.FromBytes(metadata.RawManifest).String() + } else { + // calculate chain ID for image sources where manifestDigest is not available + // https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid + input = calculateChainID(metadata.Layers) + if input == "" { + // TODO what happens here if image has no layers? + // is this case possible? + input = digest.Canonical.FromString(metadata.UserInput).String() + } + } + + if !alias.IsEmpty() { + // if the user provided an alias, we want to consider that in the artifact ID. This way if the user + // scans the same item but is considered to be logically different, then ID will express that. + aliasStr := fmt.Sprintf(":%s@%s", alias.Name, alias.Version) + input = digest.Canonical.FromString(input + aliasStr).String() + } + + return artifactIDFromDigest(input) +} + +func calculateChainID(lm []StereoscopeLayerMetadata) string { + if len(lm) < 1 { + return "" + } + + // DiffID(L0) = digest of layer 0 + // https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32 + chainID := lm[0].Digest + id := chain(chainID, lm[1:]) + + return id +} + +func chain(chainID string, layers []StereoscopeLayerMetadata) string { + if len(layers) < 1 { + return chainID + } + + chainID = digest.Canonical.FromString(layers[0].Digest + " " + chainID).String() + return chain(chainID, layers[1:]) +} + +func getImageExclusionFunction(exclusions []string) func(string) bool { + if len(exclusions) == 0 { + return nil + } + // add subpath exclusions + for _, exclusion := range exclusions { + exclusions = append(exclusions, exclusion+"/**") + } + return func(path string) bool { + for _, exclusion := range exclusions { + matches, err := doublestar.Match(exclusion, path) + if err != nil { + return false + } + if matches { + return true + } + } + return false + } +} diff --git a/syft/source/stereoscope_image_source_test.go b/syft/source/stereoscope_image_source_test.go new file mode 100644 index 00000000000..8f1b8d4b202 --- /dev/null +++ b/syft/source/stereoscope_image_source_test.go @@ -0,0 +1,243 @@ +package source + +import ( + "crypto/sha256" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/artifact" +) + +func Test_StereoscopeImage_Exclusions(t *testing.T) { + testCases := []struct { + desc string + input string + glob string + expected int + exclusions []string + }{ + // NOTE: in the Dockerfile, /target is moved to /, which makes /really a top-level dir + { + input: "image-simple", + desc: "a single path excluded", + glob: "**", + expected: 2, + exclusions: []string{"/really/**"}, + }, + { + input: "image-simple", + desc: "a directly referenced directory is excluded", + glob: "**", + expected: 2, + exclusions: []string{"/really"}, + }, + { + input: "image-simple", + desc: "a partial directory is not excluded", + glob: "**", + expected: 3, + exclusions: []string{"/reall"}, + }, + { + input: "image-simple", + desc: "exclude files deeper", + glob: "**", + expected: 2, + exclusions: []string{"**/nested/**"}, + }, + { + input: "image-simple", + desc: "files excluded with extension", + glob: "**", + expected: 2, + exclusions: []string{"**/*1.txt"}, + }, + { + input: "image-simple", + desc: "keep files with different extensions", + glob: "**", + expected: 3, + exclusions: []string{"**/target/**/*.jar"}, + }, + { + input: "image-simple", + desc: "file directly excluded", + glob: "**", + expected: 2, + exclusions: []string{"**/somefile-1.txt"}, // file-1 renamed to somefile-1 in Dockerfile + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := NewFromStereoscopeImage( + StereoscopeImageConfig{ + Reference: strings.SplitN(imagetest.PrepareFixtureImage(t, "docker-archive", test.input), ":", 2)[1], + From: image.DockerTarballSource, + Exclude: ExcludeConfig{ + Paths: test.exclusions, + }, + }, + ) + + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + res, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + contents, err := res.FilesByGlob(test.glob) + require.NoError(t, err) + + assert.Len(t, contents, test.expected) + }) + } +} + +func Test_StereoscopeImageSource_ID(t *testing.T) { + tests := []struct { + name string + alias Alias + metadata StereoscopeImageSourceMetadata + want artifact.ID + }{ + { + name: "use raw manifest over chain ID or user input", + metadata: StereoscopeImageSourceMetadata{ + UserInput: "user-input", + Layers: []StereoscopeLayerMetadata{ + { + Digest: "a", + }, + { + Digest: "b", + }, + { + Digest: "c", + }, + }, + RawManifest: []byte("raw-manifest"), + }, + want: func() artifact.ID { + hasher := sha256.New() + hasher.Write([]byte("raw-manifest")) + return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil))) + }(), + }, + { + name: "use chain ID over user input", + metadata: StereoscopeImageSourceMetadata{ + //UserInput: "user-input", + Layers: []StereoscopeLayerMetadata{ + { + Digest: "a", + }, + { + Digest: "b", + }, + { + Digest: "c", + }, + }, + }, + want: func() artifact.ID { + metadata := []StereoscopeLayerMetadata{ + { + Digest: "a", + }, + { + Digest: "b", + }, + { + Digest: "c", + }, + } + return artifact.ID(strings.TrimPrefix(calculateChainID(metadata), "sha256:")) + }(), + }, + { + name: "use user input last", + metadata: StereoscopeImageSourceMetadata{ + UserInput: "user-input", + }, + want: func() artifact.ID { + hasher := sha256.New() + hasher.Write([]byte("user-input")) + return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil))) + }(), + }, + { + name: "without alias (first)", + metadata: StereoscopeImageSourceMetadata{ + UserInput: "user-input", + Layers: []StereoscopeLayerMetadata{ + { + Digest: "a", + }, + { + Digest: "b", + }, + { + Digest: "c", + }, + }, + RawManifest: []byte("raw-manifest"), + }, + want: "85298926ecd92ed57688f13039017160cd728f04dd0d2d10a10629007106f107", + }, + { + name: "always consider alias (first)", + alias: Alias{ + Name: "alias", + Version: "version", + }, + metadata: StereoscopeImageSourceMetadata{ + UserInput: "user-input", + Layers: []StereoscopeLayerMetadata{ + { + Digest: "a", + }, + { + Digest: "b", + }, + { + Digest: "c", + }, + }, + RawManifest: []byte("raw-manifest"), + }, + want: "a8717e42449960c1dd4963f2f22bd69c7c105e7e82445be0a65aa1825d62ff0d", + }, + { + name: "without alias (last)", + metadata: StereoscopeImageSourceMetadata{ + UserInput: "user-input", + }, + want: "ab0dff627d80b9753193d7280bec8f45e8ec6b4cb0912c6fffcf7cd782d9739e", + }, + { + name: "always consider alias (last)", + alias: Alias{ + Name: "alias", + Version: "version", + }, + metadata: StereoscopeImageSourceMetadata{ + UserInput: "user-input", + }, + want: "fe86c0eecd5654d3c0c0b2176aa394aef6440347c241aa8d9b628dfdde4287cf", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, deriveIDFromStereoscopeImage(tt.alias, tt.metadata)) + }) + } +} diff --git a/syft/source/test-fixtures/file-index-filter/.1/something b/syft/source/test-fixtures/file-index-filter/.1/something new file mode 100644 index 00000000000..6b584e8ece5 --- /dev/null +++ b/syft/source/test-fixtures/file-index-filter/.1/something @@ -0,0 +1 @@ +content \ No newline at end of file diff --git a/syft/source/test-fixtures/file-index-filter/.2 b/syft/source/test-fixtures/file-index-filter/.2 new file mode 100644 index 00000000000..a459bc245bd --- /dev/null +++ b/syft/source/test-fixtures/file-index-filter/.2 @@ -0,0 +1 @@ +something \ No newline at end of file diff --git a/syft/source/test-fixtures/file-index-filter/.vimrc b/syft/source/test-fixtures/file-index-filter/.vimrc new file mode 100644 index 00000000000..7f865a925e7 --- /dev/null +++ b/syft/source/test-fixtures/file-index-filter/.vimrc @@ -0,0 +1 @@ +Another .vimrc file \ No newline at end of file diff --git a/syft/source/test-fixtures/file-index-filter/empty b/syft/source/test-fixtures/file-index-filter/empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/catalog_packages_test.go b/test/integration/catalog_packages_test.go index 2c88c0615fe..f2f153a5248 100644 --- a/test/integration/catalog_packages_test.go +++ b/test/integration/catalog_packages_test.go @@ -26,13 +26,15 @@ func BenchmarkImagePackageCatalogers(b *testing.B) { for _, c := range cataloger.ImageCatalogers(cataloger.DefaultConfig()) { // in case of future alteration where state is persisted, assume no dependency is safe to reuse userInput := "docker-archive:" + tarPath - sourceInput, err := source.ParseInput(userInput, "") + detection, err := source.Detect(userInput, source.DefaultDetectConfig()) require.NoError(b, err) - theSource, cleanupSource, err := source.New(*sourceInput, nil, nil) - b.Cleanup(cleanupSource) + theSource, err := detection.NewSource(source.DefaultDetectionSourceConfig()) if err != nil { b.Fatalf("unable to get source: %+v", err) } + b.Cleanup(func() { + theSource.Close() + }) resolver, err := theSource.FileResolver(source.SquashedScope) if err != nil { diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index 77f50045051..eeb583f6558 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -12,15 +12,17 @@ import ( "github.com/anchore/syft/syft/source" ) -func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Scope, catalogerCfg []string) (sbom.SBOM, *source.Source) { +func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Scope, catalogerCfg []string) (sbom.SBOM, source.Source) { imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) userInput := "docker-archive:" + tarPath - sourceInput, err := source.ParseInput(userInput, "") + detection, err := source.Detect(userInput, source.DefaultDetectConfig()) require.NoError(t, err) - theSource, cleanupSource, err := source.New(*sourceInput, nil, nil) - t.Cleanup(cleanupSource) + theSource, err := detection.NewSource(source.DefaultDetectionSourceConfig()) require.NoError(t, err) + t.Cleanup(func() { + theSource.Close() + }) c := cataloger.DefaultConfig() c.Catalogers = catalogerCfg @@ -37,7 +39,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco LinuxDistribution: actualDistro, }, Relationships: relationships, - Source: theSource.Metadata, + Source: theSource.Describe(), Descriptor: sbom.Descriptor{ Name: "syft", Version: "v0.42.0-bogus", @@ -50,13 +52,15 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco }, theSource } -func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, *source.Source) { +func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, source.Source) { userInput := "dir:" + dir - sourceInput, err := source.ParseInput(userInput, "") + detection, err := source.Detect(userInput, source.DefaultDetectConfig()) require.NoError(t, err) - theSource, cleanupSource, err := source.New(*sourceInput, nil, nil) - t.Cleanup(cleanupSource) + theSource, err := detection.NewSource(source.DefaultDetectionSourceConfig()) require.NoError(t, err) + t.Cleanup(func() { + theSource.Close() + }) // TODO: this would be better with functional options (after/during API refactor) c := cataloger.DefaultConfig() @@ -72,6 +76,6 @@ func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, *source.Source) { LinuxDistribution: actualDistro, }, Relationships: relationships, - Source: theSource.Metadata, + Source: theSource.Describe(), }, theSource } From 2e3c7fa158d179a06500aea7dd7464a8c33dc9b2 Mon Sep 17 00:00:00 2001 From: Marco Damiani Date: Mon, 3 Jul 2023 19:50:01 +0200 Subject: [PATCH 03/17] doc(readme): add installation section with scoop (#1909) Signed-off-by: drazen04 --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 1336b526f0d..9a63a6d5fbf 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,12 @@ The chocolatey distribution of syft is community maintained and not distributed choco install syft -y ``` +### Scoop + +```powershell +scoop install syft +``` + ### Homebrew ```bash brew install syft From 023ca1be322c1c52b0b26cf6a733bccc58ad3a54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:05:46 -0400 Subject: [PATCH 04/17] chore(deps): bump golang.org/x/term from 0.9.0 to 0.10.0 (#1913) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.9.0 to 0.10.0. - [Commits](https://github.com/golang/term/compare/v0.9.0...v0.10.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ad92efa9386..a243cc76da0 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/mod v0.11.0 golang.org/x/net v0.11.0 - golang.org/x/term v0.9.0 + golang.org/x/term v0.10.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -144,7 +144,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index d6ec23fc5f8..c6da3914dd5 100644 --- a/go.sum +++ b/go.sum @@ -882,8 +882,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -891,8 +891,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From e8f7108e6e08207cb1f23d836393083351307e39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:06:05 -0400 Subject: [PATCH 05/17] chore(deps): bump golang.org/x/mod from 0.11.0 to 0.12.0 (#1912) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.11.0 to 0.12.0. - [Commits](https://github.com/golang/mod/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/mod dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a243cc76da0..ff8d61dfcc1 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5 github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/mod v0.11.0 + golang.org/x/mod v0.12.0 golang.org/x/net v0.11.0 golang.org/x/term v0.10.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index c6da3914dd5..f321e6edd9b 100644 --- a/go.sum +++ b/go.sum @@ -720,8 +720,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 6280146c812be183d119acacefee606bbf0e31f3 Mon Sep 17 00:00:00 2001 From: "anchore-actions-token-generator[bot]" <102182147+anchore-actions-token-generator[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:06:22 -0400 Subject: [PATCH 06/17] chore(deps): update bootstrap tools to latest versions (#1908) Signed-off-by: GitHub Co-authored-by: spiffcs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f04590d8004..e2f0707e97d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ GOLANGCILINT_VERSION := v1.53.3 GOSIMPORTS_VERSION := v0.3.8 BOUNCER_VERSION := v0.4.0 CHRONICLE_VERSION := v0.6.0 -GORELEASER_VERSION := v1.19.0 +GORELEASER_VERSION := v1.19.1 YAJSV_VERSION := v1.4.1 COSIGN_VERSION := v2.1.1 QUILL_VERSION := v0.2.0 From cfbb9f703bd77c80cef5c8777236db6be6b14bbf Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 5 Jul 2023 13:47:13 -0400 Subject: [PATCH 07/17] add file source digest support (#1914) Signed-off-by: Alex Goodman --- .gitignore | 2 + README.md | 18 ++- cmd/syft/cli/attest/attest.go | 12 +- cmd/syft/cli/eventloop/tasks.go | 25 +--- cmd/syft/cli/options/packages.go | 4 +- cmd/syft/cli/packages/packages.go | 13 +- cmd/syft/cli/poweruser/poweruser.go | 4 +- internal/config/application.go | 7 +- internal/config/source.go | 17 +++ internal/file/digest.go | 76 ++++++++++ internal/file/digest_test.go | 132 ++++++++++++++++++ internal/file/test-fixtures/digest.txt | 1 + syft/file/cataloger/filedigest/cataloger.go | 7 +- .../cataloger/filedigest/cataloger_test.go | 3 +- syft/file/digest.go | 70 ---------- .../cyclonedxhelpers/external_references.go | 3 +- syft/pkg/cataloger/java/archive_parser.go | 2 +- syft/source/file_source.go | 3 +- 18 files changed, 286 insertions(+), 113 deletions(-) create mode 100644 internal/config/source.go create mode 100644 internal/file/digest.go create mode 100644 internal/file/digest_test.go create mode 100644 internal/file/test-fixtures/digest.txt diff --git a/.gitignore b/.gitignore index ab7501da491..e4c0ffcd7e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +go.work +go.work.sum /.bin CHANGELOG.md VERSION diff --git a/README.md b/README.md index 9a63a6d5fbf..70c293f6797 100644 --- a/README.md +++ b/README.md @@ -605,7 +605,7 @@ file-metadata: # SYFT_FILE_METADATA_CATALOGER_SCOPE env var scope: "squashed" - # the file digest algorithms to use when cataloging files (options: "sha256", "md5", "sha1") + # the file digest algorithms to use when cataloging files (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512") # SYFT_FILE_METADATA_DIGESTS env var digests: ["sha256"] @@ -643,11 +643,27 @@ secrets: # SYFT_SECRETS_EXCLUDE_PATTERN_NAMES env var exclude-pattern-names: [] +# options that apply to all scan sources +source: + # alias name for the source + # SYFT_SOURCE_NAME env var; --source-name flag + name: "" + + # alias version for the source + # SYFT_SOURCE_VERSION env var; --source-version flag + version: "" + + # options affecting the file source type + file: + # the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512") + digests: ["sha256"] + # options when pulling directly from a registry via the "registry:" scheme registry: # skip TLS verification when communicating with the registry # SYFT_REGISTRY_INSECURE_SKIP_TLS_VERIFY env var insecure-skip-tls-verify: false + # use http instead of https when connecting to the registry # SYFT_REGISTRY_INSECURE_USE_HTTP env var insecure-use-http: false diff --git a/cmd/syft/cli/attest/attest.go b/cmd/syft/cli/attest/attest.go index 9ed452b359c..cdd25ad3200 100644 --- a/cmd/syft/cli/attest/attest.go +++ b/cmd/syft/cli/attest/attest.go @@ -18,6 +18,7 @@ import ( "github.com/anchore/syft/cmd/syft/cli/packages" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/config" + "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/ui" "github.com/anchore/syft/syft" @@ -74,18 +75,23 @@ func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbo } } + hashers, err := file.Hashers(app.Source.File.Digests...) + if err != nil { + return nil, fmt.Errorf("invalid hash: %w", err) + } + src, err := detection.NewSource( source.DetectionSourceConfig{ Alias: source.Alias{ - Name: app.SourceName, - Version: app.SourceVersion, + Name: app.Source.Name, + Version: app.Source.Version, }, RegistryOptions: app.Registry.ToOptions(), Platform: platform, Exclude: source.ExcludeConfig{ Paths: app.Exclusions, }, - DigestAlgorithms: nil, + DigestAlgorithms: hashers, }, ) diff --git a/cmd/syft/cli/eventloop/tasks.go b/cmd/syft/cli/eventloop/tasks.go index 4c04565427c..b6121d0daa1 100644 --- a/cmd/syft/cli/eventloop/tasks.go +++ b/cmd/syft/cli/eventloop/tasks.go @@ -1,13 +1,10 @@ package eventloop import ( - "crypto" - "fmt" - "github.com/anchore/syft/internal/config" + "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file/cataloger/filecontent" "github.com/anchore/syft/syft/file/cataloger/filedigest" "github.com/anchore/syft/syft/file/cataloger/filemetadata" @@ -89,23 +86,9 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) { return nil, nil } - supportedHashAlgorithms := make(map[string]crypto.Hash) - for _, h := range []crypto.Hash{ - crypto.MD5, - crypto.SHA1, - crypto.SHA256, - } { - supportedHashAlgorithms[file.DigestAlgorithmName(h)] = h - } - - var hashes []crypto.Hash - for _, hashStr := range app.FileMetadata.Digests { - name := file.CleanDigestAlgorithmName(hashStr) - hashObj, ok := supportedHashAlgorithms[name] - if !ok { - return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr) - } - hashes = append(hashes, hashObj) + hashes, err := file.Hashers(app.FileMetadata.Digests...) + if err != nil { + return nil, err } digestsCataloger := filedigest.NewCataloger(hashes) diff --git a/cmd/syft/cli/options/packages.go b/cmd/syft/cli/options/packages.go index f6992a948c2..bea013b6922 100644 --- a/cmd/syft/cli/options/packages.go +++ b/cmd/syft/cli/options/packages.go @@ -86,11 +86,11 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error { return err } - if err := v.BindPFlag("source-name", flags.Lookup("source-name")); err != nil { + if err := v.BindPFlag("source.name", flags.Lookup("source-name")); err != nil { return err } - if err := v.BindPFlag("source-version", flags.Lookup("source-version")); err != nil { + if err := v.BindPFlag("source.version", flags.Lookup("source-version")); err != nil { return err } diff --git a/cmd/syft/cli/packages/packages.go b/cmd/syft/cli/packages/packages.go index da52581935b..53b2b860df1 100644 --- a/cmd/syft/cli/packages/packages.go +++ b/cmd/syft/cli/packages/packages.go @@ -13,6 +13,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/config" + "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/ui" "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft" @@ -77,18 +78,24 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < } } + hashers, err := file.Hashers(app.Source.File.Digests...) + if err != nil { + errs <- fmt.Errorf("invalid hash: %w", err) + return + } + src, err := detection.NewSource( source.DetectionSourceConfig{ Alias: source.Alias{ - Name: app.SourceName, - Version: app.SourceVersion, + Name: app.Source.Name, + Version: app.Source.Version, }, RegistryOptions: app.Registry.ToOptions(), Platform: platform, Exclude: source.ExcludeConfig{ Paths: app.Exclusions, }, - DigestAlgorithms: nil, + DigestAlgorithms: hashers, }, ) diff --git a/cmd/syft/cli/poweruser/poweruser.go b/cmd/syft/cli/poweruser/poweruser.go index dd1b758fe0f..e37455ece45 100644 --- a/cmd/syft/cli/poweruser/poweruser.go +++ b/cmd/syft/cli/poweruser/poweruser.go @@ -94,8 +94,8 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < src, err := detection.NewSource( source.DetectionSourceConfig{ Alias: source.Alias{ - Name: app.SourceName, - Version: app.SourceVersion, + Name: app.Source.Name, + Version: app.Source.Version, }, RegistryOptions: app.Registry.ToOptions(), Platform: platform, diff --git a/internal/config/application.go b/internal/config/application.go index 9f3274265fa..ea85410025f 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -61,8 +61,7 @@ type Application struct { Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"` Platform string `yaml:"platform" json:"platform" mapstructure:"platform"` Name string `yaml:"name" json:"name" mapstructure:"name"` - SourceName string `yaml:"source-name" json:"source-name" mapstructure:"source-name"` - SourceVersion string `yaml:"source-version" json:"source-version" mapstructure:"source-version"` + Source sourceCfg `yaml:"source" json:"source" mapstructure:"source"` Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel DefaultImagePullSource string `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"` // specify default image pull source } @@ -147,8 +146,8 @@ func (cfg *Application) parseConfigValues() error { if cfg.Name != "" { log.Warnf("name parameter is deprecated. please use: source-name. name will be removed in a future version") - if cfg.SourceName == "" { - cfg.SourceName = cfg.Name + if cfg.Source.Name == "" { + cfg.Source.Name = cfg.Name } } diff --git a/internal/config/source.go b/internal/config/source.go new file mode 100644 index 00000000000..5346f994fa7 --- /dev/null +++ b/internal/config/source.go @@ -0,0 +1,17 @@ +package config + +import "github.com/spf13/viper" + +type sourceCfg struct { + Name string `json:"name" yaml:"name" mapstructure:"name"` + Version string `json:"version" yaml:"version" mapstructure:"version"` + File fileSource `json:"file" yaml:"file" mapstructure:"file"` +} + +type fileSource struct { + Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"` +} + +func (cfg sourceCfg) loadDefaultValues(v *viper.Viper) { + v.SetDefault("source.file.digests", []string{"sha256"}) +} diff --git a/internal/file/digest.go b/internal/file/digest.go new file mode 100644 index 00000000000..4bc8c42325a --- /dev/null +++ b/internal/file/digest.go @@ -0,0 +1,76 @@ +package file + +import ( + "crypto" + "fmt" + "hash" + "io" + "strings" + + "github.com/anchore/syft/syft/file" +) + +func supportedHashAlgorithms() []crypto.Hash { + return []crypto.Hash{ + crypto.MD5, + crypto.SHA1, + crypto.SHA224, + crypto.SHA256, + crypto.SHA384, + crypto.SHA512, + } +} + +func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]file.Digest, error) { + // create a set of hasher objects tied together with a single writer to feed content into + hashers := make([]hash.Hash, len(hashes)) + writers := make([]io.Writer, len(hashes)) + for idx, hashObj := range hashes { + hashers[idx] = hashObj.New() + writers[idx] = hashers[idx] + } + + size, err := io.Copy(io.MultiWriter(writers...), closer) + if err != nil { + return nil, err + } + + if size == 0 { + return make([]file.Digest, 0), nil + } + + result := make([]file.Digest, len(hashes)) + // only capture digests when there is content. It is important to do this based on SIZE and not + // FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only + // file type but a body is still allowed. + for idx, hasher := range hashers { + result[idx] = file.Digest{ + Algorithm: CleanDigestAlgorithmName(hashes[idx].String()), + Value: fmt.Sprintf("%+x", hasher.Sum(nil)), + } + } + + return result, nil +} + +func Hashers(names ...string) ([]crypto.Hash, error) { + hashByName := make(map[string]crypto.Hash) + for _, h := range supportedHashAlgorithms() { + hashByName[CleanDigestAlgorithmName(h.String())] = h + } + + var hashers []crypto.Hash + for _, hashStr := range names { + hashObj, ok := hashByName[CleanDigestAlgorithmName(hashStr)] + if !ok { + return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr) + } + hashers = append(hashers, hashObj) + } + return hashers, nil +} + +func CleanDigestAlgorithmName(name string) string { + lower := strings.ToLower(name) + return strings.ReplaceAll(lower, "-", "") +} diff --git a/internal/file/digest_test.go b/internal/file/digest_test.go new file mode 100644 index 00000000000..df50798f2aa --- /dev/null +++ b/internal/file/digest_test.go @@ -0,0 +1,132 @@ +package file + +import ( + "crypto" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" +) + +func TestCleanDigestAlgorithmName(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "go case", + input: "SHA-256", + want: "sha256", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, CleanDigestAlgorithmName(tt.input)) + }) + } +} + +func TestNewDigestsFromFile(t *testing.T) { + require.NotEmpty(t, supportedHashAlgorithms()) + + tests := []struct { + name string + fixture string + hashes []crypto.Hash + want []file.Digest + wantErr require.ErrorAssertionFunc + }{ + { + name: "check supported hash algorithms", + fixture: "test-fixtures/digest.txt", + hashes: supportedHashAlgorithms(), + want: []file.Digest{ + { + Algorithm: "md5", + Value: "e8818a24402ae7f8b874cdd9350c1b51", + }, + { + Algorithm: "sha1", + Value: "eea4671d168c81fd52e615ed9fb3531a526f4748", + }, + { + Algorithm: "sha224", + Value: "fd993e84c7afb449d34bcae7c5ee118f5c73b50170da05171523b22c", + }, + { + Algorithm: "sha256", + Value: "cbf1a703b7e4a67529d6e17114880dfa9f879f3749872e1a9d4a20ac509165ad", + }, + { + Algorithm: "sha384", + Value: "1eaded3f17fb8d7b731c9175a0f355d3a35575c3cb6cdda46a5272b632968d7257a5e6437d0efae599a81a1b2dcc81ba", + }, + { + Algorithm: "sha512", + Value: "b49d5995456edba144dce750eaa8eae12af8fd08c076d401fcf78aac4172080feb70baaa5ed8c1b05046ec278446330fbf77e8ca9e60c03945ded761a641a7e1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + fh, err := os.Open(tt.fixture) + require.NoError(t, err) + + got, err := NewDigestsFromFile(fh, tt.hashes) + tt.wantErr(t, err) + if err != nil { + return + } + assert.Equal(t, tt.want, got) + }) + } +} + +func TestHashers(t *testing.T) { + tests := []struct { + name string + names []string + want []crypto.Hash + wantErr require.ErrorAssertionFunc + }{ + { + name: "check supported hash algorithms", + names: []string{"MD-5", "shA1", "sHa224", "sha---256", "sha384", "sha512"}, + want: []crypto.Hash{ + crypto.MD5, + crypto.SHA1, + crypto.SHA224, + crypto.SHA256, + crypto.SHA384, + crypto.SHA512, + }, + }, + { + name: "error on unsupported hash algorithm", + names: []string{"made-up"}, + wantErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + got, err := Hashers(tt.names...) + tt.wantErr(t, err) + if err != nil { + return + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/file/test-fixtures/digest.txt b/internal/file/test-fixtures/digest.txt new file mode 100644 index 00000000000..887a7ea3796 --- /dev/null +++ b/internal/file/test-fixtures/digest.txt @@ -0,0 +1 @@ +hello, file! \ No newline at end of file diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index e06c05a3514..31a4367ab1f 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -10,10 +10,11 @@ import ( stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/bus" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/file" - internal2 "github.com/anchore/syft/syft/file/cataloger/internal" + intCataloger "github.com/anchore/syft/syft/file/cataloger/internal" ) var ErrUndigestableFile = errors.New("undigestable file") @@ -33,7 +34,7 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina var locations []file.Location if len(coordinates) == 0 { - locations = internal2.AllRegularFiles(resolver) + locations = intCataloger.AllRegularFiles(resolver) } else { for _, c := range coordinates { locations = append(locations, file.NewLocationFromCoordinates(c)) @@ -82,7 +83,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati } defer internal.CloseAndLogError(contentReader, location.VirtualPath) - digests, err := file.NewDigestsFromFile(contentReader, i.hashes) + digests, err := intFile.NewDigestsFromFile(contentReader, i.hashes) if err != nil { return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} } diff --git a/syft/file/cataloger/filedigest/cataloger_test.go b/syft/file/cataloger/filedigest/cataloger_test.go index 1ed1af58878..9ebaceed007 100644 --- a/syft/file/cataloger/filedigest/cataloger_test.go +++ b/syft/file/cataloger/filedigest/cataloger_test.go @@ -13,6 +13,7 @@ import ( stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" ) @@ -40,7 +41,7 @@ func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Has h := hash.New() h.Write(b) digests[file.NewLocation(f).Coordinates] = append(digests[file.NewLocation(f).Coordinates], file.Digest{ - Algorithm: file.CleanDigestAlgorithmName(hash.String()), + Algorithm: intFile.CleanDigestAlgorithmName(hash.String()), Value: fmt.Sprintf("%x", h.Sum(nil)), }) } diff --git a/syft/file/digest.go b/syft/file/digest.go index 23219e68875..87b53dbb833 100644 --- a/syft/file/digest.go +++ b/syft/file/digest.go @@ -1,76 +1,6 @@ package file -import ( - "crypto" - "fmt" - "hash" - "io" - "strings" -) - type Digest struct { Algorithm string `json:"algorithm"` Value string `json:"value"` } - -func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]Digest, error) { - // create a set of hasher objects tied together with a single writer to feed content into - hashers := make([]hash.Hash, len(hashes)) - writers := make([]io.Writer, len(hashes)) - for idx, hashObj := range hashes { - hashers[idx] = hashObj.New() - writers[idx] = hashers[idx] - } - - size, err := io.Copy(io.MultiWriter(writers...), closer) - if err != nil { - return nil, err - } - - if size == 0 { - return make([]Digest, 0), nil - } - - result := make([]Digest, len(hashes)) - // only capture digests when there is content. It is important to do this based on SIZE and not - // FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only - // file type but a body is still allowed. - for idx, hasher := range hashers { - result[idx] = Digest{ - Algorithm: DigestAlgorithmName(hashes[idx]), - Value: fmt.Sprintf("%+x", hasher.Sum(nil)), - } - } - - return result, nil -} - -func Hashers(names ...string) ([]crypto.Hash, error) { - supportedHashAlgorithms := make(map[string]crypto.Hash) - for _, h := range []crypto.Hash{ - crypto.MD5, - crypto.SHA1, - crypto.SHA256, - } { - supportedHashAlgorithms[DigestAlgorithmName(h)] = h - } - - var hashers []crypto.Hash - for _, hashStr := range names { - hashObj, ok := supportedHashAlgorithms[CleanDigestAlgorithmName(hashStr)] - if !ok { - return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr) - } - hashers = append(hashers, hashObj) - } - return hashers, nil -} - -func DigestAlgorithmName(hash crypto.Hash) string { - return CleanDigestAlgorithmName(hash.String()) -} - -func CleanDigestAlgorithmName(name string) string { - lower := strings.ToLower(name) - return strings.ReplaceAll(lower, "-", "") -} diff --git a/syft/formats/common/cyclonedxhelpers/external_references.go b/syft/formats/common/cyclonedxhelpers/external_references.go index da657de6008..59f388717a3 100644 --- a/syft/formats/common/cyclonedxhelpers/external_references.go +++ b/syft/formats/common/cyclonedxhelpers/external_references.go @@ -6,6 +6,7 @@ import ( "github.com/CycloneDX/cyclonedx-go" + "github.com/anchore/syft/internal/file" syftFile "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) @@ -116,7 +117,7 @@ func decodeExternalReferences(c *cyclonedx.Component, metadata interface{}) { if ref.Hashes != nil { for _, hash := range *ref.Hashes { digests = append(digests, syftFile.Digest{ - Algorithm: syftFile.CleanDigestAlgorithmName(string(hash.Algorithm)), + Algorithm: file.CleanDigestAlgorithmName(string(hash.Algorithm)), Value: hash.Value, }) } diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index a1efd022d0c..ea216e9062f 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -179,7 +179,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { defer archiveCloser.Close() // grab and assign digest for the entire archive - digests, err := file.NewDigestsFromFile(archiveCloser, javaArchiveHashes) + digests, err := intFile.NewDigestsFromFile(archiveCloser, javaArchiveHashes) if err != nil { log.Warnf("failed to create digest for file=%q: %+v", j.archivePath, err) } diff --git a/syft/source/file_source.go b/syft/source/file_source.go index 2025d085676..5adc81d97ec 100644 --- a/syft/source/file_source.go +++ b/syft/source/file_source.go @@ -13,6 +13,7 @@ import ( "github.com/opencontainers/go-digest" stereoFile "github.com/anchore/stereoscope/pkg/file" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -67,7 +68,7 @@ func NewFromFile(cfg FileConfig) (*FileSource, error) { defer fh.Close() - digests, err = file.NewDigestsFromFile(fh, cfg.DigestAlgorithms) + digests, err = intFile.NewDigestsFromFile(fh, cfg.DigestAlgorithms) if err != nil { return nil, fmt.Errorf("unable to calculate digests for file=%q: %w", cfg.Path, err) } From a00a3df10c3a20566c10d6d42de6d1a4bfbcea5d Mon Sep 17 00:00:00 2001 From: "DD (Devdatta) Deshpande" Date: Thu, 6 Jul 2023 00:19:22 +0530 Subject: [PATCH 08/17] fix: use filepath.EvalSymlinks if os.Readlink fails to evaluate the link (#1884) Signed-off-by: DD (Devdatta) Deshpande Co-authored-by: Keith Zantow --- syft/internal/fileresolver/directory_indexer.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/syft/internal/fileresolver/directory_indexer.go b/syft/internal/fileresolver/directory_indexer.go index 47349a4456b..01f332a3de6 100644 --- a/syft/internal/fileresolver/directory_indexer.go +++ b/syft/internal/fileresolver/directory_indexer.go @@ -332,7 +332,19 @@ func (r directoryIndexer) addFileToIndex(p string, info os.FileInfo) error { func (r directoryIndexer) addSymlinkToIndex(p string, info os.FileInfo) (string, error) { linkTarget, err := os.Readlink(p) if err != nil { - return "", fmt.Errorf("unable to readlink for path=%q: %w", p, err) + if runtime.GOOS == WindowsOS { + p = posixToWindows(p) + } + + linkTarget, err = filepath.EvalSymlinks(p) + + if runtime.GOOS == WindowsOS { + p = windowsToPosix(p) + } + + if err != nil { + return "", fmt.Errorf("unable to readlink for path=%q: %w", p, err) + } } if filepath.IsAbs(linkTarget) { From f8b832e6c3aab3a50a369ce3de57d8b6d1f729ce Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 6 Jul 2023 09:00:46 -0400 Subject: [PATCH 09/17] Switch UI to bubbletea (#1888) * add bubbletea UI Signed-off-by: Alex Goodman * swap pipeline to go 1.20.x and add attest guard for cosign binary Signed-off-by: Alex Goodman * update note in developing.md about the required golang version Signed-off-by: Alex Goodman * fix merge conflict for windows path handling Signed-off-by: Alex Goodman * temp test for attest handler Signed-off-by: Alex Goodman * add addtional test iterations for background reader Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman Signed-off-by: Alex Goodman --- .github/actions/bootstrap/action.yaml | 2 +- .gitignore | 1 + cmd/syft/cli/attest.go | 1 + cmd/syft/cli/attest/attest.go | 15 +- cmd/syft/cli/commands.go | 2 +- cmd/syft/cli/convert.go | 1 + cmd/syft/cli/convert/convert.go | 51 +++- cmd/syft/cli/eventloop/event_loop.go | 14 +- cmd/syft/cli/eventloop/event_loop_test.go | 29 +- cmd/syft/cli/options/writer.go | 29 +- cmd/syft/cli/options/writer_test.go | 2 + cmd/syft/cli/packages.go | 1 + cmd/syft/cli/packages/packages.go | 14 +- cmd/syft/cli/poweruser.go | 1 + cmd/syft/cli/poweruser/poweruser.go | 12 +- .../handle_attestation_test.snap | 19 ++ .../handle_cataloger_task_test.snap | 16 ++ .../handle_fetch_image_test.snap | 8 + .../handle_file_digests_cataloger_test.snap | 8 + .../handle_file_indexing_test.snap | 8 + .../handle_file_metadata_cataloger_test.snap | 8 + .../handle_package_cataloger_test.snap | 16 ++ .../handle_pull_docker_image_test.snap | 12 + .../__snapshots__/handle_read_image_test.snap | 8 + .../handle_secrets_cataloger_test.snap | 8 + cmd/syft/cli/ui/handle_attestation.go | 247 ++++++++++++++++++ cmd/syft/cli/ui/handle_attestation_test.go | 133 ++++++++++ cmd/syft/cli/ui/handle_cataloger_task.go | 72 +++++ cmd/syft/cli/ui/handle_cataloger_task_test.go | 123 +++++++++ cmd/syft/cli/ui/handle_fetch_image.go | 32 +++ cmd/syft/cli/ui/handle_fetch_image_test.go | 99 +++++++ .../cli/ui/handle_file_digests_cataloger.go | 28 ++ .../ui/handle_file_digests_cataloger_test.go | 97 +++++++ cmd/syft/cli/ui/handle_file_indexing.go | 31 +++ cmd/syft/cli/ui/handle_file_indexing_test.go | 99 +++++++ .../cli/ui/handle_file_metadata_cataloger.go | 29 ++ .../ui/handle_file_metadata_cataloger_test.go | 97 +++++++ cmd/syft/cli/ui/handle_package_cataloger.go | 87 ++++++ .../cli/ui/handle_package_cataloger_test.go | 133 ++++++++++ cmd/syft/cli/ui/handle_pull_docker_image.go | 201 ++++++++++++++ .../cli/ui/handle_pull_docker_image_test.go | 163 ++++++++++++ cmd/syft/cli/ui/handle_read_image.go | 33 +++ cmd/syft/cli/ui/handle_read_image_test.go | 117 +++++++++ cmd/syft/cli/ui/handle_secrets_cataloger.go | 57 ++++ .../cli/ui/handle_secrets_cataloger_test.go | 96 +++++++ cmd/syft/cli/ui/handler.go | 68 +++++ cmd/syft/cli/ui/new_task_progress.go | 19 ++ cmd/syft/cli/ui/util_test.go | 62 +++++ .../post_ui_event_writer_test.snap | 46 ++++ cmd/syft/internal/ui/no_ui.go | 44 ++++ cmd/syft/internal/ui/post_ui_event_writer.go | 133 ++++++++++ .../internal/ui/post_ui_event_writer_test.go | 95 +++++++ {internal => cmd/syft/internal}/ui/select.go | 11 +- .../syft/internal}/ui/select_windows.go | 6 +- cmd/syft/internal/ui/ui.go | 163 ++++++++++++ go.mod | 34 ++- go.sum | 71 ++++- internal/bus/bus.go | 18 +- internal/bus/helpers.go | 32 +++ internal/log/log.go | 47 +++- internal/ui/common_event_handlers.go | 24 -- internal/ui/components/spinner.go | 42 --- internal/ui/ephemeral_terminal_ui.go | 154 ----------- internal/ui/etui_event_handlers.go | 36 --- internal/ui/logger_ui.go | 40 --- internal/ui/ui.go | 11 - syft/event/event.go | 46 ++-- syft/event/{ => monitor}/cataloger_task.go | 7 +- syft/event/monitor/generic_task.go | 2 +- syft/event/parsers/parsers.go | 68 +++-- .../fileresolver/directory_indexer.go | 9 +- syft/lib.go | 4 +- syft/pkg/cataloger/golang/cataloger.go | 4 +- syft/pkg/cataloger/golang/licenses.go | 12 +- syft/pkg/cataloger/rust/parse_audit_binary.go | 3 +- ui/{event_handlers.go => deprecated.go} | 162 ++++++++---- ui/handler.go | 85 ------ 77 files changed, 3225 insertions(+), 593 deletions(-) create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_attestation_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_fetch_image_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_file_digests_cataloger_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_file_indexing_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_file_metadata_cataloger_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_package_cataloger_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_pull_docker_image_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_read_image_test.snap create mode 100755 cmd/syft/cli/ui/__snapshots__/handle_secrets_cataloger_test.snap create mode 100644 cmd/syft/cli/ui/handle_attestation.go create mode 100644 cmd/syft/cli/ui/handle_attestation_test.go create mode 100644 cmd/syft/cli/ui/handle_cataloger_task.go create mode 100644 cmd/syft/cli/ui/handle_cataloger_task_test.go create mode 100644 cmd/syft/cli/ui/handle_fetch_image.go create mode 100644 cmd/syft/cli/ui/handle_fetch_image_test.go create mode 100644 cmd/syft/cli/ui/handle_file_digests_cataloger.go create mode 100644 cmd/syft/cli/ui/handle_file_digests_cataloger_test.go create mode 100644 cmd/syft/cli/ui/handle_file_indexing.go create mode 100644 cmd/syft/cli/ui/handle_file_indexing_test.go create mode 100644 cmd/syft/cli/ui/handle_file_metadata_cataloger.go create mode 100644 cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go create mode 100644 cmd/syft/cli/ui/handle_package_cataloger.go create mode 100644 cmd/syft/cli/ui/handle_package_cataloger_test.go create mode 100644 cmd/syft/cli/ui/handle_pull_docker_image.go create mode 100644 cmd/syft/cli/ui/handle_pull_docker_image_test.go create mode 100644 cmd/syft/cli/ui/handle_read_image.go create mode 100644 cmd/syft/cli/ui/handle_read_image_test.go create mode 100644 cmd/syft/cli/ui/handle_secrets_cataloger.go create mode 100644 cmd/syft/cli/ui/handle_secrets_cataloger_test.go create mode 100644 cmd/syft/cli/ui/handler.go create mode 100644 cmd/syft/cli/ui/new_task_progress.go create mode 100644 cmd/syft/cli/ui/util_test.go create mode 100755 cmd/syft/internal/ui/__snapshots__/post_ui_event_writer_test.snap create mode 100644 cmd/syft/internal/ui/no_ui.go create mode 100644 cmd/syft/internal/ui/post_ui_event_writer.go create mode 100644 cmd/syft/internal/ui/post_ui_event_writer_test.go rename {internal => cmd/syft/internal}/ui/select.go (72%) rename {internal => cmd/syft/internal}/ui/select_windows.go (81%) create mode 100644 cmd/syft/internal/ui/ui.go create mode 100644 internal/bus/helpers.go delete mode 100644 internal/ui/common_event_handlers.go delete mode 100644 internal/ui/components/spinner.go delete mode 100644 internal/ui/ephemeral_terminal_ui.go delete mode 100644 internal/ui/etui_event_handlers.go delete mode 100644 internal/ui/logger_ui.go delete mode 100644 internal/ui/ui.go rename syft/event/{ => monitor}/cataloger_task.go (84%) rename ui/{event_handlers.go => deprecated.go} (84%) delete mode 100644 ui/handler.go diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 112037f00e4..10ef129563c 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -4,7 +4,7 @@ inputs: go-version: description: "Go version to install" required: true - default: "1.19.x" + default: "1.20.x" use-go-cache: description: "Restore go cache" required: true diff --git a/.gitignore b/.gitignore index e4c0ffcd7e1..ddea041ed06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ go.work go.work.sum +/bin /.bin CHANGELOG.md VERSION diff --git a/cmd/syft/cli/attest.go b/cmd/syft/cli/attest.go index a826977d4b5..0234057fe8c 100644 --- a/cmd/syft/cli/attest.go +++ b/cmd/syft/cli/attest.go @@ -43,6 +43,7 @@ func Attest(v *viper.Viper, app *config.Application, ro *options.RootOptions, po RunE: func(cmd *cobra.Command, args []string) error { if app.CheckForAppUpdate { checkForApplicationUpdate() + // TODO: this is broke, the bus isn't available yet } return attest.Run(cmd.Context(), app, args) diff --git a/cmd/syft/cli/attest/attest.go b/cmd/syft/cli/attest/attest.go index cdd25ad3200..1d0dddf1c1c 100644 --- a/cmd/syft/cli/attest/attest.go +++ b/cmd/syft/cli/attest/attest.go @@ -16,11 +16,11 @@ import ( "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" "github.com/anchore/syft/cmd/syft/cli/packages" + "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/config" "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/ui" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor" @@ -39,6 +39,13 @@ func Run(_ context.Context, app *config.Application, args []string) error { // note: must be a container image userInput := args[0] + _, err = exec.LookPath("cosign") + if err != nil { + // when cosign is not installed the error will be rendered like so: + // 2023/06/30 08:31:52 error during command execution: 'syft attest' requires cosign to be installed: exec: "cosign": executable file not found in $PATH + return fmt.Errorf("'syft attest' requires cosign to be installed: %w", err) + } + eventBus := partybus.NewBus() stereoscope.SetBus(eventBus) syft.SetBus(eventBus) @@ -119,7 +126,7 @@ func execWorker(app *config.Application, userInput string) <-chan error { errs := make(chan error) go func() { defer close(errs) - defer bus.Publish(partybus.Event{Type: event.Exit}) + defer bus.Exit() s, err := buildSBOM(app, userInput, errs) if err != nil { @@ -207,8 +214,8 @@ func execWorker(app *config.Application, userInput string) <-chan error { Context: "cosign", }, Value: &monitor.ShellProgress{ - Reader: r, - Manual: mon, + Reader: r, + Progressable: mon, }, }, ) diff --git a/cmd/syft/cli/commands.go b/cmd/syft/cli/commands.go index 39e3ab01257..aa64d5a625e 100644 --- a/cmd/syft/cli/commands.go +++ b/cmd/syft/cli/commands.go @@ -125,7 +125,7 @@ func checkForApplicationUpdate() { log.Infof("new version of %s is available: %s (current version is %s)", internal.ApplicationName, newVersion, version.FromBuild().Version) bus.Publish(partybus.Event{ - Type: event.AppUpdateAvailable, + Type: event.CLIAppUpdateAvailable, Value: newVersion, }) } else { diff --git a/cmd/syft/cli/convert.go b/cmd/syft/cli/convert.go index a93cb304ae6..16c24cac52a 100644 --- a/cmd/syft/cli/convert.go +++ b/cmd/syft/cli/convert.go @@ -43,6 +43,7 @@ func Convert(v *viper.Viper, app *config.Application, ro *options.RootOptions, p RunE: func(cmd *cobra.Command, args []string) error { if app.CheckForAppUpdate { checkForApplicationUpdate() + // TODO: this is broke, the bus isn't available yet } return convert.Run(cmd.Context(), app, args) }, diff --git a/cmd/syft/cli/convert/convert.go b/cmd/syft/cli/convert/convert.go index a646bded3c7..2f0dbcedbb8 100644 --- a/cmd/syft/cli/convert/convert.go +++ b/cmd/syft/cli/convert/convert.go @@ -6,20 +6,29 @@ import ( "io" "os" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/stereoscope" + "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" + "github.com/anchore/syft/cmd/syft/internal/ui" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/config" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/formats" + "github.com/anchore/syft/syft/sbom" ) func Run(_ context.Context, app *config.Application, args []string) error { log.Warn("convert is an experimental feature, run `syft convert -h` for help") + writer, err := options.MakeSBOMWriter(app.Outputs, app.File, app.OutputTemplatePath) if err != nil { return err } - // this can only be a SBOM file + // could be an image or a directory, with or without a scheme userInput := args[0] var reader io.ReadCloser @@ -37,10 +46,40 @@ func Run(_ context.Context, app *config.Application, args []string) error { reader = f } - sbom, _, err := formats.Decode(reader) - if err != nil { - return fmt.Errorf("failed to decode SBOM: %w", err) - } + eventBus := partybus.NewBus() + stereoscope.SetBus(eventBus) + syft.SetBus(eventBus) + subscription := eventBus.Subscribe() + + return eventloop.EventLoop( + execWorker(reader, writer), + eventloop.SetupSignals(), + subscription, + stereoscope.Cleanup, + ui.Select(options.IsVerbose(app), app.Quiet)..., + ) +} + +func execWorker(reader io.Reader, writer sbom.Writer) <-chan error { + errs := make(chan error) + go func() { + defer close(errs) + defer bus.Exit() - return writer.Write(*sbom) + s, _, err := formats.Decode(reader) + if err != nil { + errs <- fmt.Errorf("failed to decode SBOM: %w", err) + return + } + + if s == nil { + errs <- fmt.Errorf("no SBOM produced") + return + } + + if err := writer.Write(*s); err != nil { + errs <- fmt.Errorf("failed to write SBOM: %w", err) + } + }() + return errs } diff --git a/cmd/syft/cli/eventloop/event_loop.go b/cmd/syft/cli/eventloop/event_loop.go index 592556ca22f..e7d008e71f5 100644 --- a/cmd/syft/cli/eventloop/event_loop.go +++ b/cmd/syft/cli/eventloop/event_loop.go @@ -8,20 +8,20 @@ import ( "github.com/hashicorp/go-multierror" "github.com/wagoodman/go-partybus" + "github.com/anchore/clio" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/ui" ) -// eventLoop listens to worker errors (from execution path), worker events (from a partybus subscription), and +// EventLoop listens to worker errors (from execution path), worker events (from a partybus subscription), and // signal interrupts. Is responsible for handling each event relative to a given UI an to coordinate eventing until // an eventual graceful exit. -func EventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription *partybus.Subscription, cleanupFn func(), uxs ...ui.UI) error { +func EventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription *partybus.Subscription, cleanupFn func(), uxs ...clio.UI) error { defer cleanupFn() events := subscription.Events() var err error - var ux ui.UI + var ux clio.UI - if ux, err = setupUI(subscription.Unsubscribe, uxs...); err != nil { + if ux, err = setupUI(subscription, uxs...); err != nil { return err } @@ -85,9 +85,9 @@ func EventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription * // during teardown. With the given UIs, the first UI which the ui.Setup() function does not return an error // will be utilized in execution. Providing a set of UIs allows for the caller to provide graceful fallbacks // when there are environmental problem (e.g. unable to setup a TUI with the current TTY). -func setupUI(unsubscribe func() error, uis ...ui.UI) (ui.UI, error) { +func setupUI(subscription *partybus.Subscription, uis ...clio.UI) (clio.UI, error) { for _, ux := range uis { - if err := ux.Setup(unsubscribe); err != nil { + if err := ux.Setup(subscription); err != nil { log.Warnf("unable to setup given UI, falling back to alternative UI: %+v", err) continue } diff --git a/cmd/syft/cli/eventloop/event_loop_test.go b/cmd/syft/cli/eventloop/event_loop_test.go index 2b76bcb65bc..495af90b8e7 100644 --- a/cmd/syft/cli/eventloop/event_loop_test.go +++ b/cmd/syft/cli/eventloop/event_loop_test.go @@ -11,34 +11,37 @@ import ( "github.com/stretchr/testify/mock" "github.com/wagoodman/go-partybus" - "github.com/anchore/syft/internal/ui" + "github.com/anchore/clio" "github.com/anchore/syft/syft/event" ) -var _ ui.UI = (*uiMock)(nil) +var _ clio.UI = (*uiMock)(nil) type uiMock struct { - t *testing.T - finalEvent partybus.Event - unsubscribe func() error + t *testing.T + finalEvent partybus.Event + subscription partybus.Unsubscribable mock.Mock } -func (u *uiMock) Setup(unsubscribe func() error) error { +func (u *uiMock) Setup(unsubscribe partybus.Unsubscribable) error { + u.t.Helper() u.t.Logf("UI Setup called") - u.unsubscribe = unsubscribe - return u.Called(unsubscribe).Error(0) + u.subscription = unsubscribe + return u.Called(unsubscribe.Unsubscribe).Error(0) } func (u *uiMock) Handle(event partybus.Event) error { + u.t.Helper() u.t.Logf("UI Handle called: %+v", event.Type) if event == u.finalEvent { - assert.NoError(u.t, u.unsubscribe()) + assert.NoError(u.t, u.subscription.Unsubscribe()) } return u.Called(event).Error(0) } func (u *uiMock) Teardown(_ bool) error { + u.t.Helper() u.t.Logf("UI Teardown called") return u.Called().Error(0) } @@ -51,7 +54,7 @@ func Test_EventLoop_gracefulExit(t *testing.T) { t.Cleanup(testBus.Close) finalEvent := partybus.Event{ - Type: event.Exit, + Type: event.CLIExit, } worker := func() <-chan error { @@ -183,7 +186,7 @@ func Test_EventLoop_unsubscribeError(t *testing.T) { t.Cleanup(testBus.Close) finalEvent := partybus.Event{ - Type: event.Exit, + Type: event.CLIExit, } worker := func() <-chan error { @@ -252,7 +255,7 @@ func Test_EventLoop_handlerError(t *testing.T) { t.Cleanup(testBus.Close) finalEvent := partybus.Event{ - Type: event.Exit, + Type: event.CLIExit, Error: fmt.Errorf("an exit error occured"), } @@ -377,7 +380,7 @@ func Test_EventLoop_uiTeardownError(t *testing.T) { t.Cleanup(testBus.Close) finalEvent := partybus.Event{ - Type: event.Exit, + Type: event.CLIExit, } worker := func() <-chan error { diff --git a/cmd/syft/cli/options/writer.go b/cmd/syft/cli/options/writer.go index 40c8a267511..1ea4ff1a205 100644 --- a/cmd/syft/cli/options/writer.go +++ b/cmd/syft/cli/options/writer.go @@ -1,6 +1,7 @@ package options import ( + "bytes" "fmt" "io" "os" @@ -10,6 +11,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/mitchellh/go-homedir" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/formats" "github.com/anchore/syft/syft/formats/table" @@ -114,14 +116,6 @@ type sbomMultiWriter struct { writers []sbom.Writer } -type nopWriteCloser struct { - io.Writer -} - -func (n nopWriteCloser) Close() error { - return nil -} - // newSBOMMultiWriter create all report writers from input options; if a file is not specified the given defaultWriter is used func newSBOMMultiWriter(options ...sbomWriterDescription) (_ *sbomMultiWriter, err error) { if len(options) == 0 { @@ -133,9 +127,8 @@ func newSBOMMultiWriter(options ...sbomWriterDescription) (_ *sbomMultiWriter, e for _, option := range options { switch len(option.Path) { case 0: - out.writers = append(out.writers, &sbomStreamWriter{ + out.writers = append(out.writers, &sbomPublisher{ format: option.Format, - out: nopWriteCloser{Writer: os.Stdout}, }) default: // create any missing subdirectories @@ -195,3 +188,19 @@ func (w *sbomStreamWriter) Close() error { } return nil } + +// sbomPublisher implements sbom.Writer that publishes results to the event bus +type sbomPublisher struct { + format sbom.Format +} + +// Write the provided SBOM to the data stream +func (w *sbomPublisher) Write(s sbom.SBOM) error { + buf := &bytes.Buffer{} + if err := w.format.Encode(buf, s); err != nil { + return fmt.Errorf("unable to encode SBOM: %w", err) + } + + bus.Report(buf.String()) + return nil +} diff --git a/cmd/syft/cli/options/writer_test.go b/cmd/syft/cli/options/writer_test.go index 2e251234e0f..643d251cd7f 100644 --- a/cmd/syft/cli/options/writer_test.go +++ b/cmd/syft/cli/options/writer_test.go @@ -191,6 +191,8 @@ func Test_newSBOMMultiWriter(t *testing.T) { if e.file != "" { assert.FileExists(t, tmp+e.file) } + case *sbomPublisher: + assert.Equal(t, string(w.format.ID()), e.format) default: t.Fatalf("unknown writer type: %T", w) } diff --git a/cmd/syft/cli/packages.go b/cmd/syft/cli/packages.go index 9f34c52bb8a..88154b19912 100644 --- a/cmd/syft/cli/packages.go +++ b/cmd/syft/cli/packages.go @@ -70,6 +70,7 @@ func Packages(v *viper.Viper, app *config.Application, ro *options.RootOptions, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { if app.CheckForAppUpdate { + // TODO: this is broke, the bus isn't available yet checkForApplicationUpdate() } return packages.Run(cmd.Context(), app, args) diff --git a/cmd/syft/cli/packages/packages.go b/cmd/syft/cli/packages/packages.go index 53b2b860df1..a84b3c09af1 100644 --- a/cmd/syft/cli/packages/packages.go +++ b/cmd/syft/cli/packages/packages.go @@ -10,15 +10,14 @@ import ( "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" + "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/config" "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/internal/ui" "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/formats/template" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" @@ -52,10 +51,12 @@ func Run(_ context.Context, app *config.Application, args []string) error { ) } +// nolint:funlen func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error { errs := make(chan error) go func() { defer close(errs) + defer bus.Exit() detection, err := source.Detect( userInput, @@ -115,12 +116,13 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < if s == nil { errs <- fmt.Errorf("no SBOM produced for %q", userInput) + return } - bus.Publish(partybus.Event{ - Type: event.Exit, - Value: func() error { return writer.Write(*s) }, - }) + if err := writer.Write(*s); err != nil { + errs <- fmt.Errorf("failed to write SBOM: %w", err) + return + } }() return errs } diff --git a/cmd/syft/cli/poweruser.go b/cmd/syft/cli/poweruser.go index f979d3afb51..e3c935d9ea7 100644 --- a/cmd/syft/cli/poweruser.go +++ b/cmd/syft/cli/poweruser.go @@ -41,6 +41,7 @@ func PowerUser(v *viper.Viper, app *config.Application, ro *options.RootOptions) RunE: func(cmd *cobra.Command, args []string) error { if app.CheckForAppUpdate { checkForApplicationUpdate() + // TODO: this is broke, the bus isn't available yet } return poweruser.Run(cmd.Context(), app, args) }, diff --git a/cmd/syft/cli/poweruser/poweruser.go b/cmd/syft/cli/poweruser/poweruser.go index e37455ece45..e9a251f3e26 100644 --- a/cmd/syft/cli/poweruser/poweruser.go +++ b/cmd/syft/cli/poweruser/poweruser.go @@ -13,14 +13,13 @@ import ( "github.com/anchore/syft/cmd/syft/cli/eventloop" "github.com/anchore/syft/cmd/syft/cli/options" "github.com/anchore/syft/cmd/syft/cli/packages" + "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/config" - "github.com/anchore/syft/internal/ui" "github.com/anchore/syft/internal/version" "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/formats/syftjson" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" @@ -59,6 +58,7 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < errs := make(chan error) go func() { defer close(errs) + defer bus.Exit() app.Secrets.Cataloger.Enabled = true app.FileMetadata.Cataloger.Enabled = true @@ -133,10 +133,10 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < s.Relationships = append(s.Relationships, packages.MergeRelationships(relationships...)...) - bus.Publish(partybus.Event{ - Type: event.Exit, - Value: func() error { return writer.Write(s) }, - }) + if err := writer.Write(s); err != nil { + errs <- fmt.Errorf("failed to write sbom: %w", err) + return + } }() return errs diff --git a/cmd/syft/cli/ui/__snapshots__/handle_attestation_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_attestation_test.snap new file mode 100755 index 00000000000..981402b26b5 --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_attestation_test.snap @@ -0,0 +1,19 @@ + +[TestHandler_handleAttestationStarted/attesting_in_progress/task_line - 1] + ⠋ Creating a thing running a thing +--- + +[TestHandler_handleAttestationStarted/attesting_in_progress/log - 1] + ░░ contents + ░░ of + ░░ stuff! + +--- + +[TestHandler_handleAttestationStarted/attesting_complete/task_line - 1] + ✔ Created a thing running a thing +--- + +[TestHandler_handleAttestationStarted/attesting_complete/log - 1] + +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap new file mode 100755 index 00000000000..aff1f474a75 --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap @@ -0,0 +1,16 @@ + +[TestHandler_handleCatalogerTaskStarted/cataloging_task_in_progress - 1] + some task title [some value] +--- + +[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_in_progress - 1] + └── some task title [some value] +--- + +[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete - 1] + ✔ └── some task done [some value] +--- + +[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_with_removal - 1] + +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_fetch_image_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_fetch_image_test.snap new file mode 100755 index 00000000000..19e0b1e693b --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_fetch_image_test.snap @@ -0,0 +1,8 @@ + +[TestHandler_handleFetchImage/fetch_image_in_progress - 1] + ⠋ Loading image ━━━━━━━━━━━━━━━━━━━━ [current] the-image +--- + +[TestHandler_handleFetchImage/fetch_image_complete - 1] + ✔ Loaded image the-image +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_file_digests_cataloger_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_file_digests_cataloger_test.snap new file mode 100755 index 00000000000..b4572c26541 --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_file_digests_cataloger_test.snap @@ -0,0 +1,8 @@ + +[TestHandler_handleFileDigestsCatalogerStarted/cataloging_in_progress - 1] + ⠋ Cataloging file digests ━━━━━━━━━━━━━━━━━━━━ [current] +--- + +[TestHandler_handleFileDigestsCatalogerStarted/cataloging_complete - 1] + ✔ Cataloged file digests +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_file_indexing_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_file_indexing_test.snap new file mode 100755 index 00000000000..2f5968e8f0c --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_file_indexing_test.snap @@ -0,0 +1,8 @@ + +[TestHandler_handleFileIndexingStarted/cataloging_in_progress - 1] + ⠋ Indexing file system ━━━━━━━━━━━━━━━━━━━━ [current] /some/path +--- + +[TestHandler_handleFileIndexingStarted/cataloging_complete - 1] + ✔ Indexed file system /some/path +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_file_metadata_cataloger_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_file_metadata_cataloger_test.snap new file mode 100755 index 00000000000..200a0dfb4ad --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_file_metadata_cataloger_test.snap @@ -0,0 +1,8 @@ + +[TestHandler_handleFileMetadataCatalogerStarted/cataloging_in_progress - 1] + ⠋ Cataloging file metadata ━━━━━━━━━━━━━━━━━━━━ [current] +--- + +[TestHandler_handleFileMetadataCatalogerStarted/cataloging_complete - 1] + ✔ Cataloged file metadata +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_package_cataloger_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_package_cataloger_test.snap new file mode 100755 index 00000000000..5d5c165e03d --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_package_cataloger_test.snap @@ -0,0 +1,16 @@ + +[TestHandler_handlePackageCatalogerStarted/cataloging_in_progress - 1] + ⠋ Cataloging packages [50 packages] +--- + +[TestHandler_handlePackageCatalogerStarted/cataloging_only_files_complete - 1] + ⠋ Cataloging packages [50 packages] +--- + +[TestHandler_handlePackageCatalogerStarted/cataloging_only_packages_complete - 1] + ⠋ Cataloging packages [100 packages] +--- + +[TestHandler_handlePackageCatalogerStarted/cataloging_complete - 1] + ✔ Cataloged packages [100 packages] +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_pull_docker_image_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_pull_docker_image_test.snap new file mode 100755 index 00000000000..6f8bf9e2960 --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_pull_docker_image_test.snap @@ -0,0 +1,12 @@ + +[Test_dockerPullStatusFormatter_Render/pulling - 1] +3 Layers▕▅▃ ▏[12 B / 30 B] +--- + +[Test_dockerPullStatusFormatter_Render/download_complete - 1] +3 Layers▕█▃ ▏[30 B] Extracting... +--- + +[Test_dockerPullStatusFormatter_Render/complete - 1] +3 Layers▕███▏[30 B] Extracting... +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_read_image_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_read_image_test.snap new file mode 100755 index 00000000000..ded3b615bee --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_read_image_test.snap @@ -0,0 +1,8 @@ + +[TestHandler_handleReadImage/read_image_in_progress - 1] + ⠋ Parsing image ━━━━━━━━━━━━━━━━━━━━ id +--- + +[TestHandler_handleReadImage/read_image_complete - 1] + ✔ Parsed image id +--- diff --git a/cmd/syft/cli/ui/__snapshots__/handle_secrets_cataloger_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_secrets_cataloger_test.snap new file mode 100755 index 00000000000..00a123ef7f0 --- /dev/null +++ b/cmd/syft/cli/ui/__snapshots__/handle_secrets_cataloger_test.snap @@ -0,0 +1,8 @@ + +[TestHandler_handleSecretsCatalogerStarted/cataloging_in_progress - 1] + ⠋ Cataloging secrets ━━━━━━━━━━━━━━━━━━━━ [64 secrets] +--- + +[TestHandler_handleSecretsCatalogerStarted/cataloging_complete - 1] + ✔ Cataloged secrets [64 secrets] +--- diff --git a/cmd/syft/cli/ui/handle_attestation.go b/cmd/syft/cli/ui/handle_attestation.go new file mode 100644 index 00000000000..26c58e3890b --- /dev/null +++ b/cmd/syft/cli/ui/handle_attestation.go @@ -0,0 +1,247 @@ +package ui + +import ( + "bufio" + "fmt" + "io" + "strings" + "sync" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/google/uuid" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + "github.com/zyedidia/generic/queue" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" +) + +var ( + _ tea.Model = (*attestLogFrame)(nil) + _ cosignOutputReader = (*backgroundLineReader)(nil) +) + +type attestLogFrame struct { + reader cosignOutputReader + prog progress.Progressable + lines []string + completed bool + failed bool + windowSize tea.WindowSizeMsg + + id uint32 + sequence int + + updateDuration time.Duration + borderStype lipgloss.Style +} + +// attestLogFrameTickMsg indicates that the timer has ticked and we should render a frame. +type attestLogFrameTickMsg struct { + Time time.Time + Sequence int + ID uint32 +} + +type cosignOutputReader interface { + Lines() []string +} + +type backgroundLineReader struct { + limit int + lines *queue.Queue[string] + lock *sync.RWMutex +} + +func (m *Handler) handleAttestationStarted(e partybus.Event) []tea.Model { + reader, prog, taskInfo, err := syftEventParsers.ParseAttestationStartedEvent(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + stage := progress.Stage{} + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: taskInfo.Title.Default, + Running: taskInfo.Title.WhileRunning, + Success: taskInfo.Title.OnSuccess, + }, + taskprogress.WithStagedProgressable( + struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &stage, + }, + ), + ) + + tsk.HideStageOnSuccess = false + + if taskInfo.Context != "" { + tsk.Context = []string{taskInfo.Context} + } + + borderStyle := tsk.HintStyle + + return []tea.Model{ + tsk, + newLogFrame(newBackgroundLineReader(m.Running, reader, &stage), prog, borderStyle), + } +} + +func newLogFrame(reader cosignOutputReader, prog progress.Progressable, borderStyle lipgloss.Style) attestLogFrame { + return attestLogFrame{ + reader: reader, + prog: prog, + id: uuid.Must(uuid.NewUUID()).ID(), + updateDuration: 250 * time.Millisecond, + borderStype: borderStyle, + } +} + +func newBackgroundLineReader(wg *sync.WaitGroup, reader io.Reader, stage *progress.Stage) *backgroundLineReader { + wg.Add(1) + r := &backgroundLineReader{ + limit: 7, + lock: &sync.RWMutex{}, + lines: queue.New[string](), + } + + go func() { + defer wg.Done() + r.read(reader, stage) + }() + + return r +} + +func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) { + s := bufio.NewScanner(reader) + + for s.Scan() { + l.lock.Lock() + + text := s.Text() + l.lines.Enqueue(text) + + if strings.Contains(text, "tlog entry created with index") { + fields := strings.SplitN(text, ":", 2) + present := text + if len(fields) == 2 { + present = fmt.Sprintf("transparency log index: %s", fields[1]) + } + stage.Current = present + } else if strings.Contains(text, "WARNING: skipping transparency log upload") { + stage.Current = "transparency log upload skipped" + } + + // only show the last X lines of the shell output + for l.lines.Len() > l.limit { + l.lines.Dequeue() + } + + l.lock.Unlock() + } +} + +func (l backgroundLineReader) Lines() []string { + l.lock.RLock() + defer l.lock.RUnlock() + + var lines []string + + l.lines.Each(func(line string) { + lines = append(lines, line) + }) + + return lines +} + +func (l attestLogFrame) Init() tea.Cmd { + // this is the periodic update of state information + return func() tea.Msg { + return attestLogFrameTickMsg{ + // The time at which the tick occurred. + Time: time.Now(), + + // The ID of the log frame that this message belongs to. This can be + // helpful when routing messages, however bear in mind that log frames + // will ignore messages that don't contain ID by default. + ID: l.id, + + Sequence: l.sequence, + } + } +} + +func (l attestLogFrame) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + l.windowSize = msg + return l, nil + + case attestLogFrameTickMsg: + l.lines = l.reader.Lines() + + l.completed = progress.IsCompleted(l.prog) + err := l.prog.Error() + l.failed = err != nil && !progress.IsErrCompleted(err) + + tickCmd := l.handleTick(msg) + + return l, tickCmd + } + + return l, nil +} + +func (l attestLogFrame) View() string { + if l.completed && !l.failed { + return "" + } + + sb := strings.Builder{} + + for _, line := range l.lines { + sb.WriteString(fmt.Sprintf(" %s %s\n", l.borderStype.Render("░░"), line)) + } + + return sb.String() +} + +func (l attestLogFrame) queueNextTick() tea.Cmd { + return tea.Tick(l.updateDuration, func(t time.Time) tea.Msg { + return attestLogFrameTickMsg{ + Time: t, + ID: l.id, + Sequence: l.sequence, + } + }) +} + +func (l *attestLogFrame) handleTick(msg attestLogFrameTickMsg) tea.Cmd { + // If an ID is set, and the ID doesn't belong to this log frame, reject the message. + if msg.ID > 0 && msg.ID != l.id { + return nil + } + + // If a sequence is set, and it's not the one we expect, reject the message. + // This prevents the log frame from receiving too many messages and + // thus updating too frequently. + if msg.Sequence > 0 && msg.Sequence != l.sequence { + return nil + } + + l.sequence++ + + // note: even if the log is completed we should still respond to stage changes and window size events + return l.queueNextTick() +} diff --git a/cmd/syft/cli/ui/handle_attestation_test.go b/cmd/syft/cli/ui/handle_attestation_test.go new file mode 100644 index 00000000000..6fcc9fbfea2 --- /dev/null +++ b/cmd/syft/cli/ui/handle_attestation_test.go @@ -0,0 +1,133 @@ +package ui + +import ( + "strings" + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" +) + +func TestHandler_handleAttestationStarted(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "attesting in progress", + // note: this model depends on a background reader. Multiple iterations ensures that the + // reader has time to at least start and process the test fixture before the runModel + // test harness completes (which is a fake event loop anyway). + iterations: 2, + eventFn: func(t *testing.T) partybus.Event { + reader := strings.NewReader("contents\nof\nstuff!") + + src := monitor.GenericTask{ + Title: monitor.Title{ + Default: "Create a thing", + WhileRunning: "Creating a thing", + OnSuccess: "Created a thing", + }, + Context: "running a thing", + } + + mon := progress.NewManual(-1) + mon.Set(50) + + value := &monitor.ShellProgress{ + Reader: reader, + Progressable: mon, + } + + return partybus.Event{ + Type: syftEvent.AttestationStarted, + Source: src, + Value: value, + } + }, + }, + { + name: "attesting complete", + // note: this model depends on a background reader. Multiple iterations ensures that the + // reader has time to at least start and process the test fixture before the runModel + // test harness completes (which is a fake event loop anyway). + iterations: 2, + eventFn: func(t *testing.T) partybus.Event { + reader := strings.NewReader("contents\nof\nstuff!") + + src := monitor.GenericTask{ + Title: monitor.Title{ + Default: "Create a thing", + WhileRunning: "Creating a thing", + OnSuccess: "Created a thing", + }, + Context: "running a thing", + } + + mon := progress.NewManual(-1) + mon.Set(50) + mon.SetCompleted() + + value := &monitor.ShellProgress{ + Reader: reader, + Progressable: mon, + } + + return partybus.Event{ + Type: syftEvent.AttestationStarted, + Source: src, + Value: value, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 2) + + t.Run("task line", func(t *testing.T) { + tsk, ok := models[0].(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + + t.Run("log", func(t *testing.T) { + log, ok := models[1].(attestLogFrame) + require.True(t, ok) + got := runModel(t, log, tt.iterations, attestLogFrameTickMsg{ + Time: time.Now(), + Sequence: log.sequence, + ID: log.id, + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + + }) + } +} diff --git a/cmd/syft/cli/ui/handle_cataloger_task.go b/cmd/syft/cli/ui/handle_cataloger_task.go new file mode 100644 index 00000000000..393bd6e7ad6 --- /dev/null +++ b/cmd/syft/cli/ui/handle_cataloger_task.go @@ -0,0 +1,72 @@ +package ui + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event/monitor" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" +) + +var _ progress.Stager = (*catalogerTaskStageAdapter)(nil) + +type catalogerTaskStageAdapter struct { + mon *monitor.CatalogerTask +} + +func newCatalogerTaskStageAdapter(mon *monitor.CatalogerTask) *catalogerTaskStageAdapter { + return &catalogerTaskStageAdapter{ + mon: mon, + } +} + +func (c catalogerTaskStageAdapter) Stage() string { + return c.mon.GetValue() +} + +func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) []tea.Model { + mon, err := syftEventParsers.ParseCatalogerTaskStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + var prefix string + if mon.SubStatus { + // TODO: support list of sub-statuses, not just a single leaf + prefix = "└── " + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + // TODO: prefix should not be part of the title, but instead a separate field that is aware of the tree structure + Default: prefix + mon.Title, + Running: prefix + mon.Title, + Success: prefix + mon.TitleOnCompletion, + }, + taskprogress.WithStagedProgressable( + struct { + progress.Stager + progress.Progressable + }{ + Progressable: mon.GetMonitor(), + Stager: newCatalogerTaskStageAdapter(mon), + }, + ), + ) + + // TODO: this isn't ideal since the model stays around after it is no longer needed, but it works for now + tsk.HideOnSuccess = mon.RemoveOnCompletion + tsk.HideStageOnSuccess = false + tsk.HideProgressOnSuccess = false + + tsk.TitleStyle = lipgloss.NewStyle() + // TODO: this is a hack to get the spinner to not show up, but ideally the component would support making the spinner optional + tsk.Spinner.Spinner.Frames = []string{" "} + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_cataloger_task_test.go b/cmd/syft/cli/ui/handle_cataloger_task_test.go new file mode 100644 index 00000000000..055694588fa --- /dev/null +++ b/cmd/syft/cli/ui/handle_cataloger_task_test.go @@ -0,0 +1,123 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" +) + +func TestHandler_handleCatalogerTaskStarted(t *testing.T) { + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "cataloging task in progress", + eventFn: func(t *testing.T) partybus.Event { + src := &monitor.CatalogerTask{ + SubStatus: false, + RemoveOnCompletion: false, + Title: "some task title", + TitleOnCompletion: "some task done", + } + + src.SetValue("some value") + + return partybus.Event{ + Type: syftEvent.CatalogerTaskStarted, + Source: src, + } + }, + }, + { + name: "cataloging sub task in progress", + eventFn: func(t *testing.T) partybus.Event { + src := &monitor.CatalogerTask{ + SubStatus: true, + RemoveOnCompletion: false, + Title: "some task title", + TitleOnCompletion: "some task done", + } + + src.SetValue("some value") + + return partybus.Event{ + Type: syftEvent.CatalogerTaskStarted, + Source: src, + } + }, + }, + { + name: "cataloging sub task complete", + eventFn: func(t *testing.T) partybus.Event { + src := &monitor.CatalogerTask{ + SubStatus: true, + RemoveOnCompletion: false, + Title: "some task title", + TitleOnCompletion: "some task done", + } + + src.SetValue("some value") + src.SetCompleted() + + return partybus.Event{ + Type: syftEvent.CatalogerTaskStarted, + Source: src, + } + }, + }, + { + name: "cataloging sub task complete with removal", + eventFn: func(t *testing.T) partybus.Event { + src := &monitor.CatalogerTask{ + SubStatus: true, + RemoveOnCompletion: true, + Title: "some task title", + TitleOnCompletion: "some task done", + } + + src.SetValue("some value") + src.SetCompleted() + + return partybus.Event{ + Type: syftEvent.CatalogerTaskStarted, + Source: src, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_fetch_image.go b/cmd/syft/cli/ui/handle_fetch_image.go new file mode 100644 index 00000000000..9821853a070 --- /dev/null +++ b/cmd/syft/cli/ui/handle_fetch_image.go @@ -0,0 +1,32 @@ +package ui + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/taskprogress" + stereoEventParsers "github.com/anchore/stereoscope/pkg/event/parsers" + "github.com/anchore/syft/internal/log" +) + +func (m *Handler) handleFetchImage(e partybus.Event) []tea.Model { + imgName, prog, err := stereoEventParsers.ParseFetchImage(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Load image", + Running: "Loading image", + Success: "Loaded image", + }, + taskprogress.WithStagedProgressable(prog), + ) + if imgName != "" { + tsk.Context = []string{imgName} + } + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_fetch_image_test.go b/cmd/syft/cli/ui/handle_fetch_image_test.go new file mode 100644 index 00000000000..c514b986542 --- /dev/null +++ b/cmd/syft/cli/ui/handle_fetch_image_test.go @@ -0,0 +1,99 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + stereoscopeEvent "github.com/anchore/stereoscope/pkg/event" +) + +func TestHandler_handleFetchImage(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "fetch image in progress", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: stereoscopeEvent.FetchImage, + Source: "the-image", + Value: mon, + } + }, + }, + { + name: "fetch image complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: stereoscopeEvent.FetchImage, + Source: "the-image", + Value: mon, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_file_digests_cataloger.go b/cmd/syft/cli/ui/handle_file_digests_cataloger.go new file mode 100644 index 00000000000..79550e9cf4a --- /dev/null +++ b/cmd/syft/cli/ui/handle_file_digests_cataloger.go @@ -0,0 +1,28 @@ +package ui + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" +) + +func (m *Handler) handleFileDigestsCatalogerStarted(e partybus.Event) []tea.Model { + prog, err := syftEventParsers.ParseFileDigestsCatalogingStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Catalog file digests", + Running: "Cataloging file digests", + Success: "Cataloged file digests", + }, taskprogress.WithStagedProgressable(prog), + ) + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_file_digests_cataloger_test.go b/cmd/syft/cli/ui/handle_file_digests_cataloger_test.go new file mode 100644 index 00000000000..2e74009a4cd --- /dev/null +++ b/cmd/syft/cli/ui/handle_file_digests_cataloger_test.go @@ -0,0 +1,97 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" +) + +func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "cataloging in progress", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: syftEvent.FileDigestsCatalogerStarted, + Value: mon, + } + }, + }, + { + name: "cataloging complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: syftEvent.FileDigestsCatalogerStarted, + Value: mon, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_file_indexing.go b/cmd/syft/cli/ui/handle_file_indexing.go new file mode 100644 index 00000000000..7d2eef9b57b --- /dev/null +++ b/cmd/syft/cli/ui/handle_file_indexing.go @@ -0,0 +1,31 @@ +package ui + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" +) + +func (m *Handler) handleFileIndexingStarted(e partybus.Event) []tea.Model { + path, prog, err := syftEventParsers.ParseFileIndexingStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Index files system", + Running: "Indexing file system", + Success: "Indexed file system", + }, + taskprogress.WithStagedProgressable(prog), + ) + + tsk.Context = []string{path} + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_file_indexing_test.go b/cmd/syft/cli/ui/handle_file_indexing_test.go new file mode 100644 index 00000000000..86473c411a4 --- /dev/null +++ b/cmd/syft/cli/ui/handle_file_indexing_test.go @@ -0,0 +1,99 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" +) + +func TestHandler_handleFileIndexingStarted(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "cataloging in progress", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: syftEvent.FileIndexingStarted, + Source: "/some/path", + Value: mon, + } + }, + }, + { + name: "cataloging complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: syftEvent.FileIndexingStarted, + Source: "/some/path", + Value: mon, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_file_metadata_cataloger.go b/cmd/syft/cli/ui/handle_file_metadata_cataloger.go new file mode 100644 index 00000000000..58535abc198 --- /dev/null +++ b/cmd/syft/cli/ui/handle_file_metadata_cataloger.go @@ -0,0 +1,29 @@ +package ui + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" +) + +func (m *Handler) handleFileMetadataCatalogerStarted(e partybus.Event) []tea.Model { + prog, err := syftEventParsers.ParseFileMetadataCatalogingStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Catalog file metadata", + Running: "Cataloging file metadata", + Success: "Cataloged file metadata", + }, + taskprogress.WithStagedProgressable(prog), + ) + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go b/cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go new file mode 100644 index 00000000000..d247001c8fd --- /dev/null +++ b/cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go @@ -0,0 +1,97 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" +) + +func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "cataloging in progress", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: syftEvent.FileMetadataCatalogerStarted, + Value: mon, + } + }, + }, + { + name: "cataloging complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + mon := struct { + progress.Progressable + progress.Stager + }{ + Progressable: prog, + Stager: &progress.Stage{ + Current: "current", + }, + } + + return partybus.Event{ + Type: syftEvent.FileMetadataCatalogerStarted, + Value: mon, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_package_cataloger.go b/cmd/syft/cli/ui/handle_package_cataloger.go new file mode 100644 index 00000000000..3aa2f9330e5 --- /dev/null +++ b/cmd/syft/cli/ui/handle_package_cataloger.go @@ -0,0 +1,87 @@ +package ui + +import ( + "fmt" + + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" + "github.com/anchore/syft/syft/pkg/cataloger" +) + +var _ progress.StagedProgressable = (*packageCatalogerProgressAdapter)(nil) + +type packageCatalogerProgressAdapter struct { + monitor *cataloger.Monitor + monitors []progress.Monitorable +} + +func newPackageCatalogerProgressAdapter(monitor *cataloger.Monitor) packageCatalogerProgressAdapter { + return packageCatalogerProgressAdapter{ + monitor: monitor, + monitors: []progress.Monitorable{ + monitor.FilesProcessed, + monitor.PackagesDiscovered, + }, + } +} + +func (p packageCatalogerProgressAdapter) Stage() string { + return fmt.Sprintf("%d packages", p.monitor.PackagesDiscovered.Current()) +} + +func (p packageCatalogerProgressAdapter) Current() int64 { + return p.monitor.PackagesDiscovered.Current() +} + +func (p packageCatalogerProgressAdapter) Error() error { + completedMonitors := 0 + for _, monitor := range p.monitors { + err := monitor.Error() + if err == nil { + continue + } + if progress.IsErrCompleted(err) { + completedMonitors++ + continue + } + // something went wrong + return err + } + if completedMonitors == len(p.monitors) && len(p.monitors) > 0 { + return p.monitors[0].Error() + } + return nil +} + +func (p packageCatalogerProgressAdapter) Size() int64 { + // this is an inherently unknown value (indeterminate total number of packages to discover) + return -1 +} + +func (m *Handler) handlePackageCatalogerStarted(e partybus.Event) []tea.Model { + monitor, err := syftEventParsers.ParsePackageCatalogerStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Catalog packages", + Running: "Cataloging packages", + Success: "Cataloged packages", + }, + taskprogress.WithStagedProgressable( + newPackageCatalogerProgressAdapter(monitor), + ), + ) + + tsk.HideStageOnSuccess = false + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_package_cataloger_test.go b/cmd/syft/cli/ui/handle_package_cataloger_test.go new file mode 100644 index 00000000000..a5a72d59d36 --- /dev/null +++ b/cmd/syft/cli/ui/handle_package_cataloger_test.go @@ -0,0 +1,133 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/pkg/cataloger" +) + +func TestHandler_handlePackageCatalogerStarted(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "cataloging in progress", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + mon := cataloger.Monitor{ + FilesProcessed: progress.NewManual(-1), + PackagesDiscovered: prog, + } + + return partybus.Event{ + Type: syftEvent.PackageCatalogerStarted, + Value: mon, + } + }, + }, + { + name: "cataloging only files complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + files := progress.NewManual(-1) + files.SetCompleted() + + mon := cataloger.Monitor{ + FilesProcessed: files, + PackagesDiscovered: prog, + } + + return partybus.Event{ + Type: syftEvent.PackageCatalogerStarted, + Value: mon, + } + }, + }, + { + name: "cataloging only packages complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + files := progress.NewManual(-1) + + mon := cataloger.Monitor{ + FilesProcessed: files, + PackagesDiscovered: prog, + } + + return partybus.Event{ + Type: syftEvent.PackageCatalogerStarted, + Value: mon, + } + }, + }, + { + name: "cataloging complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + files := progress.NewManual(-1) + files.SetCompleted() + + mon := cataloger.Monitor{ + FilesProcessed: files, + PackagesDiscovered: prog, + } + + return partybus.Event{ + Type: syftEvent.PackageCatalogerStarted, + Value: mon, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_pull_docker_image.go b/cmd/syft/cli/ui/handle_pull_docker_image.go new file mode 100644 index 00000000000..6675e3aeb94 --- /dev/null +++ b/cmd/syft/cli/ui/handle_pull_docker_image.go @@ -0,0 +1,201 @@ +package ui + +import ( + "fmt" + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/dustin/go-humanize" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + stereoscopeParsers "github.com/anchore/stereoscope/pkg/event/parsers" + "github.com/anchore/stereoscope/pkg/image/docker" + "github.com/anchore/syft/internal/log" +) + +var _ interface { + progress.Stager + progress.Progressable +} = (*dockerPullProgressAdapter)(nil) + +type dockerPullStatus interface { + Complete() bool + Layers() []docker.LayerID + Current(docker.LayerID) docker.LayerState +} + +type dockerPullProgressAdapter struct { + status dockerPullStatus + formatter dockerPullStatusFormatter +} + +type dockerPullStatusFormatter struct { + auxInfoStyle lipgloss.Style + dockerPullCompletedStyle lipgloss.Style + dockerPullDownloadStyle lipgloss.Style + dockerPullExtractStyle lipgloss.Style + dockerPullStageChars []string + layerCaps []string +} + +func (m *Handler) handlePullDockerImage(e partybus.Event) []tea.Model { + _, pullStatus, err := stereoscopeParsers.ParsePullDockerImage(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Pull image", + Running: "Pulling image", + Success: "Pulled image", + }, + taskprogress.WithStagedProgressable( + newDockerPullProgressAdapter(pullStatus), + ), + ) + + tsk.HintStyle = lipgloss.NewStyle() + tsk.HintEndCaps = nil + + return []tea.Model{tsk} +} + +func newDockerPullProgressAdapter(status dockerPullStatus) *dockerPullProgressAdapter { + return &dockerPullProgressAdapter{ + status: status, + formatter: newDockerPullStatusFormatter(), + } +} + +func newDockerPullStatusFormatter() dockerPullStatusFormatter { + return dockerPullStatusFormatter{ + auxInfoStyle: lipgloss.NewStyle().Foreground(lipgloss.Color("#777777")), + dockerPullCompletedStyle: lipgloss.NewStyle().Foreground(lipgloss.Color("#fcba03")), + dockerPullDownloadStyle: lipgloss.NewStyle().Foreground(lipgloss.Color("#777777")), + dockerPullExtractStyle: lipgloss.NewStyle().Foreground(lipgloss.Color("#ffffff")), + dockerPullStageChars: strings.Split("▁▃▄▅▆▇█", ""), + layerCaps: strings.Split("▕▏", ""), + } +} + +func (d dockerPullProgressAdapter) Size() int64 { + return -1 +} + +func (d dockerPullProgressAdapter) Current() int64 { + return 1 +} + +func (d dockerPullProgressAdapter) Error() error { + if d.status.Complete() { + return progress.ErrCompleted + } + // TODO: return intermediate error indications + return nil +} + +func (d dockerPullProgressAdapter) Stage() string { + return d.formatter.Render(d.status) +} + +// Render crafts the given docker image pull status summarized into a single line. +func (f dockerPullStatusFormatter) Render(pullStatus dockerPullStatus) string { + var size, current uint64 + + layers := pullStatus.Layers() + status := make(map[docker.LayerID]docker.LayerState) + completed := make([]string, len(layers)) + + // fetch the current state + for idx, layer := range layers { + completed[idx] = " " + status[layer] = pullStatus.Current(layer) + } + + numCompleted := 0 + for idx, layer := range layers { + prog := status[layer].PhaseProgress + curN := prog.Current() + curSize := prog.Size() + + if progress.IsCompleted(prog) { + input := f.dockerPullStageChars[len(f.dockerPullStageChars)-1] + completed[idx] = f.formatDockerPullPhase(status[layer].Phase, input) + } else if curN != 0 { + var ratio float64 + switch { + case curN == 0 || curSize < 0: + ratio = 0 + case curN >= curSize: + ratio = 1 + default: + ratio = float64(curN) / float64(curSize) + } + + i := int(ratio * float64(len(f.dockerPullStageChars)-1)) + input := f.dockerPullStageChars[i] + completed[idx] = f.formatDockerPullPhase(status[layer].Phase, input) + } + + if progress.IsErrCompleted(status[layer].DownloadProgress.Error()) { + numCompleted++ + } + } + + for _, layer := range layers { + prog := status[layer].DownloadProgress + size += uint64(prog.Size()) + current += uint64(prog.Current()) + } + + var progStr, auxInfo string + if len(layers) > 0 { + render := strings.Join(completed, "") + prefix := f.dockerPullCompletedStyle.Render(fmt.Sprintf("%d Layers", len(layers))) + auxInfo = f.auxInfoStyle.Render(fmt.Sprintf("[%s / %s]", humanize.Bytes(current), humanize.Bytes(size))) + if len(layers) == numCompleted { + auxInfo = f.auxInfoStyle.Render(fmt.Sprintf("[%s] Extracting...", humanize.Bytes(size))) + } + + progStr = fmt.Sprintf("%s%s%s%s", prefix, f.layerCap(false), render, f.layerCap(true)) + } + + return progStr + auxInfo +} + +// formatDockerPullPhase returns a single character that represents the status of a layer pull. +func (f dockerPullStatusFormatter) formatDockerPullPhase(phase docker.PullPhase, inputStr string) string { + switch phase { + case docker.WaitingPhase: + // ignore any progress related to waiting + return " " + case docker.PullingFsPhase, docker.DownloadingPhase: + return f.dockerPullDownloadStyle.Render(inputStr) + case docker.DownloadCompletePhase: + return f.dockerPullDownloadStyle.Render(f.dockerPullStageChars[len(f.dockerPullStageChars)-1]) + case docker.ExtractingPhase: + return f.dockerPullExtractStyle.Render(inputStr) + case docker.VerifyingChecksumPhase, docker.PullCompletePhase: + return f.dockerPullCompletedStyle.Render(inputStr) + case docker.AlreadyExistsPhase: + return f.dockerPullCompletedStyle.Render(f.dockerPullStageChars[len(f.dockerPullStageChars)-1]) + default: + return inputStr + } +} + +func (f dockerPullStatusFormatter) layerCap(end bool) string { + l := len(f.layerCaps) + if l == 0 { + return "" + } + if end { + return f.layerCaps[l-1] + } + return f.layerCaps[0] +} diff --git a/cmd/syft/cli/ui/handle_pull_docker_image_test.go b/cmd/syft/cli/ui/handle_pull_docker_image_test.go new file mode 100644 index 00000000000..afa986fc2b6 --- /dev/null +++ b/cmd/syft/cli/ui/handle_pull_docker_image_test.go @@ -0,0 +1,163 @@ +package ui + +import ( + "testing" + + "github.com/gkampitakis/go-snaps/snaps" + "github.com/wagoodman/go-progress" + + "github.com/anchore/stereoscope/pkg/image/docker" +) + +var _ dockerPullStatus = (*mockDockerPullStatus)(nil) + +type mockDockerPullStatus struct { + complete bool + layers []docker.LayerID + current map[docker.LayerID]docker.LayerState +} + +func (m mockDockerPullStatus) Complete() bool { + return m.complete +} + +func (m mockDockerPullStatus) Layers() []docker.LayerID { + return m.layers +} + +func (m mockDockerPullStatus) Current(id docker.LayerID) docker.LayerState { + return m.current[id] +} + +func Test_dockerPullStatusFormatter_Render(t *testing.T) { + + tests := []struct { + name string + status dockerPullStatus + }{ + { + name: "pulling", + status: func() dockerPullStatus { + complete := progress.NewManual(10) + complete.Set(10) + complete.SetCompleted() + + quarter := progress.NewManual(10) + quarter.Set(2) + + half := progress.NewManual(10) + half.Set(6) + + empty := progress.NewManual(10) + + return mockDockerPullStatus{ + complete: false, + layers: []docker.LayerID{ + "sha256:1", + "sha256:2", + "sha256:3", + }, + current: map[docker.LayerID]docker.LayerState{ + "sha256:1": { + Phase: docker.ExtractingPhase, + PhaseProgress: half, + DownloadProgress: complete, + }, + "sha256:2": { + Phase: docker.DownloadingPhase, + PhaseProgress: quarter, + DownloadProgress: quarter, + }, + "sha256:3": { + Phase: docker.WaitingPhase, + PhaseProgress: empty, + DownloadProgress: empty, + }, + }, + } + }(), + }, + { + name: "download complete", + status: func() dockerPullStatus { + complete := progress.NewManual(10) + complete.Set(10) + complete.SetCompleted() + + quarter := progress.NewManual(10) + quarter.Set(2) + + half := progress.NewManual(10) + half.Set(6) + + empty := progress.NewManual(10) + + return mockDockerPullStatus{ + complete: false, + layers: []docker.LayerID{ + "sha256:1", + "sha256:2", + "sha256:3", + }, + current: map[docker.LayerID]docker.LayerState{ + "sha256:1": { + Phase: docker.ExtractingPhase, + PhaseProgress: complete, + DownloadProgress: complete, + }, + "sha256:2": { + Phase: docker.ExtractingPhase, + PhaseProgress: quarter, + DownloadProgress: complete, + }, + "sha256:3": { + Phase: docker.ExtractingPhase, + PhaseProgress: empty, + DownloadProgress: complete, + }, + }, + } + }(), + }, + { + name: "complete", + status: func() dockerPullStatus { + complete := progress.NewManual(10) + complete.Set(10) + complete.SetCompleted() + + return mockDockerPullStatus{ + complete: true, + layers: []docker.LayerID{ + "sha256:1", + "sha256:2", + "sha256:3", + }, + current: map[docker.LayerID]docker.LayerState{ + "sha256:1": { + Phase: docker.PullCompletePhase, + PhaseProgress: complete, + DownloadProgress: complete, + }, + "sha256:2": { + Phase: docker.PullCompletePhase, + PhaseProgress: complete, + DownloadProgress: complete, + }, + "sha256:3": { + Phase: docker.PullCompletePhase, + PhaseProgress: complete, + DownloadProgress: complete, + }, + }, + } + }(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newDockerPullStatusFormatter() + snaps.MatchSnapshot(t, f.Render(tt.status)) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_read_image.go b/cmd/syft/cli/ui/handle_read_image.go new file mode 100644 index 00000000000..e55ff485e4b --- /dev/null +++ b/cmd/syft/cli/ui/handle_read_image.go @@ -0,0 +1,33 @@ +package ui + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/taskprogress" + stereoEventParsers "github.com/anchore/stereoscope/pkg/event/parsers" + "github.com/anchore/syft/internal/log" +) + +func (m *Handler) handleReadImage(e partybus.Event) []tea.Model { + imgMetadata, prog, err := stereoEventParsers.ParseReadImage(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Parse image", + Running: "Parsing image", + Success: "Parsed image", + }, + taskprogress.WithProgress(prog), + ) + + if imgMetadata != nil { + tsk.Context = []string{imgMetadata.ID} + } + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_read_image_test.go b/cmd/syft/cli/ui/handle_read_image_test.go new file mode 100644 index 00000000000..864d1e782f4 --- /dev/null +++ b/cmd/syft/cli/ui/handle_read_image_test.go @@ -0,0 +1,117 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + stereoscopeEvent "github.com/anchore/stereoscope/pkg/event" + "github.com/anchore/stereoscope/pkg/image" +) + +func TestHandler_handleReadImage(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "read image in progress", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(50) + + src := image.Metadata{ + ID: "id", + Size: 42, + Config: v1.ConfigFile{ + Architecture: "arch", + Author: "auth", + Container: "cont", + OS: "os", + OSVersion: "os-ver", + Variant: "vari", + }, + MediaType: "media", + ManifestDigest: "digest", + Architecture: "arch", + Variant: "var", + OS: "os", + } + + return partybus.Event{ + Type: stereoscopeEvent.ReadImage, + Source: src, + Value: prog, + } + }, + }, + { + name: "read image complete", + eventFn: func(t *testing.T) partybus.Event { + prog := &progress.Manual{} + prog.SetTotal(100) + prog.Set(100) + prog.SetCompleted() + + src := image.Metadata{ + ID: "id", + Size: 42, + Config: v1.ConfigFile{ + Architecture: "arch", + Author: "auth", + Container: "cont", + OS: "os", + OSVersion: "os-ver", + Variant: "vari", + }, + MediaType: "media", + ManifestDigest: "digest", + Architecture: "arch", + Variant: "var", + OS: "os", + } + + return partybus.Event{ + Type: stereoscopeEvent.ReadImage, + Source: src, + Value: prog, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handle_secrets_cataloger.go b/cmd/syft/cli/ui/handle_secrets_cataloger.go new file mode 100644 index 00000000000..95b96454bea --- /dev/null +++ b/cmd/syft/cli/ui/handle_secrets_cataloger.go @@ -0,0 +1,57 @@ +package ui + +import ( + "fmt" + + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/syft/internal/log" + syftEventParsers "github.com/anchore/syft/syft/event/parsers" + "github.com/anchore/syft/syft/file/cataloger/secrets" +) + +var _ progress.StagedProgressable = (*secretsCatalogerProgressAdapter)(nil) + +// Deprecated: will be removed in syft 1.0 +type secretsCatalogerProgressAdapter struct { + *secrets.Monitor +} + +// Deprecated: will be removed in syft 1.0 +func newSecretsCatalogerProgressAdapter(monitor *secrets.Monitor) secretsCatalogerProgressAdapter { + return secretsCatalogerProgressAdapter{ + Monitor: monitor, + } +} + +func (s secretsCatalogerProgressAdapter) Stage() string { + return fmt.Sprintf("%d secrets", s.Monitor.SecretsDiscovered.Current()) +} + +// Deprecated: will be removed in syft 1.0 +func (m *Handler) handleSecretsCatalogerStarted(e partybus.Event) []tea.Model { + mon, err := syftEventParsers.ParseSecretsCatalogingStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil + } + + tsk := m.newTaskProgress( + taskprogress.Title{ + Default: "Catalog secrets", + Running: "Cataloging secrets", + Success: "Cataloged secrets", + }, + + taskprogress.WithStagedProgressable( + newSecretsCatalogerProgressAdapter(mon), + ), + ) + + tsk.HideStageOnSuccess = false + + return []tea.Model{tsk} +} diff --git a/cmd/syft/cli/ui/handle_secrets_cataloger_test.go b/cmd/syft/cli/ui/handle_secrets_cataloger_test.go new file mode 100644 index 00000000000..3a04cbce563 --- /dev/null +++ b/cmd/syft/cli/ui/handle_secrets_cataloger_test.go @@ -0,0 +1,96 @@ +package ui + +import ( + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + "github.com/anchore/bubbly/bubbles/taskprogress" + syftEvent "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/file/cataloger/secrets" +) + +func TestHandler_handleSecretsCatalogerStarted(t *testing.T) { + + tests := []struct { + name string + eventFn func(*testing.T) partybus.Event + iterations int + }{ + { + name: "cataloging in progress", + eventFn: func(t *testing.T) partybus.Event { + stage := &progress.Stage{ + Current: "current", + } + secretsDiscovered := progress.NewManual(-1) + secretsDiscovered.Set(64) + prog := progress.NewManual(72) + prog.Set(50) + + return partybus.Event{ + Type: syftEvent.SecretsCatalogerStarted, + Source: secretsDiscovered, + Value: secrets.Monitor{ + Stager: progress.Stager(stage), + SecretsDiscovered: secretsDiscovered, + Progressable: prog, + }, + } + }, + }, + { + name: "cataloging complete", + eventFn: func(t *testing.T) partybus.Event { + stage := &progress.Stage{ + Current: "current", + } + secretsDiscovered := progress.NewManual(-1) + secretsDiscovered.Set(64) + prog := progress.NewManual(72) + prog.Set(72) + prog.SetCompleted() + + return partybus.Event{ + Type: syftEvent.SecretsCatalogerStarted, + Source: secretsDiscovered, + Value: secrets.Monitor{ + Stager: progress.Stager(stage), + SecretsDiscovered: secretsDiscovered, + Progressable: prog, + }, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := tt.eventFn(t) + handler := New(DefaultHandlerConfig()) + handler.WindowSize = tea.WindowSizeMsg{ + Width: 100, + Height: 80, + } + + models := handler.Handle(event) + require.Len(t, models, 1) + model := models[0] + + tsk, ok := model.(taskprogress.Model) + require.True(t, ok) + + got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + Time: time.Now(), + Sequence: tsk.Sequence(), + ID: tsk.ID(), + }) + t.Log(got) + snaps.MatchSnapshot(t, got) + }) + } +} diff --git a/cmd/syft/cli/ui/handler.go b/cmd/syft/cli/ui/handler.go new file mode 100644 index 00000000000..8dae89dca5b --- /dev/null +++ b/cmd/syft/cli/ui/handler.go @@ -0,0 +1,68 @@ +package ui + +import ( + "sync" + + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly" + "github.com/anchore/bubbly/bubbles/taskprogress" + stereoscopeEvent "github.com/anchore/stereoscope/pkg/event" + syftEvent "github.com/anchore/syft/syft/event" +) + +var _ bubbly.EventHandler = (*Handler)(nil) + +type HandlerConfig struct { + TitleWidth int + AdjustDefaultTask func(taskprogress.Model) taskprogress.Model +} + +type Handler struct { + WindowSize tea.WindowSizeMsg + Running *sync.WaitGroup + Config HandlerConfig + + bubbly.EventHandler +} + +func DefaultHandlerConfig() HandlerConfig { + return HandlerConfig{ + TitleWidth: 30, + } +} + +func New(cfg HandlerConfig) *Handler { + d := bubbly.NewEventDispatcher() + + h := &Handler{ + EventHandler: d, + Running: &sync.WaitGroup{}, + Config: cfg, + } + + // register all supported event types with the respective handler functions + d.AddHandlers(map[partybus.EventType]bubbly.EventHandlerFn{ + stereoscopeEvent.PullDockerImage: h.handlePullDockerImage, + stereoscopeEvent.ReadImage: h.handleReadImage, + stereoscopeEvent.FetchImage: h.handleFetchImage, + syftEvent.PackageCatalogerStarted: h.handlePackageCatalogerStarted, + syftEvent.FileDigestsCatalogerStarted: h.handleFileDigestsCatalogerStarted, + syftEvent.FileMetadataCatalogerStarted: h.handleFileMetadataCatalogerStarted, + syftEvent.FileIndexingStarted: h.handleFileIndexingStarted, + syftEvent.AttestationStarted: h.handleAttestationStarted, + syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted, + + // deprecated + syftEvent.SecretsCatalogerStarted: h.handleSecretsCatalogerStarted, + }) + + return h +} + +func (m *Handler) Update(msg tea.Msg) { + if msg, ok := msg.(tea.WindowSizeMsg); ok { + m.WindowSize = msg + } +} diff --git a/cmd/syft/cli/ui/new_task_progress.go b/cmd/syft/cli/ui/new_task_progress.go new file mode 100644 index 00000000000..036f7b37de9 --- /dev/null +++ b/cmd/syft/cli/ui/new_task_progress.go @@ -0,0 +1,19 @@ +package ui + +import "github.com/anchore/bubbly/bubbles/taskprogress" + +func (m Handler) newTaskProgress(title taskprogress.Title, opts ...taskprogress.Option) taskprogress.Model { + tsk := taskprogress.New(m.Running, opts...) + + tsk.HideProgressOnSuccess = true + tsk.HideStageOnSuccess = true + tsk.WindowSize = m.WindowSize + tsk.TitleWidth = m.Config.TitleWidth + tsk.TitleOptions = title + + if m.Config.AdjustDefaultTask != nil { + tsk = m.Config.AdjustDefaultTask(tsk) + } + + return tsk +} diff --git a/cmd/syft/cli/ui/util_test.go b/cmd/syft/cli/ui/util_test.go new file mode 100644 index 00000000000..cfd5ddf5476 --- /dev/null +++ b/cmd/syft/cli/ui/util_test.go @@ -0,0 +1,62 @@ +package ui + +import ( + "reflect" + "testing" + "unsafe" + + tea "github.com/charmbracelet/bubbletea" +) + +func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg) string { + t.Helper() + if iterations == 0 { + iterations = 1 + } + m.Init() + var cmd tea.Cmd = func() tea.Msg { + return message + } + + for i := 0; cmd != nil && i < iterations; i++ { + msgs := flatten(cmd()) + var nextCmds []tea.Cmd + var next tea.Cmd + for _, msg := range msgs { + t.Logf("Message: %+v %+v\n", reflect.TypeOf(msg), msg) + m, next = m.Update(msg) + nextCmds = append(nextCmds, next) + } + cmd = tea.Batch(nextCmds...) + } + return m.View() +} + +func flatten(p tea.Msg) (msgs []tea.Msg) { + if reflect.TypeOf(p).Name() == "batchMsg" { + partials := extractBatchMessages(p) + for _, m := range partials { + msgs = append(msgs, flatten(m)...) + } + } else { + msgs = []tea.Msg{p} + } + return msgs +} + +func extractBatchMessages(m tea.Msg) (ret []tea.Msg) { + sliceMsgType := reflect.SliceOf(reflect.TypeOf(tea.Cmd(nil))) + value := reflect.ValueOf(m) // note: this is technically unaddressable + + // make our own instance that is addressable + valueCopy := reflect.New(value.Type()).Elem() + valueCopy.Set(value) + + cmds := reflect.NewAt(sliceMsgType, unsafe.Pointer(valueCopy.UnsafeAddr())).Elem() + for i := 0; i < cmds.Len(); i++ { + item := cmds.Index(i) + r := item.Call(nil) + ret = append(ret, r[0].Interface().(tea.Msg)) + } + return ret +} diff --git a/cmd/syft/internal/ui/__snapshots__/post_ui_event_writer_test.snap b/cmd/syft/internal/ui/__snapshots__/post_ui_event_writer_test.snap new file mode 100755 index 00000000000..bf473e331d4 --- /dev/null +++ b/cmd/syft/internal/ui/__snapshots__/post_ui_event_writer_test.snap @@ -0,0 +1,46 @@ + +[Test_postUIEventWriter_write/no_events/stdout - 1] + +--- + +[Test_postUIEventWriter_write/no_events/stderr - 1] + +--- + +[Test_postUIEventWriter_write/all_events/stdout - 1] + + + + + +--- + +[Test_postUIEventWriter_write/all_events/stderr - 1] + + + + + + + + + + + + + +--- + +[Test_postUIEventWriter_write/quiet_only_shows_report/stdout - 1] + + +--- + +[Test_postUIEventWriter_write/quiet_only_shows_report/stderr - 1] + +--- diff --git a/cmd/syft/internal/ui/no_ui.go b/cmd/syft/internal/ui/no_ui.go new file mode 100644 index 00000000000..015ae821899 --- /dev/null +++ b/cmd/syft/internal/ui/no_ui.go @@ -0,0 +1,44 @@ +package ui + +import ( + "os" + + "github.com/wagoodman/go-partybus" + + "github.com/anchore/clio" + "github.com/anchore/syft/syft/event" +) + +var _ clio.UI = (*NoUI)(nil) + +type NoUI struct { + finalizeEvents []partybus.Event + subscription partybus.Unsubscribable + quiet bool +} + +func None(quiet bool) *NoUI { + return &NoUI{ + quiet: quiet, + } +} + +func (n *NoUI) Setup(subscription partybus.Unsubscribable) error { + n.subscription = subscription + return nil +} + +func (n *NoUI) Handle(e partybus.Event) error { + switch e.Type { + case event.CLIReport, event.CLINotification: + // keep these for when the UI is terminated to show to the screen (or perform other events) + n.finalizeEvents = append(n.finalizeEvents, e) + case event.CLIExit: + return n.subscription.Unsubscribe() + } + return nil +} + +func (n NoUI) Teardown(_ bool) error { + return newPostUIEventWriter(os.Stdout, os.Stderr).write(n.quiet, n.finalizeEvents...) +} diff --git a/cmd/syft/internal/ui/post_ui_event_writer.go b/cmd/syft/internal/ui/post_ui_event_writer.go new file mode 100644 index 00000000000..e3772981b06 --- /dev/null +++ b/cmd/syft/internal/ui/post_ui_event_writer.go @@ -0,0 +1,133 @@ +package ui + +import ( + "fmt" + "io" + "strings" + + "github.com/charmbracelet/lipgloss" + "github.com/hashicorp/go-multierror" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/parsers" +) + +type postUIEventWriter struct { + handles []postUIHandle +} + +type postUIHandle struct { + respectQuiet bool + event partybus.EventType + writer io.Writer + dispatch eventWriter +} + +type eventWriter func(io.Writer, ...partybus.Event) error + +func newPostUIEventWriter(stdout, stderr io.Writer) *postUIEventWriter { + return &postUIEventWriter{ + handles: []postUIHandle{ + { + event: event.CLIReport, + respectQuiet: false, + writer: stdout, + dispatch: writeReports, + }, + { + event: event.CLINotification, + respectQuiet: true, + writer: stderr, + dispatch: writeNotifications, + }, + { + event: event.CLIAppUpdateAvailable, + respectQuiet: true, + writer: stderr, + dispatch: writeAppUpdate, + }, + }, + } +} + +func (w postUIEventWriter) write(quiet bool, events ...partybus.Event) error { + var errs error + for _, h := range w.handles { + if quiet && h.respectQuiet { + continue + } + + for _, e := range events { + if e.Type != h.event { + continue + } + + if err := h.dispatch(h.writer, e); err != nil { + errs = multierror.Append(errs, err) + } + } + } + return errs +} + +func writeReports(writer io.Writer, events ...partybus.Event) error { + var reports []string + for _, e := range events { + _, report, err := parsers.ParseCLIReport(e) + if err != nil { + log.WithFields("error", err).Warn("failed to gather final report") + continue + } + + // remove all whitespace padding from the end of the report + reports = append(reports, strings.TrimRight(report, "\n ")+"\n") + } + + // prevent the double new-line at the end of the report + report := strings.Join(reports, "\n") + + if _, err := fmt.Fprint(writer, report); err != nil { + return fmt.Errorf("failed to write final report to stdout: %w", err) + } + return nil +} + +func writeNotifications(writer io.Writer, events ...partybus.Event) error { + // 13 = high intensity magenta (ANSI 16 bit code) + style := lipgloss.NewStyle().Foreground(lipgloss.Color("13")) + + for _, e := range events { + _, notification, err := parsers.ParseCLINotification(e) + if err != nil { + log.WithFields("error", err).Warn("failed to parse notification") + continue + } + + if _, err := fmt.Fprintln(writer, style.Render(notification)); err != nil { + // don't let this be fatal + log.WithFields("error", err).Warn("failed to write final notifications") + } + } + return nil +} + +func writeAppUpdate(writer io.Writer, events ...partybus.Event) error { + // 13 = high intensity magenta (ANSI 16 bit code) + italics + style := lipgloss.NewStyle().Foreground(lipgloss.Color("13")).Italic(true) + + for _, e := range events { + notice, err := parsers.ParseCLIAppUpdateAvailable(e) + if err != nil { + log.WithFields("error", err).Warn("failed to parse app update notification") + continue + } + + if _, err := fmt.Fprintln(writer, style.Render(notice)); err != nil { + // don't let this be fatal + log.WithFields("error", err).Warn("failed to write app update notification") + } + } + return nil +} diff --git a/cmd/syft/internal/ui/post_ui_event_writer_test.go b/cmd/syft/internal/ui/post_ui_event_writer_test.go new file mode 100644 index 00000000000..a5bdd5792eb --- /dev/null +++ b/cmd/syft/internal/ui/post_ui_event_writer_test.go @@ -0,0 +1,95 @@ +package ui + +import ( + "bytes" + "testing" + + "github.com/gkampitakis/go-snaps/snaps" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/syft/syft/event" +) + +func Test_postUIEventWriter_write(t *testing.T) { + + tests := []struct { + name string + quiet bool + events []partybus.Event + wantErr require.ErrorAssertionFunc + }{ + { + name: "no events", + }, + { + name: "all events", + events: []partybus.Event{ + { + Type: event.CLINotification, + Value: "\n\n\n\n", + }, + { + Type: event.CLINotification, + Value: "", + }, + { + Type: event.CLIAppUpdateAvailable, + Value: "\n\n\n\n", + }, + { + Type: event.CLINotification, + Value: "", + }, + { + Type: event.CLIReport, + Value: "\n\n\n\n", + }, + { + Type: event.CLIReport, + Value: "", + }, + }, + }, + { + name: "quiet only shows report", + quiet: true, + events: []partybus.Event{ + + { + Type: event.CLINotification, + Value: "", + }, + { + Type: event.CLIAppUpdateAvailable, + Value: "", + }, + { + Type: event.CLIReport, + Value: "", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + w := newPostUIEventWriter(stdout, stderr) + + tt.wantErr(t, w.write(tt.quiet, tt.events...)) + + t.Run("stdout", func(t *testing.T) { + snaps.MatchSnapshot(t, stdout.String()) + }) + + t.Run("stderr", func(t *testing.T) { + snaps.MatchSnapshot(t, stderr.String()) + }) + }) + } +} diff --git a/internal/ui/select.go b/cmd/syft/internal/ui/select.go similarity index 72% rename from internal/ui/select.go rename to cmd/syft/internal/ui/select.go index 2c501cacbcc..27b536192e6 100644 --- a/internal/ui/select.go +++ b/cmd/syft/internal/ui/select.go @@ -8,6 +8,9 @@ import ( "runtime" "golang.org/x/term" + + "github.com/anchore/clio" + handler "github.com/anchore/syft/cmd/syft/cli/ui" ) // Select is responsible for determining the specific UI function given select user option, the current platform @@ -15,16 +18,18 @@ import ( // is intended to be used and the UIs that follow are meant to be attempted only in a fallback posture when there // are environmental problems (e.g. cannot write to the terminal). A writer is provided to capture the output of // the final SBOM report. -func Select(verbose, quiet bool) (uis []UI) { +func Select(verbose, quiet bool) (uis []clio.UI) { isStdoutATty := term.IsTerminal(int(os.Stdout.Fd())) isStderrATty := term.IsTerminal(int(os.Stderr.Fd())) notATerminal := !isStderrATty && !isStdoutATty switch { case runtime.GOOS == "windows" || verbose || quiet || notATerminal || !isStderrATty: - uis = append(uis, NewLoggerUI()) + uis = append(uis, None(quiet)) default: - uis = append(uis, NewEphemeralTerminalUI()) + // TODO: it may make sense in the future to pass handler options into select + h := handler.New(handler.DefaultHandlerConfig()) + uis = append(uis, New(h, verbose, quiet)) } return uis diff --git a/internal/ui/select_windows.go b/cmd/syft/internal/ui/select_windows.go similarity index 81% rename from internal/ui/select_windows.go rename to cmd/syft/internal/ui/select_windows.go index cd8c79839ec..0408be53b28 100644 --- a/internal/ui/select_windows.go +++ b/cmd/syft/internal/ui/select_windows.go @@ -3,11 +3,13 @@ package ui +import "github.com/anchore/clio" + // Select is responsible for determining the specific UI function given select user option, the current platform // config values, and environment status (such as a TTY being present). The first UI in the returned slice of UIs // is intended to be used and the UIs that follow are meant to be attempted only in a fallback posture when there // are environmental problems (e.g. cannot write to the terminal). A writer is provided to capture the output of // the final SBOM report. -func Select(verbose, quiet bool) (uis []UI) { - return append(uis, NewLoggerUI()) +func Select(verbose, quiet bool) (uis []clio.UI) { + return append(uis, None(quiet)) } diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go new file mode 100644 index 00000000000..4a8ab83e5c1 --- /dev/null +++ b/cmd/syft/internal/ui/ui.go @@ -0,0 +1,163 @@ +package ui + +import ( + "os" + "sync" + + tea "github.com/charmbracelet/bubbletea" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/bubbly/bubbles/frame" + "github.com/anchore/clio" + "github.com/anchore/go-logger" + handler "github.com/anchore/syft/cmd/syft/cli/ui" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event" +) + +var _ interface { + tea.Model + partybus.Responder + clio.UI +} = (*UI)(nil) + +type UI struct { + program *tea.Program + running *sync.WaitGroup + quiet bool + subscription partybus.Unsubscribable + finalizeEvents []partybus.Event + + handler *handler.Handler + frame tea.Model +} + +func New(h *handler.Handler, _, quiet bool) *UI { + return &UI{ + handler: h, + frame: frame.New(), + running: &sync.WaitGroup{}, + quiet: quiet, + } +} + +func (m *UI) Setup(subscription partybus.Unsubscribable) error { + // we still want to collect log messages, however, we also the logger shouldn't write to the screen directly + if logWrapper, ok := log.Get().(logger.Controller); ok { + logWrapper.SetOutput(m.frame.(*frame.Frame).Footer()) + } + + m.subscription = subscription + m.program = tea.NewProgram(m, tea.WithOutput(os.Stderr), tea.WithInput(os.Stdin)) + m.running.Add(1) + + go func() { + defer m.running.Done() + if _, err := m.program.Run(); err != nil { + log.Errorf("unable to start UI: %+v", err) + m.exit() + } + }() + + return nil +} + +func (m *UI) exit() { + // stop the event loop + bus.Exit() +} + +func (m *UI) Handle(e partybus.Event) error { + if m.program != nil { + m.program.Send(e) + if e.Type == event.CLIExit { + return m.subscription.Unsubscribe() + } + } + return nil +} + +func (m *UI) Teardown(force bool) error { + if !force { + m.handler.Running.Wait() + m.program.Quit() + } else { + m.program.Kill() + } + + m.running.Wait() + + // TODO: allow for writing out the full log output to the screen (only a partial log is shown currently) + // this needs coordination to know what the last frame event is to change the state accordingly (which isn't possible now) + + return newPostUIEventWriter(os.Stdout, os.Stderr).write(m.quiet, m.finalizeEvents...) +} + +// bubbletea.Model functions + +func (m UI) Init() tea.Cmd { + return m.frame.Init() +} + +func (m UI) RespondsTo() []partybus.EventType { + return append([]partybus.EventType{ + event.CLIReport, + event.CLINotification, + event.CLIExit, + event.CLIAppUpdateAvailable, + }, m.handler.RespondsTo()...) +} + +func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + // note: we need a pointer receiver such that the same instance of UI used in Teardown is referenced (to keep finalize events) + + var cmds []tea.Cmd + + // allow for non-partybus UI updates (such as window size events). Note: these must not affect existing models, + // that is the responsibility of the frame object on this UI object. The handler is a factory of models + // which the frame is responsible for the lifecycle of. This update allows for injecting the initial state + // of the world when creating those models. + m.handler.Update(msg) + + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case "esc", "ctrl+c": + m.exit() + return m, tea.Quit + } + + case partybus.Event: + log.WithFields("component", "ui").Tracef("event: %q", msg.Type) + + switch msg.Type { + case event.CLIReport, event.CLINotification, event.CLIExit, event.CLIAppUpdateAvailable: + // keep these for when the UI is terminated to show to the screen (or perform other events) + m.finalizeEvents = append(m.finalizeEvents, msg) + + // why not return tea.Quit here for exit events? because there may be UI components that still need the update-render loop. + // for this reason we'll let the syft event loop call Teardown() which will explicitly wait for these components + return m, nil + } + + for _, newModel := range m.handler.Handle(msg) { + if newModel == nil { + continue + } + cmds = append(cmds, newModel.Init()) + m.frame.(*frame.Frame).AppendModel(newModel) + } + // intentionally fallthrough to update the frame model + } + + frameModel, cmd := m.frame.Update(msg) + m.frame = frameModel + cmds = append(cmds, cmd) + + return m, tea.Batch(cmds...) +} + +func (m UI) View() string { + return m.frame.View() +} diff --git a/go.mod b/go.mod index ff8d61dfcc1..03c6a45d1a3 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 github.com/vifraa/gopom v0.2.1 - github.com/wagoodman/go-partybus v0.0.0-20210627031916-db1f5573bbc5 + github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5 github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb github.com/xeipuuv/gojsonschema v1.2.0 @@ -52,12 +52,17 @@ require ( github.com/CycloneDX/cyclonedx-go v0.7.1 github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8 + github.com/anchore/bubbly v0.0.0-20230622134437-40226fdcc0f5 + github.com/anchore/clio v0.0.0-20230602170917-e747e60c4aa0 + github.com/anchore/go-logger v0.0.0-20230531193951-db5ae83e7dbe github.com/anchore/stereoscope v0.0.0-20230627195312-cd49355d934e + github.com/charmbracelet/bubbletea v0.24.2 + github.com/charmbracelet/lipgloss v0.7.1 github.com/dave/jennifer v1.6.1 github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da github.com/docker/docker v24.0.2+incompatible github.com/github/go-spdx/v2 v2.1.2 + github.com/gkampitakis/go-snaps v0.4.0 github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git/v5 v5.7.0 github.com/google/go-containerregistry v0.15.2 @@ -67,6 +72,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/sassoftware/go-rpmutils v0.2.0 github.com/vbatts/go-mtree v0.5.3 + github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 golang.org/x/exp v0.0.0-20230202163644-54bba9f4231b gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.23.1 @@ -80,9 +86,14 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/anchore/fangs v0.0.0-20230531202914-48a718c6b4ba // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/becheran/wildmatch-go v1.0.0 // indirect + github.com/charmbracelet/bubbles v0.16.1 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect + github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/containerd/containerd v1.7.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -93,14 +104,18 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/felixge/fgprof v0.9.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect + github.com/gkampitakis/ciinfo v0.1.1 // indirect + github.com/gkampitakis/go-diff v1.3.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-restruct/restruct v1.2.0-alpha // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect @@ -112,21 +127,32 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.1 // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pborman/indent v1.2.1 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/spf13/cast v1.5.1 // indirect diff --git a/go.sum b/go.sum index f321e6edd9b..693beaee4e6 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,14 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8 h1:imgMA0gN0TZx7PSa/pdWqXadBvrz8WsN6zySzCe4XX0= -github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8/go.mod h1:+gPap4jha079qzRTUaehv+UZ6sSdaNwkH0D3b6zhTuk= +github.com/anchore/bubbly v0.0.0-20230622134437-40226fdcc0f5 h1:ylXHybVevy9Musod3gplxsn7g9Ws7ET/XcCrWFXkuvw= +github.com/anchore/bubbly v0.0.0-20230622134437-40226fdcc0f5/go.mod h1:tBC1jAU9gk7ekAbUmBXCuRX1l5Z9sMSqgcGSgsV1ECY= +github.com/anchore/clio v0.0.0-20230602170917-e747e60c4aa0 h1:g0UqRW60JDrf5fb40RUyIwwcfQ3nAJqGj4aUCVTwFE4= +github.com/anchore/clio v0.0.0-20230602170917-e747e60c4aa0/go.mod h1:0IQVIROfgRX4WZFMfgsbNZmMgLKqW/KgByyJDYvWiDE= +github.com/anchore/fangs v0.0.0-20230531202914-48a718c6b4ba h1:tJ186HK8e0Lf+hhNWX4fJrq14yj3mw8JQkkLhA0nFhE= +github.com/anchore/fangs v0.0.0-20230531202914-48a718c6b4ba/go.mod h1:E3zNHEz7mizIFGJhuX+Ga7AbCmEN5TfzVDxmOfj7XZw= +github.com/anchore/go-logger v0.0.0-20230531193951-db5ae83e7dbe h1:Df867YMmymdMG6z5IW8pR0/2CRpLIjYnaTXLp6j+s0k= +github.com/anchore/go-logger v0.0.0-20230531193951-db5ae83e7dbe/go.mod h1:ubLFmlsv8/DFUQrZwY5syT5/8Er3ugSr4rDFwHsE3hg= github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb h1:iDMnx6LIjtjZ46C0akqveX83WFzhpTD3eqOthawb5vU= github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb/go.mod h1:DmTY2Mfcv38hsHbG78xMiTDdxFtkHpgYNVDPsF2TgHk= github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc= @@ -112,6 +118,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA= github.com/becheran/wildmatch-go v1.0.0/go.mod h1:gbMvj0NtVdJ15Mg/mH9uxk2R1QCistMyU7d9KFzroX4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -127,6 +135,14 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= +github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= +github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= +github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= +github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= +github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -146,6 +162,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg= github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= @@ -202,6 +220,8 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA= github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -211,6 +231,12 @@ github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmx github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/github/go-spdx/v2 v2.1.2 h1:p+Tv0yMgcuO0/vnMe9Qh4tmUgYhI6AsLVlakZ/Sx+DM= github.com/github/go-spdx/v2 v2.1.2/go.mod h1:hMCrsFgT0QnCwn7G8gxy/MxMpy67WgZrwFeISTn0o6w= +github.com/gkampitakis/ciinfo v0.1.1 h1:dz1LCkOd+zmZ3YYlFNpr0hRDqGY7Ox2mcaltHzdahqk= +github.com/gkampitakis/ciinfo v0.1.1/go.mod h1:bVaOGziPqf8PoeYZxatq1HmCsJUmv191hLnFboYxd9Y= +github.com/gkampitakis/go-diff v1.3.0 h1:Szdbo5w73LSQ9sQ02h+NSSf2ZlW/E8naJCI1ZzQtWgE= +github.com/gkampitakis/go-diff v1.3.0/go.mod h1:QUJDQRA0JkEX0d7tgDaBHzJv9IH6k6e91TByC+9/RFk= +github.com/gkampitakis/go-snaps v0.4.0 h1:yTMQ4RaGrQvsr70XZRoxZeJiMkmdLbZ9fWpW/vypdVk= +github.com/gkampitakis/go-snaps v0.4.0/go.mod h1:xYclGIA7Al0CoYwehW0dd/NEr6oJge+1Dl4OWWxQUWY= github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -315,7 +341,9 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -372,6 +400,7 @@ github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -414,13 +443,17 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -442,11 +475,14 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 h1:P8UmIzZMYDR+NGImiFvErt6VWfIRPuGM+vyjiEdkmIw= +github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= @@ -482,6 +518,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= +github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= @@ -494,6 +538,8 @@ github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0 github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/indent v1.2.1 h1:lFiviAbISHv3Rf0jcuh489bi06hj98JsVMtIDZQb9yM= +github.com/pborman/indent v1.2.1/go.mod h1:FitS+t35kIYtB5xWTZAPhnmrxcciEEOdbyrrpz5K6Vw= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -504,10 +550,13 @@ github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -529,11 +578,14 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= @@ -618,8 +670,8 @@ github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RV github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/vifraa/gopom v0.2.1 h1:MYVMAMyiGzXPPy10EwojzKIL670kl5Zbae+o3fFvQEM= github.com/vifraa/gopom v0.2.1/go.mod h1:oPa1dcrGrtlO37WPDBm5SqHAT+wTgF8An1Q71Z6Vv4o= -github.com/wagoodman/go-partybus v0.0.0-20210627031916-db1f5573bbc5 h1:phTLPgMRDYTizrBSKsNSOa2zthoC2KsJsaY/8sg3rD8= -github.com/wagoodman/go-partybus v0.0.0-20210627031916-db1f5573bbc5/go.mod h1:JPirS5jde/CF5qIjcK4WX+eQmKXdPc6vcZkJ/P0hfPw= +github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 h1:jIVmlAFIqV3d+DOxazTR9v+zgj8+VYuQBzPgBZvWBHA= +github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651/go.mod h1:b26F2tHLqaoRQf8DywqzVaV1MQ9yvjb0OMcNl7Nxu20= github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5 h1:lwgTsTy18nYqASnH58qyfRW/ldj7Gt2zzBvgYPzdA4s= github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb h1:Yz6VVOcLuWLAHYlJzTw7JKnWxdV/WXpug2X0quEzRnY= @@ -643,6 +695,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 h1:V+UsotZpAVvfj3X/LMoEytoLzSiP6Lg0F7wdVyu9gGg= +github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -878,6 +932,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/bus/bus.go b/internal/bus/bus.go index 0810c2fc670..c85eb77cbfb 100644 --- a/internal/bus/bus.go +++ b/internal/bus/bus.go @@ -16,20 +16,20 @@ package bus import "github.com/wagoodman/go-partybus" var publisher partybus.Publisher -var active bool -// SetPublisher sets the singleton event bus publisher. This is optional; if no bus is provided, the library will +// Set sets the singleton event bus publisher. This is optional; if no bus is provided, the library will // behave no differently than if a bus had been provided. -func SetPublisher(p partybus.Publisher) { +func Set(p partybus.Publisher) { publisher = p - if p != nil { - active = true - } +} + +func Get() partybus.Publisher { + return publisher } // Publish an event onto the bus. If there is no bus set by the calling application, this does nothing. -func Publish(event partybus.Event) { - if active { - publisher.Publish(event) +func Publish(e partybus.Event) { + if publisher != nil { + publisher.Publish(e) } } diff --git a/internal/bus/helpers.go b/internal/bus/helpers.go new file mode 100644 index 00000000000..5efacfb358b --- /dev/null +++ b/internal/bus/helpers.go @@ -0,0 +1,32 @@ +package bus + +import ( + "github.com/wagoodman/go-partybus" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event" +) + +func Exit() { + Publish(partybus.Event{ + Type: event.CLIExit, + }) +} + +func Report(report string) { + if len(report) == 0 { + return + } + report = log.Redactor.RedactString(report) + Publish(partybus.Event{ + Type: event.CLIReport, + Value: report, + }) +} + +func Notify(message string) { + Publish(partybus.Event{ + Type: event.CLINotification, + Value: message, + }) +} diff --git a/internal/log/log.go b/internal/log/log.go index 30952987ff6..242c5f0b7bb 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -6,67 +6,86 @@ package log import ( "github.com/anchore/go-logger" "github.com/anchore/go-logger/adapter/discard" + "github.com/anchore/go-logger/adapter/redact" ) -// Log is the singleton used to facilitate logging internally within syft -var Log logger.Logger = discard.New() +var ( + // log is the singleton used to facilitate logging internally within + log = discard.New() + + store = redact.NewStore() + + Redactor = store.(redact.Redactor) +) + +func Set(l logger.Logger) { + log = redact.New(l, store) +} + +func Get() logger.Logger { + return log +} + +func Redact(values ...string) { + store.Add(values...) +} // Errorf takes a formatted template string and template arguments for the error logging level. func Errorf(format string, args ...interface{}) { - Log.Errorf(format, args...) + log.Errorf(format, args...) } // Error logs the given arguments at the error logging level. func Error(args ...interface{}) { - Log.Error(args...) + log.Error(args...) } // Warnf takes a formatted template string and template arguments for the warning logging level. func Warnf(format string, args ...interface{}) { - Log.Warnf(format, args...) + log.Warnf(format, args...) } // Warn logs the given arguments at the warning logging level. func Warn(args ...interface{}) { - Log.Warn(args...) + log.Warn(args...) } // Infof takes a formatted template string and template arguments for the info logging level. func Infof(format string, args ...interface{}) { - Log.Infof(format, args...) + log.Infof(format, args...) } // Info logs the given arguments at the info logging level. func Info(args ...interface{}) { - Log.Info(args...) + log.Info(args...) } // Debugf takes a formatted template string and template arguments for the debug logging level. func Debugf(format string, args ...interface{}) { - Log.Debugf(format, args...) + log.Debugf(format, args...) } // Debug logs the given arguments at the debug logging level. func Debug(args ...interface{}) { - Log.Debug(args...) + log.Debug(args...) } // Tracef takes a formatted template string and template arguments for the trace logging level. func Tracef(format string, args ...interface{}) { - Log.Tracef(format, args...) + log.Tracef(format, args...) } // Trace logs the given arguments at the trace logging level. func Trace(args ...interface{}) { - Log.Trace(args...) + log.Trace(args...) } // WithFields returns a message logger with multiple key-value fields. func WithFields(fields ...interface{}) logger.MessageLogger { - return Log.WithFields(fields...) + return log.WithFields(fields...) } // Nested returns a new logger with hard coded key-value pairs func Nested(fields ...interface{}) logger.Logger { - return Log.Nested(fields...) + return log.Nested(fields...) } diff --git a/internal/ui/common_event_handlers.go b/internal/ui/common_event_handlers.go deleted file mode 100644 index f7dbaaa3d26..00000000000 --- a/internal/ui/common_event_handlers.go +++ /dev/null @@ -1,24 +0,0 @@ -package ui - -import ( - "fmt" - - "github.com/wagoodman/go-partybus" - - syftEventParsers "github.com/anchore/syft/syft/event/parsers" -) - -// handleExit is a UI function for processing the Exit bus event, -// and calling the given function to output the contents. -func handleExit(event partybus.Event) error { - // show the report to stdout - fn, err := syftEventParsers.ParseExit(event) - if err != nil { - return fmt.Errorf("bad CatalogerFinished event: %w", err) - } - - if err := fn(); err != nil { - return fmt.Errorf("unable to show package catalog report: %w", err) - } - return nil -} diff --git a/internal/ui/components/spinner.go b/internal/ui/components/spinner.go deleted file mode 100644 index debc8cb961e..00000000000 --- a/internal/ui/components/spinner.go +++ /dev/null @@ -1,42 +0,0 @@ -package components - -import ( - "strings" - "sync" -) - -// TODO: move me to a common module (used in multiple repos) - -const ( - SpinnerDotSet = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏" -) - -type Spinner struct { - index int - charset []string - lock sync.Mutex -} - -func NewSpinner(charset string) Spinner { - return Spinner{ - charset: strings.Split(charset, ""), - } -} - -func (s *Spinner) Current() string { - s.lock.Lock() - defer s.lock.Unlock() - - return s.charset[s.index] -} - -func (s *Spinner) Next() string { - s.lock.Lock() - defer s.lock.Unlock() - c := s.charset[s.index] - s.index++ - if s.index >= len(s.charset) { - s.index = 0 - } - return c -} diff --git a/internal/ui/ephemeral_terminal_ui.go b/internal/ui/ephemeral_terminal_ui.go deleted file mode 100644 index c5270f8b90c..00000000000 --- a/internal/ui/ephemeral_terminal_ui.go +++ /dev/null @@ -1,154 +0,0 @@ -//go:build linux || darwin || netbsd -// +build linux darwin netbsd - -package ui - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "sync" - - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/jotframe/pkg/frame" - - "github.com/anchore/go-logger" - "github.com/anchore/syft/internal/log" - syftEvent "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/ui" -) - -// ephemeralTerminalUI provides an "ephemeral" terminal user interface to display the application state dynamically. -// The terminal is placed into raw mode and the cursor is manipulated to allow for a dynamic, multi-line -// UI (provided by the jotframe lib), for this reason all other application mechanisms that write to the screen -// must be suppressed before starting (such as logs); since bytes in the device and in application memory combine to make -// a shared state, bytes coming from elsewhere to the screen will disrupt this state. -// -// This UI is primarily driven off of events from the event bus, creating single-line terminal widgets to represent a -// published element on the event bus, typically polling the element for the latest state. This allows for the UI to -// control update frequency to the screen, provide "liveness" indications that are interpolated between bus events, -// and overall loosely couple the bus events from screen interactions. -// -// By convention, all elements published on the bus should be treated as read-only, and publishers on the bus should -// attempt to enforce this when possible by wrapping complex objects with interfaces to prescribe interactions. Also by -// convention, each new event that the UI should respond to should be added either in this package as a handler function, -// or in the shared ui package as a function on the main handler object. All handler functions should be completed -// processing an event before the ETUI exits (coordinated with a sync.WaitGroup) -type ephemeralTerminalUI struct { - unsubscribe func() error - handler *ui.Handler - waitGroup *sync.WaitGroup - frame *frame.Frame - logBuffer *bytes.Buffer - uiOutput *os.File -} - -// NewEphemeralTerminalUI writes all events to a TUI and writes the final report to the given writer. -func NewEphemeralTerminalUI() UI { - return &ephemeralTerminalUI{ - handler: ui.NewHandler(), - waitGroup: &sync.WaitGroup{}, - uiOutput: os.Stderr, - } -} - -func (h *ephemeralTerminalUI) Setup(unsubscribe func() error) error { - h.unsubscribe = unsubscribe - hideCursor(h.uiOutput) - - // prep the logger to not clobber the screen from now on (logrus only) - h.logBuffer = bytes.NewBufferString("") - logController, ok := log.Log.(logger.Controller) - if ok { - logController.SetOutput(h.logBuffer) - } - - return h.openScreen() -} - -func (h *ephemeralTerminalUI) Handle(event partybus.Event) error { - ctx := context.Background() - switch { - case h.handler.RespondsTo(event): - if err := h.handler.Handle(ctx, h.frame, event, h.waitGroup); err != nil { - log.Errorf("unable to show %s event: %+v", event.Type, err) - } - - case event.Type == syftEvent.AppUpdateAvailable: - if err := handleAppUpdateAvailable(ctx, h.frame, event, h.waitGroup); err != nil { - log.Errorf("unable to show %s event: %+v", event.Type, err) - } - - case event.Type == syftEvent.Exit: - // we need to close the screen now since signaling the sbom is ready means that we - // are about to write bytes to stdout, so we should reset the terminal state first - h.closeScreen(false) - - if err := handleExit(event); err != nil { - log.Errorf("unable to show %s event: %+v", event.Type, err) - } - - // this is the last expected event, stop listening to events - return h.unsubscribe() - } - return nil -} - -func (h *ephemeralTerminalUI) openScreen() error { - config := frame.Config{ - PositionPolicy: frame.PolicyFloatForward, - // only report output to stderr, reserve report output for stdout - Output: h.uiOutput, - } - - fr, err := frame.New(config) - if err != nil { - return fmt.Errorf("failed to create the screen object: %w", err) - } - h.frame = fr - - return nil -} - -func (h *ephemeralTerminalUI) closeScreen(force bool) { - // we may have other background processes still displaying progress, wait for them to - // finish before discontinuing dynamic content and showing the final report - if !h.frame.IsClosed() { - if !force { - h.waitGroup.Wait() - } - h.frame.Close() - // TODO: there is a race condition within frame.Close() that sometimes leads to an extra blank line being output - frame.Close() - - // only flush the log on close - h.flushLog() - } -} - -func (h *ephemeralTerminalUI) flushLog() { - // flush any errors to the screen before the report - logController, ok := log.Log.(logger.Controller) - if ok { - fmt.Fprint(logController.GetOutput(), h.logBuffer.String()) - logController.SetOutput(h.uiOutput) - } else { - fmt.Fprint(h.uiOutput, h.logBuffer.String()) - } -} - -func (h *ephemeralTerminalUI) Teardown(force bool) error { - h.closeScreen(force) - showCursor(h.uiOutput) - return nil -} - -func hideCursor(output io.Writer) { - fmt.Fprint(output, "\x1b[?25l") -} - -func showCursor(output io.Writer) { - fmt.Fprint(output, "\x1b[?25h") -} diff --git a/internal/ui/etui_event_handlers.go b/internal/ui/etui_event_handlers.go deleted file mode 100644 index f1703f81311..00000000000 --- a/internal/ui/etui_event_handlers.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build linux || darwin || netbsd -// +build linux darwin netbsd - -package ui - -import ( - "context" - "fmt" - "io" - "sync" - - "github.com/gookit/color" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/jotframe/pkg/frame" - - "github.com/anchore/syft/internal" - syftEventParsers "github.com/anchore/syft/syft/event/parsers" -) - -// handleAppUpdateAvailable is a UI handler function to display a new application version to the top of the screen. -func handleAppUpdateAvailable(_ context.Context, fr *frame.Frame, event partybus.Event, _ *sync.WaitGroup) error { - newVersion, err := syftEventParsers.ParseAppUpdateAvailable(event) - if err != nil { - return fmt.Errorf("bad AppUpdateAvailable event: %w", err) - } - - line, err := fr.Prepend() - if err != nil { - return err - } - - message := color.Magenta.Sprintf("New version of %s is available: %s", internal.ApplicationName, newVersion) - _, _ = io.WriteString(line, message) - - return nil -} diff --git a/internal/ui/logger_ui.go b/internal/ui/logger_ui.go deleted file mode 100644 index 48f5c1ed6bf..00000000000 --- a/internal/ui/logger_ui.go +++ /dev/null @@ -1,40 +0,0 @@ -package ui - -import ( - "github.com/wagoodman/go-partybus" - - "github.com/anchore/syft/internal/log" - syftEvent "github.com/anchore/syft/syft/event" -) - -type loggerUI struct { - unsubscribe func() error -} - -// NewLoggerUI writes all events to the common application logger and writes the final report to the given writer. -func NewLoggerUI() UI { - return &loggerUI{} -} - -func (l *loggerUI) Setup(unsubscribe func() error) error { - l.unsubscribe = unsubscribe - return nil -} - -func (l loggerUI) Handle(event partybus.Event) error { - // ignore all events except for the final event - if event.Type != syftEvent.Exit { - return nil - } - - if err := handleExit(event); err != nil { - log.Warnf("unable to show catalog image finished event: %+v", err) - } - - // this is the last expected event, stop listening to events - return l.unsubscribe() -} - -func (l loggerUI) Teardown(_ bool) error { - return nil -} diff --git a/internal/ui/ui.go b/internal/ui/ui.go deleted file mode 100644 index cb551f1cfcb..00000000000 --- a/internal/ui/ui.go +++ /dev/null @@ -1,11 +0,0 @@ -package ui - -import ( - "github.com/wagoodman/go-partybus" -) - -type UI interface { - Setup(unsubscribe func() error) error - partybus.Handler - Teardown(force bool) error -} diff --git a/syft/event/event.go b/syft/event/event.go index 1e9e6f40076..4c2a29296b4 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -4,37 +4,51 @@ defined here there should be a corresponding event parser defined in the parsers */ package event -import "github.com/wagoodman/go-partybus" +import ( + "github.com/wagoodman/go-partybus" + + "github.com/anchore/syft/internal" +) const ( - // AppUpdateAvailable is a partybus event that occurs when an application update is available - AppUpdateAvailable partybus.EventType = "syft-app-update-available" + typePrefix = internal.ApplicationName + cliTypePrefix = typePrefix + "-cli" + + // Events from the syft library // PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun - PackageCatalogerStarted partybus.EventType = "syft-package-cataloger-started-event" + PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event" //nolint:gosec // SecretsCatalogerStarted is a partybus event that occurs when the secrets cataloging has begun - SecretsCatalogerStarted partybus.EventType = "syft-secrets-cataloger-started-event" + SecretsCatalogerStarted partybus.EventType = typePrefix + "-secrets-cataloger-started-event" // FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun - FileMetadataCatalogerStarted partybus.EventType = "syft-file-metadata-cataloger-started-event" + FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event" // FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun - FileDigestsCatalogerStarted partybus.EventType = "syft-file-digests-cataloger-started-event" + FileDigestsCatalogerStarted partybus.EventType = typePrefix + "-file-digests-cataloger-started-event" // FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem - FileIndexingStarted partybus.EventType = "syft-file-indexing-started-event" - - // Exit is a partybus event that occurs when an analysis result is ready for final presentation - Exit partybus.EventType = "syft-exit-event" - - // ImportStarted is a partybus event that occurs when an SBOM upload process has begun - ImportStarted partybus.EventType = "syft-import-started-event" + FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event" // AttestationStarted is a partybus event that occurs when starting an SBOM attestation process - AttestationStarted partybus.EventType = "syft-attestation-started-event" + AttestationStarted partybus.EventType = typePrefix + "-attestation-started-event" // CatalogerTaskStarted is a partybus event that occurs when starting a task within a cataloger - CatalogerTaskStarted partybus.EventType = "syft-cataloger-task-started" + CatalogerTaskStarted partybus.EventType = typePrefix + "-cataloger-task-started" + + // Events exclusively for the CLI + + // CLIAppUpdateAvailable is a partybus event that occurs when an application update is available + CLIAppUpdateAvailable partybus.EventType = cliTypePrefix + "-app-update-available" + + // CLIReport is a partybus event that occurs when an analysis result is ready for final presentation to stdout + CLIReport partybus.EventType = cliTypePrefix + "-report" + + // CLINotification is a partybus event that occurs when auxiliary information is ready for presentation to stderr + CLINotification partybus.EventType = cliTypePrefix + "-notification" + + // CLIExit is a partybus event that occurs when an analysis result is ready for final presentation + CLIExit partybus.EventType = cliTypePrefix + "-exit-event" ) diff --git a/syft/event/cataloger_task.go b/syft/event/monitor/cataloger_task.go similarity index 84% rename from syft/event/cataloger_task.go rename to syft/event/monitor/cataloger_task.go index 49fa1cdee83..4a06132e79e 100644 --- a/syft/event/cataloger_task.go +++ b/syft/event/monitor/cataloger_task.go @@ -1,12 +1,15 @@ -package event +package monitor import ( "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/syft/event" ) +// TODO: this should be refactored to support read-only/write-only access using idioms of the progress lib + type CatalogerTask struct { prog *progress.Manual // Title @@ -25,7 +28,7 @@ func (e *CatalogerTask) init() { e.prog = progress.NewManual(-1) bus.Publish(partybus.Event{ - Type: CatalogerTaskStarted, + Type: event.CatalogerTaskStarted, Source: e, }) } diff --git a/syft/event/monitor/generic_task.go b/syft/event/monitor/generic_task.go index 6deb31e368a..cf5a6ea6df2 100644 --- a/syft/event/monitor/generic_task.go +++ b/syft/event/monitor/generic_task.go @@ -8,7 +8,7 @@ import ( type ShellProgress struct { io.Reader - *progress.Manual + progress.Progressable } type Title struct { diff --git a/syft/event/parsers/parsers.go b/syft/event/parsers/parsers.go index 3d0c8bfb85b..2e6cbeca93b 100644 --- a/syft/event/parsers/parsers.go +++ b/syft/event/parsers/parsers.go @@ -23,7 +23,7 @@ type ErrBadPayload struct { } func (e *ErrBadPayload) Error() string { - return fmt.Sprintf("event='%s' has bad event payload field='%v': '%+v'", string(e.Type), e.Field, e.Value) + return fmt.Sprintf("event='%s' has bad event payload field=%q: %q", string(e.Type), e.Field, e.Value) } func newPayloadErr(t partybus.EventType, field string, value interface{}) error { @@ -111,12 +111,12 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress return path, prog, nil } -func ParseCatalogerTaskStarted(e partybus.Event) (*event.CatalogerTask, error) { +func ParseCatalogerTaskStarted(e partybus.Event) (*monitor.CatalogerTask, error) { if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil { return nil, err } - source, ok := e.Source.(*event.CatalogerTask) + source, ok := e.Source.(*monitor.CatalogerTask) if !ok { return nil, newPayloadErr(e.Type, "Source", e.Source) } @@ -124,8 +124,28 @@ func ParseCatalogerTaskStarted(e partybus.Event) (*event.CatalogerTask, error) { return source, nil } -func ParseExit(e partybus.Event) (func() error, error) { - if err := checkEventType(e.Type, event.Exit); err != nil { +func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) { + if err := checkEventType(e.Type, event.AttestationStarted); err != nil { + return nil, nil, nil, err + } + + source, ok := e.Source.(monitor.GenericTask) + if !ok { + return nil, nil, nil, newPayloadErr(e.Type, "Source", e.Source) + } + + sp, ok := e.Value.(*monitor.ShellProgress) + if !ok { + return nil, nil, nil, newPayloadErr(e.Type, "Value", e.Value) + } + + return sp.Reader, sp.Progressable, &source, nil +} + +// CLI event types + +func ParseCLIExit(e partybus.Event) (func() error, error) { + if err := checkEventType(e.Type, event.CLIExit); err != nil { return nil, err } @@ -137,8 +157,8 @@ func ParseExit(e partybus.Event) (func() error, error) { return fn, nil } -func ParseAppUpdateAvailable(e partybus.Event) (string, error) { - if err := checkEventType(e.Type, event.AppUpdateAvailable); err != nil { +func ParseCLIAppUpdateAvailable(e partybus.Event) (string, error) { + if err := checkEventType(e.Type, event.CLIAppUpdateAvailable); err != nil { return "", err } @@ -150,38 +170,40 @@ func ParseAppUpdateAvailable(e partybus.Event) (string, error) { return newVersion, nil } -func ParseImportStarted(e partybus.Event) (string, progress.StagedProgressable, error) { - if err := checkEventType(e.Type, event.ImportStarted); err != nil { - return "", nil, err +func ParseCLIReport(e partybus.Event) (string, string, error) { + if err := checkEventType(e.Type, event.CLIReport); err != nil { + return "", "", err } - host, ok := e.Source.(string) + context, ok := e.Source.(string) if !ok { - return "", nil, newPayloadErr(e.Type, "Source", e.Source) + // this is optional + context = "" } - prog, ok := e.Value.(progress.StagedProgressable) + report, ok := e.Value.(string) if !ok { - return "", nil, newPayloadErr(e.Type, "Value", e.Value) + return "", "", newPayloadErr(e.Type, "Value", e.Value) } - return host, prog, nil + return context, report, nil } -func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) { - if err := checkEventType(e.Type, event.AttestationStarted); err != nil { - return nil, nil, nil, err +func ParseCLINotification(e partybus.Event) (string, string, error) { + if err := checkEventType(e.Type, event.CLINotification); err != nil { + return "", "", err } - source, ok := e.Source.(monitor.GenericTask) + context, ok := e.Source.(string) if !ok { - return nil, nil, nil, newPayloadErr(e.Type, "Source", e.Source) + // this is optional + context = "" } - sp, ok := e.Value.(*monitor.ShellProgress) + notification, ok := e.Value.(string) if !ok { - return nil, nil, nil, newPayloadErr(e.Type, "Value", e.Value) + return "", "", newPayloadErr(e.Type, "Value", e.Value) } - return sp.Reader, sp.Manual, &source, nil + return context, notification, nil } diff --git a/syft/internal/fileresolver/directory_indexer.go b/syft/internal/fileresolver/directory_indexer.go index 01f332a3de6..c9b5567a93d 100644 --- a/syft/internal/fileresolver/directory_indexer.go +++ b/syft/internal/fileresolver/directory_indexer.go @@ -332,14 +332,15 @@ func (r directoryIndexer) addFileToIndex(p string, info os.FileInfo) error { func (r directoryIndexer) addSymlinkToIndex(p string, info os.FileInfo) (string, error) { linkTarget, err := os.Readlink(p) if err != nil { - if runtime.GOOS == WindowsOS { - p = posixToWindows(p) + isOnWindows := windows.HostRunningOnWindows() + if isOnWindows { + p = windows.FromPosix(p) } linkTarget, err = filepath.EvalSymlinks(p) - if runtime.GOOS == WindowsOS { - p = windowsToPosix(p) + if isOnWindows { + p = windows.ToPosix(p) } if err != nil { diff --git a/syft/lib.go b/syft/lib.go index 849584ab728..b4530701218 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -94,10 +94,10 @@ func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []a // SetLogger sets the logger object used for all syft logging calls. func SetLogger(logger logger.Logger) { - log.Log = logger + log.Set(logger) } // SetBus sets the event bus for all syft library bus publish events onto (in-library subscriptions are not allowed). func SetBus(b *partybus.Bus) { - bus.SetPublisher(b) + bus.Set(b) } diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index bde2a9b5715..ee936da9682 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -6,7 +6,7 @@ package golang import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" @@ -37,7 +37,7 @@ func NewGoModuleBinaryCataloger(opts GoCatalogerOpts) pkg.Cataloger { } type progressingCataloger struct { - progress *event.CatalogerTask + progress *monitor.CatalogerTask cataloger *generic.Cataloger } diff --git a/syft/pkg/cataloger/golang/licenses.go b/syft/pkg/cataloger/golang/licenses.go index 829a73dd3f3..cce84772e2f 100644 --- a/syft/pkg/cataloger/golang/licenses.go +++ b/syft/pkg/cataloger/golang/licenses.go @@ -21,7 +21,7 @@ import ( "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/pkg" @@ -30,14 +30,14 @@ import ( type goLicenses struct { opts GoCatalogerOpts localModCacheResolver file.WritableResolver - progress *event.CatalogerTask + progress *monitor.CatalogerTask } func newGoLicenses(opts GoCatalogerOpts) goLicenses { return goLicenses{ opts: opts, localModCacheResolver: modCacheResolver(opts.localModCacheDir), - progress: &event.CatalogerTask{ + progress: &monitor.CatalogerTask{ SubStatus: true, RemoveOnCompletion: true, Title: "Downloading go mod", @@ -195,7 +195,7 @@ func processCaps(s string) string { }) } -func getModule(progress *event.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) { +func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) { for _, proxy := range proxies { u, _ := url.Parse(proxy) if proxy == "direct" { @@ -217,7 +217,7 @@ func getModule(progress *event.CatalogerTask, proxies []string, moduleName, modu return } -func getModuleProxy(progress *event.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) { +func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) { u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion) progress.SetValue(u) // get the module zip @@ -265,7 +265,7 @@ func findVersionPath(f fs.FS, dir string) string { return "" } -func getModuleRepository(progress *event.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) { +func getModuleRepository(progress *monitor.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) { repoName := moduleName parts := strings.Split(moduleName, "/") if len(parts) > 2 { diff --git a/syft/pkg/cataloger/rust/parse_audit_binary.go b/syft/pkg/cataloger/rust/parse_audit_binary.go index de894006b56..5b74389281e 100644 --- a/syft/pkg/cataloger/rust/parse_audit_binary.go +++ b/syft/pkg/cataloger/rust/parse_audit_binary.go @@ -49,8 +49,7 @@ func parseAuditBinaryEntry(reader unionreader.UnionReader, filename string) []ru // binary, we should not show warnings/logs in this case. return nil } - // Use an Info level log here like golang/scan_bin.go - log.Infof("rust cataloger: unable to read dependency information (file=%q): %v", filename, err) + log.Tracef("rust cataloger: unable to read dependency information (file=%q): %v", filename, err) return nil } diff --git a/ui/event_handlers.go b/ui/deprecated.go similarity index 84% rename from ui/event_handlers.go rename to ui/deprecated.go index 66f6277d00e..fb542ca5d5c 100644 --- a/ui/event_handlers.go +++ b/ui/deprecated.go @@ -1,3 +1,8 @@ +/* +Package ui provides all public UI elements intended to be repurposed in other applications. Specifically, a single +Handler object is provided to allow consuming applications (such as grype) to check if there are UI elements the handler +can respond to (given a specific event type) and handle the event in context of the given screen frame object. +*/ package ui import ( @@ -18,15 +23,16 @@ import ( "github.com/wagoodman/go-progress/format" "github.com/wagoodman/jotframe/pkg/frame" + stereoscopeEvent "github.com/anchore/stereoscope/pkg/event" stereoEventParsers "github.com/anchore/stereoscope/pkg/event/parsers" "github.com/anchore/stereoscope/pkg/image/docker" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/ui/components" + syftEvent "github.com/anchore/syft/syft/event" syftEventParsers "github.com/anchore/syft/syft/event/parsers" ) const maxBarWidth = 50 -const statusSet = components.SpinnerDotSet +const statusSet = SpinnerDotSet const completedStatus = "✔" const failedStatus = "✘" const titleFormat = color.Bold @@ -46,16 +52,118 @@ var ( subStatusTitleTemplate = fmt.Sprintf(" └── %%-%ds ", StatusTitleColumn-3) ) +// Handler is an aggregated event handler for the set of supported events (PullDockerImage, ReadImage, FetchImage, PackageCatalogerStarted) +// Deprecated: use the bubbletea event handler in cmd/syft/ui/handler.go instead. +type Handler struct { +} + +// NewHandler returns an empty Handler +// Deprecated: use the bubbletea event handler in cmd/syft/ui/handler.go instead. +func NewHandler() *Handler { + return &Handler{} +} + +// RespondsTo indicates if the handler is capable of handling the given event. +// Deprecated: use the bubbletea event handler in cmd/syft/ui/handler.go instead. +func (r *Handler) RespondsTo(event partybus.Event) bool { + switch event.Type { + case stereoscopeEvent.PullDockerImage, + stereoscopeEvent.ReadImage, + stereoscopeEvent.FetchImage, + syftEvent.PackageCatalogerStarted, + syftEvent.SecretsCatalogerStarted, + syftEvent.FileDigestsCatalogerStarted, + syftEvent.FileMetadataCatalogerStarted, + syftEvent.FileIndexingStarted, + syftEvent.AttestationStarted, + syftEvent.CatalogerTaskStarted: + return true + default: + return false + } +} + +// Handle calls the specific event handler for the given event within the context of the screen frame. +// Deprecated: use the bubbletea event handler in cmd/syft/ui/handler.go instead. +func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { + switch event.Type { + case stereoscopeEvent.PullDockerImage: + return PullDockerImageHandler(ctx, fr, event, wg) + + case stereoscopeEvent.ReadImage: + return ReadImageHandler(ctx, fr, event, wg) + + case stereoscopeEvent.FetchImage: + return FetchImageHandler(ctx, fr, event, wg) + + case syftEvent.PackageCatalogerStarted: + return PackageCatalogerStartedHandler(ctx, fr, event, wg) + + case syftEvent.SecretsCatalogerStarted: + return SecretsCatalogerStartedHandler(ctx, fr, event, wg) + + case syftEvent.FileDigestsCatalogerStarted: + return FileDigestsCatalogerStartedHandler(ctx, fr, event, wg) + + case syftEvent.FileMetadataCatalogerStarted: + return FileMetadataCatalogerStartedHandler(ctx, fr, event, wg) + + case syftEvent.FileIndexingStarted: + return FileIndexingStartedHandler(ctx, fr, event, wg) + + case syftEvent.AttestationStarted: + return AttestationStartedHandler(ctx, fr, event, wg) + + case syftEvent.CatalogerTaskStarted: + return CatalogerTaskStartedHandler(ctx, fr, event, wg) + } + return nil +} + +const ( + SpinnerDotSet = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏" +) + +type spinner struct { + index int + charset []string + lock sync.Mutex +} + +func newSpinner(charset string) spinner { + return spinner{ + charset: strings.Split(charset, ""), + } +} + +func (s *spinner) Current() string { + s.lock.Lock() + defer s.lock.Unlock() + + return s.charset[s.index] +} + +func (s *spinner) Next() string { + s.lock.Lock() + defer s.lock.Unlock() + c := s.charset[s.index] + s.index++ + if s.index >= len(s.charset) { + s.index = 0 + } + return c +} + // startProcess is a helper function for providing common elements for long-running UI elements (such as a // progress bar formatter and status spinner) -func startProcess() (format.Simple, *components.Spinner) { +func startProcess() (format.Simple, *spinner) { width, _ := frame.GetTerminalSize() barWidth := int(0.25 * float64(width)) if barWidth > maxBarWidth { barWidth = maxBarWidth } formatter := format.NewSimpleWithTheme(barWidth, format.HeavyNoBarTheme, format.ColorCompleted, format.ColorTodo) - spinner := components.NewSpinner(statusSet) + spinner := newSpinner(statusSet) return formatter, &spinner } @@ -82,7 +190,7 @@ func formatDockerPullPhase(phase docker.PullPhase, inputStr string) string { } // formatDockerImagePullStatus writes the docker image pull status summarized into a single line for the given state. -func formatDockerImagePullStatus(pullStatus *docker.PullStatus, spinner *components.Spinner, line *frame.Line) { +func formatDockerImagePullStatus(pullStatus *docker.PullStatus, spinner *spinner, line *frame.Line) { var size, current uint64 title := titleFormat.Sprint("Pulling image") @@ -491,50 +599,6 @@ func FileDigestsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, ev return err } -// ImportStartedHandler shows the intermittent upload progress to Anchore Enterprise. -func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { - host, prog, err := syftEventParsers.ParseImportStarted(event) - if err != nil { - return fmt.Errorf("bad %s event: %w", event.Type, err) - } - - line, err := fr.Append() - if err != nil { - return err - } - wg.Add(1) - - formatter, spinner := startProcess() - stream := progress.Stream(ctx, prog, interval) - title := titleFormat.Sprint("Uploading image") - - formatFn := func(p progress.Progress) { - progStr, err := formatter.Format(p) - spin := color.Magenta.Sprint(spinner.Next()) - if err != nil { - _, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err)) - } else { - auxInfo := auxInfoFormat.Sprintf("[%s]", prog.Stage()) - _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s %s", spin, title, progStr, auxInfo)) - } - } - - go func() { - defer wg.Done() - - formatFn(progress.Progress{}) - for p := range stream { - formatFn(p) - } - - spin := color.Green.Sprint(completedStatus) - title = titleFormat.Sprint("Uploaded image") - auxInfo := auxInfoFormat.Sprintf("[%s]", host) - _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo)) - }() - return err -} - // AttestationStartedHandler takes bytes from a event.ShellOutput and publishes them to the frame. // //nolint:funlen,gocognit diff --git a/ui/handler.go b/ui/handler.go deleted file mode 100644 index b5e3a8fd692..00000000000 --- a/ui/handler.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Package ui provides all public UI elements intended to be repurposed in other applications. Specifically, a single -Handler object is provided to allow consuming applications (such as grype) to check if there are UI elements the handler -can respond to (given a specific event type) and handle the event in context of the given screen frame object. -*/ -package ui - -import ( - "context" - "sync" - - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/jotframe/pkg/frame" - - stereoscopeEvent "github.com/anchore/stereoscope/pkg/event" - syftEvent "github.com/anchore/syft/syft/event" -) - -// Handler is an aggregated event handler for the set of supported events (PullDockerImage, ReadImage, FetchImage, PackageCatalogerStarted) -type Handler struct { -} - -// NewHandler returns an empty Handler -func NewHandler() *Handler { - return &Handler{} -} - -// RespondsTo indicates if the handler is capable of handling the given event. -func (r *Handler) RespondsTo(event partybus.Event) bool { - switch event.Type { - case stereoscopeEvent.PullDockerImage, - stereoscopeEvent.ReadImage, - stereoscopeEvent.FetchImage, - syftEvent.PackageCatalogerStarted, - syftEvent.SecretsCatalogerStarted, - syftEvent.FileDigestsCatalogerStarted, - syftEvent.FileMetadataCatalogerStarted, - syftEvent.FileIndexingStarted, - syftEvent.ImportStarted, - syftEvent.AttestationStarted, - syftEvent.CatalogerTaskStarted: - return true - default: - return false - } -} - -// Handle calls the specific event handler for the given event within the context of the screen frame. -func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { - switch event.Type { - case stereoscopeEvent.PullDockerImage: - return PullDockerImageHandler(ctx, fr, event, wg) - - case stereoscopeEvent.ReadImage: - return ReadImageHandler(ctx, fr, event, wg) - - case stereoscopeEvent.FetchImage: - return FetchImageHandler(ctx, fr, event, wg) - - case syftEvent.PackageCatalogerStarted: - return PackageCatalogerStartedHandler(ctx, fr, event, wg) - - case syftEvent.SecretsCatalogerStarted: - return SecretsCatalogerStartedHandler(ctx, fr, event, wg) - - case syftEvent.FileDigestsCatalogerStarted: - return FileDigestsCatalogerStartedHandler(ctx, fr, event, wg) - - case syftEvent.FileMetadataCatalogerStarted: - return FileMetadataCatalogerStartedHandler(ctx, fr, event, wg) - - case syftEvent.FileIndexingStarted: - return FileIndexingStartedHandler(ctx, fr, event, wg) - - case syftEvent.ImportStarted: - return ImportStartedHandler(ctx, fr, event, wg) - - case syftEvent.AttestationStarted: - return AttestationStartedHandler(ctx, fr, event, wg) - - case syftEvent.CatalogerTaskStarted: - return CatalogerTaskStartedHandler(ctx, fr, event, wg) - } - return nil -} From 8ce88e11fdcd2356309370e6a734d7eacf710032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:02:44 -0400 Subject: [PATCH 10/17] chore(deps): bump golang.org/x/net from 0.11.0 to 0.12.0 (#1916) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.11.0 to 0.12.0. - [Commits](https://github.com/golang/net/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 03c6a45d1a3..6c5fd681676 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/mod v0.12.0 - golang.org/x/net v0.11.0 + golang.org/x/net v0.12.0 golang.org/x/term v0.10.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -171,7 +171,7 @@ require ( github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect @@ -195,7 +195,7 @@ require ( // go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption github.com/andybalholm/brotli v1.0.4 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/crypto v0.10.0 // indirect + golang.org/x/crypto v0.11.0 // indirect ) retract ( diff --git a/go.sum b/go.sum index 693beaee4e6..0de4c414fc4 100644 --- a/go.sum +++ b/go.sum @@ -731,8 +731,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -824,8 +824,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -960,8 +960,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 81d8019207ccc7dc44462b7c95bb973dbd82e07c Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Thu, 6 Jul 2023 16:12:55 -0400 Subject: [PATCH 11/17] Remove erroneous Java CPEs from generation (#1918) Signed-off-by: Dan Luhring --- .../common/cpe/candidate_by_package_type.go | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go b/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go index 5481108f852..875b6dad02c 100644 --- a/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go +++ b/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go @@ -377,6 +377,90 @@ var defaultCandidateRemovals = buildCandidateRemovalLookup( candidateKey{PkgName: "docker"}, candidateRemovals{VendorsToRemove: []string{"docker"}}, }, + // Java packages + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-builder-support"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-model"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-repository-metadata"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-settings"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-settings-builder"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-api"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-connector-basic"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-impl"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-named-locks"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-spi"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-transport-file"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-transport-http"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-transport-wagon"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-resolver-util"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "maven-shared-utils"}, + candidateRemovals{ProductsToRemove: []string{"maven"}}, + }, + { + pkg.JavaPkg, + candidateKey{PkgName: "gradle-enterprise"}, + candidateRemovals{ + ProductsToRemove: []string{"gradle-enterprise"}, + VendorsToRemove: []string{"gradle"}, + }, + }, }) // buildCandidateLookup is a convenience function for creating the defaultCandidateAdditions set From 376c42893b38a68e9703470d9e625bf98612a1d4 Mon Sep 17 00:00:00 2001 From: Lorenzo Orsatti <49567430+lorsatti@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:56:07 +0200 Subject: [PATCH 12/17] fix(install): return with right error code (#1915) This resolves #1566. Signed-off-by: Lorenzo Orsatti <49567430+lorsatti@users.noreply.github.com> Co-authored-by: Christopher Phillips --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index f118257ed68..9d49f9c8cb2 100755 --- a/install.sh +++ b/install.sh @@ -694,6 +694,6 @@ main() ( set +u if [ -z "${TEST_INSTALL_SH}" ]; then set -u - main "$@" + exit $(main "$@") fi set -u \ No newline at end of file From c0c089ffd5a5560859f49fe3e968f4850b4e6ae9 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Mon, 10 Jul 2023 10:24:42 -0400 Subject: [PATCH 13/17] fix: Don't use the actual redis or grpc CPEs for gems (#1926) Signed-off-by: Dan Luhring --- .../cataloger/common/cpe/candidate_by_package_type.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go b/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go index 875b6dad02c..6588ea8bb18 100644 --- a/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go +++ b/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go @@ -461,6 +461,17 @@ var defaultCandidateRemovals = buildCandidateRemovalLookup( VendorsToRemove: []string{"gradle"}, }, }, + // Ruby packages + { + pkg.GemPkg, + candidateKey{PkgName: "redis"}, + candidateRemovals{ProductsToRemove: []string{"redis"}}, + }, + { + pkg.GemPkg, + candidateKey{PkgName: "grpc"}, + candidateRemovals{ProductsToRemove: []string{"grpc"}}, + }, }) // buildCandidateLookup is a convenience function for creating the defaultCandidateAdditions set From d5d95da3b609f76497e4918b8d43a78323cb6d0d Mon Sep 17 00:00:00 2001 From: "anchore-actions-token-generator[bot]" <102182147+anchore-actions-token-generator[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:03:09 -0400 Subject: [PATCH 14/17] chore(deps): update bootstrap tools to latest versions (#1922) Signed-off-by: GitHub Co-authored-by: spiffcs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e2f0707e97d..5e2d700a004 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ GOLANGCILINT_VERSION := v1.53.3 GOSIMPORTS_VERSION := v0.3.8 BOUNCER_VERSION := v0.4.0 CHRONICLE_VERSION := v0.6.0 -GORELEASER_VERSION := v1.19.1 +GORELEASER_VERSION := v1.19.2 YAJSV_VERSION := v1.4.1 COSIGN_VERSION := v2.1.1 QUILL_VERSION := v0.2.0 From d21fa843352f88154cc8d20d56dc22f07dbfe46f Mon Sep 17 00:00:00 2001 From: Christopher Angelo Phillips <32073428+spiffcs@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:44:54 -0400 Subject: [PATCH 15/17] chore: update iterations to protect against race (#1927) * chore: update iterations to protect against race --------- Signed-off-by: Christopher Phillips --- cmd/syft/cli/ui/handle_attestation_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/syft/cli/ui/handle_attestation_test.go b/cmd/syft/cli/ui/handle_attestation_test.go index 6fcc9fbfea2..12f6bf4c314 100644 --- a/cmd/syft/cli/ui/handle_attestation_test.go +++ b/cmd/syft/cli/ui/handle_attestation_test.go @@ -28,7 +28,7 @@ func TestHandler_handleAttestationStarted(t *testing.T) { // note: this model depends on a background reader. Multiple iterations ensures that the // reader has time to at least start and process the test fixture before the runModel // test harness completes (which is a fake event loop anyway). - iterations: 2, + iterations: 100, eventFn: func(t *testing.T) partybus.Event { reader := strings.NewReader("contents\nof\nstuff!") @@ -61,7 +61,7 @@ func TestHandler_handleAttestationStarted(t *testing.T) { // note: this model depends on a background reader. Multiple iterations ensures that the // reader has time to at least start and process the test fixture before the runModel // test harness completes (which is a fake event loop anyway). - iterations: 2, + iterations: 100, eventFn: func(t *testing.T) partybus.Event { reader := strings.NewReader("contents\nof\nstuff!") From 9744f4c009e84a675815f3d718fec0c8480c9e3d Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Mon, 10 Jul 2023 11:54:19 -0400 Subject: [PATCH 16/17] Fix CPE gen for k8s python client (#1921) Signed-off-by: Dan Luhring Co-authored-by: Christopher Phillips --- syft/pkg/cataloger/common/cpe/candidate_by_package_type.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go b/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go index 6588ea8bb18..bc62d390984 100644 --- a/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go +++ b/syft/pkg/cataloger/common/cpe/candidate_by_package_type.go @@ -356,6 +356,11 @@ var defaultCandidateRemovals = buildCandidateRemovalLookup( candidateKey{PkgName: "redis"}, candidateRemovals{VendorsToRemove: []string{"redis"}}, }, + { + pkg.PythonPkg, + candidateKey{PkgName: "kubernetes"}, + candidateRemovals{ProductsToRemove: []string{"kubernetes"}}, + }, // NPM packages { pkg.NpmPkg, From 4ab9f393fc4f20db3f7c48f9e82341e500b9e4bf Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Mon, 10 Jul 2023 20:36:41 +0300 Subject: [PATCH 17/17] feat: CLI flag for directory base (#1867) Signed-off-by: Avi Deitcher Signed-off-by: Keith Zantow Co-authored-by: Keith Zantow --- cmd/syft/cli/attest/attest.go | 1 + cmd/syft/cli/options/packages.go | 8 ++++++++ cmd/syft/cli/packages/packages.go | 1 + cmd/syft/cli/poweruser/poweruser.go | 1 + internal/config/application.go | 1 + syft/source/detection.go | 7 ++++++- 6 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cmd/syft/cli/attest/attest.go b/cmd/syft/cli/attest/attest.go index 1d0dddf1c1c..758af670635 100644 --- a/cmd/syft/cli/attest/attest.go +++ b/cmd/syft/cli/attest/attest.go @@ -99,6 +99,7 @@ func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbo Paths: app.Exclusions, }, DigestAlgorithms: hashers, + BasePath: app.BasePath, }, ) diff --git a/cmd/syft/cli/options/packages.go b/cmd/syft/cli/options/packages.go index bea013b6922..259a787811c 100644 --- a/cmd/syft/cli/options/packages.go +++ b/cmd/syft/cli/options/packages.go @@ -23,6 +23,7 @@ type PackagesOptions struct { Catalogers []string SourceName string SourceVersion string + BasePath string } var _ Interface = (*PackagesOptions)(nil) @@ -59,6 +60,9 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error { cmd.Flags().StringVarP(&o.SourceVersion, "source-version", "", "", "set the name of the target being analyzed") + cmd.Flags().StringVarP(&o.BasePath, "base-path", "", "", + "base directory for scanning, no links will be followed above this directory, and all paths will be reported relative to this directory") + return bindPackageConfigOptions(cmd.Flags(), v) } @@ -106,5 +110,9 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error { return err } + if err := v.BindPFlag("base-path", flags.Lookup("base-path")); err != nil { + return err + } + return nil } diff --git a/cmd/syft/cli/packages/packages.go b/cmd/syft/cli/packages/packages.go index a84b3c09af1..a7c1d5521dc 100644 --- a/cmd/syft/cli/packages/packages.go +++ b/cmd/syft/cli/packages/packages.go @@ -97,6 +97,7 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < Paths: app.Exclusions, }, DigestAlgorithms: hashers, + BasePath: app.BasePath, }, ) diff --git a/cmd/syft/cli/poweruser/poweruser.go b/cmd/syft/cli/poweruser/poweruser.go index e9a251f3e26..cfc10e1bcc1 100644 --- a/cmd/syft/cli/poweruser/poweruser.go +++ b/cmd/syft/cli/poweruser/poweruser.go @@ -103,6 +103,7 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) < Paths: app.Exclusions, }, DigestAlgorithms: nil, + BasePath: app.BasePath, }, ) diff --git a/internal/config/application.go b/internal/config/application.go index ea85410025f..e7c726134d8 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -64,6 +64,7 @@ type Application struct { Source sourceCfg `yaml:"source" json:"source" mapstructure:"source"` Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel DefaultImagePullSource string `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"` // specify default image pull source + BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths } func (cfg Application) ToCatalogerConfig() cataloger.Config { diff --git a/syft/source/detection.go b/syft/source/detection.go index 3d301f14da5..f96dc0023f5 100644 --- a/syft/source/detection.go +++ b/syft/source/detection.go @@ -87,6 +87,7 @@ type DetectionSourceConfig struct { Platform *image.Platform Exclude ExcludeConfig DigestAlgorithms []crypto.Hash + BasePath string } func DefaultDetectionSourceConfig() DetectionSourceConfig { @@ -117,10 +118,14 @@ func (d Detection) NewSource(cfg DetectionSourceConfig) (Source, error) { }, ) case directoryType: + base := cfg.BasePath + if base == "" { + base = d.location + } src, err = NewFromDirectory( DirectoryConfig{ Path: d.location, - Base: d.location, + Base: base, Exclude: cfg.Exclude, Alias: cfg.Alias, },