Refactor source API (#1846)

* refactor source API and syft json source block Signed-off-by: Alex Goodman <[email protected]> * update source detection and format test utils Signed-off-by: Alex Goodman <[email protected]> * generate list of all source metadata types Signed-off-by: Alex Goodman <[email protected]> * extract base and root normalization into helper functions Signed-off-by: Alex Goodman <[email protected]> * preserve syftjson model package name import ref Signed-off-by: Alex Goodman <[email protected]> * alias should not be a pointer Signed-off-by: Alex Goodman <[email protected]> --------- Signed-off-by: Alex Goodman <[email protected]> Signed-off-by: Alex Goodman <[email protected]>
anchore · Jun 30, 2023 · 4da3be8 · 4da3be8
1 parent 608dbde
commit 4da3be8
Show file tree

Hide file tree

Showing 126 changed files with 7,384 additions and 3,190 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+/.bin
 CHANGELOG.md
 VERSION
 /test/results

diff --git a/Makefile b/Makefile
@@ -302,7 +302,7 @@ compare-test-rpm-package-install: $(TEMP_DIR) $(SNAPSHOT_DIR)
 
 .PHONY: generate-json-schema
 generate-json-schema:  ## Generate a new json schema
-	cd schema/json && go generate . && go run .
+	cd syft/internal && go generate . && cd jsonschema && go run .
 
 .PHONY: generate-license-list
 generate-license-list:  ## Generate an updated spdx license list

diff --git a/cmd/syft/cli/attest/attest.go b/cmd/syft/cli/attest/attest.go
@@ -12,6 +12,7 @@ import (
 	"golang.org/x/exp/slices"
 
 	"github.com/anchore/stereoscope"
+	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/cmd/syft/cli/eventloop"
 	"github.com/anchore/syft/cmd/syft/cli/options"
 	"github.com/anchore/syft/cmd/syft/cli/packages"
@@ -34,39 +35,65 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 		return err
 	}
 
-	// could be an image or a directory, with or without a scheme
-	// TODO: validate that source is image
+	// note: must be a container image
 	userInput := args[0]
-	si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
-	if err != nil {
-		return fmt.Errorf("could not generate source input for packages command: %w", err)
-	}
-
-	if si.Scheme != source.ImageScheme {
-		return fmt.Errorf("attestations are only supported for oci images at this time")
-	}
 
 	eventBus := partybus.NewBus()
 	stereoscope.SetBus(eventBus)
 	syft.SetBus(eventBus)
 	subscription := eventBus.Subscribe()
 
 	return eventloop.EventLoop(
-		execWorker(app, *si),
+		execWorker(app, userInput),
 		eventloop.SetupSignals(),
 		subscription,
 		stereoscope.Cleanup,
 		ui.Select(options.IsVerbose(app), app.Quiet)...,
 	)
 }
 
-func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom.SBOM, error) {
-	src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
-	if cleanup != nil {
-		defer cleanup()
+func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbom.SBOM, error) {
+	cfg := source.DetectConfig{
+		DefaultImageSource: app.DefaultImagePullSource,
+	}
+	detection, err := source.Detect(userInput, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("could not deteremine source: %w", err)
+	}
+
+	if detection.IsContainerImage() {
+		return nil, fmt.Errorf("attestations are only supported for oci images at this time")
+	}
+
+	var platform *image.Platform
+
+	if app.Platform != "" {
+		platform, err = image.NewPlatform(app.Platform)
+		if err != nil {
+			return nil, fmt.Errorf("invalid platform: %w", err)
+		}
+	}
+
+	src, err := detection.NewSource(
+		source.DetectionSourceConfig{
+			Alias: source.Alias{
+				Name:    app.SourceName,
+				Version: app.SourceVersion,
+			},
+			RegistryOptions: app.Registry.ToOptions(),
+			Platform:        platform,
+			Exclude: source.ExcludeConfig{
+				Paths: app.Exclusions,
+			},
+			DigestAlgorithms: nil,
+		},
+	)
+
+	if src != nil {
+		defer src.Close()
 	}
 	if err != nil {
-		return nil, fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
+		return nil, fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
 	}
 
 	s, err := packages.GenerateSBOM(src, errs, app)
@@ -75,20 +102,20 @@ func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom
 	}
 
 	if s == nil {
-		return nil, fmt.Errorf("no SBOM produced for %q", si.UserInput)
+		return nil, fmt.Errorf("no SBOM produced for %q", userInput)
 	}
 
 	return s, nil
 }
 
 //nolint:funlen
-func execWorker(app *config.Application, si source.Input) <-chan error {
+func execWorker(app *config.Application, userInput string) <-chan error {
 	errs := make(chan error)
 	go func() {
 		defer close(errs)
 		defer bus.Publish(partybus.Event{Type: event.Exit})
 
-		s, err := buildSBOM(app, si, errs)
+		s, err := buildSBOM(app, userInput, errs)
 		if err != nil {
 			errs <- fmt.Errorf("unable to build SBOM: %w", err)
 			return
@@ -136,7 +163,7 @@ func execWorker(app *config.Application, si source.Input) <-chan error {
 			predicateType = "custom"
 		}
 
-		args := []string{"attest", si.UserInput, "--predicate", f.Name(), "--type", predicateType}
+		args := []string{"attest", userInput, "--predicate", f.Name(), "--type", predicateType}
 		if app.Attest.Key != "" {
 			args = append(args, "--key", app.Attest.Key)
 		}

diff --git a/cmd/syft/cli/eventloop/tasks.go b/cmd/syft/cli/eventloop/tasks.go
@@ -16,7 +16,7 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-type Task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
+type Task func(*sbom.Artifacts, source.Source) ([]artifact.Relationship, error)
 
 func Tasks(app *config.Application) ([]Task, error) {
 	var tasks []Task
@@ -48,7 +48,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) {
 		return nil, nil
 	}
 
-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig())
 
 		results.Packages = packageCatalog
@@ -67,7 +67,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) {
 
 	metadataCataloger := filemetadata.NewCataloger()
 
-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -110,7 +110,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {
 
 	digestsCataloger := filedigest.NewCataloger(hashes)
 
-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -142,7 +142,7 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
 		return nil, err
 	}
 
-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.Secrets.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -169,7 +169,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
 		return nil, err
 	}
 
-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.FileContents.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -186,7 +186,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
 	return task, nil
 }
 
-func RunTask(t Task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
+func RunTask(t Task, a *sbom.Artifacts, src source.Source, c chan<- artifact.Relationship, errs chan<- error) {
 	defer close(c)
 
 	relationships, err := t(a, src)

diff --git a/cmd/syft/cli/packages/packages.go b/cmd/syft/cli/packages/packages.go
@@ -7,6 +7,7 @@ import (
 	"github.com/wagoodman/go-partybus"
 
 	"github.com/anchore/stereoscope"
+	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/cmd/syft/cli/eventloop"
 	"github.com/anchore/syft/cmd/syft/cli/options"
 	"github.com/anchore/syft/internal"
@@ -35,36 +36,67 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 
 	// could be an image or a directory, with or without a scheme
 	userInput := args[0]
-	si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
-	if err != nil {
-		return fmt.Errorf("could not generate source input for packages command: %w", err)
-	}
 
 	eventBus := partybus.NewBus()
 	stereoscope.SetBus(eventBus)
 	syft.SetBus(eventBus)
 	subscription := eventBus.Subscribe()
 
 	return eventloop.EventLoop(
-		execWorker(app, *si, writer),
+		execWorker(app, userInput, writer),
 		eventloop.SetupSignals(),
 		subscription,
 		stereoscope.Cleanup,
 		ui.Select(options.IsVerbose(app), app.Quiet)...,
 	)
 }
 
-func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error {
+func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error {
 	errs := make(chan error)
 	go func() {
 		defer close(errs)
 
-		src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
-		if cleanup != nil {
-			defer cleanup()
+		detection, err := source.Detect(
+			userInput,
+			source.DetectConfig{
+				DefaultImageSource: app.DefaultImagePullSource,
+			},
+		)
+		if err != nil {
+			errs <- fmt.Errorf("could not deteremine source: %w", err)
+			return
+		}
+
+		var platform *image.Platform
+
+		if app.Platform != "" {
+			platform, err = image.NewPlatform(app.Platform)
+			if err != nil {
+				errs <- fmt.Errorf("invalid platform: %w", err)
+				return
+			}
+		}
+
+		src, err := detection.NewSource(
+			source.DetectionSourceConfig{
+				Alias: source.Alias{
+					Name:    app.SourceName,
+					Version: app.SourceVersion,
+				},
+				RegistryOptions: app.Registry.ToOptions(),
+				Platform:        platform,
+				Exclude: source.ExcludeConfig{
+					Paths: app.Exclusions,
+				},
+				DigestAlgorithms: nil,
+			},
+		)
+
+		if src != nil {
+			defer src.Close()
 		}
 		if err != nil {
-			errs <- fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
+			errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
 			return
 		}
 
@@ -75,7 +107,7 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
 		}
 
 		if s == nil {
-			errs <- fmt.Errorf("no SBOM produced for %q", si.UserInput)
+			errs <- fmt.Errorf("no SBOM produced for %q", userInput)
 		}
 
 		bus.Publish(partybus.Event{
@@ -86,14 +118,14 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
 	return errs
 }
 
-func GenerateSBOM(src *source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
+func GenerateSBOM(src source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
 	tasks, err := eventloop.Tasks(app)
 	if err != nil {
 		return nil, err
 	}
 
 	s := sbom.SBOM{
-		Source: src.Metadata,
+		Source: src.Describe(),
 		Descriptor: sbom.Descriptor{
 			Name:          internal.ApplicationName,
 			Version:       version.FromBuild().Version,
@@ -106,7 +138,7 @@ func GenerateSBOM(src *source.Source, errs chan error, app *config.Application)
 	return &s, nil
 }
 
-func buildRelationships(s *sbom.SBOM, src *source.Source, tasks []eventloop.Task, errs chan error) {
+func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task, errs chan error) {
 	var relationships []<-chan artifact.Relationship
 	for _, task := range tasks {
 		c := make(chan artifact.Relationship)