feat: --enrich flag for data enrichment feature enablement

cmd/syft/internal/options/catalog.go

@@ -36,6 +36,7 @@ type Catalog struct {
 	Scope             string              `yaml:"scope" json:"scope" mapstructure:"scope"`
 	Parallelism       int                 `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
 	Relationships     relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"`
+	Enrich            []string            `yaml:"enrich" json:"enrich" mapstructure:"enrich"`
 
 	// ecosystem-specific cataloger configuration
 	Golang      golangConfig      `yaml:"golang" json:"golang" mapstructure:"golang"`
@@ -132,7 +133,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
 		Golang: golang.DefaultCatalogerConfig().
 			WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses).
 			WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
-			WithSearchRemoteLicenses(cfg.Golang.SearchRemoteLicenses).
+			WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, "golang", "go"), cfg.Golang.SearchRemoteLicenses)).
 			WithProxy(cfg.Golang.Proxy).
 			WithNoProxy(cfg.Golang.NoProxy).
 			WithMainModuleVersion(
@@ -142,7 +143,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
 					WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags),
 			),
 		JavaScript: javascript.DefaultCatalogerConfig().
-			WithSearchRemoteLicenses(cfg.JavaScript.SearchRemoteLicenses).
+			WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, "javascript", "js"), cfg.JavaScript.SearchRemoteLicenses)).
 			WithNpmBaseURL(cfg.JavaScript.NpmBaseURL),
 		LinuxKernel: kernel.LinuxKernelCatalogerConfig{
 			CatalogModules: cfg.LinuxKernel.CatalogModules,
@@ -153,7 +154,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
 		JavaArchive: java.DefaultArchiveCatalogerConfig().
 			WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository).
 			WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
-			WithUseNetwork(cfg.Java.UseNetwork).
+			WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, "java", "maven"), cfg.Java.UseNetwork)).
 			WithMavenBaseURL(cfg.Java.MavenURL).
 			WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth),
 	}
@@ -193,6 +194,9 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
 	flags.StringArrayVarP(&cfg.SelectCatalogers, "select-catalogers", "",
 		"add, remove, and filter the catalogers to be used")
 
+	flags.StringArrayVarP(&cfg.Enrich, "enrich", "",
+		"enable data enrichment operations, which can utilize services such as Maven Central and NPM")
+
 	flags.StringVarP(&cfg.Source.Name, "source-name", "",
 		"set the name of the target being analyzed")
 
@@ -205,6 +209,9 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
 
 func (cfg *Catalog) DescribeFields(descriptions fangs.FieldDescriptionSet) {
 	descriptions.Add(&cfg.Parallelism, "number of cataloger workers to run in parallel")
+
+	descriptions.Add(&cfg.Enrich, `enable/disable data enrichment operations. By default all enrichment is disabled, use: all to enable everything.
+Prefixing with a minus will remove the enrichment, e.g. enrich: [all, -java] would enable all except java enrichment. `)
 }
 
 func (cfg *Catalog) PostLoad() error {
@@ -215,23 +222,12 @@ func (cfg *Catalog) PostLoad() error {
 		return fmt.Errorf("cannot use both 'catalogers' and 'select-catalogers'/'default-catalogers' flags")
 	}
 
-	flatten := func(l []string) []string {
-		var out []string
-		for _, v := range l {
-			for _, s := range strings.Split(v, ",") {
-				out = append(out, strings.TrimSpace(s))
-			}
-		}
-		sort.Strings(out)
-
-		return out
-	}
-
 	cfg.From = flatten(cfg.From)
 
 	cfg.Catalogers = flatten(cfg.Catalogers)
 	cfg.DefaultCatalogers = flatten(cfg.DefaultCatalogers)
 	cfg.SelectCatalogers = flatten(cfg.SelectCatalogers)
+	cfg.Enrich = flatten(cfg.Enrich)
 
 	// for backwards compatibility
 	cfg.DefaultCatalogers = append(cfg.DefaultCatalogers, cfg.Catalogers...)
@@ -243,3 +239,61 @@ func (cfg *Catalog) PostLoad() error {
 
 	return nil
 }
+
+func flatten(commaSeparatedEntries []string) []string {
+	var out []string
+	for _, v := range commaSeparatedEntries {
+		for _, s := range strings.Split(v, ",") {
+			out = append(out, strings.TrimSpace(s))
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+func enrichmentEnabled(enrichDirectives []string, features ...string) *bool {
+	if len(enrichDirectives) == 0 {
+		return nil
+	}
+
+	enabled := func(features ...string) *bool {
+		for _, directive := range enrichDirectives {
+			enable := true
+			directive = strings.TrimPrefix(directive, "+") // +java and java are equivalent
+			if strings.HasPrefix(directive, "-") {
+				directive = directive[1:]
+				enable = false
+			}
+			for _, feature := range features {
+				if directive == feature {
+					return &enable
+				}
+			}
+		}
+		return nil
+	}
+
+	enableAll := enabled("all")
+	disableAll := enabled("none")
+
+	if disableAll != nil {
+		if enableAll != nil {
+			log.Warn("you have specified to both enable and disable all network functionality, defaulting to disabled")
+		}
+		enableAll = ptr(!*disableAll)
+	}
+
+	// check for explicit enable/disable of feature names
+	for _, feat := range features {
+		enableFeature := enabled(feat)
+		if enableFeature != nil {
+			return enableFeature
+		}
+	}
+
+	return enableAll
+}
+
+func ptr[T any](val T) *T {
+	return &val
+}

cmd/syft/internal/options/catalog_test.go

@@ -70,3 +70,69 @@ func TestCatalog_PostLoad(t *testing.T) {
 		})
 	}
 }
+
+func Test_enrichmentEnabled(t *testing.T) {
+	tests := []struct {
+		directives string
+		test       string
+		expected   *bool
+	}{
+		{
+			directives: "",
+			test:       "java",
+			expected:   nil,
+		},
+		{
+			directives: "none",
+			test:       "java",
+			expected:   ptr(false),
+		},
+		{
+			directives: "none,+java",
+			test:       "java",
+			expected:   ptr(true),
+		},
+		{
+			directives: "all,none",
+			test:       "java",
+			expected:   ptr(false),
+		},
+		{
+			directives: "all",
+			test:       "java",
+			expected:   ptr(true),
+		},
+		{
+			directives: "golang,js",
+			test:       "java",
+			expected:   nil,
+		},
+		{
+			directives: "golang,-js,java",
+			test:       "java",
+			expected:   ptr(true),
+		},
+		{
+			directives: "golang,js,-java",
+			test:       "java",
+			expected:   ptr(false),
+		},
+		{
+			directives: "all",
+			test:       "java",
+			expected:   ptr(true),
+		},
+		{
+			directives: "all,-java",
+			test:       "java",
+			expected:   ptr(false),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.directives, func(t *testing.T) {
+			got := enrichmentEnabled(flatten([]string{test.directives}), test.test)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}

cmd/syft/internal/options/golang.go

@@ -10,7 +10,7 @@ import (
 type golangConfig struct {
 	SearchLocalModCacheLicenses bool                          `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
 	LocalModCacheDir            string                        `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
-	SearchRemoteLicenses        bool                          `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
+	SearchRemoteLicenses        *bool                         `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
 	Proxy                       string                        `json:"proxy" yaml:"proxy" mapstructure:"proxy"`
 	NoProxy                     string                        `json:"no-proxy" yaml:"no-proxy" mapstructure:"no-proxy"`
 	MainModuleVersion           golangMainModuleVersionConfig `json:"main-module-version" yaml:"main-module-version" mapstructure:"main-module-version"`
@@ -49,7 +49,7 @@ func defaultGolangConfig() golangConfig {
 	return golangConfig{
 		SearchLocalModCacheLicenses: def.SearchLocalModCacheLicenses,
 		LocalModCacheDir:            def.LocalModCacheDir,
-		SearchRemoteLicenses:        def.SearchRemoteLicenses,
+		SearchRemoteLicenses:        nil, // this defaults to false, which is the API default
 		Proxy:                       strings.Join(def.Proxies, ","),
 		NoProxy:                     strings.Join(def.NoProxy, ","),
 		MainModuleVersion: golangMainModuleVersionConfig{

cmd/syft/internal/options/java.go

@@ -6,7 +6,7 @@ import (
 )
 
 type javaConfig struct {
-	UseNetwork              bool   `yaml:"use-network" json:"use-network" mapstructure:"use-network"`
+	UseNetwork              *bool  `yaml:"use-network" json:"use-network" mapstructure:"use-network"`
 	UseMavenLocalRepository bool   `yaml:"use-maven-local-repository" json:"use-maven-local-repository" mapstructure:"use-maven-local-repository"`
 	MavenLocalRepositoryDir string `yaml:"maven-local-repository-dir" json:"maven-local-repository-dir" mapstructure:"maven-local-repository-dir"`
 	MavenURL                string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"`
@@ -17,7 +17,7 @@ func defaultJavaConfig() javaConfig {
 	def := java.DefaultArchiveCatalogerConfig()
 
 	return javaConfig{
-		UseNetwork:              def.UseNetwork,
+		UseNetwork:              nil, // this defaults to "false", which is the API default
 		MaxParentRecursiveDepth: def.MaxParentRecursiveDepth,
 		UseMavenLocalRepository: def.UseMavenLocalRepository,
 		MavenLocalRepositoryDir: def.MavenLocalRepositoryDir,

cmd/syft/internal/options/javascript.go

@@ -3,7 +3,7 @@ package options
 import "github.com/anchore/clio"
 
 type javaScriptConfig struct {
-	SearchRemoteLicenses bool   `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
+	SearchRemoteLicenses *bool  `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
 	NpmBaseURL           string `json:"npm-base-url" yaml:"npm-base-url" mapstructure:"npm-base-url"`
 }
 

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/acobaugh/osrelease v0.1.0
 	github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
 	github.com/anchore/clio v0.0.0-20240522144804-d81e109008aa
-	github.com/anchore/fangs v0.0.0-20240508143433-f016b099950f
+	github.com/anchore/fangs v0.0.0-20240903175602-e716ef12c23d
 	github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537
 	github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
 	github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb
@@ -84,7 +84,7 @@ require (
 	modernc.org/sqlite v1.32.0
 )
 
-require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
+require google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
 
 require (
 	github.com/BurntSushi/toml v1.4.0
@@ -137,13 +137,13 @@ require (
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
 	github.com/gkampitakis/ciinfo v0.3.0 // indirect
 	github.com/gkampitakis/go-diff v1.3.2 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-restruct/restruct v1.2.0-alpha // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -187,7 +187,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pborman/indent v1.2.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.19 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -206,7 +206,7 @@ require (
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.18.2 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/sylabs/sif/v2 v2.17.1 // indirect
 	github.com/sylabs/squashfs v1.0.0 // indirect
@@ -223,10 +223,10 @@ require (
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
-	go.opentelemetry.io/otel v1.19.0 // indirect
-	go.opentelemetry.io/otel/metric v1.19.0 // indirect
-	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.26.0 // indirect
@@ -236,8 +236,8 @@ require (
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
-	google.golang.org/grpc v1.59.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect
+	google.golang.org/grpc v1.62.1 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect