Skip to content

Commit

Permalink
feat: implement network selector similar to cataloger selector
Browse files Browse the repository at this point in the history
Signed-off-by: Keith Zantow <[email protected]>
  • Loading branch information
kzantow committed Sep 4, 2024
1 parent 437ea7d commit aa35bc9
Show file tree
Hide file tree
Showing 11 changed files with 185 additions and 42 deletions.
9 changes: 5 additions & 4 deletions cmd/syft/internal/commands/attest.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ const (
type attestOptions struct {
options.Config `yaml:",inline" mapstructure:",squash"`
options.Output `yaml:",inline" mapstructure:",squash"`
options.Network `yaml:",inline" mapstructure:",squash"`
options.UpdateCheck `yaml:",inline" mapstructure:",squash"`
options.Catalog `yaml:",inline" mapstructure:",squash"`
Attest options.Attest `yaml:"attest" mapstructure:"attest"`
Expand All @@ -63,7 +64,7 @@ func Attest(app clio.Application) *cobra.Command {
"command": "attest",
}),
Args: validateScanArgs,
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck),
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck, &opts.Network),
RunE: func(cmd *cobra.Command, args []string) error {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
Expand Down Expand Up @@ -113,7 +114,7 @@ func runAttest(ctx context.Context, id clio.Identification, opts *attestOptions,
}
defer os.Remove(f.Name())

s, err := generateSBOMForAttestation(ctx, id, &opts.Catalog, userInput)
s, err := generateSBOMForAttestation(ctx, id, &opts.Catalog, opts.Network, userInput)
if err != nil {
return fmt.Errorf("unable to build SBOM: %w", err)
}
Expand Down Expand Up @@ -247,7 +248,7 @@ func predicateType(outputName string) string {
}
}

func generateSBOMForAttestation(ctx context.Context, id clio.Identification, opts *options.Catalog, userInput string) (*sbom.SBOM, error) {
func generateSBOMForAttestation(ctx context.Context, id clio.Identification, opts *options.Catalog, net options.Network, userInput string) (*sbom.SBOM, error) {
if len(opts.From) > 1 || (len(opts.From) == 1 && opts.From[0] != stereoscope.RegistryTag) {
return nil, fmt.Errorf("attest requires use of an OCI registry directly, one or more of the specified sources is unsupported: %v", opts.From)
}
Expand All @@ -266,7 +267,7 @@ func generateSBOMForAttestation(ctx context.Context, id clio.Identification, opt
}
}()

s, err := generateSBOM(ctx, id, src, opts)
s, err := generateSBOM(ctx, id, src, opts, net)
if err != nil {
return nil, err
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/syft/internal/commands/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const (
type ConvertOptions struct {
options.Config `yaml:",inline" mapstructure:",squash"`
options.Output `yaml:",inline" mapstructure:",squash"`
options.Network `yaml:",inline" mapstructure:",squash"`
options.UpdateCheck `yaml:",inline" mapstructure:",squash"`
}

Expand All @@ -45,7 +46,7 @@ func Convert(app clio.Application) *cobra.Command {
"command": "convert",
}),
Args: validateConvertArgs,
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck),
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck, &opts.Network),
RunE: func(_ *cobra.Command, args []string) error {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
Expand Down
2 changes: 1 addition & 1 deletion cmd/syft/internal/commands/packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func Packages(app clio.Application, scanCmd *cobra.Command) *cobra.Command {
"appName": id.Name,
"command": "packages",
}),
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck),
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck, &opts.Network),
RunE: func(cmd *cobra.Command, args []string) error {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
Expand Down
2 changes: 1 addition & 1 deletion cmd/syft/internal/commands/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func Root(app clio.Application, packagesCmd *cobra.Command) *cobra.Command {
Long: packagesCmd.Long,
Args: packagesCmd.Args,
Example: packagesCmd.Example,
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck),
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck, &opts.Network),
RunE: func(cmd *cobra.Command, args []string) error {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
Expand Down
9 changes: 5 additions & 4 deletions cmd/syft/internal/commands/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ const (
type scanOptions struct {
options.Config `yaml:",inline" mapstructure:",squash"`
options.Output `yaml:",inline" mapstructure:",squash"`
options.Network `yaml:",inline" mapstructure:",squash"`
options.UpdateCheck `yaml:",inline" mapstructure:",squash"`
options.Catalog `yaml:",inline" mapstructure:",squash"`
Cache options.Cache `json:"-" yaml:"cache" mapstructure:"cache"`
Expand Down Expand Up @@ -94,7 +95,7 @@ func Scan(app clio.Application) *cobra.Command {
"command": "scan",
}),
Args: validateScanArgs,
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck),
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck, &opts.Network),
RunE: func(cmd *cobra.Command, args []string) error {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
Expand Down Expand Up @@ -196,7 +197,7 @@ func runScan(ctx context.Context, id clio.Identification, opts *scanOptions, use
}
}()

s, err := generateSBOM(ctx, id, src, &opts.Catalog)
s, err := generateSBOM(ctx, id, src, &opts.Catalog, opts.Network)
if err != nil {
return err
}
Expand Down Expand Up @@ -253,8 +254,8 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou
return src, nil
}

func generateSBOM(ctx context.Context, id clio.Identification, src source.Source, opts *options.Catalog) (*sbom.SBOM, error) {
s, err := syft.CreateSBOM(ctx, src, opts.ToSBOMConfig(id))
func generateSBOM(ctx context.Context, id clio.Identification, src source.Source, opts *options.Catalog, net options.Network) (*sbom.SBOM, error) {
s, err := syft.CreateSBOM(ctx, src, opts.ToSBOMConfig(id, net))
if err != nil {
expErrs := filterExpressionErrors(err)
notifyExpressionErrors(expErrs)
Expand Down
4 changes: 2 additions & 2 deletions cmd/syft/internal/commands/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ var latestAppVersionURL = struct {
path: "/syft/releases/latest/VERSION",
}

func applicationUpdateCheck(id clio.Identification, check *options.UpdateCheck) func(cmd *cobra.Command, args []string) error {
func applicationUpdateCheck(id clio.Identification, check *options.UpdateCheck, net *options.Network) func(cmd *cobra.Command, args []string) error {
return func(_ *cobra.Command, _ []string) error {
if check.CheckForAppUpdate {
if check.DoCheckForAppUpdate(*net) {
checkForApplicationUpdate(id)
}
return nil
Expand Down
30 changes: 6 additions & 24 deletions cmd/syft/internal/options/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package options

import (
"fmt"
"sort"
"strings"

"github.com/iancoleman/strcase"

Expand Down Expand Up @@ -36,7 +34,6 @@ type Catalog struct {
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"`
UseNetwork *bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"`

// ecosystem-specific cataloger configuration
Golang golangConfig `yaml:"golang" json:"golang" mapstructure:"golang"`
Expand Down Expand Up @@ -73,13 +70,13 @@ func DefaultCatalog() Catalog {
}
}

func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
func (cfg Catalog) ToSBOMConfig(id clio.Identification, net Network) *syft.CreateSBOMConfig {
return syft.DefaultCreateSBOMConfig().
WithTool(id.Name, id.Version).
WithParallelism(cfg.Parallelism).
WithRelationshipsConfig(cfg.ToRelationshipsConfig()).
WithSearchConfig(cfg.ToSearchConfig()).
WithPackagesConfig(cfg.ToPackagesConfig()).
WithPackagesConfig(cfg.ToPackagesConfig(net)).
WithFilesConfig(cfg.ToFilesConfig()).
WithCatalogerSelection(
pkgcataloging.NewSelectionRequest().
Expand Down Expand Up @@ -123,7 +120,7 @@ func (cfg Catalog) ToFilesConfig() filecataloging.Config {
}
}

func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
func (cfg Catalog) ToPackagesConfig(net Network) pkgcataloging.Config {
archiveSearch := cataloging.ArchiveSearchConfig{
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
Expand All @@ -133,7 +130,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
Golang: golang.DefaultCatalogerConfig().
WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses).
WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
WithSearchRemoteLicenses(*multiLevelOption(false, cfg.UseNetwork, cfg.Golang.SearchRemoteLicenses)).
WithSearchRemoteLicenses(*multiLevelOption(false, net.Enabled("golang", "go"), cfg.Golang.SearchRemoteLicenses)).
WithProxy(cfg.Golang.Proxy).
WithNoProxy(cfg.Golang.NoProxy).
WithMainModuleVersion(
Expand All @@ -143,7 +140,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags),
),
JavaScript: javascript.DefaultCatalogerConfig().
WithSearchRemoteLicenses(*multiLevelOption(false, cfg.UseNetwork, cfg.JavaScript.SearchRemoteLicenses)).
WithSearchRemoteLicenses(*multiLevelOption(false, net.Enabled("javascript", "js"), cfg.JavaScript.SearchRemoteLicenses)).
WithNpmBaseURL(cfg.JavaScript.NpmBaseURL),
LinuxKernel: kernel.LinuxKernelCatalogerConfig{
CatalogModules: cfg.LinuxKernel.CatalogModules,
Expand All @@ -154,7 +151,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
JavaArchive: java.DefaultArchiveCatalogerConfig().
WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository).
WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
WithUseNetwork(*multiLevelOption(false, cfg.UseNetwork, cfg.Java.UseNetwork)).
WithUseNetwork(*multiLevelOption(false, net.Enabled("java", "maven"), cfg.Java.UseNetwork)).
WithMavenBaseURL(cfg.Java.MavenURL).
WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth),
}
Expand Down Expand Up @@ -202,9 +199,6 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {

flags.StringVarP(&cfg.Source.BasePath, "base-path", "",
"base directory for scanning, no links will be followed above this directory, and all paths will be reported relative to this directory")

flags.BoolPtrVarP(&cfg.UseNetwork, "use-network", "",
"use the network to fetch and augment package information")
}

func (cfg *Catalog) DescribeFields(descriptions fangs.FieldDescriptionSet) {
Expand All @@ -219,18 +213,6 @@ func (cfg *Catalog) PostLoad() error {
return fmt.Errorf("cannot use both 'catalogers' and 'select-catalogers'/'default-catalogers' flags")
}

flatten := func(l []string) []string {
var out []string
for _, v := range l {
for _, s := range strings.Split(v, ",") {
out = append(out, strings.TrimSpace(s))
}
}
sort.Strings(out)

return out
}

cfg.From = flatten(cfg.From)

cfg.Catalogers = flatten(cfg.Catalogers)
Expand Down
98 changes: 98 additions & 0 deletions cmd/syft/internal/options/network.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package options

import (
"sort"
"strings"

"github.com/anchore/clio"
"github.com/anchore/fangs"
"github.com/anchore/syft/internal/log"
)

type Network struct {
Directives []string `yaml:"network" json:"network" mapstructure:"network"`
}

func (n *Network) PostLoad() error {
n.Directives = flatten(n.Directives)
return nil
}

func (n *Network) AddFlags(flags clio.FlagSet) {
flags.StringArrayVarP(&n.Directives, "network", "",
"use the network to fetch and augment package information")

// if pfp, ok := flags.(fangs.PFlagSetProvider); ok {
// flagSet := pfp.PFlagSet()
// flag := flagSet.Lookup("network")
// flag.NoOptDefVal = "all"
//}
}

func (n *Network) Enabled(features ...string) *bool {
return networkEnabled(n.Directives, features...)
}

var _ interface {
fangs.PostLoader
fangs.FlagAdder
} = (*Network)(nil)

func networkEnabled(networkDirectives []string, features ...string) *bool {
if len(networkDirectives) == 0 {
return nil
}

enabled := func(features ...string) *bool {
for _, directive := range networkDirectives {
enable := true
directive = strings.TrimPrefix(directive, "+") // +java and java are equivalent
if strings.HasPrefix(directive, "-") {
directive = directive[1:]
enable = false
}
for _, feature := range features {
if directive == feature {
return &enable
}
}
}
return nil
}

enableAll := enabled("all", "yes", "on", "enable", "enabled")
disableAll := enabled("none", "no", "off", "disable", "disabled")

if disableAll != nil {
if enableAll != nil {
log.Warn("you have specified to both enable and disable all network functionality, defaulting to disabled")
} else {
enableAll = ptr(!*disableAll)
}
}

// check for explicit enable/disable of each particular feature, in order
for _, feat := range features {
enableFeature := enabled(feat)
if enableFeature != nil {
return enableFeature
}
}

return enableAll
}

func ptr[T any](val T) *T {
return &val
}

func flatten(commaSeparatedEntries []string) []string {
var out []string
for _, v := range commaSeparatedEntries {
for _, s := range strings.Split(v, ",") {
out = append(out, strings.TrimSpace(s))
}
}
sort.Strings(out)
return out
}
58 changes: 58 additions & 0 deletions cmd/syft/internal/options/network_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package options

import (
"testing"

"github.com/stretchr/testify/require"
)

func Test_networkEnabled(t *testing.T) {
tests := []struct {
directives string
test string
expected *bool
}{
{
directives: "",
test: "java",
expected: nil,
},
{
directives: "none",
test: "java",
expected: ptr(false),
},
{
directives: "none,+java",
test: "java",
expected: ptr(true),
},
{
directives: "all",
test: "java",
expected: ptr(true),
},
{
directives: "on",
test: "java",
expected: ptr(true),
},
{
directives: "on,-java",
test: "java",
expected: ptr(false),
},
}

for _, test := range tests {
t.Run(test.directives, func(t *testing.T) {
n := Network{
Directives: []string{test.directives},
}
require.NoError(t, n.PostLoad())

got := n.Enabled(test.test)
require.Equal(t, test.expected, got)
})
}
}
Loading

0 comments on commit aa35bc9

Please sign in to comment.