refactor: Isolate O.S. matchers to re-use by node scanning (#904)

pkg/matcher/matcher.go

@@ -4,10 +4,12 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/stackrox/rox/pkg/set"
 	"github.com/stackrox/scanner/pkg/whiteout"
 )
 
@@ -20,25 +22,84 @@ type Matcher interface {
 	Match(fullPath string, fileInfo os.FileInfo, contents io.ReaderAt) (matches bool, extract bool)
 }
 
+// PrefixMatcher is a matcher that uses file prefixes.
+type PrefixMatcher interface {
+	Matcher
+
+	// GetCommonPrefixDirs list all directories from all the prefixes used in this
+	// matcher, and returns a list of common directories in all of them, up to one
+	// level below the root dir, e.g. prefixes are {"a/b/f", "a/c/f", "b/c/"} the
+	// common prefix list is {"a/", "b/c/"}. The returned directories will always be
+	// terminated with /. If a name is not terminated by a slash it is considered a
+	// file and ignored. Example:
+	//
+	// Prefixes:
+	//   - var/lib/rpm/
+	//   - var/lib/dpkg/
+	//   - root/buildinfo/
+	//   - usr/bin
+	//   - usr/bin/bash
+	//   - etc/apt.sources
+	//
+	// Output:
+	//   - var/lib/
+	//   - root/buildinfo/
+	//   - usr/
+	//   - etc/
+	GetCommonPrefixDirs() []string
+}
+
 type allowlistMatcher struct {
 	allowlist []string
 }
 
-// NewPrefixAllowlistMatcher returns a matcher that matches all filenames which have any
-// of the passed paths as a prefix.
-func NewPrefixAllowlistMatcher(allowlist ...string) Matcher {
+// NewPrefixAllowlistMatcher returns a prefix matcher that matches all filenames
+// which have any of the passed paths as a prefix.
+func NewPrefixAllowlistMatcher(allowlist ...string) PrefixMatcher {
 	return &allowlistMatcher{allowlist: allowlist}
 }
 
-func (w *allowlistMatcher) Match(fullPath string, _ os.FileInfo, _ io.ReaderAt) (matches bool, extract bool) {
-	for _, s := range w.allowlist {
+func (m *allowlistMatcher) Match(fullPath string, _ os.FileInfo, _ io.ReaderAt) (matches bool, extract bool) {
+	for _, s := range m.allowlist {
 		if strings.HasPrefix(fullPath, s) {
 			return true, true
 		}
 	}
 	return false, false
 }
 
+func (m *allowlistMatcher) GetCommonPrefixDirs() []string {
+	return findCommonDirPrefixes(m.allowlist)
+}
+
+// findCommonDirPrefixes goes over all prefixes, steps one level down from the
+// root directory, and returns exactly one common prefix per first level dir
+// referenced. It does it by doing creating a trie-like structure with the
+// directory tree filtering paths with only single-children nodes.
+func findCommonDirPrefixes(prefixes []string) []string {
+	prefixToSubdirs := make(map[string]set.StringSet)
+	for _, d := range prefixes {
+		for d != "" {
+			p, _ := path.Split(strings.TrimSuffix(d, "/"))
+			s := prefixToSubdirs[p]
+			s.Add(d)
+			prefixToSubdirs[p] = s
+			d = p
+		}
+	}
+	// Work on one step below root.
+	firstLevelDirs := prefixToSubdirs[""].AsSlice()
+	ret := firstLevelDirs[:0]
+	for _, d := range firstLevelDirs {
+		for len(prefixToSubdirs[d]) == 1 {
+			d = prefixToSubdirs[d].GetArbitraryElem()
+		}
+		d, _ := path.Split(d)
+		ret = append(ret, d)
+	}
+	return ret
+}
+
 type whiteoutMatcher struct{}
 
 // NewWhiteoutMatcher returns a matcher that matches all whiteout files

pkg/matcher/matcher_test.go

@@ -199,3 +199,82 @@ func TestAndMatcher(t *testing.T) {
 	assert.False(t, match)
 	assert.False(t, extract)
 }
+
+func Test_findCommonDirPrefixes(t *testing.T) {
+	tests := []struct {
+		name     string
+		prefixes []string
+		want     []string
+	}{
+		{
+			name: "happy case",
+			prefixes: []string{
+				"bin/[",
+				"bin/busybox",
+				"etc/alpine-release",
+				"etc/apt/sources.list",
+				"etc/centos-release",
+				"etc/lsb-release",
+				"etc/oracle-release",
+				"etc/os-release",
+				"etc/os-release",
+				"etc/redhat-release",
+				"etc/system-release",
+				"lib/apk/db/installed",
+				"root/buildinfo/content_manifests",
+				"usr/lib/os-release",
+				"var/lib/dpkg/status",
+				"var/lib/rpm/Packages",
+				"var/lib/rpm/Packages",
+			},
+			want: []string{
+				"bin/",
+				"etc/",
+				"lib/apk/db/",
+				"root/buildinfo/",
+				"usr/lib/",
+				"var/lib/",
+			},
+		},
+		{
+			name: "prefixes with directories",
+			prefixes: []string{
+				"foo/bar/",
+				"foo/bar/ok/",
+				"foo/bar/nook/",
+				"foo/bar/nook/",
+			},
+			want: []string{"foo/bar/"},
+		},
+		{
+			name: "non-slash are considered files",
+			prefixes: []string{
+				"usr/bin",
+				"usr/bin/",
+			},
+			want: []string{"usr/"},
+		},
+		{
+			name: "example from doc comment",
+			prefixes: []string{
+				"var/lib/rpm/",
+				"var/lib/dpkg/",
+				"root/buildinfo/",
+				"usr/bin",
+				"usr/bin/bash",
+				"etc/apt.sources",
+			},
+			want: []string{
+				"var/lib/",
+				"root/buildinfo/",
+				"usr/",
+				"etc/",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.ElementsMatch(t, tt.want, findCommonDirPrefixes(tt.prefixes))
+		})
+	}
+}

singletons/requiredfilenames/matcher.go

@@ -11,30 +11,36 @@ import (
 )
 
 var (
+	osMatcher     matcher.PrefixMatcher
+	osMatcherOnce sync.Once
+
+	activeVulnMatcher     matcher.Matcher
+	activeVulnMatcherOnce sync.Once
+
 	instance matcher.Matcher
 	once     sync.Once
+
 	// dynamicLibRegexp matches all dynamic libraries.
 	dynamicLibRegexp = regexp.MustCompile(`(^|/)(lib|ld-)[^/.-][^/]*\.so(\.[^/.]+)*$`)
 	// libraryDirRegexp matches all files under directories where the dynamic libraries are commonly found.
 	// This is to filter for symbolic links needed to resolve dynamic library paths.
 	libraryDirRegexp = regexp.MustCompile(`^(usr/(local/)?)?lib(32|64)?(/.+|$)`)
 )
 
-// SingletonMatcher returns the singleton matcher instance to use for extracting
-// files to be analyzed for operating system features.
-// Note: language-level analyzers implement a different interface, and do not require
-// extraction of files into a `FileMap`. Therefore, the respective files do not need
-// to be matched here.
-func SingletonMatcher() matcher.Matcher {
-	once.Do(func() {
+// SingletonOSMatcher returns the singleton matcher instance for extracting files
+// for OS package analysis.
+func SingletonOSMatcher() matcher.PrefixMatcher {
+	osMatcherOnce.Do(func() {
 		allFileNames := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...)
-		clairMatcher := matcher.NewPrefixAllowlistMatcher(allFileNames...)
-		whiteoutMatcher := matcher.NewWhiteoutMatcher()
-
-		allMatchers := make([]matcher.Matcher, 0, 6)
-		allMatchers = append(allMatchers, clairMatcher, whiteoutMatcher)
+		osMatcher = matcher.NewPrefixAllowlistMatcher(allFileNames...)
+	})
+	return osMatcher
+}
 
-		// Active Vuln Mgmt related matchers.
+// SingletonActiveVulnMatcher returns the singleton matcher instance for
+// extracting files for active vulnerability analysis.
+func SingletonActiveVulnMatcher() matcher.Matcher {
+	activeVulnMatcherOnce.Do(func() {
 		dpkgFilenamesMatcher := matcher.NewRegexpMatcher(dpkg.FilenamesListRegexp, true)
 		dynamicLibMatcher := matcher.NewRegexpMatcher(dynamicLibRegexp, false)
 		libDirSymlinkMatcher := matcher.NewAndMatcher(matcher.NewRegexpMatcher(libraryDirRegexp, false), matcher.NewSymbolicLinkMatcher())
@@ -44,9 +50,27 @@ func SingletonMatcher() matcher.Matcher {
 		// remaining executable files which went unmatched otherwise.
 		// Therefore, this matcher MUST be the last matcher.
 		executableMatcher := matcher.NewExecutableMatcher()
-		allMatchers = append(allMatchers, dpkgFilenamesMatcher, dynamicLibMatcher, libDirSymlinkMatcher, executableMatcher)
+		activeVulnMatcher = matcher.NewOrMatcher(
+			dpkgFilenamesMatcher,
+			dynamicLibMatcher,
+			libDirSymlinkMatcher,
+			executableMatcher,
+		)
+	})
+	return activeVulnMatcher
+}
 
-		instance = matcher.NewOrMatcher(allMatchers...)
+// SingletonMatcher returns the singleton matcher instance to use for extracting
+// files for analyzing image container. It includes matching for OS features
+// and active vulnerability. Note: language-level analyzers implement a different
+// interface, and do not require extraction of files. Therefore, the respective
+// files do not need to be matched here.
+func SingletonMatcher() matcher.Matcher {
+	once.Do(func() {
+		instance = matcher.NewOrMatcher(
+			matcher.NewWhiteoutMatcher(),
+			SingletonOSMatcher(),
+			SingletonActiveVulnMatcher())
 	})
 	return instance
 }