diff --git a/benchmarks/detectcontent/detect_content_test.go b/benchmarks/detectcontent/detect_content_test.go index 8d0a04c48..216de2219 100644 --- a/benchmarks/detectcontent/detect_content_test.go +++ b/benchmarks/detectcontent/detect_content_test.go @@ -7,6 +7,7 @@ import ( "github.com/stackrox/scanner/benchmarks" "github.com/stackrox/scanner/database" "github.com/stackrox/scanner/pkg/features" + "github.com/stackrox/scanner/pkg/tarutil" "github.com/stackrox/scanner/pkg/testutils" "github.com/stretchr/testify/require" @@ -56,8 +57,9 @@ func runBenchmarkDetectContent(b *testing.B, imageName string) { for i := 0; i < b.N; i++ { var namespace *database.Namespace var err error + var files *tarutil.LayerFiles for _, l := range layers { - namespace, _, _, _, _, _, err = clair.DetectContentFromReader(l, "Docker", l.Name, &database.Layer{Namespace: namespace}, false) + namespace, _, _, _, _, files, err = clair.DetectContentFromReader(l, "Docker", l.Name, &database.Layer{Namespace: namespace}, files, false) require.NoError(b, err) } } diff --git a/e2etests/testcase_test.go b/e2etests/testcase_test.go index 598ee2eb7..26582dc7e 100644 --- a/e2etests/testcase_test.go +++ b/e2etests/testcase_test.go @@ -176,6 +176,50 @@ var testCases = []testCase{ {Name: "apt", Version: "1.0.9.8.4"}, }, }, + { + Path: "/usr/lib/apt/methods/xz", + RequiredFeatures: []*v1.FeatureNameVersion{ + {Name: "apt", Version: "1.0.9.8.4"}, + {Name: "bzip2", Version: "1.0.6-7"}, + {Name: "gcc-4.9", Version: "4.9.2-10"}, + {Name: "glibc", Version: "2.19-18+deb8u10"}, + {Name: "xz-utils", Version: "5.1.1alpha+20120614-2"}, + {Name: "zlib", Version: "1:1.2.8.dfsg-2"}, + }, + }, + { + Path: "/usr/lib/apt/methods/bzip2", + RequiredFeatures: []*v1.FeatureNameVersion{ + {Name: "apt", Version: "1.0.9.8.4"}, + {Name: "bzip2", Version: "1.0.6-7"}, + {Name: "gcc-4.9", Version: "4.9.2-10"}, + {Name: "glibc", Version: "2.19-18+deb8u10"}, + {Name: "xz-utils", Version: "5.1.1alpha+20120614-2"}, + {Name: "zlib", Version: "1:1.2.8.dfsg-2"}, + }, + }, + { + Path: "/usr/lib/apt/methods/lzma", + RequiredFeatures: []*v1.FeatureNameVersion{ + {Name: "apt", Version: "1.0.9.8.4"}, + {Name: "bzip2", Version: "1.0.6-7"}, + {Name: "gcc-4.9", Version: "4.9.2-10"}, + {Name: "glibc", Version: "2.19-18+deb8u10"}, + {Name: "xz-utils", Version: "5.1.1alpha+20120614-2"}, + {Name: "zlib", Version: "1:1.2.8.dfsg-2"}, + }, + }, + { + Path: "/usr/lib/apt/methods/ssh", + RequiredFeatures: []*v1.FeatureNameVersion{ + {Name: "apt", Version: "1.0.9.8.4"}, + {Name: "bzip2", Version: "1.0.6-7"}, + {Name: "gcc-4.9", Version: "4.9.2-10"}, + {Name: "glibc", Version: "2.19-18+deb8u10"}, + {Name: "xz-utils", Version: "5.1.1alpha+20120614-2"}, + {Name: "zlib", Version: "1:1.2.8.dfsg-2"}, + }, + }, { Path: "/usr/lib/dpkg/methods/apt/update", RequiredFeatures: []*v1.FeatureNameVersion{ @@ -493,9 +537,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -511,9 +558,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -529,9 +579,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -547,9 +600,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -565,9 +621,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -583,9 +642,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -601,9 +663,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -619,9 +684,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "ncurses-libs", Version: "5.9-14.20130511.el7_4"}, @@ -638,9 +706,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -656,9 +727,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -674,9 +748,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "ncurses-libs", Version: "5.9-14.20130511.el7_4"}, @@ -693,9 +770,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -711,9 +791,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -729,9 +812,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -747,9 +833,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "ncurses-libs", Version: "5.9-14.20130511.el7_4"}, @@ -766,9 +855,33 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, + {Name: "libattr", Version: "2.4.46-13.el7"}, + {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, + {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, + {Name: "libselinux", Version: "2.5-14.1.el7"}, + {Name: "lz4", Version: "1.7.5-3.el7"}, + {Name: "pcre", Version: "8.32-17.el7"}, + {Name: "procps-ng", Version: "3.3.10-26.el7"}, + {Name: "systemd-libs", Version: "219-67.el7_7.1"}, + {Name: "xz-libs", Version: "5.2.2-1.el7"}, + {Name: "zlib", Version: "1.2.7-18.el7"}, + }, + }, + { + Path: "/usr/lib64/libprocps.so.4", + RequiredFeatures: []*v1.FeatureNameVersion{ + {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, + {Name: "elfutils-libelf", Version: "0.176-2.el7"}, + {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -784,9 +897,12 @@ var testCases = []testCase{ {Name: "bzip2-libs", Version: "1.0.6-13.el7"}, {Name: "elfutils-libelf", Version: "0.176-2.el7"}, {Name: "elfutils-libs", Version: "0.176-2.el7"}, + {Name: "glibc", Version: "2.17-292.el7"}, {Name: "libattr", Version: "2.4.46-13.el7"}, {Name: "libcap", Version: "2.22-10.el7"}, + {Name: "libgcc", Version: "4.8.5-39.el7"}, {Name: "libgcrypt", Version: "1.5.3-14.el7"}, + {Name: "libgpg-error", Version: "1.12-3.el7"}, {Name: "libselinux", Version: "2.5-14.1.el7"}, {Name: "lz4", Version: "1.7.5-3.el7"}, {Name: "pcre", Version: "8.32-17.el7"}, @@ -1868,6 +1984,8 @@ var testCases = []testCase{ { Path: "/usr/bin/vi", RequiredFeatures: []*v1.FeatureNameVersion{ + {Name: "glibc", Version: "2.17-307.el7.1.i686"}, + {Name: "glibc", Version: "2.17-307.el7.1.x86_64"}, {Name: "libacl", Version: "2.2.51-15.el7.x86_64"}, {Name: "libattr", Version: "2.4.46-13.el7.i686"}, {Name: "libattr", Version: "2.4.46-13.el7.x86_64"}, diff --git a/ext/featurefmt/apk/apk.go b/ext/featurefmt/apk/apk.go index 6f1226b16..597ec95ce 100644 --- a/ext/featurefmt/apk/apk.go +++ b/ext/featurefmt/apk/apk.go @@ -41,8 +41,8 @@ func init() { type lister struct{} -func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { - file, exists := files[dbPath] +func (l lister) ListFeatures(files tarutil.LayerFiles) ([]database.FeatureVersion, error) { + file, exists := files.Get(dbPath) if !exists { return []database.FeatureVersion{}, nil } @@ -106,8 +106,11 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, dir = line[2:] case line[:2] == "R:" && features.ActiveVulnMgmt.Enabled(): filename := fmt.Sprintf("/%s/%s", dir, line[2:]) - // The first character is always "/", which is removed when inserted into the files maps. - featurefmt.AddToDependencyMap(filename, files[filename[1:]], execToDeps, libToDeps) + // The first character is always "/", which is removed when inserted into the layer files. + fileData, hasFile := files.Get(filename[1:]) + if hasFile { + featurefmt.AddToDependencyMap(filename, fileData, execToDeps, libToDeps) + } } } diff --git a/ext/featurefmt/apk/apk_test.go b/ext/featurefmt/apk/apk_test.go index 0357ba5ab..90ed9ddf4 100644 --- a/ext/featurefmt/apk/apk_test.go +++ b/ext/featurefmt/apk/apk_test.go @@ -78,9 +78,9 @@ func TestAPKFeatureDetection(t *testing.T) { Version: "0.7-r0", }, }, - Files: tarutil.FilesMap{ - "lib/apk/db/installed": tarutil.FileData{Contents: featurefmt.LoadFileForTest("apk/testdata/installed")}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "lib/apk/db/installed": {Contents: featurefmt.LoadFileForTest("apk/testdata/installed")}, + }), }, } featurefmt.TestLister(t, &lister{}, testData) @@ -154,14 +154,14 @@ func TestAPKFeatureDetectionWithActiveVulnMgmt(t *testing.T) { Version: "0.7-r0", }, }, - Files: tarutil.FilesMap{ - "lib/apk/db/installed": tarutil.FileData{Contents: featurefmt.LoadFileForTest("apk/testdata/installed")}, - "lib/libc.musl-x86_64.so.1": tarutil.FileData{Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{"c.so.1"}, ImportedLibraries: []string{"ld.so.1"}}}, - "lib/ld-musl-x86_64.so.1": tarutil.FileData{Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{"ld.so.1"}}}, - "bin/busybox": tarutil.FileData{Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{}, ImportedLibraries: []string{"c.so.1", "ld.so.1"}}}, - "etc/hosts": tarutil.FileData{Executable: true}, - "etc/crontabs/root": tarutil.FileData{Executable: true}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "lib/apk/db/installed": {Contents: featurefmt.LoadFileForTest("apk/testdata/installed")}, + "lib/libc.musl-x86_64.so.1": {Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{"c.so.1"}, ImportedLibraries: []string{"ld.so.1"}}}, + "lib/ld-musl-x86_64.so.1": {Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{"ld.so.1"}}}, + "bin/busybox": {Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{}, ImportedLibraries: []string{"c.so.1", "ld.so.1"}}}, + "etc/hosts": {Executable: true}, + "etc/crontabs/root": {Executable: true}, + }), }, } featurefmt.TestLister(t, &lister{}, testData) diff --git a/ext/featurefmt/dpkg/dpkg.go b/ext/featurefmt/dpkg/dpkg.go index 668723613..6fa046699 100644 --- a/ext/featurefmt/dpkg/dpkg.go +++ b/ext/featurefmt/dpkg/dpkg.go @@ -67,7 +67,7 @@ type componentMetadata struct { arch string } -func (l lister) parseComponents(files tarutil.FilesMap, file []byte, packagesMap map[featurefmt.PackageKey]*database.FeatureVersion, removedPackages set.StringSet, distroless bool) error { +func (l lister) parseComponents(files tarutil.LayerFiles, file []byte, packagesMap map[featurefmt.PackageKey]*database.FeatureVersion, removedPackages set.StringSet, distroless bool) error { pkgFmt := `dpkg` if distroless { pkgFmt = `distroless` @@ -147,7 +147,7 @@ func (l lister) parseComponents(files tarutil.FilesMap, file []byte, packagesMap return scanner.Err() } -func handleComponent(files tarutil.FilesMap, pkgMetadata *componentMetadata, packagesMap map[featurefmt.PackageKey]*database.FeatureVersion, removedPackages set.StringSet, distroless bool) { +func handleComponent(files tarutil.LayerFiles, pkgMetadata *componentMetadata, packagesMap map[featurefmt.PackageKey]*database.FeatureVersion, removedPackages set.StringSet, distroless bool) { // Debian and Ubuntu vulnerability feeds only have entries for source packages, // and not the package binaries, though usually only the binaries are installed. pkgName := stringutils.FirstNonEmpty(pkgMetadata.sourceName, pkgMetadata.name) @@ -168,17 +168,20 @@ func handleComponent(files tarutil.FilesMap, pkgMetadata *componentMetadata, pac filenamesArchList := dpkgInfoPrefix + pkgMetadata.name + ":" + pkgMetadata.arch + dpkgFilenamesSuffix // Read the list of files provided by the current package. - filenamesFileData := files[filenamesList] - if len(filenamesFileData.Contents) == 0 { - filenamesFileData = files[filenamesArchList] + filenamesFileData, hasFile := files.Get(filenamesList) + if !hasFile || len(filenamesFileData.Contents) == 0 { + filenamesFileData, _ = files.Get(filenamesArchList) } filenamesFileScanner := bufio.NewScanner(bytes.NewReader(filenamesFileData.Contents)) for filenamesFileScanner.Scan() { filename := filenamesFileScanner.Text() - // The first character is always "/", which is removed when inserted into the files maps. - featurefmt.AddToDependencyMap(filename, files[filename[1:]], execToDeps, libToDeps) + // The first character is always "/", which is removed when inserted into the layer files. + fileData, hasFile := files.Get(filename[1:]) + if hasFile { + featurefmt.AddToDependencyMap(filename, fileData, execToDeps, libToDeps) + } } if err := filenamesFileScanner.Err(); err != nil { log.WithError(err).WithFields(log.Fields{"name": pkgMetadata.name, "version": pkgMetadata.version}).Warning("could not parse provided file list") @@ -219,20 +222,20 @@ func handleComponent(files tarutil.FilesMap, pkgMetadata *componentMetadata, pac } } -func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { +func (l lister) ListFeatures(files tarutil.LayerFiles) ([]database.FeatureVersion, error) { // Create a map to store packages and ensure their uniqueness packagesMap := make(map[featurefmt.PackageKey]*database.FeatureVersion) // Create a set to store removed packages to ensure it holds between files. // TODO: This may not be needed cross-file... removedPackages := set.NewStringSet() // For general images using dpkg. - if f, hasFile := files[statusFile]; hasFile { + if f, hasFile := files.Get(statusFile); hasFile { if err := l.parseComponents(files, f.Contents, packagesMap, removedPackages, false); err != nil { return []database.FeatureVersion{}, errors.Wrapf(err, "parsing %s", statusFile) } } - for filename, file := range files { + for filename, file := range files.GetFilesMap() { // For distroless images, which are based on Debian, but also useful for // all images using dpkg. // The var/lib/dpkg/status.d directory holds the files which define packages. diff --git a/ext/featurefmt/dpkg/dpkg_test.go b/ext/featurefmt/dpkg/dpkg_test.go index 1fa9d1fba..a5d38d0ac 100644 --- a/ext/featurefmt/dpkg/dpkg_test.go +++ b/ext/featurefmt/dpkg/dpkg_test.go @@ -60,12 +60,12 @@ func TestDpkgFeatureDetection(t *testing.T) { Version: "1.1.8-3.1ubuntu3", }, }, - Files: tarutil.FilesMap{ - "var/lib/dpkg/status": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/status")}, - "var/lib/dpkg/status.d": tarutil.FileData{}, - "var/lib/dpkg/status.d/base": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-base")}, - "var/lib/dpkg/status.d/netbase": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-netbase")}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/dpkg/status": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/status")}, + "var/lib/dpkg/status.d": {}, + "var/lib/dpkg/status.d/base": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-base")}, + "var/lib/dpkg/status.d/netbase": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-netbase")}, + }), }, } @@ -112,28 +112,30 @@ func TestDpkgFeatureDetectionWithActiveVulnMgmt(t *testing.T) { LibraryToDependencies: database.StringToStringsMap{"somelib.so.1": {"gcc5.so.1": {}}}, }, }, - Files: tarutil.FilesMap{ - "var/lib/dpkg/status": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/status")}, - "var/lib/dpkg/status.d": tarutil.FileData{}, - "var/lib/dpkg/status.d/base": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-base")}, - "var/lib/dpkg/info/base-files.list": tarutil.FileData{Contents: []byte{}}, - "var/lib/dpkg/status.d/netbase": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-netbase")}, - "var/lib/dpkg/info/netbase.list": tarutil.FileData{Contents: []byte{}}, - "var/lib/dpkg/info/libpam-runtime.list": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/libpam-runtime.list")}, - "var/lib/dpkg/info/libpam-modules-bin.list": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/libpam-modules-bin.list")}, - "var/lib/dpkg/info/libgcc1:amd64.list": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/libgcc1:amd64.list")}, - "test/executable": tarutil.FileData{Executable: true}, - "another/one": tarutil.FileData{Executable: true}, - "i/am/an/executable": tarutil.FileData{Executable: true}, - "var/lib/dpkg/info/pkg-source.list": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/pkg-source.list")}, - "var/lib/dpkg/info/pkg1:amd64.list": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/pkg1:amd64.list")}, - "var/lib/dpkg/info/pkg2.list": tarutil.FileData{Contents: featurefmt.LoadFileForTest("dpkg/testdata/pkg2.list")}, - "exec-me": tarutil.FileData{Executable: true}, - "exec-me-2": tarutil.FileData{Executable: true, ELFMetadata: &elf.Metadata{ImportedLibraries: []string{"gcc5.so.1"}}}, - "my-jar.jar": tarutil.FileData{Contents: []byte("jar contents")}, - "lib/linux/libgcc5.so.1": tarutil.FileData{ELFMetadata: &elf.Metadata{Sonames: []string{"gcc5.so.1"}}}, - "lib/linux/libsomelib.so.1": tarutil.FileData{ELFMetadata: &elf.Metadata{Sonames: []string{"somelib.so.1"}, ImportedLibraries: []string{"gcc5.so.1"}}}, - }, + Files: tarutil.CreateNewLayerFiles( + map[string]tarutil.FileData{ + "var/lib/dpkg/status": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/status")}, + "var/lib/dpkg/status.d": {}, + "var/lib/dpkg/status.d/base": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-base")}, + "var/lib/dpkg/info/base-files.list": {Contents: []byte{}}, + "var/lib/dpkg/status.d/netbase": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/statusd-netbase")}, + "var/lib/dpkg/info/netbase.list": {Contents: []byte{}}, + "var/lib/dpkg/info/libpam-runtime.list": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/libpam-runtime.list")}, + "var/lib/dpkg/info/libpam-modules-bin.list": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/libpam-modules-bin.list")}, + "var/lib/dpkg/info/libgcc1:amd64.list": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/libgcc1:amd64.list")}, + "test/executable": {Executable: true}, + "another/one": {Executable: true}, + "i/am/an/executable": {Executable: true}, + "var/lib/dpkg/info/pkg-source.list": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/pkg-source.list")}, + "var/lib/dpkg/info/pkg1:amd64.list": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/pkg1:amd64.list")}, + "var/lib/dpkg/info/pkg2.list": {Contents: featurefmt.LoadFileForTest("dpkg/testdata/pkg2.list")}, + "exec-me": {Executable: true}, + "exec-me-2": {Executable: true, ELFMetadata: &elf.Metadata{ImportedLibraries: []string{"gcc5.so.1"}}}, + "my-jar.jar": {Contents: []byte("jar contents")}, + "lib/linux/libgcc5.so.1": {ELFMetadata: &elf.Metadata{Sonames: []string{"gcc5.so.1"}}}, + "lib/linux/libsomelib.so.1": {ELFMetadata: &elf.Metadata{Sonames: []string{"somelib.so.1"}, ImportedLibraries: []string{"gcc5.so.1"}}}, + }, + ), }, } diff --git a/ext/featurefmt/driver.go b/ext/featurefmt/driver.go index 83e662c11..d16825a97 100644 --- a/ext/featurefmt/driver.go +++ b/ext/featurefmt/driver.go @@ -44,9 +44,9 @@ type PackageKey struct { // Lister represents an ability to list the features present in an image layer. type Lister interface { // ListFeatures produces a list of FeatureVersions present in an image layer. - ListFeatures(tarutil.FilesMap) ([]database.FeatureVersion, error) + ListFeatures(tarutil.LayerFiles) ([]database.FeatureVersion, error) - // RequiredFilenames returns the list of files required to be in the FilesMap + // RequiredFilenames returns the list of files required to be in the LayerFiles // provided to the ListFeatures method. // // Filenames must not begin with "/". @@ -77,7 +77,7 @@ func RegisterLister(name string, l Lister) { // ListFeatures produces the list of FeatureVersions in an image layer using // every registered Lister. -func ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { +func ListFeatures(files tarutil.LayerFiles) ([]database.FeatureVersion, error) { listersM.RLock() defer listersM.RUnlock() @@ -110,7 +110,7 @@ func RequiredFilenames() (files []string) { // TestData represents the data used to test an implementation of Lister. type TestData struct { - Files tarutil.FilesMap + Files tarutil.LayerFiles FeatureVersions []database.FeatureVersion } diff --git a/ext/featurefmt/rpm/rpm.go b/ext/featurefmt/rpm/rpm.go index d9b19c30f..7acb9bd4a 100644 --- a/ext/featurefmt/rpm/rpm.go +++ b/ext/featurefmt/rpm/rpm.go @@ -54,8 +54,8 @@ func init() { featurefmt.RegisterLister("rpm", &lister{}) } -func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { - f, hasFile := files[dbPath] +func (l lister) ListFeatures(files tarutil.LayerFiles) ([]database.FeatureVersion, error) { + f, hasFile := files.Get(dbPath) if !hasFile { return []database.FeatureVersion{}, nil } @@ -114,7 +114,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, return featureVersions, nil } -func parseFeatures(r io.Reader, files tarutil.FilesMap) ([]database.FeatureVersion, error) { +func parseFeatures(r io.Reader, files tarutil.LayerFiles) ([]database.FeatureVersion, error) { var featureVersions []database.FeatureVersion var fv database.FeatureVersion @@ -165,8 +165,11 @@ func parseFeatures(r io.Reader, files tarutil.FilesMap) ([]database.FeatureVersi // Rename to make it clear what the line represents. filename := line - // The first character is always "/", which is removed when inserted into the files maps. - rpm.AddToDependencyMap(filename, files[filename[1:]], execToDeps, libToDeps) + // The first character is always "/", which is removed when inserted into the layer files. + fileData, hasFile := files.Get(filename[1:]) + if hasFile { + rpm.AddToDependencyMap(filename, fileData, execToDeps, libToDeps) + } } } diff --git a/ext/featurefmt/rpm/rpm_test.go b/ext/featurefmt/rpm/rpm_test.go index a5e7c2229..0db7005c9 100644 --- a/ext/featurefmt/rpm/rpm_test.go +++ b/ext/featurefmt/rpm/rpm_test.go @@ -50,9 +50,9 @@ func TestRpmFeatureDetection(t *testing.T) { Version: "0.0.1-el7", }, }, - Files: tarutil.FilesMap{ - "var/lib/rpm/Packages": tarutil.FileData{Contents: featurefmt.LoadFileForTest("rpm/testdata/Packages")}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/rpm/Packages": {Contents: featurefmt.LoadFileForTest("rpm/testdata/Packages")}, + }), }, } @@ -97,16 +97,16 @@ func TestRpmFeatureDetectionWithActiveVulnMgmt(t *testing.T) { }, }, }, - Files: tarutil.FilesMap{ - "var/lib/rpm/Packages": tarutil.FileData{Contents: featurefmt.LoadFileForTest("rpm/testdata/Packages")}, - "etc/centos-release": tarutil.FileData{Executable: true}, - "usr/games": tarutil.FileData{Executable: true, ELFMetadata: &elf.Metadata{ImportedLibraries: []string{"base.so.1", "mock.so.1.0"}}}, - "usr/include": tarutil.FileData{Executable: true}, - "usr/lib/debug": tarutil.FileData{Executable: true}, - "usr/bin/mock_exec": tarutil.FileData{Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{}}}, - "usr/lib64/libmock.so.1": tarutil.FileData{ELFMetadata: &elf.Metadata{Sonames: []string{"mock.so.1", "mock.so.1.0"}, ImportedLibraries: []string{"base.so.1"}}}, - "usr/lib64/libbase.so.1": tarutil.FileData{ELFMetadata: &elf.Metadata{Sonames: []string{"base.so.1"}}}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/rpm/Packages": {Contents: featurefmt.LoadFileForTest("rpm/testdata/Packages")}, + "etc/centos-release": {Executable: true}, + "usr/games": {Executable: true, ELFMetadata: &elf.Metadata{ImportedLibraries: []string{"base.so.1", "mock.so.1.0"}}}, + "usr/include": {Executable: true}, + "usr/lib/debug": {Executable: true}, + "usr/bin/mock_exec": {Executable: true, ELFMetadata: &elf.Metadata{Sonames: []string{}}}, + "usr/lib64/libmock.so.1": {ELFMetadata: &elf.Metadata{Sonames: []string{"mock.so.1", "mock.so.1.0"}, ImportedLibraries: []string{"base.so.1"}}}, + "usr/lib64/libbase.so.1": {ELFMetadata: &elf.Metadata{Sonames: []string{"base.so.1"}}}, + }), }, } diff --git a/ext/featurens/alpinerelease/alpinerelease.go b/ext/featurens/alpinerelease/alpinerelease.go index e7a8a93a2..39da85ed0 100644 --- a/ext/featurens/alpinerelease/alpinerelease.go +++ b/ext/featurens/alpinerelease/alpinerelease.go @@ -43,8 +43,8 @@ func init() { type detector struct{} -func (d detector) Detect(files tarutil.FilesMap, _ *featurens.DetectorOptions) *database.Namespace { - file, exists := files[alpineReleasePath] +func (d detector) Detect(files tarutil.LayerFiles, _ *featurens.DetectorOptions) *database.Namespace { + file, exists := files.Get(alpineReleasePath) if !exists { return nil } @@ -64,7 +64,7 @@ func (d detector) Detect(files tarutil.FilesMap, _ *featurens.DetectorOptions) * // It is possible this is an alpine:edge image. // Verify this. - file, exists = files[osReleasePath] + file, exists = files.Get(osReleasePath) if !exists { return nil } diff --git a/ext/featurens/alpinerelease/alpinerelease_test.go b/ext/featurens/alpinerelease/alpinerelease_test.go index 77b72fb77..9c34fa14e 100644 --- a/ext/featurens/alpinerelease/alpinerelease_test.go +++ b/ext/featurens/alpinerelease/alpinerelease_test.go @@ -27,38 +27,38 @@ func TestDetector(t *testing.T) { testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "alpine:v3.3", VersionFormat: apk.ParserName}, - Files: tarutil.FilesMap{"etc/alpine-release": tarutil.FileData{Contents: []byte(`3.3.4`)}}, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{"etc/alpine-release": {Contents: []byte(`3.3.4`)}}), }, { ExpectedNamespace: &database.Namespace{Name: "alpine:v3.4", VersionFormat: apk.ParserName}, - Files: tarutil.FilesMap{"etc/alpine-release": tarutil.FileData{Contents: []byte(`3.4.0`)}}, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{"etc/alpine-release": {Contents: []byte(`3.4.0`)}}), }, { ExpectedNamespace: &database.Namespace{Name: "alpine:v0.3", VersionFormat: apk.ParserName}, - Files: tarutil.FilesMap{"etc/alpine-release": tarutil.FileData{Contents: []byte(`0.3.4`)}}, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{"etc/alpine-release": {Contents: []byte(`0.3.4`)}}), }, { ExpectedNamespace: &database.Namespace{Name: "alpine:v0.3", VersionFormat: apk.ParserName}, - Files: tarutil.FilesMap{"etc/alpine-release": tarutil.FileData{Contents: []byte(` + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{"etc/alpine-release": {Contents: []byte(` 0.3.4 -`)}}, +`)}}), }, { ExpectedNamespace: &database.Namespace{Name: "alpine:edge", VersionFormat: apk.ParserName}, - Files: tarutil.FilesMap{ - "etc/alpine-release": tarutil.FileData{Contents: []byte(`3.14.0_alpha20210212`)}, - "etc/os-release": tarutil.FileData{Contents: []byte( + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/alpine-release": {Contents: []byte(`3.14.0_alpha20210212`)}, + "etc/os-release": {Contents: []byte( `NAME="Alpine Linux" ID=alpine VERSION_ID=3.14.0_alpha20210212 PRETTY_NAME="Alpine Linux edge" HOME_URL="https://alpinelinux.org/" BUG_REPORT_URL="https://bugs.alpinelinux.org/"`)}, - }, + }), }, { ExpectedNamespace: nil, - Files: tarutil.FilesMap{}, + Files: tarutil.CreateNewLayerFiles(nil), }, } diff --git a/ext/featurens/aptsources/aptsources.go b/ext/featurens/aptsources/aptsources.go index 118956846..8651297fb 100644 --- a/ext/featurens/aptsources/aptsources.go +++ b/ext/featurens/aptsources/aptsources.go @@ -35,8 +35,8 @@ func init() { featurens.RegisterDetector("apt-sources", &detector{}) } -func (d detector) Detect(files tarutil.FilesMap, _ *featurens.DetectorOptions) *database.Namespace { - f, hasFile := files["etc/apt/sources.list"] +func (d detector) Detect(files tarutil.LayerFiles, _ *featurens.DetectorOptions) *database.Namespace { + f, hasFile := files.Get("etc/apt/sources.list") if !hasFile { return nil } diff --git a/ext/featurens/aptsources/aptsources_test.go b/ext/featurens/aptsources/aptsources_test.go index eaa51140b..5a7215095 100644 --- a/ext/featurens/aptsources/aptsources_test.go +++ b/ext/featurens/aptsources/aptsources_test.go @@ -27,20 +27,19 @@ func TestDetector(t *testing.T) { testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "debian:unstable", VersionFormat: dpkg.ParserName}, - Files: tarutil.FilesMap{ - "etc/os-release": tarutil.FileData{Contents: []byte( + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/os-release": {Contents: []byte( `PRETTY_NAME="Debian GNU/Linux stretch/sid" NAME="Debian GNU/Linux" ID=debian HOME_URL="https://www.debian.org/" SUPPORT_URL="https://www.debian.org/support/" BUG_REPORT_URL="https://bugs.debian.org/"`)}, - "etc/apt/sources.list": tarutil.FileData{Contents: []byte(`deb http://httpredir.debian.org/debian unstable main`)}, - }, + "etc/apt/sources.list": {Contents: []byte(`deb http://httpredir.debian.org/debian unstable main`)}}), }, { ExpectedNamespace: nil, - Files: tarutil.FilesMap{}, + Files: tarutil.CreateNewLayerFiles(nil), }, } diff --git a/ext/featurens/driver.go b/ext/featurens/driver.go index 059347806..81136362a 100644 --- a/ext/featurens/driver.go +++ b/ext/featurens/driver.go @@ -34,11 +34,11 @@ var ( // Detector represents an ability to detect a namespace used for organizing // features present in an image layer. type Detector interface { - // Detect attempts to determine a Namespace from a FilesMap of an image + // Detect attempts to determine a Namespace from a LayerFiles of an image // layer. - Detect(tarutil.FilesMap, *DetectorOptions) *database.Namespace + Detect(tarutil.LayerFiles, *DetectorOptions) *database.Namespace - // RequiredFilenames returns the list of files required to be in the FilesMap + // RequiredFilenames returns the list of files required to be in the LayerFiles // provided to the Detect method. // // Filenames must not begin with "/". @@ -69,7 +69,7 @@ func RegisterDetector(name string, d Detector) { // Detect iterators through all registered Detectors and returns the first // non-nil detected namespace. -func Detect(files tarutil.FilesMap, opts *DetectorOptions) *database.Namespace { +func Detect(files tarutil.LayerFiles, opts *DetectorOptions) *database.Namespace { detectorsM.RLock() defer detectorsM.RUnlock() @@ -100,7 +100,7 @@ func RequiredFilenames() (files []string) { // TestData represents the data used to test an implementation of Detector. type TestData struct { - Files tarutil.FilesMap + Files tarutil.LayerFiles ExpectedNamespace *database.Namespace Options *DetectorOptions } diff --git a/ext/featurens/lsbrelease/lsbrelease.go b/ext/featurens/lsbrelease/lsbrelease.go index 64c996a3c..7a8103c33 100644 --- a/ext/featurens/lsbrelease/lsbrelease.go +++ b/ext/featurens/lsbrelease/lsbrelease.go @@ -42,8 +42,8 @@ func init() { featurens.RegisterDetector("lsb-release", &detector{}) } -func (d detector) Detect(files tarutil.FilesMap, _ *featurens.DetectorOptions) *database.Namespace { - f, hasFile := files["etc/lsb-release"] +func (d detector) Detect(files tarutil.LayerFiles, _ *featurens.DetectorOptions) *database.Namespace { + f, hasFile := files.Get("etc/lsb-release") if !hasFile { return nil } diff --git a/ext/featurens/lsbrelease/lsbrelease_test.go b/ext/featurens/lsbrelease/lsbrelease_test.go index db56e391e..b57848115 100644 --- a/ext/featurens/lsbrelease/lsbrelease_test.go +++ b/ext/featurens/lsbrelease/lsbrelease_test.go @@ -27,31 +27,31 @@ func TestDetector(t *testing.T) { testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "ubuntu:12.04", VersionFormat: dpkg.ParserName}, - Files: tarutil.FilesMap{ - "etc/lsb-release": tarutil.FileData{ + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/lsb-release": { Contents: []byte( `DISTRIB_ID=Ubuntu DISTRIB_RELEASE=12.04 DISTRIB_CODENAME=precise DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`), }, - }, + }), }, { // We don't care about the minor version of Debian ExpectedNamespace: &database.Namespace{Name: "debian:7", VersionFormat: dpkg.ParserName}, - Files: tarutil.FilesMap{ - "etc/lsb-release": tarutil.FileData{ + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/lsb-release": { Contents: []byte( `DISTRIB_ID=Debian DISTRIB_RELEASE=7.1 DISTRIB_CODENAME=wheezy DISTRIB_DESCRIPTION="Debian 7.1"`), }, - }, + }), }, { ExpectedNamespace: nil, - Files: tarutil.FilesMap{}, + Files: tarutil.LayerFiles{}, }, } diff --git a/ext/featurens/osrelease/osrelease.go b/ext/featurens/osrelease/osrelease.go index 36916d3c0..a325eedf2 100644 --- a/ext/featurens/osrelease/osrelease.go +++ b/ext/featurens/osrelease/osrelease.go @@ -47,17 +47,17 @@ func init() { featurens.RegisterDetector("os-release", &detector{}) } -func (d detector) Detect(files tarutil.FilesMap, _ *featurens.DetectorOptions) *database.Namespace { +func (d detector) Detect(files tarutil.LayerFiles, _ *featurens.DetectorOptions) *database.Namespace { var OS, version string for _, filePath := range blocklistFilenames { - if _, hasFile := files[filePath]; hasFile { + if _, hasFile := files.Get(filePath); hasFile { return nil } } for _, filePath := range d.RequiredFilenames() { - f, hasFile := files[filePath] + f, hasFile := files.Get(filePath) if !hasFile { continue } diff --git a/ext/featurens/osrelease/osrelease_test.go b/ext/featurens/osrelease/osrelease_test.go index e76bdff39..1273fcf4b 100644 --- a/ext/featurens/osrelease/osrelease_test.go +++ b/ext/featurens/osrelease/osrelease_test.go @@ -28,8 +28,8 @@ func TestDetector(t *testing.T) { testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "debian:8", VersionFormat: dpkg.ParserName}, - Files: tarutil.FilesMap{ - "etc/os-release": tarutil.FileData{Contents: []byte( + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/os-release": {Contents: []byte( `PRETTY_NAME="Debian GNU/Linux 8 (jessie)" NAME="Debian GNU/Linux" VERSION_ID="8" @@ -38,12 +38,12 @@ ID=debian HOME_URL="http://www.debian.org/" SUPPORT_URL="http://www.debian.org/support/" BUG_REPORT_URL="https://bugs.debian.org/"`)}, - }, + }), }, { ExpectedNamespace: &database.Namespace{Name: "ubuntu:15.10", VersionFormat: dpkg.ParserName}, - Files: tarutil.FilesMap{ - "etc/os-release": tarutil.FileData{Contents: []byte( + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/os-release": {Contents: []byte( `NAME="Ubuntu" VERSION="15.10 (Wily Werewolf)" ID=ubuntu @@ -53,12 +53,12 @@ VERSION_ID="15.10" HOME_URL="http://www.ubuntu.com/" SUPPORT_URL="http://help.ubuntu.com/" BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`)}, - }, + }), }, { // Doesn't have quotes around VERSION_ID ExpectedNamespace: &database.Namespace{Name: "fedora:20", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/os-release": tarutil.FileData{Contents: []byte( + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/os-release": {Contents: []byte( `NAME=Fedora VERSION="20 (Heisenbug)" ID=fedora @@ -72,11 +72,11 @@ REDHAT_BUGZILLA_PRODUCT="Fedora" REDHAT_BUGZILLA_PRODUCT_VERSION=20 REDHAT_SUPPORT_PRODUCT="Fedora" REDHAT_SUPPORT_PRODUCT_VERSION=20`)}, - }, + }), }, { ExpectedNamespace: nil, - Files: tarutil.FilesMap{}, + Files: tarutil.CreateNewLayerFiles(nil), }, } diff --git a/ext/featurens/redhatrelease/redhatrelease.go b/ext/featurens/redhatrelease/redhatrelease.go index 366e34f7b..d6c68fff4 100644 --- a/ext/featurens/redhatrelease/redhatrelease.go +++ b/ext/featurens/redhatrelease/redhatrelease.go @@ -45,9 +45,9 @@ func init() { featurens.RegisterDetector("redhat-release", &detector{}) } -func (d detector) Detect(files tarutil.FilesMap, opts *featurens.DetectorOptions) *database.Namespace { +func (d detector) Detect(files tarutil.LayerFiles, opts *featurens.DetectorOptions) *database.Namespace { for _, filePath := range d.RequiredFilenames() { - f, hasFile := files[filePath] + f, hasFile := files.Get(filePath) if !hasFile { continue } diff --git a/ext/featurens/redhatrelease/redhatrelease_test.go b/ext/featurens/redhatrelease/redhatrelease_test.go index df50d819e..5e113f3a7 100644 --- a/ext/featurens/redhatrelease/redhatrelease_test.go +++ b/ext/featurens/redhatrelease/redhatrelease_test.go @@ -27,68 +27,68 @@ func TestDetector(t *testing.T) { testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "amzn:2", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/system-release": tarutil.FileData{Contents: []byte(`Amazon Linux release 2 (Karoo)`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/system-release": {Contents: []byte(`Amazon Linux release 2 (Karoo)`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "amzn:2018.03", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/system-release": tarutil.FileData{Contents: []byte(`Amazon Linux AMI release 2018.03`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/system-release": {Contents: []byte(`Amazon Linux AMI release 2018.03`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "oracle:6", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/oracle-release": tarutil.FileData{Contents: []byte(`Oracle Linux Server release 6.8`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/oracle-release": {Contents: []byte(`Oracle Linux Server release 6.8`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "oracle:7", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/oracle-release": tarutil.FileData{Contents: []byte(`Oracle Linux Server release 7.2`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/oracle-release": {Contents: []byte(`Oracle Linux Server release 7.2`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "centos:6", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/centos-release": tarutil.FileData{Contents: []byte(`CentOS release 6.6 (Final)`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/centos-release": {Contents: []byte(`CentOS release 6.6 (Final)`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "rhel:7", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/redhat-release": tarutil.FileData{Contents: []byte(`Red Hat Enterprise Linux Server release 7.2 (Maipo)`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/redhat-release": {Contents: []byte(`Red Hat Enterprise Linux Server release 7.2 (Maipo)`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "rhel:8", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/redhat-release": tarutil.FileData{Contents: []byte(`Red Hat Enterprise Linux release 8.0 (Ootpa)`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/redhat-release": {Contents: []byte(`Red Hat Enterprise Linux release 8.0 (Ootpa)`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "centos:8", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/redhat-release": tarutil.FileData{Contents: []byte(`Red Hat Enterprise Linux release 8.0 (Ootpa)`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/redhat-release": {Contents: []byte(`Red Hat Enterprise Linux release 8.0 (Ootpa)`)}, + }), Options: &featurens.DetectorOptions{UncertifiedRHEL: true}, }, { ExpectedNamespace: &database.Namespace{Name: "centos:8", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/redhat-release": tarutil.FileData{Contents: []byte(`CentOS Linux release 8.3.2011`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/redhat-release": {Contents: []byte(`CentOS Linux release 8.3.2011`)}, + }), }, { ExpectedNamespace: &database.Namespace{Name: "centos:7", VersionFormat: rpm.ParserName}, - Files: tarutil.FilesMap{ - "etc/system-release": tarutil.FileData{Contents: []byte(`CentOS Linux release 7.1.1503 (Core)`)}, - }, + Files: tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "etc/system-release": {Contents: []byte(`CentOS Linux release 7.1.1503 (Core)`)}, + }), }, { ExpectedNamespace: nil, - Files: tarutil.FilesMap{}, + Files: tarutil.CreateNewLayerFiles(nil), }, } diff --git a/ext/imagefmt/docker/docker.go b/ext/imagefmt/docker/docker.go index b16c20796..a591ffa0b 100644 --- a/ext/imagefmt/docker/docker.go +++ b/ext/imagefmt/docker/docker.go @@ -30,6 +30,6 @@ func init() { imagefmt.RegisterExtractor("docker", &format{}) } -func (f format) ExtractFiles(layerReader io.ReadCloser, filenameMatcher matcher.Matcher) (tarutil.FilesMap, error) { +func (f format) ExtractFiles(layerReader io.ReadCloser, filenameMatcher matcher.Matcher) (tarutil.LayerFiles, error) { return tarutil.ExtractFiles(layerReader, filenameMatcher) } diff --git a/ext/imagefmt/driver.go b/ext/imagefmt/driver.go index 80bf38309..3ac44c8ab 100644 --- a/ext/imagefmt/driver.go +++ b/ext/imagefmt/driver.go @@ -51,8 +51,8 @@ var ( // Extractor represents an ability to extract files from a particular container // image format. type Extractor interface { - // ExtractFiles produces a tarutil.FilesMap from a image layer. - ExtractFiles(layer io.ReadCloser, filenameMatcher matcher.Matcher) (tarutil.FilesMap, error) + // ExtractFiles produces a tarutil.LayerFiles from a image layer. + ExtractFiles(layer io.ReadCloser, filenameMatcher matcher.Matcher) (tarutil.LayerFiles, error) } // RegisterExtractor makes an extractor available by the provided name. @@ -102,7 +102,7 @@ func UnregisterExtractor(name string) { } // ExtractFromReader extracts the files from a reader which is in the format of a .tar.gz -func ExtractFromReader(reader io.ReadCloser, format string, filenameMatcher matcher.Matcher) (tarutil.FilesMap, error) { +func ExtractFromReader(reader io.ReadCloser, format string, filenameMatcher matcher.Matcher) (*tarutil.LayerFiles, error) { defer reader.Close() if extractor, exists := Extractors()[strings.ToLower(format)]; exists { @@ -110,7 +110,7 @@ func ExtractFromReader(reader io.ReadCloser, format string, filenameMatcher matc if err != nil { return nil, err } - return files, nil + return &files, nil } return nil, commonerr.NewBadRequestError(fmt.Sprintf("unsupported image format %q", format)) @@ -118,7 +118,7 @@ func ExtractFromReader(reader io.ReadCloser, format string, filenameMatcher matc // Extract streams an image layer from disk or over HTTP, determines the // image format, then extracts the files specified. -func Extract(format, path string, headers map[string]string, filenameMatcher matcher.Matcher) (tarutil.FilesMap, error) { +func Extract(format, path string, headers map[string]string, filenameMatcher matcher.Matcher) (*tarutil.LayerFiles, error) { var layerReader io.ReadCloser if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { // Create a new HTTP request object. diff --git a/localdev/main.go b/localdev/main.go index 7f118efe6..bc25c7726 100644 --- a/localdev/main.go +++ b/localdev/main.go @@ -83,12 +83,13 @@ func analyzeLocalImage(path string) { panic(err) } - if _, ok := filemap["manifest.json"]; !ok { + if _, ok := filemap.Get("manifest.json"); !ok { panic("malformed .tar does not contain manifest.json") } var configs []Config - if err := json.Unmarshal(filemap["manifest.json"].Contents, &configs); err != nil { + fileData, _ := filemap.Get("manifest.json") + if err := json.Unmarshal(fileData.Contents, &configs); err != nil { panic(err) } if len(configs) == 0 { @@ -99,21 +100,25 @@ func analyzeLocalImage(path string) { // detect namespace var namespace *database.Namespace for _, l := range config.Layers { - layerTarReader := io.NopCloser(bytes.NewBuffer(filemap[l].Contents)) + fileData, _ = filemap.Get(l) + layerTarReader := io.NopCloser(bytes.NewBuffer(fileData.Contents)) files, err := imagefmt.ExtractFromReader(layerTarReader, "Docker", requiredfilenames.SingletonMatcher()) if err != nil { panic(err) } - namespace = clair.DetectNamespace(l, files, nil, false) + namespace = clair.DetectNamespace(l, *files, nil, false) if namespace != nil { break } } fmt.Println(namespace) var total time.Duration + var baseMap *tarutil.LayerFiles for _, l := range config.Layers { - layerTarReader := io.NopCloser(bytes.NewBuffer(filemap[l].Contents)) - _, _, _, rhelv2Components, languageComponents, removedComponents, err := clair.DetectContentFromReader(layerTarReader, "Docker", l, &database.Layer{Namespace: namespace}, false) + fileData, _ = filemap.Get(l) + layerTarReader := io.NopCloser(bytes.NewBuffer(fileData.Contents)) + _, _, _, rhelv2Components, languageComponents, files, err := clair.DetectContentFromReader(layerTarReader, "Docker", l, &database.Layer{Namespace: namespace}, baseMap, false) + baseMap = files if err != nil { fmt.Println(err.Error()) return @@ -123,7 +128,7 @@ func analyzeLocalImage(path string) { fmt.Printf("RHELv2 Components (%d): %s\n", len(rhelv2Components.Packages), rhelv2Components) } - fmt.Printf("Removed components: %v\n", removedComponents) + fmt.Printf("Removed components: %v\n", baseMap.GetRemovedFiles()) languageComponents = filterComponentsByName(languageComponents, "") diff --git a/pkg/matcher/matcher.go b/pkg/matcher/matcher.go index fd60277fc..bab7b68b6 100644 --- a/pkg/matcher/matcher.go +++ b/pkg/matcher/matcher.go @@ -2,6 +2,7 @@ package matcher import ( "io" + "io/fs" "os" "path/filepath" "regexp" @@ -94,3 +95,38 @@ func (o *orMatcher) Match(fullPath string, fileInfo os.FileInfo, contents io.Rea func NewOrMatcher(subMatchers ...Matcher) Matcher { return &orMatcher{matchers: subMatchers} } + +type symlinkMatcher struct{} + +func (o *symlinkMatcher) Match(fullPath string, fileInfo os.FileInfo, _ io.ReaderAt) (matches bool, extract bool) { + if fileInfo.Mode()&fs.ModeSymlink != 0 { + return true, false + } + return false, false +} + +// NewSymbolicLinkMatcher returns a matcher that matches symbolic links +func NewSymbolicLinkMatcher(subMatchers ...Matcher) Matcher { + return &symlinkMatcher{} +} + +type andMatcher struct { + matchers []Matcher +} + +func (a *andMatcher) Match(fullPath string, fileInfo os.FileInfo, contents io.ReaderAt) (matches bool, extract bool) { + extract = true + for _, subMatcher := range a.matchers { + match, extractable := subMatcher.Match(fullPath, fileInfo, contents) + if !match { + return false, false + } + extract = extract && extractable + } + return true, extract +} + +// NewAndMatcher returns a matcher that matches if all the passed submatchers match. +func NewAndMatcher(subMatchers ...Matcher) Matcher { + return &andMatcher{matchers: subMatchers} +} diff --git a/pkg/rhelv2/rpm/bench_test.go b/pkg/rhelv2/rpm/bench_test.go index 401e67c05..fdb9ac32e 100644 --- a/pkg/rhelv2/rpm/bench_test.go +++ b/pkg/rhelv2/rpm/bench_test.go @@ -26,10 +26,10 @@ func BenchmarkListFeaturesNoActiveVulnMgmt(b *testing.B) { envIsolator.Setenv(features.ActiveVulnMgmt.EnvVar(), "false") defer envIsolator.RestoreAll() - filemap := tarutil.FilesMap{ - "var/lib/rpm/Packages": tarutil.FileData{Contents: d}, - "root/buildinfo/content_manifests/test.json": tarutil.FileData{Contents: manifest}, - } + filemap := tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/rpm/Packages": {Contents: d}, + "root/buildinfo/content_manifests/test.json": {Contents: manifest}, + }) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -51,18 +51,18 @@ func BenchmarkListFeatures(b *testing.B) { envIsolator.Setenv(features.ActiveVulnMgmt.EnvVar(), "true") defer envIsolator.RestoreAll() - filemap := tarutil.FilesMap{ - "var/lib/rpm/Packages": tarutil.FileData{Contents: d}, - "root/buildinfo/content_manifests/test.json": tarutil.FileData{Contents: manifest}, - "usr/lib64/libz.so.1": tarutil.FileData{Executable: true}, - "usr/lib64/libz.so.1.2.11": tarutil.FileData{Executable: true}, - "usr/lib64/libform.so.6": tarutil.FileData{Executable: true}, - "usr/lib64/libncursesw.so.6.1": tarutil.FileData{Executable: true}, - "usr/lib64/libpanelw.so.6": tarutil.FileData{Executable: true}, - "etc/redhat-release": tarutil.FileData{Executable: true}, - "etc/os-release": tarutil.FileData{Executable: true}, - "usr/lib/redhat-release": tarutil.FileData{Executable: true}, - } + filemap := tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/rpm/Packages": {Contents: d}, + "root/buildinfo/content_manifests/test.json": {Contents: manifest}, + "usr/lib64/libz.so.1": {Executable: true}, + "usr/lib64/libz.so.1.2.11": {Executable: true}, + "usr/lib64/libform.so.6": {Executable: true}, + "usr/lib64/libncursesw.so.6.1": {Executable: true}, + "usr/lib64/libpanelw.so.6": {Executable: true}, + "etc/redhat-release": {Executable: true}, + "etc/os-release": {Executable: true}, + "usr/lib/redhat-release": {Executable: true}, + }) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/pkg/rhelv2/rpm/rpm.go b/pkg/rhelv2/rpm/rpm.go index 0ebd9fbd7..39608d3c1 100644 --- a/pkg/rhelv2/rpm/rpm.go +++ b/pkg/rhelv2/rpm/rpm.go @@ -78,20 +78,20 @@ func init() { // ListFeatures returns the features found from the given files. // returns a slice of packages found via rpm and a slice of CPEs found in // /root/buildinfo/content_manifests. -func ListFeatures(files tarutil.FilesMap) ([]*database.RHELv2Package, []string, error) { +func ListFeatures(files tarutil.LayerFiles) ([]*database.RHELv2Package, []string, error) { if features.ActiveVulnMgmt.Enabled() { return listFeatures(files, queryFmtActiveVulnMgmt) } return listFeatures(files, queryFmt) } -func listFeatures(files tarutil.FilesMap, queryFmt string) ([]*database.RHELv2Package, []string, error) { +func listFeatures(files tarutil.LayerFiles, queryFmt string) ([]*database.RHELv2Package, []string, error) { cpes, err := getCPEsUsingEmbeddedContentSets(files) if err != nil { return nil, nil, err } - f, hasFile := files[dbPath] + f, hasFile := files.Get(dbPath) if !hasFile { return nil, cpes, nil } @@ -147,7 +147,7 @@ func listFeatures(files tarutil.FilesMap, queryFmt string) ([]*database.RHELv2Pa return pkgs, cpes, nil } -func parsePackages(r io.Reader, files tarutil.FilesMap) ([]*database.RHELv2Package, error) { +func parsePackages(r io.Reader, files tarutil.LayerFiles) ([]*database.RHELv2Package, error) { var pkgs []*database.RHELv2Package p := &database.RHELv2Package{} @@ -206,8 +206,11 @@ func parsePackages(r io.Reader, files tarutil.FilesMap) ([]*database.RHELv2Packa // Rename to make it clear what the line represents. filename := line - // The first character is always "/", which is removed when inserted into the files maps. - AddToDependencyMap(filename, files[filename[1:]], execToDeps, libToDeps) + // The first character is always "/", which is removed when inserted into the layer files. + fileData, hasFile := files.Get(filename[1:]) + if hasFile { + AddToDependencyMap(filename, fileData, execToDeps, libToDeps) + } } } @@ -216,7 +219,7 @@ func parsePackages(r io.Reader, files tarutil.FilesMap) ([]*database.RHELv2Packa // AddToDependencyMap checks and adds files to executable and library dependency for RHEL package func AddToDependencyMap(filename string, fileData tarutil.FileData, execToDeps, libToDeps database.StringToStringsMap) { - // The first character is always "/", which is removed when inserted into the files maps. + // The first character is always "/", which is removed when inserted into the layer files. if fileData.Executable && !AllRHELRequiredFiles.Contains(filename[1:]) { deps := set.NewStringSet() if fileData.ELFMetadata != nil { @@ -236,7 +239,7 @@ func AddToDependencyMap(filename string, fileData tarutil.FileData, execToDeps, } } -func getCPEsUsingEmbeddedContentSets(files tarutil.FilesMap) ([]string, error) { +func getCPEsUsingEmbeddedContentSets(files tarutil.LayerFiles) ([]string, error) { defer metrics.ObserveListFeaturesTime(pkgFmt, "cpes", time.Now()) // Get CPEs using embedded content-set files. @@ -255,8 +258,8 @@ func getCPEsUsingEmbeddedContentSets(files tarutil.FilesMap) ([]string, error) { return repo2cpe.Singleton().Get(contentManifest.ContentSets) } -func getContentManifestFileContents(files tarutil.FilesMap) []byte { - for file, contents := range files { +func getContentManifestFileContents(files tarutil.LayerFiles) []byte { + for file, contents := range files.GetFilesMap() { if !contentManifestPattern.MatchString(file) { continue } diff --git a/pkg/rhelv2/rpm/rpm_language_analyzer.go b/pkg/rhelv2/rpm/rpm_language_analyzer.go index 302268c11..388e25eef 100644 --- a/pkg/rhelv2/rpm/rpm_language_analyzer.go +++ b/pkg/rhelv2/rpm/rpm_language_analyzer.go @@ -12,11 +12,11 @@ import ( // AnnotateComponentsWithPackageManagerInfo checks for each component if it was installed by the package manager, // and sets the `FromPackageManager` attribute accordingly. -func AnnotateComponentsWithPackageManagerInfo(files tarutil.FilesMap, components []*component.Component) error { +func AnnotateComponentsWithPackageManagerInfo(files tarutil.LayerFiles, components []*component.Component) error { if len(components) == 0 { return nil } - f, hasFile := files[dbPath] + f, hasFile := files.Get(dbPath) if !hasFile { return nil } diff --git a/pkg/rhelv2/rpm/rpm_test.go b/pkg/rhelv2/rpm/rpm_test.go index cb642a086..8e6a4cbca 100644 --- a/pkg/rhelv2/rpm/rpm_test.go +++ b/pkg/rhelv2/rpm/rpm_test.go @@ -16,7 +16,7 @@ import ( ) // ListFeaturesTest does the same as ListFeatures but should only be used for testing. -func ListFeaturesTest(files tarutil.FilesMap) ([]*database.RHELv2Package, []string, error) { +func ListFeaturesTest(files tarutil.LayerFiles) ([]*database.RHELv2Package, []string, error) { if features.ActiveVulnMgmt.Enabled() { return listFeatures(files, queryFmtActiveVulnMgmtTest) } @@ -75,10 +75,10 @@ func TestRPMFeatureDetection(t *testing.T) { cpesDir := filepath.Join(filepath.Dir(filename), "/testdata") envIsolator.Setenv("REPO_TO_CPE_DIR", cpesDir) - pkgs, cpes, err := ListFeaturesTest(tarutil.FilesMap{ - "var/lib/rpm/Packages": tarutil.FileData{Contents: d}, - "root/buildinfo/content_manifests/test.json": tarutil.FileData{Contents: manifest}, - }) + pkgs, cpes, err := ListFeaturesTest(tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/rpm/Packages": {Contents: d}, + "root/buildinfo/content_manifests/test.json": {Contents: manifest}, + })) assert.NoError(t, err) assert.ElementsMatch(t, cpes, expectedCPEs) assert.Subset(t, pkgs, sampleExpectedPkgs) @@ -146,18 +146,18 @@ func TestRPMFeatureDetectionWithActiveVulnMgmt(t *testing.T) { cpesDir := filepath.Join(filepath.Dir(filename), "/testdata") envIsolator.Setenv("REPO_TO_CPE_DIR", cpesDir) - pkgs, cpes, err := ListFeaturesTest(tarutil.FilesMap{ - "var/lib/rpm/Packages": tarutil.FileData{Contents: d}, - "root/buildinfo/content_manifests/test.json": tarutil.FileData{Contents: manifest}, - "usr/lib64/libz.so.1": tarutil.FileData{Executable: true}, - "usr/lib64/libz.so.1.2.11": tarutil.FileData{Executable: true}, - "usr/lib64/libform.so.6": tarutil.FileData{Executable: true}, - "usr/lib64/libncursesw.so.6.1": tarutil.FileData{Executable: true}, - "usr/lib64/libpanelw.so.6": tarutil.FileData{Executable: true}, - "etc/redhat-release": tarutil.FileData{Executable: true}, - "etc/os-release": tarutil.FileData{Executable: true}, - "usr/lib/redhat-release": tarutil.FileData{Executable: true}, - }) + pkgs, cpes, err := ListFeaturesTest(tarutil.CreateNewLayerFiles(map[string]tarutil.FileData{ + "var/lib/rpm/Packages": {Contents: d}, + "root/buildinfo/content_manifests/test.json": {Contents: manifest}, + "usr/lib64/libz.so.1": {Executable: true}, + "usr/lib64/libz.so.1.2.11": {Executable: true}, + "usr/lib64/libform.so.6": {Executable: true}, + "usr/lib64/libncursesw.so.6.1": {Executable: true}, + "usr/lib64/libpanelw.so.6": {Executable: true}, + "etc/redhat-release": {Executable: true}, + "etc/os-release": {Executable: true}, + "usr/lib/redhat-release": {Executable: true}, + })) assert.NoError(t, err) assert.ElementsMatch(t, cpes, expectedCPEs) assert.Subset(t, pkgs, sampleExpectedPkgs) diff --git a/pkg/scan/scan.go b/pkg/scan/scan.go index 194786444..5f2912681 100644 --- a/pkg/scan/scan.go +++ b/pkg/scan/scan.go @@ -17,6 +17,7 @@ import ( clair "github.com/stackrox/scanner" "github.com/stackrox/scanner/database" "github.com/stackrox/scanner/pkg/clairify/types" + "github.com/stackrox/scanner/pkg/tarutil" ) const ( @@ -28,6 +29,7 @@ func analyzeLayers(storage database.Datastore, registry types.Registry, image *t var prevLayer string var prevLineage, lineage string + var baseFiles *tarutil.LayerFiles h := sha256.New() for _, layer := range layers { layerReadCloser := &LayerDownloadReadCloser{ @@ -36,7 +38,9 @@ func analyzeLayers(storage database.Datastore, registry types.Registry, image *t }, } - err := clair.ProcessLayerFromReader(storage, "Docker", layer, lineage, prevLayer, prevLineage, layerReadCloser, uncertifiedRHEL) + var err error + // baseFiles tracks the files from previous layer to help resolve paths + baseFiles, err = clair.ProcessLayerFromReader(storage, "Docker", layer, lineage, prevLayer, prevLineage, layerReadCloser, baseFiles, uncertifiedRHEL) if err != nil { logrus.Errorf("Error analyzing layer: %v", err) return "", err diff --git a/pkg/tarutil/layer_files.go b/pkg/tarutil/layer_files.go new file mode 100644 index 000000000..fccd138e2 --- /dev/null +++ b/pkg/tarutil/layer_files.go @@ -0,0 +1,121 @@ +package tarutil + +import ( + "fmt" + "path" + "strings" + + "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/scanner/pkg/elf" + "github.com/stackrox/scanner/pkg/whiteout" +) + +// FileData is the contents of a file and relevant metadata. +type FileData struct { + // Contents is the contents of the file. + Contents []byte + // Executable indicates if the file is executable. + Executable bool + // ELFMetadata contains the dynamic library dependency metadata if the file is in ELF format. + ELFMetadata *elf.Metadata +} + +// LayerFiles represent the files in an image layer. +// It contains a map of the files' paths to their data and the information to resolve them. +type LayerFiles struct { + data map[string]FileData + // links maps a symbolic link to link target. + links map[string]string + removed set.StringSet +} + +// CreateNewLayerFiles creates a LayerFiles +func CreateNewLayerFiles(data map[string]FileData) LayerFiles { + if data == nil { + data = make(map[string]FileData) + } + return LayerFiles{data: data, links: make(map[string]string), removed: set.NewStringSet()} +} + +// GetFilesMap returns the map of files to their data +func (f LayerFiles) GetFilesMap() map[string]FileData { + return f.data +} + +// Get resolves and gets FileData for the path +func (f LayerFiles) Get(path string) (FileData, bool) { + resolved := f.resolve(path) + if !strings.HasSuffix(resolved, "/") && strings.HasSuffix(path, "/") { + resolved += "/" + } + fileData, exists := f.data[resolved] + return fileData, exists +} + +// MergeBaseAndResolveSymlinks merges a base LayerFiles to this and resolves all symbolic links +// The symbolic links are merged only for resolving paths and the files' data are not merged. +func (f LayerFiles) MergeBaseAndResolveSymlinks(base *LayerFiles) { + if base != nil { + for fileName, linkTo := range base.links { + if f.removed.Contains(fileName) { + continue + } + if _, exists := f.links[fileName]; exists { + continue + } + f.links[fileName] = linkTo + } + } + for fileName, linkTo := range f.links { + f.links[fileName] = f.resolve(linkTo) + } +} + +// GetRemovedFiles returns the files removed +func (f LayerFiles) GetRemovedFiles() []string { + return f.removed.AsSlice() +} + +func (f LayerFiles) detectRemovedFiles() { + for filePath := range f.data { + base := path.Base(filePath) + if base == whiteout.OpaqueDirectory { + // The entire directory does not exist in lower layers. + f.removed.Add(path.Dir(filePath)) + } else if strings.HasPrefix(base, whiteout.Prefix) { + removed := base[len(whiteout.Prefix):] + // Only prepend path.Dir if the directory is not `./`. + if filePath != base { + // We assume we only have Linux containers, so the path separator will be `/`. + removed = fmt.Sprintf("%s/%s", path.Dir(filePath), removed) + } + f.removed.Add(removed) + } + } +} + +// Resolve a path with symbolic links to its cleaned equivalent without +// symbolic links if it is resolvable. +// Eg. symlink -> file, and dirlink -> dir +// Resolve /dir/symlink to /dir/file and /dirlink/symlink to /dir/file +func (f LayerFiles) resolve(symLink string) string { + resolved := symLink + visited := set.NewStringSet(resolved) + for curr, list := ".", strings.Split(symLink, "/"); len(list) > 0; { + curr = path.Clean(curr + "/" + list[0]) + list = list[1:] + + if linkTo, ok := f.links[curr]; ok { + list = append(strings.Split(linkTo, "/"), list...) + curr = "." + resolved = strings.Join(list, "/") + if visited.Contains(resolved) { + // Detect a loop and return its current resolved path as best effort + // like symlink1 <=> symlink2 + return resolved + } + visited.Add(resolved) + } + } + return resolved +} diff --git a/pkg/tarutil/tarutil.go b/pkg/tarutil/tarutil.go index a54471542..3776bd749 100644 --- a/pkg/tarutil/tarutil.go +++ b/pkg/tarutil/tarutil.go @@ -23,6 +23,7 @@ import ( "compress/gzip" "io" "os/exec" + "path" "strings" "github.com/pkg/errors" @@ -59,23 +60,10 @@ func SetMaxExtractableFileSize(val int64) { maxExtractableFileSize = val } -// FileData is the contents of a file and relevant metadata. -type FileData struct { - // Contents is the contents of the file. - Contents []byte - // Executable indicates if the file is executable. - Executable bool - // ELFMetadata contains the dynamic library dependency metadata if the file is in ELF format. - ELFMetadata *elf.Metadata -} - -// FilesMap is a map of files' paths to their contents. -type FilesMap map[string]FileData - // ExtractFiles decompresses and extracts only the specified files from an // io.Reader representing an archive. -func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (FilesMap, error) { - data := make(map[string]FileData) +func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (LayerFiles, error) { + files := CreateNewLayerFiles(nil) // executableMatcher indicates if the given file is executable // for the FileData struct. @@ -84,7 +72,7 @@ func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (FilesMap, error // Decompress the archive. tr, err := NewTarReadCloser(r) if err != nil { - return data, errors.Wrap(err, "could not extract tar archive") + return files, errors.Wrap(err, "could not extract tar archive") } defer tr.Close() @@ -100,7 +88,7 @@ func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (FilesMap, error break } if err != nil { - return data, errors.Wrap(err, "could not advance in the tar archive") + return files, errors.Wrap(err, "could not advance in the tar archive") } numFiles++ @@ -133,7 +121,8 @@ func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (FilesMap, error } // Extract the element - if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink || hdr.Typeflag == tar.TypeReg { + switch hdr.Typeflag { + case tar.TypeReg, tar.TypeLink: var fileData FileData fileData.ELFMetadata, err = elf.GetExecutableMetadata(contents) @@ -144,7 +133,7 @@ func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (FilesMap, error executable, _ := executableMatcher.Match(filename, hdr.FileInfo(), contents) if !extractContents || hdr.Typeflag != tar.TypeReg { fileData.Executable = executable - data[filename] = fileData + files.data[filename] = fileData continue } @@ -157,23 +146,25 @@ func ExtractFiles(r io.Reader, filenameMatcher matcher.Matcher) (FilesMap, error // Put the file directly fileData.Contents = d fileData.Executable = executable - data[filename] = fileData + files.data[filename] = fileData numExtractedContentBytes += len(d) - } - if hdr.Typeflag == tar.TypeDir { + case tar.TypeSymlink: + files.links[filename] = path.Clean(path.Join(path.Dir(filename), hdr.Linkname)) + case tar.TypeDir: // Do not bother saving the contents, // and directories are NOT considered executable. // However, add to the map, so the entry will exist. - data[filename] = FileData{} + files.data[filename] = FileData{} } } + files.detectRemovedFiles() metrics.ObserveFileCount(numFiles) metrics.ObserveMatchedFileCount(numMatchedFiles) metrics.ObserveExtractedContentBytes(numExtractedContentBytes) - return data, nil + return files, nil } // XzReader implements io.ReadCloser for data compressed via `xz`. diff --git a/pkg/tarutil/tarutil_test.go b/pkg/tarutil/tarutil_test.go index 2a2875b1b..b0c47aae1 100644 --- a/pkg/tarutil/tarutil_test.go +++ b/pkg/tarutil/tarutil_test.go @@ -42,18 +42,18 @@ func testfilepath(filename string) string { func TestExtract(t *testing.T) { for _, filename := range testTarballs { f, err := os.Open(testfilepath(filename)) - assert.Nil(t, err) + assert.NoError(t, err) defer f.Close() data, err := ExtractFiles(f, matcher.NewPrefixAllowlistMatcher("test/")) - assert.Nil(t, err) + assert.NoError(t, err) - if c, n := data["test/test.txt"]; !n { + if c, n := data.Get("test/test.txt"); !n { assert.Fail(t, "test/test.txt should have been extracted") } else { assert.NotEqual(t, 0, len(c.Contents) > 0, "test/test.txt file is empty") } - if _, n := data["test.txt"]; n { + if _, n := data.Get("test.txt"); n { assert.Fail(t, "test.txt should not be extracted") } } @@ -62,7 +62,7 @@ func TestExtract(t *testing.T) { func TestExtractUncompressedData(t *testing.T) { for _, filename := range testTarballs { f, err := os.Open(testfilepath(filename)) - assert.Nil(t, err) + assert.NoError(t, err) defer f.Close() _, err = ExtractFiles(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), matcher.NewPrefixAllowlistMatcher()) @@ -72,15 +72,66 @@ func TestExtractUncompressedData(t *testing.T) { func TestMaxExtractableFileSize(t *testing.T) { f, err := os.Open(testfilepath("utils_test.tar.gz")) - assert.Nil(t, err) + assert.NoError(t, err) defer utils.IgnoreError(f.Close) - contents, err := ExtractFiles(f, matcher.NewPrefixAllowlistMatcher("test_big.txt")) + files, err := ExtractFiles(f, matcher.NewPrefixAllowlistMatcher("test_big.txt")) assert.NoError(t, err) // test_big.txt is of size 57 bytes. - assert.Contains(t, contents, "test_big.txt") + assert.Contains(t, files.data, "test_big.txt") SetMaxExtractableFileSize(50) - contents, err = ExtractFiles(f, matcher.NewPrefixAllowlistMatcher("test_big.txt")) + files, err = ExtractFiles(f, matcher.NewPrefixAllowlistMatcher("test_big.txt")) + assert.NoError(t, err) + assert.Empty(t, files.data) +} + +func TestExtractWithSymlink(t *testing.T) { + f, err := os.Open(testfilepath("symlink.tar.gz")) + assert.NoError(t, err) + defer utils.IgnoreError(f.Close) + expected := map[string]string{ + // Link to directory + "dirlink": "dir", + "opt/dirlink": "dir", + // Link to files + "opt/symlink": "dir/dir_file", + "dir/symlink": "dir/dir_file", + "1/2/3/4/symlink": "dir/dir_file", + // Multiple level symlinks + "link/symlink": "dir/dir_file", + // This is a loop of symlinks + "link/link1": "link/link2", + "link/link2": "link/link1", + "l1": "1", + "1/l2": "1/2", + "1/2/l3": "1/2/3", + "1/2/3/l4": "1/2/3/4", + "l4": "1/2/3/4", + "lib64": "1", + } + + files, err := ExtractFiles(f, matcher.NewPrefixAllowlistMatcher("")) + base := LayerFiles{data: make(map[string]FileData), links: map[string]string{"lib64": "l1"}} + files.MergeBaseAndResolveSymlinks(&base) assert.NoError(t, err) - assert.Empty(t, contents) + assert.Len(t, files.data, 9) + assert.Len(t, files.links, 14) + + for fileName, linkTo := range files.links { + if target, ok := expected[fileName]; ok { + assert.Equal(t, target, linkTo) + } + } + verifyContent(t, files, "opt/dirlink/symlink") + verifyContent(t, files, "l1/l2/l3/l4/symlink") + verifyContent(t, files, "l1/2/l3/4/symlink") + verifyContent(t, files, "opt/dirlink/dir_file") + + verifyContent(t, files, "lib64/2/l3/4/symlink") +} + +func verifyContent(t *testing.T, files LayerFiles, p string) { + fileData, exists := files.Get(p) + assert.True(t, exists) + assert.Equal(t, "test\n", string(fileData.Contents)) } diff --git a/pkg/tarutil/testdata/symlink.tar.gz b/pkg/tarutil/testdata/symlink.tar.gz new file mode 100644 index 000000000..f0a2c264f Binary files /dev/null and b/pkg/tarutil/testdata/symlink.tar.gz differ diff --git a/singletons/requiredfilenames/matcher.go b/singletons/requiredfilenames/matcher.go index ba77061f9..962e556cb 100644 --- a/singletons/requiredfilenames/matcher.go +++ b/singletons/requiredfilenames/matcher.go @@ -12,9 +12,13 @@ import ( ) var ( - instance matcher.Matcher - once sync.Once + instance matcher.Matcher + once sync.Once + // dynamicLibRegexp matches all dynamic libraries. dynamicLibRegexp = regexp.MustCompile(`(^|/)(lib|ld-)[^/.-][^/]*\.so(\.[^/.]+)*$`) + // libraryDirRegexp matches all files under directories where the dynamic libraries are commonly found. + // This is to filter for symbolic links needed to resolve dynamic library paths. + libraryDirRegexp = regexp.MustCompile(`^(usr/(local/)?)?lib(32|64)?(/.+|$)`) ) // SingletonMatcher returns the singleton matcher instance to use for extracting @@ -29,12 +33,13 @@ func SingletonMatcher() matcher.Matcher { whiteoutMatcher := matcher.NewWhiteoutMatcher() // Allocate extra spaces for the feature-flagged matchers. - allMatchers := make([]matcher.Matcher, 0, 5) + allMatchers := make([]matcher.Matcher, 0, 6) allMatchers = append(allMatchers, clairMatcher, whiteoutMatcher) if features.ActiveVulnMgmt.Enabled() { dpkgFilenamesMatcher := matcher.NewRegexpMatcher(dpkg.FilenamesListRegexp) dynamicLibMatcher := matcher.NewRegexpMatcher(dynamicLibRegexp) + libDirSymlinkMatcher := matcher.NewAndMatcher(matcher.NewRegexpMatcher(libraryDirRegexp), matcher.NewSymbolicLinkMatcher()) // All other matchers take precedence over this matcher. // For example, an executable python file should be matched by // the Python matcher. This matcher should be used for any @@ -42,7 +47,7 @@ func SingletonMatcher() matcher.Matcher { // Therefore, this matcher MUST be the last matcher. executableMatcher := matcher.NewExecutableMatcher() - allMatchers = append(allMatchers, dpkgFilenamesMatcher, dynamicLibMatcher, executableMatcher) + allMatchers = append(allMatchers, dpkgFilenamesMatcher, dynamicLibMatcher, libDirSymlinkMatcher, executableMatcher) } instance = matcher.NewOrMatcher(allMatchers...) diff --git a/singletons/requiredfilenames/matcher_test.go b/singletons/requiredfilenames/matcher_test.go index 57fbcd4e9..c9350bc84 100644 --- a/singletons/requiredfilenames/matcher_test.go +++ b/singletons/requiredfilenames/matcher_test.go @@ -24,5 +24,27 @@ func TestDynamicLibraryRegex(t *testing.T) { assert.True(t, dynamicLibRegexp.MatchString("lib/x86_64-linux-gnu/ld-linux-x86-64.so.2")) assert.True(t, dynamicLibRegexp.MatchString("lib/x86_64-linux-gnu/libdl.so.2")) assert.True(t, dynamicLibRegexp.MatchString("lib/x86_64-linux-gnu/libdl-2.23.so")) +} + +func TestLibraryDirRegex(t *testing.T) { + assert.True(t, libraryDirRegexp.MatchString("lib")) + assert.True(t, libraryDirRegexp.MatchString("lib32")) + assert.True(t, libraryDirRegexp.MatchString("lib64")) + assert.True(t, libraryDirRegexp.MatchString("lib64/abc")) + + assert.True(t, libraryDirRegexp.MatchString("usr/lib")) + assert.True(t, libraryDirRegexp.MatchString("usr/lib32")) + assert.True(t, libraryDirRegexp.MatchString("usr/lib64")) + assert.True(t, libraryDirRegexp.MatchString("usr/lib32/abc")) + + assert.True(t, libraryDirRegexp.MatchString("usr/local/lib")) + assert.True(t, libraryDirRegexp.MatchString("usr/local/lib32")) + assert.True(t, libraryDirRegexp.MatchString("usr/local/lib64")) + assert.True(t, libraryDirRegexp.MatchString("usr/local/lib/abc/d")) + assert.False(t, libraryDirRegexp.MatchString("usr/local/abc")) + assert.False(t, libraryDirRegexp.MatchString("lib/")) + assert.False(t, libraryDirRegexp.MatchString("/lib/abc")) + assert.False(t, libraryDirRegexp.MatchString("libxyz/abc")) + assert.False(t, libraryDirRegexp.MatchString("local/lib64/abc")) } diff --git a/worker.go b/worker.go index a729c9dcf..a4807cc34 100644 --- a/worker.go +++ b/worker.go @@ -15,11 +15,8 @@ package clair import ( - "fmt" "io" "os" - "path/filepath" - "strings" log "github.com/sirupsen/logrus" "github.com/stackrox/scanner/database" @@ -35,7 +32,6 @@ import ( rhelv2 "github.com/stackrox/scanner/pkg/rhelv2/rpm" "github.com/stackrox/scanner/pkg/tarutil" namespaces "github.com/stackrox/scanner/pkg/wellknownnamespaces" - "github.com/stackrox/scanner/pkg/whiteout" "github.com/stackrox/scanner/singletons/analyzers" "github.com/stackrox/scanner/singletons/requiredfilenames" ) @@ -112,22 +108,22 @@ func preProcessLayer(datastore database.Datastore, imageFormat, name, lineage, p // // TODO(Quentin-M): We could have a goroutine that looks for layers that have // been analyzed with an older engine version and that processes them. -func ProcessLayerFromReader(datastore database.Datastore, imageFormat, name, lineage, parentName, parentLineage string, reader io.ReadCloser, uncertifiedRHEL bool) error { +func ProcessLayerFromReader(datastore database.Datastore, imageFormat, name, lineage, parentName, parentLineage string, reader io.ReadCloser, base *tarutil.LayerFiles, uncertifiedRHEL bool) (*tarutil.LayerFiles, error) { layer, exists, err := preProcessLayer(datastore, imageFormat, name, lineage, parentName, parentLineage, uncertifiedRHEL) if err != nil { - return err + return nil, err } if exists { - return nil + return nil, nil } // Analyze the content. var rhelv2Components *database.RHELv2Components var languageComponents []*component.Component - var removedFiles []string - layer.Namespace, layer.Distroless, layer.Features, rhelv2Components, languageComponents, removedFiles, err = DetectContentFromReader(reader, imageFormat, name, layer.Parent, uncertifiedRHEL) + var files *tarutil.LayerFiles + layer.Namespace, layer.Distroless, layer.Features, rhelv2Components, languageComponents, files, err = DetectContentFromReader(reader, imageFormat, name, layer.Parent, base, uncertifiedRHEL) if err != nil { - return err + return nil, err } if rhelv2Components != nil { @@ -145,7 +141,7 @@ func ProcessLayerFromReader(datastore database.Datastore, imageFormat, name, lin } if err := datastore.InsertRHELv2Layer(rhelv2Layer); err != nil { - return err + return nil, err } } @@ -157,15 +153,15 @@ func ProcessLayerFromReader(datastore database.Datastore, imageFormat, name, lin // relies on the original layer table. if err := datastore.InsertLayer(layer, lineage, opts); err != nil { if err == commonerr.ErrNoNeedToInsert { - return nil + return nil, nil } - return err + return nil, err } - return datastore.InsertLayerComponents(layer.Name, lineage, languageComponents, removedFiles, opts) + return files, datastore.InsertLayerComponents(layer.Name, lineage, languageComponents, files.GetRemovedFiles(), opts) } -func detectFromFiles(files tarutil.FilesMap, name string, parent *database.Layer, languageComponents []*component.Component, uncertifiedRHEL bool) (*database.Namespace, bool, []database.FeatureVersion, *database.RHELv2Components, []*component.Component, []string, error) { +func detectFromFiles(files tarutil.LayerFiles, name string, parent *database.Layer, languageComponents []*component.Component, uncertifiedRHEL bool) (*database.Namespace, bool, []database.FeatureVersion, *database.RHELv2Components, []*component.Component, error) { namespace := DetectNamespace(name, files, parent, uncertifiedRHEL) distroless := isDistroless(files) || (parent != nil && parent.Distroless) @@ -178,7 +174,7 @@ func detectFromFiles(files tarutil.FilesMap, name string, parent *database.Layer // Use the RHELv2 scanner instead. packages, cpes, err := rhelv2.ListFeatures(files) if err != nil { - return nil, distroless, nil, nil, nil, nil, err + return nil, distroless, nil, nil, nil, err } rhelfeatures = &database.RHELv2Components{ Dist: namespace.Name, @@ -194,7 +190,7 @@ func detectFromFiles(files tarutil.FilesMap, name string, parent *database.Layer // Detect features. featureVersions, err = detectFeatureVersions(name, files, namespace, parent) if err != nil { - return nil, distroless, nil, nil, nil, nil, err + return nil, distroless, nil, nil, nil, err } if len(featureVersions) > 0 { log.WithFields(log.Fields{logLayerName: name, "feature count": len(featureVersions)}).Debug("detected features") @@ -202,27 +198,10 @@ func detectFromFiles(files tarutil.FilesMap, name string, parent *database.Layer } if !env.LanguageVulns.Enabled() { - return namespace, distroless, featureVersions, rhelfeatures, nil, nil, nil + return namespace, distroless, featureVersions, rhelfeatures, nil, nil } - var removedFiles []string - for filePath := range files { - base := filepath.Base(filePath) - if base == whiteout.OpaqueDirectory { - // The entire directory does not exist in lower layers. - removedFiles = append(removedFiles, filepath.Dir(filePath)) - } else if strings.HasPrefix(base, whiteout.Prefix) { - removed := base[len(whiteout.Prefix):] - // Only prepend filepath.Dir if the directory is not `./`. - if filePath != base { - // We assume we only have Linux containers, so the path separator will be `/`. - removed = fmt.Sprintf("%s/%s", filepath.Dir(filePath), removed) - } - removedFiles = append(removedFiles, removed) - } - } - - return namespace, distroless, featureVersions, rhelfeatures, languageComponents, removedFiles, nil + return namespace, distroless, featureVersions, rhelfeatures, languageComponents, nil } // analyzingMatcher is a Matcher implementation that calls ProcessFile on each analyzer, @@ -242,7 +221,7 @@ func (m *analyzingMatcher) Match(filePath string, fi os.FileInfo, contents io.Re } // DetectContentFromReader detects scanning content in the given reader. -func DetectContentFromReader(reader io.ReadCloser, format, name string, parent *database.Layer, uncertifiedRHEL bool) (*database.Namespace, bool, []database.FeatureVersion, *database.RHELv2Components, []*component.Component, []string, error) { +func DetectContentFromReader(reader io.ReadCloser, format, name string, parent *database.Layer, base *tarutil.LayerFiles, uncertifiedRHEL bool) (*database.Namespace, bool, []database.FeatureVersion, *database.RHELv2Components, []*component.Component, *tarutil.LayerFiles, error) { // Create a "matcher" that actually calls `ProcessFile` on each analyzer, before delegating // to the actual matcher for operating system-level feature extraction. // TODO: this is ugly. A matcher should not have side-effects; but the `analyzingMatcher`s @@ -257,21 +236,23 @@ func DetectContentFromReader(reader io.ReadCloser, format, name string, parent * if err != nil { return nil, false, nil, nil, nil, nil, err } + files.MergeBaseAndResolveSymlinks(base) if len(m.components) > 0 { log.WithFields(log.Fields{logLayerName: name, "component count": len(m.components)}).Debug("detected components") } - return detectFromFiles(files, name, parent, m.components, uncertifiedRHEL) + namespace, distroless, features, rhelv2Components, languageComponents, err := detectFromFiles(*files, name, parent, m.components, uncertifiedRHEL) + return namespace, distroless, features, rhelv2Components, languageComponents, files, err } -func isDistroless(filesMap tarutil.FilesMap) bool { - _, ok := filesMap["var/lib/dpkg/status.d/"] +func isDistroless(filesMap tarutil.LayerFiles) bool { + _, ok := filesMap.Get("var/lib/dpkg/status.d/") return ok } // DetectNamespace detects the layer's namespace. -func DetectNamespace(name string, files tarutil.FilesMap, parent *database.Layer, uncertifiedRHEL bool) *database.Namespace { +func DetectNamespace(name string, files tarutil.LayerFiles, parent *database.Layer, uncertifiedRHEL bool) *database.Namespace { namespace := featurens.Detect(files, &featurens.DetectorOptions{ UncertifiedRHEL: uncertifiedRHEL, }) @@ -292,7 +273,7 @@ func DetectNamespace(name string, files tarutil.FilesMap, parent *database.Layer return nil } -func detectFeatureVersions(name string, files tarutil.FilesMap, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) { +func detectFeatureVersions(name string, files tarutil.LayerFiles, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) { // TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that // some detectors would need it in order to produce the entire feature list (if they can only // detect a diff). Also, we should probably pass the detected namespace so detectors could diff --git a/worker_test.go b/worker_test.go index 1531b2c32..eb2d2ce85 100644 --- a/worker_test.go +++ b/worker_test.go @@ -91,9 +91,12 @@ func TestProcessWithDistUpgrade(t *testing.T) { // wheezy.tar: FROM debian:wheezy // jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && // apt-get -y dist-upgrade - assert.Nil(t, ProcessLayerFromReader(datastore, "Docker", "blank", "", "", "", getTestDataReader(t, testDataPath+"blank.tar.gz"), false)) - assert.Nil(t, ProcessLayerFromReader(datastore, "Docker", "wheezy", "", "blank", "", getTestDataReader(t, testDataPath+"wheezy.tar.gz"), false)) - assert.Nil(t, ProcessLayerFromReader(datastore, "Docker", "jessie", "", "wheezy", "", getTestDataReader(t, testDataPath+"jessie.tar.gz"), false)) + _, err := ProcessLayerFromReader(datastore, "Docker", "blank", "", "", "", getTestDataReader(t, testDataPath+"blank.tar.gz"), nil, false) + assert.Nil(t, err) + _, err = ProcessLayerFromReader(datastore, "Docker", "wheezy", "", "blank", "", getTestDataReader(t, testDataPath+"wheezy.tar.gz"), nil, false) + assert.Nil(t, err) + _, err = ProcessLayerFromReader(datastore, "Docker", "jessie", "", "wheezy", "", getTestDataReader(t, testDataPath+"jessie.tar.gz"), nil, false) + assert.Nil(t, err) // Ensure that the 'wheezy' layer has the expected namespace and features. wheezy, ok := datastore.layers["wheezy"]