From 4ace979750077ba20542f3755162edfd63259fdc Mon Sep 17 00:00:00 2001 From: crozzy Date: Fri, 8 Dec 2023 11:39:09 -0800 Subject: [PATCH] spdx: Add converter for index reports Adding a function to be able to convert index reports into SPDX documents and SPDX documents into index reports. Signed-off-by: crozzy --- go.mod | 2 + go.sum | 13 +- pkg/sbom/spdx/spdx.go | 314 +++++++++++++++++++++++++++++++++++++ pkg/sbom/spdx/spdx_test.go | 132 ++++++++++++++++ 4 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 pkg/sbom/spdx/spdx.go create mode 100644 pkg/sbom/spdx/spdx_test.go diff --git a/go.mod b/go.mod index 57c06f928..fb6803741 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/quay/zlog v1.1.8 github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319 github.com/rs/zerolog v1.30.0 + github.com/spdx/tools-golang v0.5.3 github.com/ulikunitz/xz v0.5.11 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 @@ -35,6 +36,7 @@ require ( ) require ( + github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index e307c6729..bd7e8ab6b 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -170,15 +172,22 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spdx/gordf v0.0.0-20201111095634-7098f93598fb/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= +github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY= +github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= @@ -290,6 +299,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -310,3 +320,4 @@ modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/sbom/spdx/spdx.go b/pkg/sbom/spdx/spdx.go new file mode 100644 index 000000000..80cc9def6 --- /dev/null +++ b/pkg/sbom/spdx/spdx.go @@ -0,0 +1,314 @@ +package spdx + +import ( + "fmt" + "runtime/debug" + "time" + + "github.com/google/uuid" + "github.com/spdx/tools-golang/spdx/v2/common" + spdxtools "github.com/spdx/tools-golang/spdx/v2/v2_3" + + "github.com/quay/claircore" + "github.com/quay/claircore/pkg/cpe" +) + +func ParseSPDXDocument(sd *spdxtools.Document) (*claircore.IndexReport, error) { + pkgMap := map[string]*spdxtools.Package{} + for _, p := range sd.Packages { + pkgMap[string(p.PackageSPDXIdentifier)] = p + } + digest, err := claircore.ParseDigest(sd.DocumentName) + if err != nil { + return nil, fmt.Errorf("cannot parse document name as a digest: %w", err) + } + out := &claircore.IndexReport{ + Hash: digest, + Repositories: map[string]*claircore.Repository{}, + Packages: map[string]*claircore.Package{}, + Distributions: map[string]*claircore.Distribution{}, + Environments: map[string][]*claircore.Environment{}, + Success: true, + } + for _, r := range sd.Relationships { + aPkg := pkgMap[string(r.RefA.ElementRefID)] + bPkg := pkgMap[string(r.RefB.ElementRefID)] + + if r.Relationship == "CONTAINED_BY" { + if bPkg.PackageSummary == "repository" { + // Create repository + repo := &claircore.Repository{ + ID: string(bPkg.PackageSPDXIdentifier), + Name: bPkg.PackageName, + } + for _, er := range bPkg.PackageExternalReferences { + switch er.RefType { + case "cpe23Type": + if er.Locator == "" { + continue + } + repo.CPE, err = cpe.Unbind(er.Locator) + if err != nil { + return nil, fmt.Errorf("error unbinding repository CPE: %w", err) + } + case "url": + repo.URI = er.Locator + case "key": + repo.Key = er.Locator + } + } + out.Repositories[string(bPkg.PackageSPDXIdentifier)] = repo + if _, ok := out.Packages[string(aPkg.PackageSPDXIdentifier)]; !ok { + out.Packages[string(aPkg.PackageSPDXIdentifier)] = &claircore.Package{ + ID: string(aPkg.PackageSPDXIdentifier), + Name: aPkg.PackageName, + Version: aPkg.PackageVersion, + Kind: claircore.BINARY, + } + } + } + if bPkg.PackageSummary == "distribution" { + if _, ok := out.Distributions[string(bPkg.PackageSPDXIdentifier)]; !ok { + dist := &claircore.Distribution{ + ID: string(bPkg.PackageSPDXIdentifier), + Name: bPkg.PackageName, + Version: bPkg.PackageVersion, + } + for _, er := range bPkg.PackageExternalReferences { + switch er.RefType { + case "cpe23Type": + if er.Locator == "" { + continue + } + dist.CPE, err = cpe.Unbind(er.Locator) + if err != nil { + return nil, fmt.Errorf("error unbinding distribution CPE: %w", err) + } + case "did": + dist.DID = er.Locator + case "version_id": + dist.VersionID = er.Locator + case "pretty_name": + dist.PrettyName = er.Locator + } + } + out.Distributions[string(bPkg.PackageSPDXIdentifier)] = dist + } + } + } + // Make or get environment for package + envs, ok := out.Environments[string(aPkg.PackageSPDXIdentifier)] + if !ok { + envs = append(envs, &claircore.Environment{ + PackageDB: aPkg.PackageFileName, + }) + } + if r.Relationship == "CONTAINED_BY" { + switch bPkg.PackageSummary { + case "layer": + envs[0].IntroducedIn = claircore.MustParseDigest(bPkg.PackageName) + case "repository": + envs[0].RepositoryIDs = append(envs[0].RepositoryIDs, string(bPkg.PackageSPDXIdentifier)) + case "distribution": + envs[0].DistributionID = string(bPkg.PackageSPDXIdentifier) + } + } + out.Environments[string(aPkg.PackageSPDXIdentifier)] = envs + } + // Go through and add the source packages + for _, r := range sd.Relationships { + aPkg := pkgMap[string(r.RefA.ElementRefID)] + bPkg := pkgMap[string(r.RefB.ElementRefID)] + if r.Relationship == "GENERATED_FROM" { + out.Packages[string(aPkg.PackageSPDXIdentifier)].Source = &claircore.Package{ + ID: string(bPkg.PackageSPDXIdentifier), + Name: bPkg.PackageName, + Version: bPkg.PackageVersion, + Kind: claircore.SOURCE, + } + } + } + return out, nil +} + +func ParseIndexReport(ir *claircore.IndexReport) (*spdxtools.Document, error) { + // Initial metadata + out := &spdxtools.Document{ + SPDXVersion: spdxtools.Version, + DataLicense: spdxtools.DataLicense, + SPDXIdentifier: "DOCUMENT", + DocumentName: ir.Hash.String(), + // This would be nice to have but don't know how we'd get context w/o + // having to accept it as an argument. + // DocumentNamespace: "https://clairproject.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + CreationInfo: &spdxtools.CreationInfo{ + Creators: []common.Creator{ + {CreatorType: "Tool", Creator: "Claircore"}, + {CreatorType: "Organization", Creator: "Clair"}, + }, + Created: time.Now().Format("2006-01-02T15:04:05Z"), + }, + DocumentComment: fmt.Sprintf("This document was created using claircore (%s).", getVersion()), + } + + rels := []*spdxtools.Relationship{} + repoMap := map[string]*spdxtools.Package{} + distMap := map[string]*spdxtools.Package{} + for _, r := range ir.IndexRecords() { + pkgDB := "" + for _, e := range ir.Environments[r.Package.ID] { + if e.PackageDB != "" { + pkgDB = e.PackageDB + } + } + pkg := &spdxtools.Package{ + PackageName: r.Package.Name, + PackageSPDXIdentifier: common.ElementID(r.Package.ID), + PackageVersion: r.Package.Version, + PackageFileName: pkgDB, + PackageDownloadLocation: "NOASSERTION", + FilesAnalyzed: true, + } + out.Packages = append(out.Packages, pkg) + if r.Package.Source != nil { + srcPkg := &spdxtools.Package{ + PackageName: r.Package.Source.Name, + PackageSPDXIdentifier: common.ElementID(r.Package.Source.ID), + PackageVersion: r.Package.Source.Version, + } + out.Packages = append(out.Packages, srcPkg) + rels = append(rels, &spdxtools.Relationship{ + RefA: common.MakeDocElementID("", string(pkg.PackageSPDXIdentifier)), + RefB: common.MakeDocElementID("", string(srcPkg.PackageSPDXIdentifier)), + Relationship: "GENERATED_FROM", + }) + } + if r.Repository != nil { + repo, ok := repoMap[r.Repository.ID] + if !ok { + repo = &spdxtools.Package{ + PackageName: r.Repository.Name, + PackageSPDXIdentifier: common.ElementID(r.Repository.ID), + FilesAnalyzed: true, + PackageSummary: "repository", + PackageExternalReferences: []*spdxtools.PackageExternalReference{ + { + Category: "SECURITY", + // TODO: always cpe:2.3? + RefType: "cpe23Type", + Locator: r.Repository.CPE.String(), + }, + { + Category: "OTHER", + RefType: "url", + Locator: r.Repository.URI, + }, + { + Category: "OTHER", + RefType: "key", + Locator: r.Repository.Key, + }, + }, + } + repoMap[r.Repository.ID] = repo + } + out.Packages = append(out.Packages, repo) + rel := &spdxtools.Relationship{ + RefA: common.MakeDocElementID("", string(pkg.PackageSPDXIdentifier)), + RefB: common.MakeDocElementID("", string(repo.PackageSPDXIdentifier)), + Relationship: "CONTAINED_BY", + } + rels = append(rels, rel) + } + if r.Distribution != nil { + dist, ok := distMap[r.Distribution.ID] + if !ok { + dist = &spdxtools.Package{ + PackageName: r.Distribution.Name, + PackageSPDXIdentifier: common.ElementID(r.Distribution.ID), + PackageVersion: r.Distribution.Version, + FilesAnalyzed: true, + PackageSummary: "distribution", + PackageExternalReferences: []*spdxtools.PackageExternalReference{ + { + Category: "SECURITY", + // TODO: always cpe:2.3? + RefType: "cpe23Type", + Locator: r.Distribution.CPE.String(), + }, + { + Category: "OTHER", + RefType: "did", + Locator: r.Distribution.DID, + }, + { + Category: "OTHER", + RefType: "version_id", + Locator: r.Distribution.VersionID, + }, + { + Category: "OTHER", + RefType: "pretty_name", + Locator: r.Distribution.PrettyName, + }, + }, + } + distMap[r.Distribution.ID] = dist + } + out.Packages = append(out.Packages, dist) + rel := &spdxtools.Relationship{ + RefA: common.MakeDocElementID("", string(pkg.PackageSPDXIdentifier)), + RefB: common.MakeDocElementID("", string(dist.PackageSPDXIdentifier)), + Relationship: "CONTAINED_BY", + } + rels = append(rels, rel) + } + } + + layerMap := map[string]*spdxtools.Package{} + for pkgID, envs := range ir.Environments { + for _, e := range envs { + pkg, ok := layerMap[e.IntroducedIn.String()] + if !ok { + pkg = &spdxtools.Package{ + PackageName: e.IntroducedIn.String(), + PackageSPDXIdentifier: common.ElementID(uuid.New().String()), + FilesAnalyzed: true, + PackageSummary: "layer", + } + out.Packages = append(out.Packages, pkg) + layerMap[e.IntroducedIn.String()] = pkg + } + rel := &spdxtools.Relationship{ + RefA: common.MakeDocElementID("", pkgID), + RefB: common.MakeDocElementID("", string(pkg.PackageSPDXIdentifier)), + Relationship: "CONTAINED_BY", + } + rels = append(rels, rel) + } + } + out.Relationships = rels + return out, nil +} + +// GetVersion is copied from Clair and can hopefully give some +// context as to which revision of claircore was used. +func getVersion() string { + info, infoOK := debug.ReadBuildInfo() + var core string + if infoOK { + for _, m := range info.Deps { + if m.Path != "github.com/quay/claircore" { + continue + } + core = m.Version + if m.Replace != nil && m.Replace.Version != m.Version { + core = m.Replace.Version + } + } + } + if core == "" { + core = "unknown revision" + } + return core +} diff --git a/pkg/sbom/spdx/spdx_test.go b/pkg/sbom/spdx/spdx_test.go new file mode 100644 index 000000000..43eedb89f --- /dev/null +++ b/pkg/sbom/spdx/spdx_test.go @@ -0,0 +1,132 @@ +package spdx + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/quay/claircore" + "github.com/quay/claircore/pkg/cpe" + + "github.com/spdx/tools-golang/tagvalue" +) + +func TestParseRoundTrip(t *testing.T) { + opts := cmp.Options{ + cmp.AllowUnexported(claircore.IndexReport{}), + cmpopts.IgnoreFields(claircore.IndexReport{}, "Hash"), + cmpopts.IgnoreFields(claircore.Environment{}, "IntroducedIn"), + } + for _, tt := range testIndexReports { + t.Run(tt.name, func(t *testing.T) { + + s, err := ParseIndexReport(tt.indexReport) + if err != nil { + t.Fatal(err) + } + w := &bytes.Buffer{} + err = tagvalue.Write(s, w) + if err != nil { + t.Fatal(err) + } + sReport, err := json.MarshalIndent(s, "", " ") + if err != nil { + t.Fatal(err) + } + t.Log(string(sReport)) + ir, err := ParseSPDXDocument(s) + if err != nil { + t.Fatal(err) + } + if !cmp.Equal(ir, tt.indexReport, opts) { + t.Error(cmp.Diff(tt.indexReport, ir, opts)) + } + iReport, err := json.MarshalIndent(ir, "", " ") + if err != nil { + t.Fatal(err) + } + t.Log(string(iReport)) + + }) + } +} + +type testcase struct { + name string + indexReport *claircore.IndexReport +} + +var testIndexReports = []testcase{ + { + name: "simple index report", + indexReport: &claircore.IndexReport{ + Hash: claircore.MustParseDigest(`sha256:` + strings.Repeat(`a`, 64)), + Packages: map[string]*claircore.Package{ + "123": { + ID: "123", + Name: "package A", + Version: "v1.0.0", + Source: &claircore.Package{ + ID: "122", + Name: "package B source", + Kind: claircore.SOURCE, + Version: "v1.0.0", + }, + Kind: claircore.BINARY, + }, + "456": { + ID: "456", + Name: "package B", + Version: "v2.0.0", + Kind: claircore.BINARY, + }, + }, + Environments: map[string][]*claircore.Environment{ + "123": { + { + PackageDB: "bdb:var/lib/rpm", + IntroducedIn: claircore.MustParseDigest(`sha256:` + strings.Repeat(`b`, 64)), + RepositoryIDs: []string{"11"}, + DistributionID: "13", + }, + }, + "456": { + { + PackageDB: "maven:opt/couchbase/lib/cbas/repo/eventstream-1.0.1.jar", + IntroducedIn: claircore.MustParseDigest(`sha256:` + strings.Repeat(`c`, 64)), + RepositoryIDs: []string{"12"}, + }, + }, + }, + Repositories: map[string]*claircore.Repository{ + "11": { + ID: "11", + Name: "cpe:/a:redhat:rhel_eus:8.6::appstream", + Key: "rhel-cpe-repository", + CPE: cpe.MustUnbind("cpe:2.3:a:redhat:rhel_eus:8.6:*:appstream:*:*:*:*:*"), + }, + "12": { + ID: "12", + Name: "maven", + URI: "https://repo1.maven.apache.org/maven2", + }, + }, + Distributions: map[string]*claircore.Distribution{ + "13": { + ID: "13", + DID: "rhel", + Name: "Red Hat Enterprise Linux Server", + Version: "7", + VersionID: "7", + CPE: cpe.MustUnbind("cpe:2.3:o:redhat:enterprise_linux:7:*:*:*:*:*:*:*"), + PrettyName: "Red Hat Enterprise Linux Server 7", + }, + }, + Success: true, + }, + }, +}