From fd0064058e9584ad0e0a0c2e16a3e7c5a8d9df54 Mon Sep 17 00:00:00 2001 From: robnester-rh Date: Tue, 13 Aug 2024 19:53:32 -0400 Subject: [PATCH] EC-724 provide sha/image digest for policy sources This commit adds the ability for ec to log the git sha or the image digest for a policy. Signed-off-by: robnester-rh --- cmd/inspect/inspect_policy_test.go | 5 +- go.mod | 22 ++++----- go.sum | 36 +++++++------- internal/downloader/downloader.go | 25 +++++----- internal/downloader/downloader_test.go | 2 +- internal/policy/source/source.go | 65 ++++++++++++++++++++------ internal/policy/source/source_test.go | 11 +++-- 7 files changed, 102 insertions(+), 64 deletions(-) diff --git a/cmd/inspect/inspect_policy_test.go b/cmd/inspect/inspect_policy_test.go index 3ea14ce42..930a09b50 100644 --- a/cmd/inspect/inspect_policy_test.go +++ b/cmd/inspect/inspect_policy_test.go @@ -23,6 +23,7 @@ import ( "fmt" "testing" + "github.com/enterprise-contract/go-gather/metadata" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -38,10 +39,10 @@ type mockDownloader struct { mock.Mock } -func (m *mockDownloader) Download(_ context.Context, dest string, sourceUrl string, showMsg bool) error { +func (m *mockDownloader) Download(_ context.Context, dest string, sourceUrl string, showMsg bool) (metadata.Metadata, error) { args := m.Called(dest, sourceUrl, showMsg) - return args.Error(0) + return nil, args.Error(0) } func TestFetchSourcesFromPolicy(t *testing.T) { diff --git a/go.mod b/go.mod index f5b1ff295..17231347c 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,11 @@ require ( github.com/MakeNowJust/heredoc v1.0.0 github.com/Maldris/go-billy-afero v0.0.0-20200815120323-e9d3de59c99a github.com/enterprise-contract/enterprise-contract-controller/api v0.1.51 - github.com/enterprise-contract/go-gather/gather v0.0.2 + github.com/enterprise-contract/go-gather/gather v0.0.3 github.com/enterprise-contract/go-gather/metadata v0.0.2 + github.com/enterprise-contract/go-gather/metadata/file v0.0.1 + github.com/enterprise-contract/go-gather/metadata/git v0.0.2 + github.com/enterprise-contract/go-gather/metadata/oci v0.0.3 github.com/evanphx/json-patch v5.9.0+incompatible github.com/gkampitakis/go-snaps v0.5.7 github.com/go-git/go-git/v5 v5.12.0 @@ -41,7 +44,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/stuart-warren/yamlfmt v0.2.0 github.com/tektoncd/pipeline v0.54.0 - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/net v0.28.0 k8s.io/apiextensions-apiserver v0.29.7 k8s.io/apimachinery v0.29.7 @@ -132,7 +135,7 @@ require ( github.com/chainguard-dev/git-urls v1.0.2 // indirect github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect github.com/clbanning/mxj/v2 v2.7.0 // indirect - github.com/cloudflare/circl v1.3.8 // indirect + github.com/cloudflare/circl v1.3.9 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect github.com/containerd/containerd v1.7.20 // indirect @@ -161,13 +164,10 @@ require ( github.com/enterprise-contract/go-gather v0.0.3 // indirect github.com/enterprise-contract/go-gather/expander v0.0.1 // indirect github.com/enterprise-contract/go-gather/gather/file v0.0.1 // indirect - github.com/enterprise-contract/go-gather/gather/git v0.0.4 // indirect - github.com/enterprise-contract/go-gather/gather/http v0.0.1 // indirect - github.com/enterprise-contract/go-gather/gather/oci v0.0.3 // indirect - github.com/enterprise-contract/go-gather/metadata/file v0.0.1 // indirect - github.com/enterprise-contract/go-gather/metadata/git v0.0.2 // indirect + github.com/enterprise-contract/go-gather/gather/git v0.0.5 // indirect + github.com/enterprise-contract/go-gather/gather/http v0.0.2 // indirect + github.com/enterprise-contract/go-gather/gather/oci v0.0.4 // indirect github.com/enterprise-contract/go-gather/metadata/http v0.0.1 // indirect - github.com/enterprise-contract/go-gather/metadata/oci v0.0.3 // indirect github.com/enterprise-contract/go-gather/saver v0.0.1 // indirect github.com/enterprise-contract/go-gather/saver/file v0.0.1 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect @@ -272,7 +272,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/segmentio/ksuid v1.0.4 // indirect @@ -329,7 +329,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/mod v0.17.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.23.0 // indirect diff --git a/go.sum b/go.sum index 4dfecc412..289c4d0cd 100644 --- a/go.sum +++ b/go.sum @@ -441,8 +441,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudevents/sdk-go/v2 v2.14.0 h1:Nrob4FwVgi5L4tV9lhjzZcjYqFVyJzsA56CwPaPfv6s= github.com/cloudevents/sdk-go/v2 v2.14.0/go.mod h1:xDmKfzNjM8gBvjaF8ijFjM1VYOVUEeUfapHMUX1T5To= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= -github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= +github.com/cloudflare/circl v1.3.9 h1:QFrlgFYf2Qpi8bSpVPK1HBvWpx16v/1TZivyo7pGuBE= +github.com/cloudflare/circl v1.3.9/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -539,16 +539,16 @@ github.com/enterprise-contract/go-gather v0.0.3 h1:Qh4CJhOPdMit4Z/BK3rv7S3GkZ5XL github.com/enterprise-contract/go-gather v0.0.3/go.mod h1:gXqnYRW9uTD06xli3pE+9cwtPVcIdqyPIqBcKQ+kK8I= github.com/enterprise-contract/go-gather/expander v0.0.1 h1:CRJX7crqNyuuo82DtFbyIpJB/2hV62zWof4t1dOmCC0= github.com/enterprise-contract/go-gather/expander v0.0.1/go.mod h1:bZ7oijDzlpY3gGc+H48YSsxbCEGxmsqQj+PxnYjtrjg= -github.com/enterprise-contract/go-gather/gather v0.0.2 h1:ndkHJHJemunUF6Ik8XG2u3sPHHOm8nNIiS2hj6jFPLA= -github.com/enterprise-contract/go-gather/gather v0.0.2/go.mod h1:Ap0QGx0fKVMgNuzOugZTp20KfIlvGPz2gq7fqx0syyI= +github.com/enterprise-contract/go-gather/gather v0.0.3 h1:rSOLqY+ydsqMtqx4IZ+jQKZjmaQuU/ZbrTJCiceoutA= +github.com/enterprise-contract/go-gather/gather v0.0.3/go.mod h1:chij1Nq6vbJP/2sii0sxCuwvrJCJV3hEbH1PpJ/DhwQ= github.com/enterprise-contract/go-gather/gather/file v0.0.1 h1:UOec5Gc7+Q9u3x0Cyw8l2JDYCH7RTtHVnC57Rqs0Nyg= github.com/enterprise-contract/go-gather/gather/file v0.0.1/go.mod h1:tHsShLa5XpNSZbH8paHZR3Ltgu/7wtxpdCTVP8EXk/U= -github.com/enterprise-contract/go-gather/gather/git v0.0.4 h1:UcQkvq2Cm7MFONvtECoNIbPe1R85+xraJRungQym78o= -github.com/enterprise-contract/go-gather/gather/git v0.0.4/go.mod h1:8pSBY25NpofcTrEsnmRv4h1s30ukwkgFwew1iA19Fnc= -github.com/enterprise-contract/go-gather/gather/http v0.0.1 h1:qMRcMNWiOEE/oFZJfD8Jj7jihNZpS3MwtmHq5qh38vQ= -github.com/enterprise-contract/go-gather/gather/http v0.0.1/go.mod h1:Fx0Anvh8Os39BaeTxxcvOwX1E9xisXehEueOkQ+qK3I= -github.com/enterprise-contract/go-gather/gather/oci v0.0.3 h1:AH7HfogbGBuP8pc06vOB3mdWQV76bXLIxcYilVWtUSA= -github.com/enterprise-contract/go-gather/gather/oci v0.0.3/go.mod h1:Bm3WiT2L8sm4+1mYyYqlu7GF1uflAXyKo2nnqEnozt0= +github.com/enterprise-contract/go-gather/gather/git v0.0.5 h1:fWz0D42Lv2AjKkQCTfv+i5bJOOVD0RusaLTJuGfcIvc= +github.com/enterprise-contract/go-gather/gather/git v0.0.5/go.mod h1:jEtaBsc8a4AsbXRwZaDPQAW9t5bNU/GvwNgYLAgwDhE= +github.com/enterprise-contract/go-gather/gather/http v0.0.2 h1:Cwx+R2ECbpFZlYDD62h5lRrV/zyspxt2TOuWH8HlqYE= +github.com/enterprise-contract/go-gather/gather/http v0.0.2/go.mod h1:Vdghi2YICVhShDK/MuTlZIrVvh0mBQrhKlstoENWrwM= +github.com/enterprise-contract/go-gather/gather/oci v0.0.4 h1:THirSwb3gkBWJgJjOTzHhw5kbhSRqjClfGau9RW5uFc= +github.com/enterprise-contract/go-gather/gather/oci v0.0.4/go.mod h1:pDBkLXvyeOiUim7xaO5H1NSzTUnbFd83Tugxp4neC50= github.com/enterprise-contract/go-gather/metadata v0.0.2 h1:BxPXXZFjX7lrYnlJosPmvISgjF13HpawEtZTDxjnjcQ= github.com/enterprise-contract/go-gather/metadata v0.0.2/go.mod h1:m2HxByQBWZyc99HDs/Lqy7QzU9+XQ2tU0X/mzkCPgPw= github.com/enterprise-contract/go-gather/metadata/file v0.0.1 h1:DRhTGKRXFRh/FVn2LNX8yIJZHHYKc5x5260hnYxQ4DY= @@ -1134,8 +1134,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -1422,8 +1422,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1452,8 +1452,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1755,8 +1755,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/downloader/downloader.go b/internal/downloader/downloader.go index e37ecc5f4..d0bd0f267 100644 --- a/internal/downloader/downloader.go +++ b/internal/downloader/downloader.go @@ -26,6 +26,7 @@ import ( "sync" "github.com/enterprise-contract/go-gather/gather" + "github.com/enterprise-contract/go-gather/metadata" "github.com/open-policy-agent/conftest/downloader" log "github.com/sirupsen/logrus" @@ -53,9 +54,9 @@ func WithDownloadImpl(ctx context.Context, d downloadImpl) context.Context { // // Note that it handles just one url at a time even though the equivalent // Conftest function can take a list of source urls. -func Download(ctx context.Context, destDir string, sourceUrl string, showMsg bool) (err error) { +func Download(ctx context.Context, destDir string, sourceUrl string, showMsg bool) (metadata.Metadata, error) { if !isSecure(sourceUrl) { - return fmt.Errorf("attempting to download from insecure source: %s", sourceUrl) + return nil, fmt.Errorf("attempting to download from insecure source: %s", sourceUrl) } msg := fmt.Sprintf("Downloading %s to %s", sourceUrl, destDir) @@ -64,41 +65,41 @@ func Download(ctx context.Context, destDir string, sourceUrl string, showMsg boo fmt.Println(msg) } - dl := func(ctx context.Context, sourceUrl, destDir string) error { + dl := func(ctx context.Context, sourceUrl, destDir string) (metadata.Metadata, error) { // conftest's Download function leverages oras under the hood to fetch from OCI. It uses the // global oras client and sets the user agent to "conftest". This is not a thread safe // operation. Here we get around this limitation by ensuring a single download happens at a // time. dlMutex.Lock() defer dlMutex.Unlock() - return downloader.Download(ctx, destDir, []string{sourceUrl}) + return nil, downloader.Download(ctx, destDir, []string{sourceUrl}) } if utils.UseGoGather() { - dl = func(ctx context.Context, sourceUrl, destDir string) error { - _, err := gatherFunc(ctx, sourceUrl, destDir) + dl = func(ctx context.Context, sourceUrl, destDir string) (metadata.Metadata, error) { + m, err := gatherFunc(ctx, sourceUrl, destDir) if err != nil { log.Debug("Download failed!") } - return err + return m, err } } else if d, ok := ctx.Value(downloadImplKey).(downloadImpl); ok { - dl = func(ctx context.Context, sourceUrl, destDir string) error { - err = d.Download(ctx, destDir, []string{sourceUrl}) + dl = func(ctx context.Context, sourceUrl, destDir string) (metadata.Metadata, error) { + err := d.Download(ctx, destDir, []string{sourceUrl}) if err != nil { log.Debug("Download failed!") } - return err + return nil, err } } - err = dl(ctx, sourceUrl, destDir) + m, err := dl(ctx, sourceUrl, destDir) if err != nil { log.Debug("Download failed!") } - return + return m, err } // matches insecure protocols, such as `git::http://...` diff --git a/internal/downloader/downloader_test.go b/internal/downloader/downloader_test.go index 9c507566d..85958eda1 100644 --- a/internal/downloader/downloader_test.go +++ b/internal/downloader/downloader_test.go @@ -103,7 +103,7 @@ func TestDownloader_Download(t *testing.T) { d.On("Download", ctx, tt.dest, []string{tt.source}).Return(tt.err) } - err := Download(ctx, tt.dest, tt.source, false) + _, err := Download(ctx, tt.dest, tt.source, false) if tt.errExpected { assert.Error(t, err) diff --git a/internal/policy/source/source.go b/internal/policy/source/source.go index c51cd5f0f..0ffd9d653 100644 --- a/internal/policy/source/source.go +++ b/internal/policy/source/source.go @@ -32,6 +32,10 @@ import ( "time" ecc "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" + "github.com/enterprise-contract/go-gather/metadata" + fileMetadata "github.com/enterprise-contract/go-gather/metadata/file" + gitMetadata "github.com/enterprise-contract/go-gather/metadata/git" + ociMetadata "github.com/enterprise-contract/go-gather/metadata/oci" log "github.com/sirupsen/logrus" "github.com/spf13/afero" @@ -52,7 +56,7 @@ const ( ) type downloaderFunc interface { - Download(context.Context, string, string, bool) error + Download(context.Context, string, string, bool) (metadata.Metadata, error) } // PolicySource in an interface representing the location a policy source. @@ -73,23 +77,31 @@ type PolicyUrl struct { // downloadCache is a concurrent map used to cache downloaded files. var downloadCache sync.Map -func getPolicyThroughCache(ctx context.Context, s PolicySource, workDir string, dl func(string, string) error) (string, error) { +type cacheContent struct { + sourceUrl string + metadata metadata.Metadata + err error +} + +func getPolicyThroughCache(ctx context.Context, s PolicySource, workDir string, dl func(string, string) (metadata.Metadata, error)) (string, error) { sourceUrl := s.PolicyUrl() dest := uniqueDestination(workDir, s.Subdir(), sourceUrl) // Load or store the downloaded policy file from the given source URL. // If the file is already in the download cache, it is loaded from there. // Otherwise, it is downloaded from the source URL and stored in the cache. - dfn, _ := downloadCache.LoadOrStore(sourceUrl, sync.OnceValues(func() (string, error) { + dfn, _ := downloadCache.LoadOrStore(sourceUrl, sync.OnceValues(func() (string, cacheContent) { log.Debugf("Download cache miss: %s", sourceUrl) // Checkout policy repo into work directory. log.Debugf("Downloading policy files from source url %s to destination %s", sourceUrl, dest) - return dest, dl(sourceUrl, dest) + m, err := dl(sourceUrl, dest) + c := &cacheContent{sourceUrl, m, err} + return dest, *c })) - d, err := dfn.(func() (string, error))() - if err != nil { - return "", err + d, c := dfn.(func() (string, cacheContent))() + if c.err != nil { + return "", c.err } // If the destination directory is different from the source directory, we @@ -106,21 +118,39 @@ func getPolicyThroughCache(ctx context.Context, s PolicySource, workDir string, if err := symlinkableFS.SymlinkIfPossible(d, dest); err != nil { return "", err } - + if c.metadata != nil { + if _, ok := c.metadata.(*gitMetadata.GitMetadata); ok { + log.Debugf("SHA for source(%s): %s\n", s.PolicyUrl(), c.metadata.(*gitMetadata.GitMetadata).LatestCommit) + } + if _, ok := c.metadata.(*ociMetadata.OCIMetadata); ok { + log.Debugf("Image digest for source(%s): %s\n", s.PolicyUrl(), c.metadata.(*ociMetadata.OCIMetadata).Digest) + } + } return dest, nil } else { log.Debugf("Filesystem does not support symlinking: %q, re-downloading instead", fs.Name()) - - return dest, dl(sourceUrl, dest) + m, err := dl(sourceUrl, dest) + if _, ok := m.(*gitMetadata.GitMetadata); ok { + log.Debugf("SHA for source(%s): %s\n", s.PolicyUrl(), m.(*gitMetadata.GitMetadata).LatestCommit) + } + return dest, err } } - return d, err + if c.metadata != nil { + if _, ok := c.metadata.(*gitMetadata.GitMetadata); ok { + log.Debugf("SHA for source(%s): %s\n", s.PolicyUrl(), c.metadata.(*gitMetadata.GitMetadata).LatestCommit) + } + if _, ok := c.metadata.(*ociMetadata.OCIMetadata); ok { + log.Debugf("Image digest for source(%s): %s\n", s.PolicyUrl(), c.metadata.(*ociMetadata.OCIMetadata).Digest) + } + } + return d, c.err } // GetPolicies clones the repository for a given PolicyUrl func (p *PolicyUrl) GetPolicy(ctx context.Context, workDir string, showMsg bool) (string, error) { - dl := func(source string, dest string) error { + dl := func(source string, dest string) (metadata.Metadata, error) { x := ctx.Value(DownloaderFuncKey) if dl, ok := x.(downloaderFunc); ok { return dl.Download(ctx, dest, source, showMsg) @@ -159,16 +189,21 @@ func InlineData(source []byte) PolicySource { } func (s inlineData) GetPolicy(ctx context.Context, workDir string, showMsg bool) (string, error) { - dl := func(source string, dest string) error { + dl := func(source string, dest string) (metadata.Metadata, error) { fs := utils.FS(ctx) if err := fs.MkdirAll(dest, 0755); err != nil { - return err + return nil, err } f := path.Join(dest, "rule_data.json") + m := &fileMetadata.FileMetadata{ + Path: dest, + Size: int64(len(dest)), + SHA: "", + } - return afero.WriteFile(fs, f, s.source, 0400) + return m, afero.WriteFile(fs, f, s.source, 0400) } return getPolicyThroughCache(ctx, s, workDir, dl) diff --git a/internal/policy/source/source_test.go b/internal/policy/source/source_test.go index 635af712a..27bf0bda5 100644 --- a/internal/policy/source/source_test.go +++ b/internal/policy/source/source_test.go @@ -28,6 +28,7 @@ import ( "testing" ecc "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" + "github.com/enterprise-contract/go-gather/metadata" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -45,10 +46,10 @@ type mockDownloader struct { mock.Mock } -func (m *mockDownloader) Download(_ context.Context, dest string, sourceUrl string, showMsg bool) error { +func (m *mockDownloader) Download(_ context.Context, dest string, sourceUrl string, showMsg bool) (metadata.Metadata, error) { args := m.Called(dest, sourceUrl, showMsg) - return args.Error(0) + return nil, args.Error(0) } func TestGetPolicy(t *testing.T) { @@ -209,13 +210,13 @@ func TestGetPolicyThroughCache(t *testing.T) { invocations := 0 data := []byte("hello") - dl := func(source, dest string) error { + dl := func(source, dest string) (metadata.Metadata, error) { invocations++ if err := fs.MkdirAll(dest, 0755); err != nil { - return err + return nil, err } - return afero.WriteFile(fs, filepath.Join(dest, "data.json"), data, 0400) + return nil, afero.WriteFile(fs, filepath.Join(dest, "data.json"), data, 0400) } s1, err := getPolicyThroughCache(ctx, &mockPolicySource{}, "/workdir1", dl)