diff --git a/.tool-versions b/.tool-versions index 83d3f70dc..0e6ce8755 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -golang 1.21.4 +golang 1.21.9 diff --git a/go.mod b/go.mod index 382d345a0..0ac6b7b72 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,14 @@ module github.com/enterprise-contract/ec-cli -go 1.21.4 +go 1.21.9 require ( cuelang.org/go v0.9.2 github.com/MakeNowJust/heredoc v1.0.0 github.com/Maldris/go-billy-afero v0.0.0-20200815120323-e9d3de59c99a github.com/enterprise-contract/enterprise-contract-controller/api v0.1.48 + github.com/enterprise-contract/go-gather/gather v0.0.2 + github.com/enterprise-contract/go-gather/metadata v0.0.2 github.com/evanphx/json-patch v5.9.0+incompatible github.com/gkampitakis/go-snaps v0.5.4 github.com/go-git/go-git/v5 v5.12.0 @@ -130,7 +132,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect github.com/clbanning/mxj/v2 v2.7.0 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.3.8 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect github.com/containerd/containerd v1.7.18 // indirect @@ -141,7 +143,7 @@ require ( github.com/coreos/go-oidc/v3 v3.10.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20231011164504-785e29786b46 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/badger/v3 v3.2103.5 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect @@ -157,6 +159,18 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/enterprise-contract/go-gather v0.0.3 // indirect + github.com/enterprise-contract/go-gather/expander v0.0.1 // indirect + github.com/enterprise-contract/go-gather/gather/file v0.0.1 // indirect + github.com/enterprise-contract/go-gather/gather/git v0.0.3 // indirect + github.com/enterprise-contract/go-gather/gather/http v0.0.1 // indirect + github.com/enterprise-contract/go-gather/gather/oci v0.0.3 // indirect + github.com/enterprise-contract/go-gather/metadata/file v0.0.1 // indirect + github.com/enterprise-contract/go-gather/metadata/git v0.0.1 // indirect + github.com/enterprise-contract/go-gather/metadata/http v0.0.1 // indirect + github.com/enterprise-contract/go-gather/metadata/oci v0.0.2 // indirect + github.com/enterprise-contract/go-gather/saver v0.0.1 // indirect + github.com/enterprise-contract/go-gather/saver/file v0.0.1 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -290,6 +304,7 @@ require ( github.com/transparency-dev/merkle v0.0.2 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/vbatts/tar-split v0.11.5 // indirect + github.com/whilp/git-urls v1.0.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xanzy/go-gitlab v0.102.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -339,7 +354,7 @@ require ( knative.dev/pkg v0.0.0-20231023150739-56bfe0dd9626 // indirect muzzammil.xyz/jsonc v1.0.0 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect - oras.land/oras-go/v2 v2.4.0 // indirect + oras.land/oras-go/v2 v2.5.0 // indirect sigs.k8s.io/controller-runtime v0.17.5 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/release-utils v0.7.7 // indirect diff --git a/go.sum b/go.sum index 2510908b6..19cc69097 100644 --- a/go.sum +++ b/go.sum @@ -439,8 +439,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudevents/sdk-go/v2 v2.14.0 h1:Nrob4FwVgi5L4tV9lhjzZcjYqFVyJzsA56CwPaPfv6s= github.com/cloudevents/sdk-go/v2 v2.14.0/go.mod h1:xDmKfzNjM8gBvjaF8ijFjM1VYOVUEeUfapHMUX1T5To= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -484,8 +484,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberphone/json-canonicalization v0.0.0-20231011164504-785e29786b46 h1:2Dx4IHfC1yHWI12AxQDJM1QbRCDfk6M+blLzlZCXdrc= github.com/cyberphone/json-canonicalization v0.0.0-20231011164504-785e29786b46/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= +github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -535,6 +535,34 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/enterprise-contract/enterprise-contract-controller/api v0.1.48 h1:jE4szEZ0kQS9N08b95T/znlATGSlIIur1pt8TjT3NPs= github.com/enterprise-contract/enterprise-contract-controller/api v0.1.48/go.mod h1:QgUezi253+QIvTWVlMX+gqyfAek1eKV3qChzm++kpI8= +github.com/enterprise-contract/go-gather v0.0.3 h1:Qh4CJhOPdMit4Z/BK3rv7S3GkZ5XLzAlAus1eMKLDA4= +github.com/enterprise-contract/go-gather v0.0.3/go.mod h1:gXqnYRW9uTD06xli3pE+9cwtPVcIdqyPIqBcKQ+kK8I= +github.com/enterprise-contract/go-gather/expander v0.0.1 h1:CRJX7crqNyuuo82DtFbyIpJB/2hV62zWof4t1dOmCC0= +github.com/enterprise-contract/go-gather/expander v0.0.1/go.mod h1:bZ7oijDzlpY3gGc+H48YSsxbCEGxmsqQj+PxnYjtrjg= +github.com/enterprise-contract/go-gather/gather v0.0.2 h1:ndkHJHJemunUF6Ik8XG2u3sPHHOm8nNIiS2hj6jFPLA= +github.com/enterprise-contract/go-gather/gather v0.0.2/go.mod h1:Ap0QGx0fKVMgNuzOugZTp20KfIlvGPz2gq7fqx0syyI= +github.com/enterprise-contract/go-gather/gather/file v0.0.1 h1:UOec5Gc7+Q9u3x0Cyw8l2JDYCH7RTtHVnC57Rqs0Nyg= +github.com/enterprise-contract/go-gather/gather/file v0.0.1/go.mod h1:tHsShLa5XpNSZbH8paHZR3Ltgu/7wtxpdCTVP8EXk/U= +github.com/enterprise-contract/go-gather/gather/git v0.0.3 h1:5YAjbgFjJPjiDaWtfLqLdK6mqL163Lo9GLhY1YoiLNk= +github.com/enterprise-contract/go-gather/gather/git v0.0.3/go.mod h1:KOVdeeJrG1XNq/juDJXQUNaEthx9j2rHjgmM50GyQd8= +github.com/enterprise-contract/go-gather/gather/http v0.0.1 h1:qMRcMNWiOEE/oFZJfD8Jj7jihNZpS3MwtmHq5qh38vQ= +github.com/enterprise-contract/go-gather/gather/http v0.0.1/go.mod h1:Fx0Anvh8Os39BaeTxxcvOwX1E9xisXehEueOkQ+qK3I= +github.com/enterprise-contract/go-gather/gather/oci v0.0.3 h1:AH7HfogbGBuP8pc06vOB3mdWQV76bXLIxcYilVWtUSA= +github.com/enterprise-contract/go-gather/gather/oci v0.0.3/go.mod h1:Bm3WiT2L8sm4+1mYyYqlu7GF1uflAXyKo2nnqEnozt0= +github.com/enterprise-contract/go-gather/metadata v0.0.2 h1:BxPXXZFjX7lrYnlJosPmvISgjF13HpawEtZTDxjnjcQ= +github.com/enterprise-contract/go-gather/metadata v0.0.2/go.mod h1:m2HxByQBWZyc99HDs/Lqy7QzU9+XQ2tU0X/mzkCPgPw= +github.com/enterprise-contract/go-gather/metadata/file v0.0.1 h1:DRhTGKRXFRh/FVn2LNX8yIJZHHYKc5x5260hnYxQ4DY= +github.com/enterprise-contract/go-gather/metadata/file v0.0.1/go.mod h1:4PckwLejZstUEBp2QUAdQYQ0O+h5tijrs48j+7OY4OY= +github.com/enterprise-contract/go-gather/metadata/git v0.0.1 h1:tBzKlmAxmkrI0IbfrDjQwzucZJh7eTMVwaz+nY2cLaU= +github.com/enterprise-contract/go-gather/metadata/git v0.0.1/go.mod h1:Gb6z7fKBr5hiF4lbkJbRupS+zHe0imCPn3ukTDlg+dc= +github.com/enterprise-contract/go-gather/metadata/http v0.0.1 h1:ebhT9h93v/Et+5c1t5PJzGj6V2g18elm1VDrQg6y63A= +github.com/enterprise-contract/go-gather/metadata/http v0.0.1/go.mod h1:VjjTqsJ+sM7MVsVkEFgpcJzY9hur9pIBEMptrVvAwoI= +github.com/enterprise-contract/go-gather/metadata/oci v0.0.2 h1:FkXQTKnGr5Hci+4mrfXJLhmjMqHTJvgtubVsFB3KgJI= +github.com/enterprise-contract/go-gather/metadata/oci v0.0.2/go.mod h1:kJSMxYth37g604Cvsa18ibgTU33zagsadeQ7p1DYIak= +github.com/enterprise-contract/go-gather/saver v0.0.1 h1:f+oHdg83kwbVDqIs6Or9BatytNKm+ISO9ChztDcnqXA= +github.com/enterprise-contract/go-gather/saver v0.0.1/go.mod h1:uOt8X/CztOGi0YC5jERopBQpjXqkU6UPUqPellgBBG8= +github.com/enterprise-contract/go-gather/saver/file v0.0.1 h1:rLDMb7AW5kJLqRaKXazZroT8wfqy43tth6O6XLKY0MY= +github.com/enterprise-contract/go-gather/saver/file v0.0.1/go.mod h1:qnNStNDYPJGjJunKANv6jq93ynndcfxmUoeYeBEnZEY= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -1265,6 +1293,8 @@ github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= github.com/vektah/gqlparser v1.2.0/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= @@ -2025,8 +2055,8 @@ muzzammil.xyz/jsonc v1.0.0 h1:B6kaT3wHueZ87mPz3q1nFuM1BlL32IG0wcq0/uOsQ18= muzzammil.xyz/jsonc v1.0.0/go.mod h1:rFv8tUUKe+QLh7v02BhfxXEf4ZHhYD7unR93HL/1Uvo= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= -oras.land/oras-go/v2 v2.4.0 h1:i+Wt5oCaMHu99guBD0yuBjdLvX7Lz8ukPbwXdR7uBMs= -oras.land/oras-go/v2 v2.4.0/go.mod h1:osvtg0/ClRq1KkydMAEu/IxFieyjItcsQ4ut4PPF+f8= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/downloader/downloader.go b/internal/downloader/downloader.go index ae29bfbda..e37ecc5f4 100644 --- a/internal/downloader/downloader.go +++ b/internal/downloader/downloader.go @@ -25,8 +25,11 @@ import ( "strings" "sync" + "github.com/enterprise-contract/go-gather/gather" "github.com/open-policy-agent/conftest/downloader" log "github.com/sirupsen/logrus" + + "github.com/enterprise-contract/ec-cli/internal/utils" ) type key int @@ -37,6 +40,8 @@ type downloadImpl interface { Download(context.Context, string, []string) error } +var gatherFunc = gather.Gather + var dlMutex sync.Mutex // WithDownloadImpl replaces the downloadImpl implementation used @@ -59,18 +64,36 @@ func Download(ctx context.Context, destDir string, sourceUrl string, showMsg boo fmt.Println(msg) } - if d, ok := ctx.Value(downloadImplKey).(downloadImpl); ok { - err = d.Download(ctx, destDir, []string{sourceUrl}) - } else { + dl := func(ctx context.Context, sourceUrl, destDir string) error { // conftest's Download function leverages oras under the hood to fetch from OCI. It uses the // global oras client and sets the user agent to "conftest". This is not a thread safe // operation. Here we get around this limitation by ensuring a single download happens at a // time. dlMutex.Lock() - err = downloader.Download(ctx, destDir, []string{sourceUrl}) - dlMutex.Unlock() + defer dlMutex.Unlock() + return downloader.Download(ctx, destDir, []string{sourceUrl}) } + if utils.UseGoGather() { + dl = func(ctx context.Context, sourceUrl, destDir string) error { + _, err := gatherFunc(ctx, sourceUrl, destDir) + if err != nil { + log.Debug("Download failed!") + } + return err + } + } else if d, ok := ctx.Value(downloadImplKey).(downloadImpl); ok { + dl = func(ctx context.Context, sourceUrl, destDir string) error { + err = d.Download(ctx, destDir, []string{sourceUrl}) + if err != nil { + log.Debug("Download failed!") + } + return err + } + } + + err = dl(ctx, sourceUrl, destDir) + if err != nil { log.Debug("Download failed!") } diff --git a/internal/downloader/downloader_test.go b/internal/downloader/downloader_test.go index 0362ff5ed..9c507566d 100644 --- a/internal/downloader/downloader_test.go +++ b/internal/downloader/downloader_test.go @@ -21,8 +21,10 @@ package downloader import ( "context" "errors" + "os" "testing" + "github.com/enterprise-contract/go-gather/metadata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -39,27 +41,45 @@ func (m *mockDownloader) Download(ctx context.Context, dest string, sourceUrls [ func TestDownloader_Download(t *testing.T) { tests := []struct { - name string - dest string - source string - err error + name string + dest string + source string + errExpected bool + err error + useGoGather bool }{ { name: "Downloads", dest: "dir", - source: "example.com/repo.git", + source: "https://example.com/org/repo.git", }, { - name: "Fails to download", - dest: "dir", - source: "example.com/repo.git", - err: errors.New("expected"), + name: "Downloads with go-gather", + dest: "dir", + source: "https://example.com/org/repo.git", + useGoGather: true, }, { - name: "insecure download", - dest: "dir", - source: "http://example.com", - err: errors.New("attempting to download from insecure source: http://example.com"), + name: "Fails to download with go-gather", + dest: "dir", + source: "https://example.com/org/repo.git", + errExpected: true, + err: errors.New("expected error with go-gather"), + useGoGather: true, + }, + { + name: "Fails to download", + dest: "dir", + source: "https://example.com/org/repo.git", + errExpected: true, + err: errors.New("expected error"), + }, + { + name: "insecure download", + dest: "dir", + source: "http://example.com/org/repo.git", + errExpected: true, + err: errors.New("attempting to download from insecure source: http://example.com/org/repo.git"), }, } @@ -67,14 +87,30 @@ func TestDownloader_Download(t *testing.T) { t.Run(tt.name, func(t *testing.T) { d := mockDownloader{} ctx := WithDownloadImpl(context.TODO(), &d) - d.On("Download", ctx, tt.dest, []string{tt.source}).Return(tt.err) + + originalGatherFunction := gatherFunc + defer func() { + gatherFunc = originalGatherFunction + }() + + if tt.useGoGather { + t.Setenv("USEGOGATHER", "1") + gatherFunc = func(_ context.Context, _ string, _ string) (metadata.Metadata, error) { + return nil, tt.err + } + } else { + os.Unsetenv("USEGOGATHER") + d.On("Download", ctx, tt.dest, []string{tt.source}).Return(tt.err) + } err := Download(ctx, tt.dest, tt.source, false) - if tt.err == nil { + + if tt.errExpected { + assert.Error(t, err) + assert.EqualError(t, err, tt.err.Error()) + } else { assert.NoError(t, err) mock.AssertExpectationsForObjects(t, &d) - } else { - assert.True(t, err.Error() == tt.err.Error()) } }) } diff --git a/internal/utils/helpers.go b/internal/utils/helpers.go index 7235a6094..cf0284397 100644 --- a/internal/utils/helpers.go +++ b/internal/utils/helpers.go @@ -124,6 +124,10 @@ func Experimental() bool { return os.Getenv("EC_EXPERIMENTAL") == "1" } +func UseGoGather() bool { + return os.Getenv("USEGOGATHER") == "1" +} + // detect if the string is json func IsJson(data string) bool { var jsMsg json.RawMessage