From 2db31303e7d251f8555ce2d7da255cecb5af2b5c Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Tue, 13 Jun 2023 09:40:26 +0200 Subject: [PATCH 01/18] chore(Dockerfile): use COPY instead of ADD Co-authored-by: Philippe Scorsolini Co-authored-by: AdamKorcz Signed-off-by: Philippe Scorsolini (cherry picked from commit b89eafb431e28a9d8bea1653c4d5ab0405f78e6e) --- cluster/images/crossplane/Dockerfile | 8 ++++---- cluster/images/xfn/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster/images/crossplane/Dockerfile b/cluster/images/crossplane/Dockerfile index 71934c167..8e30eb9c1 100644 --- a/cluster/images/crossplane/Dockerfile +++ b/cluster/images/crossplane/Dockerfile @@ -3,9 +3,9 @@ FROM gcr.io/distroless/static@sha256:7198a357ff3a8ef750b041324873960cf2153c11cc5 ARG TARGETOS ARG TARGETARCH -ADD bin/$TARGETOS\_$TARGETARCH/crossplane /usr/local/bin/ -ADD crds /crds -ADD webhookconfigurations /webhookconfigurations +COPY bin/$TARGETOS\_$TARGETARCH/crossplane /usr/local/bin/ +COPY crds /crds +COPY webhookconfigurations /webhookconfigurations EXPOSE 8080 USER 65532 -ENTRYPOINT ["crossplane"] \ No newline at end of file +ENTRYPOINT ["crossplane"] diff --git a/cluster/images/xfn/Dockerfile b/cluster/images/xfn/Dockerfile index 8966515db..7c509fc88 100644 --- a/cluster/images/xfn/Dockerfile +++ b/cluster/images/xfn/Dockerfile @@ -11,7 +11,7 @@ ARG TARGETARCH # time. RUN apt-get update && apt-get install -y ca-certificates crun && rm -rf /var/lib/apt/lists/* -ADD bin/${TARGETOS}\_${TARGETARCH}/xfn /usr/local/bin/ +COPY bin/${TARGETOS}\_${TARGETARCH}/xfn /usr/local/bin/ # We run xfn as root in order to grant it CAP_SETUID and CAP_SETGID, which are # required in order to create a user namespace with more than one available UID From 8386b6bd029d96eb9b8151adabbe279db5dab1fe Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 00:22:26 +0000 Subject: [PATCH 02/18] Update debian:bookworm-slim Docker digest to d8f9d38 --- cluster/images/xfn/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/images/xfn/Dockerfile b/cluster/images/xfn/Dockerfile index 7c509fc88..0d6c9b2d0 100644 --- a/cluster/images/xfn/Dockerfile +++ b/cluster/images/xfn/Dockerfile @@ -1,5 +1,5 @@ # This is debian:bookworm-slim (i.e. Debian 12, testing), which has crun v1.5. -FROM debian:bookworm-slim@sha256:e1a80fdca0e09f557a2bef15c7f601c49915e8977ca16eead985e541afeb5770 +FROM debian:bookworm-slim@sha256:d8f9d38c21495b04d1cca99805fbb383856e19794265684019bf193c3b7d67f9 ARG TARGETOS ARG TARGETARCH From afd6ebdca58360622ee497778970b5120f6f53b6 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Wed, 21 Jun 2023 17:20:01 +0200 Subject: [PATCH 03/18] fix(crank): copy to tar file one chunk at a time Co-authored-by: Philippe Scorsolini Co-authored-by: AdamKorcz Signed-off-by: Philippe Scorsolini (cherry picked from commit 1d7537693084b6f4b88ebee098636900ff16cb0d) --- internal/xpkg/build.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/xpkg/build.go b/internal/xpkg/build.go index fe318a405..1e941c2d6 100644 --- a/internal/xpkg/build.go +++ b/internal/xpkg/build.go @@ -109,7 +109,8 @@ func Build(ctx context.Context, b parser.Backend, p parser.Parser, l parser.Lint if err := tw.WriteHeader(hdr); err != nil { return nil, errors.Wrap(err, errTarFromStream) } - if _, err = io.Copy(tw, buf); err != nil { + // copy chunks of 1MB to avoid loading the entire package into memory twice + if _, err = copyChunks(tw, buf, 1024*1024); err != nil { return nil, errors.Wrap(err, errTarFromStream) } if err := tw.Close(); err != nil { @@ -132,3 +133,23 @@ func Build(ctx context.Context, b parser.Backend, p parser.Parser, l parser.Lint // Append layer to to scratch image. return mutate.AppendLayers(empty.Image, layer) } + +// copyChunks pleases gosec per https://github.com/securego/gosec/pull/433. +// Like Copy it reads from src until EOF, it does not treat an EOF from Read as +// an error to be reported. +// +// NOTE(negz): This rule confused me at first because io.Copy appears to use a +// buffer, but in fact it bypasses it if src/dst is an io.WriterTo/ReaderFrom. +func copyChunks(dst io.Writer, src io.Reader, chunkSize int64) (int64, error) { + var written int64 + for { + w, err := io.CopyN(dst, src, chunkSize) + written += w + if errors.Is(err, io.EOF) { + return written, nil + } + if err != nil { + return written, err + } + } +} From f403dca36410d31ed869aa59d2a042b06dbd8097 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Tue, 20 Jun 2023 19:37:16 +0200 Subject: [PATCH 04/18] fix: limit xfn stdout and stderr Signed-off-by: Philippe Scorsolini (cherry picked from commit 43325354c08313f7556c9ed0f7ab4974cbd2513d) --- cmd/xfn/spark/spark.go | 38 +++++++++++++++++++++++++++++++-- internal/xfn/container_linux.go | 7 ++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/cmd/xfn/spark/spark.go b/cmd/xfn/spark/spark.go index 3f315aff1..e9690cb9b 100644 --- a/cmd/xfn/spark/spark.go +++ b/cmd/xfn/spark/spark.go @@ -69,6 +69,9 @@ const ociRuntimeRoot = "runtime" // the RunFunctionRequest. const defaultTimeout = 25 * time.Second +// The maximum size of stdout/stderr to avoid OOM +const maxStdioBytes = 100 << 20 // 100 MB + // Command runs a containerized Composition Function. type Command struct { CacheDir string `short:"c" help:"Directory used for caching function images and containers." default:"/xfn"` @@ -157,16 +160,47 @@ func (c *Command) Run() error { //nolint:gocyclo // TODO(negz): Refactor some of cmd := exec.CommandContext(ctx, c.Runtime, "--root="+root, "run", "--bundle="+b.Path(), runID) cmd.Stdin = bytes.NewReader(req.GetInput()) - out, err := cmd.Output() + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + _ = b.Cleanup() + return errors.Wrap(err, errRuntime) + } + stderrPipe, err := cmd.StderrPipe() + if err != nil { + _ = b.Cleanup() + return errors.Wrap(err, errRuntime) + } + + if err := cmd.Start(); err != nil { + _ = b.Cleanup() + return errors.Wrap(err, errRuntime) + } + + stdout, err := io.ReadAll(io.LimitReader(stdoutPipe, maxStdioBytes)) + if err != nil { + _ = b.Cleanup() + return errors.Wrap(err, errRuntime) + } + stderr, err := io.ReadAll(io.LimitReader(stderrPipe, maxStdioBytes)) if err != nil { _ = b.Cleanup() return errors.Wrap(err, errRuntime) } + + if err := cmd.Wait(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitErr.Stderr = stderr + } + _ = b.Cleanup() + return errors.Wrap(err, errRuntime) + } + if err := b.Cleanup(); err != nil { return errors.Wrap(err, errCleanupBundle) } - rsp := &v1alpha1.RunFunctionResponse{Output: out} + rsp := &v1alpha1.RunFunctionResponse{Output: stdout} pb, err = proto.Marshal(rsp) if err != nil { return errors.Wrap(err, errMarshalResponse) diff --git a/internal/xfn/container_linux.go b/internal/xfn/container_linux.go index 5a82240ac..180d2e581 100644 --- a/internal/xfn/container_linux.go +++ b/internal/xfn/container_linux.go @@ -58,6 +58,7 @@ const ( const ( UserNamespaceUIDs = 65536 UserNamespaceGIDs = 65536 + MaxStdioBytes = 100 << 20 // 100 MB ) // The subcommand of xfn to invoke - i.e. "xfn spark " @@ -160,12 +161,14 @@ func (r *ContainerRunner) RunFunction(ctx context.Context, req *v1alpha1.RunFunc // We must read all of stdout and stderr before calling cmd.Wait, which // closes the underlying pipes. - stdout, err := io.ReadAll(stdio.Stdout) + // Limited to MaxStdioBytes to avoid OOMing if the function writes a lot of + // data to stdout or stderr. + stdout, err := io.ReadAll(io.LimitReader(stdio.Stdout, MaxStdioBytes)) if err != nil { return nil, errors.Wrap(err, errReadStdout) } - stderr, err := io.ReadAll(stdio.Stderr) + stderr, err := io.ReadAll(io.LimitReader(stdio.Stderr, MaxStdioBytes)) if err != nil { return nil, errors.Wrap(err, errReadStderr) } From 2ac2efc28831cc9801cd2557c5be1219dc73ed35 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Wed, 21 Jun 2023 08:49:27 +0200 Subject: [PATCH 05/18] refactor: move max stdio bytes to spark flag Signed-off-by: Philippe Scorsolini (cherry picked from commit 0be527693644727e2eda44d99ecd0e7cd504b4e2) --- cmd/xfn/spark/spark.go | 19 ++++++++++++------- internal/xfn/container_linux.go | 3 ++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/xfn/spark/spark.go b/cmd/xfn/spark/spark.go index e9690cb9b..32c4f70b7 100644 --- a/cmd/xfn/spark/spark.go +++ b/cmd/xfn/spark/spark.go @@ -69,13 +69,11 @@ const ociRuntimeRoot = "runtime" // the RunFunctionRequest. const defaultTimeout = 25 * time.Second -// The maximum size of stdout/stderr to avoid OOM -const maxStdioBytes = 100 << 20 // 100 MB - // Command runs a containerized Composition Function. type Command struct { - CacheDir string `short:"c" help:"Directory used for caching function images and containers." default:"/xfn"` - Runtime string `help:"OCI runtime binary to invoke." default:"crun"` + CacheDir string `short:"c" help:"Directory used for caching function images and containers." default:"/xfn"` + Runtime string `help:"OCI runtime binary to invoke." default:"crun"` + MaxStdioBytes int64 `help:"Maximum size of stdout and stderr for functions." default:"0"` } // Run a Composition Function inside an unprivileged user namespace. Reads a @@ -176,12 +174,12 @@ func (c *Command) Run() error { //nolint:gocyclo // TODO(negz): Refactor some of return errors.Wrap(err, errRuntime) } - stdout, err := io.ReadAll(io.LimitReader(stdoutPipe, maxStdioBytes)) + stdout, err := io.ReadAll(limitReaderIfNonZero(stdoutPipe, c.MaxStdioBytes)) if err != nil { _ = b.Cleanup() return errors.Wrap(err, errRuntime) } - stderr, err := io.ReadAll(io.LimitReader(stderrPipe, maxStdioBytes)) + stderr, err := io.ReadAll(limitReaderIfNonZero(stderrPipe, c.MaxStdioBytes)) if err != nil { _ = b.Cleanup() return errors.Wrap(err, errRuntime) @@ -209,6 +207,13 @@ func (c *Command) Run() error { //nolint:gocyclo // TODO(negz): Refactor some of return errors.Wrap(err, errWriteResponse) } +func limitReaderIfNonZero(r io.Reader, limit int64) io.Reader { + if limit == 0 { + return r + } + return io.LimitReader(r, limit) +} + // FromImagePullConfig configures an image client with options derived from the // supplied ImagePullConfig. func FromImagePullConfig(cfg *v1alpha1.ImagePullConfig) oci.ImageClientOption { diff --git a/internal/xfn/container_linux.go b/internal/xfn/container_linux.go index 180d2e581..675198830 100644 --- a/internal/xfn/container_linux.go +++ b/internal/xfn/container_linux.go @@ -21,6 +21,7 @@ package xfn import ( "bytes" "context" + "fmt" "io" "os" "os/exec" @@ -97,7 +98,7 @@ func (r *ContainerRunner) RunFunction(ctx context.Context, req *v1alpha1.RunFunc bundle, then then executes an OCI runtime in order to actually execute the function. */ - cmd := exec.CommandContext(ctx, os.Args[0], spark, "--cache-dir="+r.cache) //nolint:gosec // We're intentionally executing with variable input. + cmd := exec.CommandContext(ctx, os.Args[0], spark, "--cache-dir="+r.cache, fmt.Sprintf("--max-stdio-bytes=%d", MaxStdioBytes)) //nolint:gosec // We're intentionally executing with variable input. cmd.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNS, UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: r.rootUID, Size: 1}}, From 580c91649066d89a217bfd681bd791d89dcbebf4 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Mon, 19 Jun 2023 15:19:03 +0200 Subject: [PATCH 06/18] fix(xfn): set max layers number limit for images Co-authored-by: AdamKorcz Co-authored-by: Philippe Scorsolini Signed-off-by: Philippe Scorsolini (cherry picked from commit 4b1f4c4e8b51aabed3b67da1cda641f88fe83112) --- internal/oci/store/overlay/store_overlay.go | 7 +++++-- internal/oci/store/store.go | 12 +++++++++++- .../oci/store/uncompressed/store_uncompressed.go | 4 ++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/internal/oci/store/overlay/store_overlay.go b/internal/oci/store/overlay/store_overlay.go index f0e70c125..8478853ea 100644 --- a/internal/oci/store/overlay/store_overlay.go +++ b/internal/oci/store/overlay/store_overlay.go @@ -173,8 +173,11 @@ func (c *CachingBundler) Bundle(ctx context.Context, i ociv1.Image, id string, o if err != nil { return nil, errors.Wrap(err, errGetLayers) } - - lowerPaths := make([]string, len(layers)) + nLayers := len(layers) + if nLayers > store.MaxLayers { + return nil, errors.Errorf(store.ErrFmtTooManyLayers, nLayers, store.MaxLayers) + } + lowerPaths := make([]string, nLayers) for i := range layers { p, err := c.layer.Resolve(ctx, layers[i], layers[:i]...) if err != nil { diff --git a/internal/oci/store/store.go b/internal/oci/store/store.go index be46c846c..97e3161f8 100644 --- a/internal/oci/store/store.go +++ b/internal/oci/store/store.go @@ -77,6 +77,12 @@ const ( errOpenLayer = "cannot open layer" errStatLayer = "cannot stat layer" errCheckExistence = "cannot determine whether layer exists" + ErrFmtTooManyLayers = "image has too many layers: %d (max %d)" +) + +var ( + // MaxLayers is the maximum number of layers an image can have. + MaxLayers = 256 ) // A Bundler prepares OCI runtime bundles for use by an OCI runtime. @@ -158,7 +164,7 @@ func (i *Image) Image(h ociv1.Hash) (ociv1.Image, error) { } // WriteImage writes the supplied image to the store. -func (i *Image) WriteImage(img ociv1.Image) error { +func (i *Image) WriteImage(img ociv1.Image) error { //nolint:gocyclo // TODO(phisco): Refactor to reduce complexity. d, err := img.Digest() if err != nil { return errors.Wrap(err, errGetDigest) @@ -203,6 +209,10 @@ func (i *Image) WriteImage(img ociv1.Image) error { return errors.Wrap(err, errGetLayers) } + if nLayers := len(layers); nLayers > MaxLayers { + return errors.Errorf(ErrFmtTooManyLayers, nLayers, MaxLayers) + } + g := &errgroup.Group{} for _, l := range layers { l := l // Pin loop var. diff --git a/internal/oci/store/uncompressed/store_uncompressed.go b/internal/oci/store/uncompressed/store_uncompressed.go index 33b040524..2b689064c 100644 --- a/internal/oci/store/uncompressed/store_uncompressed.go +++ b/internal/oci/store/uncompressed/store_uncompressed.go @@ -104,6 +104,10 @@ func (c *Bundler) Bundle(ctx context.Context, i ociv1.Image, id string, o ...spe } b := Bundle{path: path} + if nLayers := len(layers); nLayers > store.MaxLayers { + return nil, errors.Errorf(store.ErrFmtTooManyLayers, nLayers, store.MaxLayers) + } + for _, l := range layers { tb, err := l.Uncompressed() if err != nil { From 88c9da70d8174d93cff2c13ca8ff3a700c349d53 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Tue, 20 Jun 2023 08:15:06 +0200 Subject: [PATCH 07/18] chore: refactor to Validate function Signed-off-by: Philippe Scorsolini (cherry picked from commit 7fd862047d6aa81d1811214db55fde3966bf26ed) --- internal/oci/store/overlay/store_overlay.go | 11 ++++++----- internal/oci/store/store.go | 19 ++++++++++++++++--- .../store/uncompressed/store_uncompressed.go | 4 ++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/internal/oci/store/overlay/store_overlay.go b/internal/oci/store/overlay/store_overlay.go index 8478853ea..d87ee503a 100644 --- a/internal/oci/store/overlay/store_overlay.go +++ b/internal/oci/store/overlay/store_overlay.go @@ -169,15 +169,16 @@ func (c *CachingBundler) Bundle(ctx context.Context, i ociv1.Image, id string, o return nil, errors.Wrap(err, errReadConfigFile) } + if err := store.Validate(i); err != nil { + return nil, err + } + layers, err := i.Layers() if err != nil { return nil, errors.Wrap(err, errGetLayers) } - nLayers := len(layers) - if nLayers > store.MaxLayers { - return nil, errors.Errorf(store.ErrFmtTooManyLayers, nLayers, store.MaxLayers) - } - lowerPaths := make([]string, nLayers) + + lowerPaths := make([]string, len(layers)) for i := range layers { p, err := c.layer.Resolve(ctx, layers[i], layers[:i]...) if err != nil { diff --git a/internal/oci/store/store.go b/internal/oci/store/store.go index 97e3161f8..a67a33346 100644 --- a/internal/oci/store/store.go +++ b/internal/oci/store/store.go @@ -77,7 +77,7 @@ const ( errOpenLayer = "cannot open layer" errStatLayer = "cannot stat layer" errCheckExistence = "cannot determine whether layer exists" - ErrFmtTooManyLayers = "image has too many layers: %d (max %d)" + errFmtTooManyLayers = "image has too many layers: %d (max %d)" ) var ( @@ -209,8 +209,8 @@ func (i *Image) WriteImage(img ociv1.Image) error { //nolint:gocyclo // TODO(phi return errors.Wrap(err, errGetLayers) } - if nLayers := len(layers); nLayers > MaxLayers { - return errors.Errorf(ErrFmtTooManyLayers, nLayers, MaxLayers) + if err := Validate(img); err != nil { + return err } g := &errgroup.Group{} @@ -355,3 +355,16 @@ func copyChunks(dst io.Writer, src io.Reader, chunkSize int64) (int64, error) { } } } + +// Validate returns an error if the supplied image is invalid, +// e.g. the number of layers is above the maximum allowed. +func Validate(img ociv1.Image) error { + layers, err := img.Layers() + if err != nil { + return errors.Wrap(err, errGetLayers) + } + if nLayers := len(layers); nLayers > MaxLayers { + return errors.Errorf(errFmtTooManyLayers, nLayers, MaxLayers) + } + return nil +} diff --git a/internal/oci/store/uncompressed/store_uncompressed.go b/internal/oci/store/uncompressed/store_uncompressed.go index 2b689064c..e3f1d6453 100644 --- a/internal/oci/store/uncompressed/store_uncompressed.go +++ b/internal/oci/store/uncompressed/store_uncompressed.go @@ -104,8 +104,8 @@ func (c *Bundler) Bundle(ctx context.Context, i ociv1.Image, id string, o ...spe } b := Bundle{path: path} - if nLayers := len(layers); nLayers > store.MaxLayers { - return nil, errors.Errorf(store.ErrFmtTooManyLayers, nLayers, store.MaxLayers) + if err := store.Validate(i); err != nil { + return nil, err } for _, l := range layers { From c90ffb7088745fafb3a961dc99fee0be07fa4381 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 21 Jun 2023 20:59:45 -0700 Subject: [PATCH 08/18] Run xfn with unconfined AppArmor profile https://kubernetes.io/docs/tutorials/security/apparmor/#securing-a-pod The xfn container needs the unshare syscall in order to run rootless containers (i.e. create user namespaces). Most default apparmor and seccomp profiles don't allow this. We already run the container unconfined by seccomp, but must also run it unconfined by apparmor. Signed-off-by: Nic Cope (cherry picked from commit 966436b9a0d54495e81f9c2e64534730e9e12ca4) --- cluster/charts/crossplane/templates/deployment.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cluster/charts/crossplane/templates/deployment.yaml b/cluster/charts/crossplane/templates/deployment.yaml index 2d5ab80c1..d3981618b 100644 --- a/cluster/charts/crossplane/templates/deployment.yaml +++ b/cluster/charts/crossplane/templates/deployment.yaml @@ -20,7 +20,7 @@ spec: type: {{ .Values.deploymentStrategy }} template: metadata: - {{- if or .Values.metrics.enabled .Values.customAnnotations }} + {{- if or .Values.metrics.enabled .Values.xfn.enabled .Values.customAnnotations }} annotations: {{- end }} {{- if .Values.metrics.enabled }} @@ -28,6 +28,9 @@ spec: prometheus.io/port: "8080" prometheus.io/scrape: "true" {{- end }} + {{- if .Values.xfn.enabled }} + container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}-xfn: unconfined + {{- end }} {{- with .Values.customAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} From ac590d6ef9e0c906ed3f364ec52d57765f3d57dd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 03:21:48 +0000 Subject: [PATCH 09/18] Update debian:bookworm-slim Docker digest to 9bd077d --- cluster/images/xfn/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/images/xfn/Dockerfile b/cluster/images/xfn/Dockerfile index 0d6c9b2d0..32fde905a 100644 --- a/cluster/images/xfn/Dockerfile +++ b/cluster/images/xfn/Dockerfile @@ -1,5 +1,5 @@ # This is debian:bookworm-slim (i.e. Debian 12, testing), which has crun v1.5. -FROM debian:bookworm-slim@sha256:d8f9d38c21495b04d1cca99805fbb383856e19794265684019bf193c3b7d67f9 +FROM debian:bookworm-slim@sha256:9bd077d2f77c754f4f7f5ee9e6ded9ff1dff92c6dce877754da21b917c122c77 ARG TARGETOS ARG TARGETARCH From fa45d494b29797719cd6eb8cac5399d5a1cdd60d Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Thu, 13 Jul 2023 14:36:11 +0200 Subject: [PATCH 10/18] fix: limit max number of layers for Packages Signed-off-by: Philippe Scorsolini (cherry picked from commit abcc64ec48a7c63ff6c14be35cd404b05be72be8) --- internal/controller/pkg/revision/imageback.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/controller/pkg/revision/imageback.go b/internal/controller/pkg/revision/imageback.go index 38bfb570a..1bfc53219 100644 --- a/internal/controller/pkg/revision/imageback.go +++ b/internal/controller/pkg/revision/imageback.go @@ -39,11 +39,14 @@ const ( errGetUncompressed = "failed to get uncompressed contents from layer" errMultipleAnnotatedLayers = "package is invalid due to multiple annotated base layers" errOpenPackageStream = "failed to open package stream file" + errFmtMaxManifestLayers = "package has %d layers, but only %d are allowed" ) const ( layerAnnotation = "io.crossplane.xpkg" baseAnnotationValue = "base" + // maxLayers is the maximum number of layers an image can have. + maxLayers = 256 ) // ImageBackend is a backend for parser. @@ -101,6 +104,12 @@ func (i *ImageBackend) Init(ctx context.Context, bo ...parser.BackendOption) (io if err != nil { return nil, errors.Wrap(err, errGetManifest) } + + // Check that the image has less than the maximum allowed number of layers. + if nLayers := len(manifest.Layers); nLayers > maxLayers { + return nil, errors.Errorf(errFmtMaxManifestLayers, nLayers, maxLayers) + } + // Determine if the image is using annotated layers. var tarc io.ReadCloser foundAnnotated := false From 2d30759d5b46c8647cbc962de3bc10b9a5141e4a Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Thu, 13 Jul 2023 14:43:28 +0200 Subject: [PATCH 11/18] fix: stop rbac manager's rule expansion on timeout Signed-off-by: Philippe Scorsolini (cherry picked from commit 65fc720d862bf562f52f8cb4926d7188325d177f) --- .../rbac/provider/roles/requests.go | 35 ++- .../rbac/provider/roles/requests_test.go | 226 ++++++++++++++---- 2 files changed, 204 insertions(+), 57 deletions(-) diff --git a/internal/controller/rbac/provider/roles/requests.go b/internal/controller/rbac/provider/roles/requests.go index 1a8262425..939a901d9 100644 --- a/internal/controller/rbac/provider/roles/requests.go +++ b/internal/controller/rbac/provider/roles/requests.go @@ -29,7 +29,9 @@ import ( // Error strings. const ( - errGetClusterRole = "cannot get ClusterRole" + errGetClusterRole = "cannot get ClusterRole" + errExpandClusterRoleRules = "cannot expand ClusterRole rules" + errExpandPermissionRequests = "cannot expand PermissionRequests" ) const ( @@ -126,11 +128,17 @@ func (r Rule) path() path { } // Expand RBAC policy rules into our granular rules. -func Expand(rs ...rbacv1.PolicyRule) []Rule { +func Expand(ctx context.Context, rs ...rbacv1.PolicyRule) ([]Rule, error) { //nolint:gocyclo // Granular rules are inherently complex. out := make([]Rule, 0, len(rs)) for _, r := range rs { for _, u := range r.NonResourceURLs { for _, v := range r.Verbs { + // exit if ctx is done + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } out = append(out, Rule{NonResourceURL: u, Verb: v}) } } @@ -147,13 +155,18 @@ func Expand(rs ...rbacv1.PolicyRule) []Rule { for _, rsc := range r.Resources { for _, n := range names { for _, v := range r.Verbs { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } out = append(out, Rule{APIGroup: g, Resource: rsc, ResourceName: n, Verb: v}) } } } } } - return out + return out, nil } // A ClusterRoleBackedValidator is a PermissionRequestsValidator that validates @@ -170,7 +183,7 @@ func NewClusterRoleBackedValidator(c client.Client, roleName string) *ClusterRol return &ClusterRoleBackedValidator{client: c, name: roleName} } -// ValidatePermissionRequests against the ClusterRole. +// ValidatePermissionRequests against the ClusterRole, returning the list of rejected rules. func (v *ClusterRoleBackedValidator) ValidatePermissionRequests(ctx context.Context, requests ...rbacv1.PolicyRule) ([]Rule, error) { cr := &rbacv1.ClusterRole{} if err := v.client.Get(ctx, types.NamespacedName{Name: v.name}, cr); err != nil { @@ -178,12 +191,20 @@ func (v *ClusterRoleBackedValidator) ValidatePermissionRequests(ctx context.Cont } t := newNode() - for _, rule := range Expand(cr.Rules...) { + expandedCrRules, err := Expand(ctx, cr.Rules...) + if err != nil { + return nil, errors.Wrap(err, errExpandClusterRoleRules) + } + for _, rule := range expandedCrRules { t.Allow(rule.path()) } rejected := make([]Rule, 0) - for _, rule := range Expand(requests...) { + expandedRequests, err := Expand(ctx, requests...) + if err != nil { + return nil, errors.Wrap(err, errExpandPermissionRequests) + } + for _, rule := range expandedRequests { if !t.Allowed(rule.path()) { rejected = append(rejected, rule) } @@ -195,5 +216,5 @@ func (v *ClusterRoleBackedValidator) ValidatePermissionRequests(ctx context.Cont // VerySecureValidator is a PermissionRequestsValidatorFn that rejects all // requested permissions. func VerySecureValidator(ctx context.Context, requests ...rbacv1.PolicyRule) ([]Rule, error) { - return Expand(requests...), nil + return Expand(ctx, requests...) } diff --git a/internal/controller/rbac/provider/roles/requests_test.go b/internal/controller/rbac/provider/roles/requests_test.go index be1535577..d4de22b16 100644 --- a/internal/controller/rbac/provider/roles/requests_test.go +++ b/internal/controller/rbac/provider/roles/requests_test.go @@ -83,79 +83,129 @@ func TestAllowed(t *testing.T) { } func TestExpand(t *testing.T) { + type args struct { + rs []rbacv1.PolicyRule + ctx context.Context + } + type want struct { + err error + rules []Rule + } cases := map[string]struct { reason string - rs []rbacv1.PolicyRule - want []Rule + args + want }{ "SimpleURL": { reason: "It should be possible to expand a simple, granular non-resource RBAC rule.", - rs: []rbacv1.PolicyRule{{ - NonResourceURLs: []string{"/api"}, - Verbs: []string{"get"}, - }}, - want: []Rule{{ - NonResourceURL: "/api", - Verb: "get", - }}, + args: args{ + rs: []rbacv1.PolicyRule{{ + NonResourceURLs: []string{"/api"}, + Verbs: []string{"get"}, + }}, + }, + want: want{ + rules: []Rule{{ + NonResourceURL: "/api", + Verb: "get", + }}, + }, }, "SimpleResource": { reason: "It should be possible to expand a simple, granular resource RBAC rule.", - rs: []rbacv1.PolicyRule{{ - APIGroups: []string{""}, - Resources: []string{"*"}, - Verbs: []string{"get"}, - }}, - want: []Rule{{ - APIGroup: "", - Resource: "*", - ResourceName: "*", - Verb: "get", - }}, + args: args{ + rs: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"*"}, + Verbs: []string{"get"}, + }}, + }, + want: want{ + rules: []Rule{{ + APIGroup: "", + Resource: "*", + ResourceName: "*", + Verb: "get", + }}, + }, }, "ComplexResource": { reason: "It should be possible to expand a more complex resource RBAC rule.", - rs: []rbacv1.PolicyRule{ - {APIGroups: []string{""}, Resources: []string{"*"}, Verbs: []string{"get", "list", "watch"}}, - {APIGroups: []string{"example"}, Resources: []string{"examples", "others"}, ResourceNames: []string{"barry", "hank"}, Verbs: []string{"get"}}, + args: args{ + rs: []rbacv1.PolicyRule{ + {APIGroups: []string{""}, Resources: []string{"*"}, Verbs: []string{"get", "list", "watch"}}, + {APIGroups: []string{"example"}, Resources: []string{"examples", "others"}, ResourceNames: []string{"barry", "hank"}, Verbs: []string{"get"}}, + }, }, - want: []Rule{ - {APIGroup: "", Resource: "*", ResourceName: "*", Verb: "get"}, - {APIGroup: "", Resource: "*", ResourceName: "*", Verb: "list"}, - {APIGroup: "", Resource: "*", ResourceName: "*", Verb: "watch"}, - {APIGroup: "example", Resource: "examples", ResourceName: "barry", Verb: "get"}, - {APIGroup: "example", Resource: "examples", ResourceName: "hank", Verb: "get"}, - {APIGroup: "example", Resource: "others", ResourceName: "barry", Verb: "get"}, - {APIGroup: "example", Resource: "others", ResourceName: "hank", Verb: "get"}, + want: want{ + rules: []Rule{ + {APIGroup: "", Resource: "*", ResourceName: "*", Verb: "get"}, + {APIGroup: "", Resource: "*", ResourceName: "*", Verb: "list"}, + {APIGroup: "", Resource: "*", ResourceName: "*", Verb: "watch"}, + {APIGroup: "example", Resource: "examples", ResourceName: "barry", Verb: "get"}, + {APIGroup: "example", Resource: "examples", ResourceName: "hank", Verb: "get"}, + {APIGroup: "example", Resource: "others", ResourceName: "barry", Verb: "get"}, + {APIGroup: "example", Resource: "others", ResourceName: "hank", Verb: "get"}, + }, }, }, "Combo": { reason: "We should faithfully expand a rule with both URLs and resources. This is invalid, but we let Kubernetes police that.", - rs: []rbacv1.PolicyRule{{ - APIGroups: []string{""}, - Resources: []string{"*"}, - NonResourceURLs: []string{"/api"}, - Verbs: []string{"get"}, - }}, - want: []Rule{ - { - NonResourceURL: "/api", - Verb: "get", - }, - { - APIGroup: "", - Resource: "*", - ResourceName: "*", - Verb: "get", + args: args{ + rs: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"*"}, + NonResourceURLs: []string{"/api"}, + Verbs: []string{"get"}, + }}, + }, + want: want{ + rules: []Rule{ + { + NonResourceURL: "/api", + Verb: "get", + }, + { + APIGroup: "", + Resource: "*", + ResourceName: "*", + Verb: "get", + }, }, }, }, + "ComboCtxCancelled": { + reason: "We should return an error if the context is cancelled.", + args: args{ + rs: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"*"}, + NonResourceURLs: []string{"/api"}, + Verbs: []string{"get"}, + }}, + ctx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }(), + }, + want: want{ + err: context.Canceled, + }, + }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { - got := Expand(tc.rs...) - if diff := cmp.Diff(tc.want, got); diff != "" { + ctx := tc.args.ctx + if ctx == nil { + ctx = context.Background() + } + got, err := Expand(ctx, tc.rs...) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nExpand(...): -want error, +got error:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.rules, got); diff != "" { t.Errorf("\n%s\nExpand(...): -want, +got:\n%s", tc.reason, diff) } }) @@ -229,6 +279,7 @@ func TestValidatePermissionRequests(t *testing.T) { }, }, args: args{ + ctx: context.Background(), requests: []rbacv1.PolicyRule{ // Allowed - we allow * on secrets. { @@ -270,6 +321,81 @@ func TestValidatePermissionRequests(t *testing.T) { }, }, }, + "SuccessfulRejectEvenWithTimeout": { + fields: fields{ + c: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + cr := obj.(*rbacv1.ClusterRole) + cr.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps", "events"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"apps", "extensions"}, + Resources: []string{"deployments"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + ResourceNames: []string{"this-one-really-cool-pod"}, + Verbs: []string{"*"}, + }, + } + return nil + }), + }, + }, + args: args{ + ctx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }(), + requests: []rbacv1.PolicyRule{ + // Allowed - we allow * on secrets. + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"*"}, + }, + // Allowed - we allow * on configmaps. + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Rejected - we don't allow get on extensions/deployments. + { + APIGroups: []string{"extensions"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list"}, + }, + // Allowed - we allow get and list on apps/deployments. + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list"}, + }, + // Rejected - we only allow access to really cool pods. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list"}, + }, + }, + }, + want: want{ + err: errors.Wrap(context.Canceled, errExpandClusterRoleRules), + }, + }, } for name, tc := range cases { From 93c793ec2aac3fefc04883e6d784a6a034a8cc05 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Thu, 20 Jul 2023 13:24:26 +0200 Subject: [PATCH 12/18] chore: bump go-containerregistry to v0.15.3-0.20230625233257-b8504803389b Signed-off-by: Philippe Scorsolini --- go.mod | 28 ++++++++++++++-------------- go.sum | 55 ++++++++++++++++++++++++++++--------------------------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 4ccb9c3e0..8ee7a83b6 100644 --- a/go.mod +++ b/go.mod @@ -9,15 +9,15 @@ require ( github.com/crossplane/crossplane-runtime v0.19.2 github.com/cyphar/filepath-securejoin v0.2.3 github.com/google/go-cmp v0.5.9 - github.com/google/go-containerregistry v0.15.2 + github.com/google/go-containerregistry v0.15.3-0.20230625233257-b8504803389b github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20220517194345-84eb52633e96 github.com/imdario/mergo v0.3.12 github.com/jmattheis/goverter v0.10.1 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.1 github.com/spf13/afero v1.8.0 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.2.0 k8s.io/api v0.26.1 k8s.io/apiextensions-apiserver v0.26.1 k8s.io/apimachinery v0.26.1 @@ -32,8 +32,8 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8 github.com/google/uuid v1.3.0 - golang.org/x/sys v0.7.0 - google.golang.org/grpc v1.54.0 + golang.org/x/sys v0.8.0 + google.golang.org/grpc v1.55.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.30.0 kernel.org/pub/linux/libs/security/libcap/cap v1.2.66 @@ -42,7 +42,7 @@ require ( require cloud.google.com/go/compute/metadata v0.2.3 // indirect require ( - cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go/compute v1.19.3 // indirect github.com/Azure/azure-sdk-for-go v64.1.0+incompatible // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -83,9 +83,9 @@ require ( github.com/dave/jennifer v1.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/cli v23.0.5+incompatible // indirect + github.com/docker/cli v24.0.0+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v23.0.5+incompatible // indirect + github.com/docker/docker v24.0.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -181,17 +181,17 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.7.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.8.0 // indirect + golang.org/x/tools v0.9.1 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7e740cb4b..bf2486d2b 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -194,12 +194,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE= -github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v24.0.0+incompatible h1:0+1VshNwBQzQAx9lOl+OYCTCEAD8fKs/qeXMx3O0wqM= +github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v23.0.5+incompatible h1:DaxtlTJjFSnLOXVNUBU1+6kXGz2lpDoEAH6QoxaSg8k= -github.com/docker/docker v23.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.0+incompatible h1:z4bf8HvONXX9Tde5lGBMQ7yCJgNahmJumdrStZAbeY4= +github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= @@ -340,8 +340,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= -github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= +github.com/google/go-containerregistry v0.15.3-0.20230625233257-b8504803389b h1:nEV+eNboccNOCOkad/Zkx7nYG/xsk8cceS2Zew/VPzk= +github.com/google/go-containerregistry v0.15.3-0.20230625233257-b8504803389b/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20220517194345-84eb52633e96 h1:QwDEEOmXlkBRv+SYO9aghUBpQdUrfzzaeVJMCgfQ6FI= github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20220517194345-84eb52633e96/go.mod h1:sHz+xxnG66stcZJwgNibU5o9p4PbFavngIL0vgfHbBY= github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20220516044946-14395f1b4b4e h1:xvh8QS12VXVDYdHG/AjEEKVjeFU9AP90kfLHJmySG7Y= @@ -607,8 +607,9 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= +github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= @@ -687,8 +688,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -768,8 +769,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -781,8 +782,8 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -794,8 +795,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -856,12 +857,12 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -932,8 +933,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1007,8 +1008,8 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 h1:VmCWItVXcKboEMCwZaWge+1JLiTCQSngZeINF+wzO+g= -google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1030,8 +1031,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From ec47867d35baf6f4a88e1ea38efa6710b41fa727 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Thu, 13 Jul 2023 16:11:35 +0200 Subject: [PATCH 13/18] fix: max size of package parsed limited to 200MB Signed-off-by: Philippe Scorsolini (cherry picked from commit e44f73f2a73512671c5f6a25d4ad0ac193394657) --- internal/controller/pkg/revision/reconciler.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/controller/pkg/revision/reconciler.go b/internal/controller/pkg/revision/reconciler.go index 32bd897c2..e84aac654 100644 --- a/internal/controller/pkg/revision/reconciler.go +++ b/internal/controller/pkg/revision/reconciler.go @@ -55,6 +55,8 @@ import ( const ( reconcileTimeout = 3 * time.Minute + // the max size of a package parsed by the parser + maxPackageSize = 200 << 20 // 100 MB ) const ( @@ -475,7 +477,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco } // Parse package contents. - pkg, err := r.parser.Parse(ctx, rc) + pkg, err := r.parser.Parse(ctx, struct { + io.Reader + io.Closer + }{ + Reader: io.LimitReader(rc, maxPackageSize), + Closer: rc, + }) // Wait until we finish writing to cache. Parser closes the reader. if err := <-cacheWrite; err != nil { // If we failed to cache we want to cleanup, but we don't abort unless From d577b71940deb26e60d32ad03c0ba6339d3865e9 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Thu, 13 Jul 2023 14:39:00 +0200 Subject: [PATCH 14/18] fix: validate Package images against manifest on pull Signed-off-by: Philippe Scorsolini (cherry picked from commit 4feb58e7b6f8a30d1fc311f6169855465577e3f2) --- internal/controller/pkg/revision/imageback.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/controller/pkg/revision/imageback.go b/internal/controller/pkg/revision/imageback.go index 1bfc53219..7d70e59ab 100644 --- a/internal/controller/pkg/revision/imageback.go +++ b/internal/controller/pkg/revision/imageback.go @@ -23,6 +23,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/validate" "github.com/crossplane/crossplane-runtime/pkg/errors" "github.com/crossplane/crossplane-runtime/pkg/parser" @@ -40,6 +41,8 @@ const ( errMultipleAnnotatedLayers = "package is invalid due to multiple annotated base layers" errOpenPackageStream = "failed to open package stream file" errFmtMaxManifestLayers = "package has %d layers, but only %d are allowed" + errValidateLayer = "invalid package layer" + errValidateImage = "invalid package image" ) const ( @@ -130,6 +133,9 @@ func (i *ImageBackend) Init(ctx context.Context, bo ...parser.BackendOption) (io if err != nil { return nil, errors.Wrap(err, errFetchLayer) } + if err := validate.Layer(layer); err != nil { + return nil, errors.Wrap(err, errValidateLayer) + } tarc, err = layer.Uncompressed() if err != nil { return nil, errors.Wrap(err, errGetUncompressed) @@ -138,6 +144,9 @@ func (i *ImageBackend) Init(ctx context.Context, bo ...parser.BackendOption) (io // If we still don't have content then we need to flatten image filesystem. if !foundAnnotated { + if err := validate.Image(img); err != nil { + return nil, errors.Wrap(err, errValidateImage) + } tarc = mutate.Extract(img) } From 30dc1ea2b48b399c10d98f32854d91e5e1b0cb33 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Mon, 24 Jul 2023 19:25:40 +0200 Subject: [PATCH 15/18] tests: remove successfull package fetch test case Signed-off-by: Philippe Scorsolini (cherry picked from commit 2c3db49a3967de74fbd983f8fc6e1237b8956417) --- .../controller/pkg/revision/imageback_test.go | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/internal/controller/pkg/revision/imageback_test.go b/internal/controller/pkg/revision/imageback_test.go index 5d3db208b..b73c76ff1 100644 --- a/internal/controller/pkg/revision/imageback_test.go +++ b/internal/controller/pkg/revision/imageback_test.go @@ -17,18 +17,14 @@ limitations under the License. package revision import ( - "archive/tar" - "bytes" "context" "io" - "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/crossplane/crossplane-runtime/pkg/errors" @@ -56,23 +52,24 @@ func TestImageBackend(t *testing.T) { }, }) - streamCont := "somestreamofyaml" - tarBuf := new(bytes.Buffer) - tw := tar.NewWriter(tarBuf) - hdr := &tar.Header{ - Name: xpkg.StreamFile, - Mode: int64(xpkg.StreamFileMode), - Size: int64(len(streamCont)), - } - _ = tw.WriteHeader(hdr) - _, _ = io.Copy(tw, strings.NewReader(streamCont)) - _ = tw.Close() - packLayer, _ := tarball.LayerFromOpener(func() (io.ReadCloser, error) { - // NOTE(hasheddan): we must construct a new reader each time as we - // ingest packImg in multiple tests below. - return io.NopCloser(bytes.NewReader(tarBuf.Bytes())), nil - }) - packImg, _ := mutate.AppendLayers(empty.Image, packLayer) + // TODO(phisco): uncomment when https://github.com/google/go-containerregistry/pull/1758 is merged + // streamCont := "somestreamofyaml" + // tarBuf := new(bytes.Buffer) + // tw := tar.NewWriter(tarBuf) + // hdr := &tar.Header{ + // Name: xpkg.StreamFile, + // Mode: int64(xpkg.StreamFileMode), + // Size: int64(len(streamCont)), + // } + // _ = tw.WriteHeader(hdr) + // _, _ = io.Copy(tw, strings.NewReader(streamCont)) + // _ = tw.Close() + // packLayer, _ := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + // // NOTE(hasheddan): we must construct a new reader each time as we + // // ingest packImg in multiple tests below. + // return io.NopCloser(bytes.NewReader(tarBuf.Bytes())), nil + // }) + // packImg, _ := mutate.AppendLayers(empty.Image, packLayer) type args struct { f xpkg.Fetcher @@ -151,19 +148,20 @@ func TestImageBackend(t *testing.T) { }, want: errors.Wrap(errBoom, errFetchPackage), }, - "SuccessFetchPackage": { - reason: "Should not return error is package is not in cache but is fetched successfully.", - args: args{ - f: &fake.MockFetcher{ - MockFetch: fake.NewMockFetchFn(packImg, nil), - }, - opts: []parser.BackendOption{PackageRevision(&v1.ProviderRevision{ - Spec: v1.PackageRevisionSpec{ - Package: "test/test:latest", - }, - })}, - }, - }, + // TODO(phisco): uncomment when https://github.com/google/go-containerregistry/pull/1758 is merged + // "SuccessFetchPackage": { + // reason: "Should not return error is package is not in cache but is fetched successfully.", + // args: args{ + // f: &fake.MockFetcher{ + // MockFetch: fake.NewMockFetchFn(packImg, nil), + // }, + // opts: []parser.BackendOption{PackageRevision(&v1.ProviderRevision{ + // Spec: v1.PackageRevisionSpec{ + // Package: "test/test:latest", + // }, + // })}, + // }, + // }, } for name, tc := range cases { From cfda072a65492d2a243ac9682306436af5eb0a72 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Tue, 25 Jul 2023 12:59:04 +0100 Subject: [PATCH 16/18] composite: fix nil-dereference Signed-off-by: AdamKorcz (cherry picked from commit 0cd9fd320f7671f60c07bf4a5473e684e6f7c6b3) --- .../apiextensions/composite/composition_transforms.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/controller/apiextensions/composite/composition_transforms.go b/internal/controller/apiextensions/composite/composition_transforms.go index dfc5023cb..dde189290 100644 --- a/internal/controller/apiextensions/composite/composition_transforms.go +++ b/internal/controller/apiextensions/composite/composition_transforms.go @@ -146,7 +146,14 @@ func ResolveMap(t v1.MapTransform, input any) (any, error) { } return val, nil default: - return nil, errors.Errorf(errFmtMapTypeNotSupported, reflect.TypeOf(input).String()) + var inputType string + if input == nil { + inputType = "nil" + } else { + inputType = reflect.TypeOf(input).String() + } + + return nil, errors.Errorf(errFmtMapTypeNotSupported, inputType) } } From 4895c201d304bc2bec3f6af75e6c633aa7b8cb54 Mon Sep 17 00:00:00 2001 From: ezgidemirel Date: Thu, 27 Jul 2023 14:47:14 +0300 Subject: [PATCH 17/18] Update ci.yml with new hashes Signed-off-by: ezgidemirel --- .github/workflows/ci.yml | 85 ++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a032cfb01..8817c5dba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,28 +27,28 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true - name: Setup Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - name: Cache the Go Build Cache - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-check-diff- - name: Cache Go Dependencies - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} @@ -81,28 +81,28 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true - name: Setup Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - name: Cache the Go Build Cache - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-lint- - name: Cache Go Dependencies - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} @@ -115,7 +115,7 @@ jobs: # this action because it leaves 'annotations' (i.e. it comments on PRs to # point out linter violations). - name: Lint - uses: golangci/golangci-lint-action@07db5389c99593f11ad7b44463c2d4233066a9b1 # v3 + uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299 # v3 with: version: ${{ env.GOLANGCI_VERSION }} skip-cache: true # We do our own caching. @@ -127,28 +127,28 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true - name: Setup Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - name: Cache the Go Build Cache - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-check-diff- - name: Cache Go Dependencies - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} @@ -158,12 +158,12 @@ jobs: run: make vendor vendor.check - name: Initialize CodeQL - uses: github/codeql-action/init@3ebbd71c74ef574dbc558c82f70e52732c8b44fe # v2 + uses: github/codeql-action/init@1813ca74c3faaa3a2da2070b9b8a0b3e7373a0d8 # v2 with: languages: go - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3ebbd71c74ef574dbc558c82f70e52732c8b44fe # v2 + uses: github/codeql-action/analyze@1813ca74c3faaa3a2da2070b9b8a0b3e7373a0d8 # v2 trivy-scan-fs: runs-on: ubuntu-22.04 @@ -171,17 +171,18 @@ jobs: if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true - name: Run Trivy vulnerability scanner in fs mode - uses: aquasecurity/trivy-action@9ab158e8597f3b310480b9a69402b419bc03dbd5 # 0.8.0 + uses: aquasecurity/trivy-action@41f05d9ecffa2ed3f1580af306000f734b733e54 # 0.11.2 with: scan-type: 'fs' ignore-unfixed: true skip-dirs: design scan-ref: '.' + exit-code: '1' severity: 'CRITICAL,HIGH' unit-tests: @@ -191,7 +192,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true @@ -199,23 +200,23 @@ jobs: run: git fetch --prune --unshallow - name: Setup Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - name: Cache the Go Build Cache - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-unit-tests-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-unit-tests- - name: Cache Go Dependencies - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} @@ -228,7 +229,7 @@ jobs: run: make -j2 test - name: Publish Unit Test Coverage - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3 + uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3 with: flags: unittests file: _output/tests/linux_amd64/coverage.txt @@ -240,18 +241,18 @@ jobs: steps: - name: Setup QEMU - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2 with: platforms: all - name: Setup Docker Buildx - uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2 + uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2 with: version: ${{ env.DOCKER_BUILDX_VERSION }} install: true - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true @@ -259,23 +260,23 @@ jobs: run: git fetch --prune --unshallow - name: Setup Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - name: Cache the Go Build Cache - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-e2e-tests-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-e2e-tests- - name: Cache Go Dependencies - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} @@ -302,18 +303,18 @@ jobs: steps: - name: Setup QEMU - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2 with: platforms: all - name: Setup Docker Buildx - uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2 + uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2 with: version: ${{ env.DOCKER_BUILDX_VERSION }} install: true - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 with: submodules: true @@ -321,23 +322,23 @@ jobs: run: git fetch --prune --unshallow - name: Setup Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT - name: Cache the Go Build Cache - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: ${{ steps.go.outputs.cache }} key: ${{ runner.os }}-build-publish-artifacts-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-publish-artifacts- - name: Cache Go Dependencies - uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} @@ -360,14 +361,14 @@ jobs: path: _output/** - name: Login to DockerHub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2 if: env.DOCKER_USR != '' with: username: ${{ secrets.DOCKER_USR }} password: ${{ secrets.DOCKER_PSW }} - name: Login to Upbound - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2 if: env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != '' with: registry: xpkg.upbound.io From f3621cb2feb84841bf2c309255560e550cbce075 Mon Sep 17 00:00:00 2001 From: ezgidemirel Date: Thu, 27 Jul 2023 16:03:26 +0300 Subject: [PATCH 18/18] add "Cleanup Disk" step to publish-artifacts job Signed-off-by: ezgidemirel --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8817c5dba..859a4c5c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -302,6 +302,16 @@ jobs: if: needs.detect-noop.outputs.noop != 'true' steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@main + with: + android: true + dotnet: true + haskell: true + tool-cache: true + large-packages: false + swap-storage: false + - name: Setup QEMU uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2 with: