diff --git a/cmd/crane/cmd/append.go b/cmd/crane/cmd/append.go index 3555a7864..6642e58a1 100644 --- a/cmd/crane/cmd/append.go +++ b/cmd/crane/cmd/append.go @@ -117,6 +117,5 @@ container image.`, appendCmd.MarkFlagsMutuallyExclusive("oci-empty-base", "base") appendCmd.MarkFlagRequired("new_tag") - appendCmd.MarkFlagRequired("new_layer") return appendCmd } diff --git a/cmd/crane/cmd/export.go b/cmd/crane/cmd/export.go index 497eb8e09..d5ec75d7f 100644 --- a/cmd/crane/cmd/export.go +++ b/cmd/crane/cmd/export.go @@ -70,20 +70,14 @@ func NewCmdExport(options *[]crane.Option) *cobra.Command { return fmt.Errorf("reading tarball from stdin: %w", err) } } else { - desc, err := crane.Get(src, *options...) + desc, err := crane.Artifact(src, *options...) if err != nil { return fmt.Errorf("pulling %s: %w", src, err) } - if desc.MediaType.IsSchema1() { - img, err = desc.Schema1() - if err != nil { - return fmt.Errorf("pulling schema 1 image %s: %w", src, err) - } - } else { - img, err = desc.Image() - if err != nil { - return fmt.Errorf("pulling Image %s: %w", src, err) - } + var ok bool + img, ok = desc.(v1.Image) + if !ok { + return fmt.Errorf("pulling schema 1 image %s: %w", src, err) } } diff --git a/cmd/crane/cmd/flatten.go b/cmd/crane/cmd/flatten.go index ea642e791..007514f71 100644 --- a/cmd/crane/cmd/flatten.go +++ b/cmd/crane/cmd/flatten.go @@ -91,26 +91,23 @@ func NewCmdFlatten(options *[]crane.Option) *cobra.Command { } func flatten(ref name.Reference, repo name.Repository, use string, o crane.Options) (partial.Describable, error) { - desc, err := remote.Get(ref, o.Remote...) + desc, err := remote.Artifact(ref, o.Remote...) if err != nil { return nil, fmt.Errorf("pulling %s: %w", ref, err) } - if desc.MediaType.IsIndex() { - idx, err := desc.ImageIndex() - if err != nil { - return nil, err - } + if idx, ok := desc.(v1.ImageIndex); ok { return flattenIndex(idx, repo, use, o) - } else if desc.MediaType.IsImage() { - img, err := desc.Image() - if err != nil { - return nil, err - } + } else if img, ok := desc.(v1.Image); ok { return flattenImage(img, repo, use, o) } - return nil, fmt.Errorf("can't flatten %s", desc.MediaType) + mt, err := desc.MediaType() + if err != nil { + return nil, err + } + + return nil, fmt.Errorf("can't flatten %s", mt) } func push(flat partial.Describable, ref name.Reference, o crane.Options) error { diff --git a/cmd/crane/cmd/index.go b/cmd/crane/cmd/index.go index 64e098409..12241d95a 100644 --- a/cmd/crane/cmd/index.go +++ b/cmd/crane/cmd/index.go @@ -70,16 +70,19 @@ func NewCmdIndexFilter(options *[]crane.Option) *cobra.Command { if err != nil { return err } - desc, err := remote.Get(ref, o.Remote...) + desc, err := remote.Artifact(ref, o.Remote...) if err != nil { return fmt.Errorf("pulling %s: %w", baseRef, err) } - if !desc.MediaType.IsIndex() { - return fmt.Errorf("expected %s to be an index, got %q", baseRef, desc.MediaType) - } - base, err := desc.ImageIndex() + mt, err := desc.MediaType() if err != nil { - return nil + return err + } + + var base v1.ImageIndex + var ok bool + if base, ok = desc.(v1.ImageIndex); !ok { + return fmt.Errorf("expected %s to be an index, got %q", baseRef, mt) } idx := filterIndex(base, platforms.platforms) @@ -142,6 +145,7 @@ The platform for appended manifests is inferred from the config file or omitted var ( base v1.ImageIndex err error + ok bool ref name.Reference ) @@ -160,16 +164,16 @@ The platform for appended manifests is inferred from the config file or omitted if err != nil { return err } - desc, err := remote.Get(ref, o.Remote...) + desc, err := remote.Artifact(ref, o.Remote...) if err != nil { return fmt.Errorf("pulling %s: %w", baseRef, err) } - if !desc.MediaType.IsIndex() { - return fmt.Errorf("expected %s to be an index, got %q", baseRef, desc.MediaType) - } - base, err = desc.ImageIndex() + mt, err := desc.MediaType() if err != nil { - return err + return fmt.Errorf("getting media type %s: %w", baseRef, err) + } + if base, ok = desc.(v1.ImageIndex); !ok { + return fmt.Errorf("expected %s to be an index, got %q", baseRef, mt) } } @@ -180,16 +184,11 @@ The platform for appended manifests is inferred from the config file or omitted if err != nil { return err } - desc, err := remote.Get(ref, o.Remote...) + desc, err := remote.Artifact(ref, o.Remote...) if err != nil { return err } - if desc.MediaType.IsImage() { - img, err := desc.Image() - if err != nil { - return err - } - + if img, ok := desc.(v1.Image); ok { cf, err := img.ConfigFile() if err != nil { return err @@ -203,11 +202,7 @@ The platform for appended manifests is inferred from the config file or omitted Add: img, Descriptor: *newDesc, }) - } else if desc.MediaType.IsIndex() { - idx, err := desc.ImageIndex() - if err != nil { - return err - } + } else if idx, ok := desc.(v1.ImageIndex); ok { if flatten { im, err := idx.IndexManifest() if err != nil { @@ -243,7 +238,11 @@ The platform for appended manifests is inferred from the config file or omitted }) } } else { - return fmt.Errorf("saw unexpected MediaType %q for %q", desc.MediaType, m) + mt, err := desc.MediaType() + if err != nil { + return fmt.Errorf("getting media type %s: %w", baseRef, err) + } + return fmt.Errorf("saw unexpected MediaType %q for %q", mt, m) } } diff --git a/cmd/crane/cmd/root.go b/cmd/crane/cmd/root.go index 8fc8ccefa..c754d5fad 100644 --- a/cmd/crane/cmd/root.go +++ b/cmd/crane/cmd/root.go @@ -29,6 +29,7 @@ import ( "github.com/google/go-containerregistry/internal/cmd" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/cobra" ) @@ -47,6 +48,7 @@ func New(use, short string, options []crane.Option) *cobra.Command { insecure := false ndlayers := false platform := &platformValue{} + uselocal := "" wt := &warnTransport{} @@ -68,6 +70,10 @@ func New(use, short string, options []crane.Option) *cobra.Command { if ndlayers { options = append(options, crane.WithNondistributable()) } + if uselocal != "" { + p, _ := layout.FromPath(uselocal) + options = append(options, crane.WithSource(layout.NewSource(p)), crane.WithSink(layout.NewSink(p))) + } if Version != "" { binary := "crane" if len(os.Args[0]) != 0 { @@ -137,7 +143,8 @@ func New(use, short string, options []crane.Option) *cobra.Command { root.PersistentFlags().BoolVar(&insecure, "insecure", false, "Allow image references to be fetched without TLS") root.PersistentFlags().BoolVar(&ndlayers, "allow-nondistributable-artifacts", false, "Allow pushing non-distributable (foreign) layers") root.PersistentFlags().Var(platform, "platform", "Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64).") - + root.PersistentFlags().StringVar(&uselocal, "local", "", "Use a local oci-layout as remote registry") + root.PersistentFlags().MarkHidden("local") return root } diff --git a/cmd/crane/cmd/validate.go b/cmd/crane/cmd/validate.go index 411772f52..13b70c7ef 100644 --- a/cmd/crane/cmd/validate.go +++ b/cmd/crane/cmd/validate.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/validate" "github.com/spf13/cobra" @@ -52,7 +53,7 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command { } if remoteRef != "" { - rmt, err := crane.Get(remoteRef, *options...) + rmt, err := crane.Artifact(remoteRef, *options...) if err != nil { return fmt.Errorf("failed to read image %s: %w", remoteRef, err) } @@ -63,24 +64,27 @@ func NewCmdValidate(options *[]crane.Option) *cobra.Command { if fast { opt = append(opt, validate.Fast) } - if rmt.MediaType.IsIndex() && o.Platform == nil { - idx, err := rmt.ImageIndex() - if err != nil { - return fmt.Errorf("reading index: %w", err) - } + var ( + idx v1.ImageIndex + img v1.Image + ok bool + ) + if idx, ok = rmt.(v1.ImageIndex); ok && o.Platform == nil { if err := validate.Index(idx, opt...); err != nil { fmt.Fprintf(cmd.OutOrStdout(), "FAIL: %s: %v\n", remoteRef, err) return err } - } else { - img, err := rmt.Image() - if err != nil { - return fmt.Errorf("reading image: %w", err) - } + } else if img, ok = rmt.(v1.Image); ok { if err := validate.Image(img, opt...); err != nil { fmt.Fprintf(cmd.OutOrStdout(), "FAIL: %s: %v\n", remoteRef, err) return err } + } else { + mt, err := rmt.MediaType() + if err != nil { + return err + } + return fmt.Errorf("failed to validate ref %s: uknown media type %s", remoteRef, mt) } fmt.Fprintf(cmd.OutOrStdout(), "PASS: %s\n", remoteRef) } diff --git a/internal/cmd/edit.go b/internal/cmd/edit.go index 907a8371d..8c68e9f13 100644 --- a/internal/cmd/edit.go +++ b/internal/cmd/edit.go @@ -280,14 +280,19 @@ func editManifest(in io.Reader, out io.Writer, src string, dst string, mt string return nil, err } - desc, err := remote.Get(ref, o.Remote...) + desc, err := remote.Artifact(ref, o.Remote...) + if err != nil { + return nil, err + } + + manifest, err := desc.RawManifest() if err != nil { return nil, err } var edited []byte if interactive(in, out) { - edited, err = editor.Edit(bytes.NewReader(desc.Manifest), ".json") + edited, err = editor.Edit(bytes.NewReader(manifest), ".json") if err != nil { return nil, err } @@ -317,7 +322,11 @@ func editManifest(in io.Reader, out io.Writer, src string, dst string, mt string if mt == "" { // If --media-type is unset, use Content-Type by default. - mt = string(desc.MediaType) + mediatype, err := desc.MediaType() + if err != nil { + return nil, err + } + mt = string(mediatype) // If document contains mediaType, default to that. wmt := withMediaType{} diff --git a/pkg/crane/config.go b/pkg/crane/config.go index 3e55cc93a..648d7b41e 100644 --- a/pkg/crane/config.go +++ b/pkg/crane/config.go @@ -16,7 +16,7 @@ package crane // Config returns the config file for the remote image ref. func Config(ref string, opt ...Option) ([]byte, error) { - i, _, err := getImage(ref, opt...) + i, err := GetImage(ref, opt...) if err != nil { return nil, err } diff --git a/pkg/crane/delete.go b/pkg/crane/delete.go index 58a8be1f0..26189ffc3 100644 --- a/pkg/crane/delete.go +++ b/pkg/crane/delete.go @@ -18,7 +18,6 @@ import ( "fmt" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" ) // Delete deletes the remote reference at src. @@ -29,5 +28,5 @@ func Delete(src string, opt ...Option) error { return fmt.Errorf("parsing reference %q: %w", src, err) } - return remote.Delete(ref, o.Remote...) + return o.sink.Delete(o.ctx, ref) } diff --git a/pkg/crane/digest.go b/pkg/crane/digest.go index 868a57010..e26ff8403 100644 --- a/pkg/crane/digest.go +++ b/pkg/crane/digest.go @@ -14,26 +14,19 @@ package crane -import "github.com/google/go-containerregistry/pkg/logs" +import ( + "github.com/google/go-containerregistry/pkg/logs" +) // Digest returns the sha256 hash of the remote image at ref. func Digest(ref string, opt ...Option) (string, error) { o := makeOptions(opt...) if o.Platform != nil { - desc, err := getManifest(ref, opt...) + desc, err := getArtifact(ref, opt...) if err != nil { return "", err } - if !desc.MediaType.IsIndex() { - return desc.Digest.String(), nil - } - - // TODO: does not work for indexes which contain schema v1 manifests - img, err := desc.Image() - if err != nil { - return "", err - } - digest, err := img.Digest() + digest, err := desc.Digest() if err != nil { return "", err } @@ -42,11 +35,15 @@ func Digest(ref string, opt ...Option) (string, error) { desc, err := Head(ref, opt...) if err != nil { logs.Warn.Printf("HEAD request failed, falling back on GET: %v", err) - rdesc, err := getManifest(ref, opt...) + rdesc, err := getArtifact(ref, opt...) + if err != nil { + return "", err + } + hash, err := rdesc.Digest() if err != nil { return "", err } - return rdesc.Digest.String(), nil + return hash.String(), nil } return desc.Digest.String(), nil } diff --git a/pkg/crane/get.go b/pkg/crane/get.go index 98a2e8933..b5a0234e3 100644 --- a/pkg/crane/get.go +++ b/pkg/crane/get.go @@ -19,29 +19,26 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote" ) -func getImage(r string, opt ...Option) (v1.Image, name.Reference, error) { +func getManifest(r string, opt ...Option) (*remote.Descriptor, error) { o := makeOptions(opt...) ref, err := name.ParseReference(r, o.Name...) if err != nil { - return nil, nil, fmt.Errorf("parsing reference %q: %w", r, err) - } - img, err := remote.Image(ref, o.Remote...) - if err != nil { - return nil, nil, fmt.Errorf("reading image %q: %w", ref, err) + return nil, fmt.Errorf("parsing reference %q: %w", r, err) } - return img, ref, nil + return remote.Get(ref, o.Remote...) } -func getManifest(r string, opt ...Option) (*remote.Descriptor, error) { +func getArtifact(r string, opt ...Option) (partial.Artifact, error) { o := makeOptions(opt...) ref, err := name.ParseReference(r, o.Name...) if err != nil { return nil, fmt.Errorf("parsing reference %q: %w", r, err) } - return remote.Get(ref, o.Remote...) + return o.source.Artifact(o.ctx, ref) } // Get calls remote.Get and returns an uninterpreted response. @@ -49,6 +46,10 @@ func Get(r string, opt ...Option) (*remote.Descriptor, error) { return getManifest(r, opt...) } +func Artifact(r string, opt ...Option) (partial.Artifact, error) { + return getArtifact(r, opt...) +} + // Head performs a HEAD request for a manifest and returns a content descriptor // based on the registry's response. func Head(r string, opt ...Option) (*v1.Descriptor, error) { @@ -57,5 +58,5 @@ func Head(r string, opt ...Option) (*v1.Descriptor, error) { if err != nil { return nil, err } - return remote.Head(ref, o.Remote...) + return o.source.Head(o.ctx, ref) } diff --git a/pkg/crane/image.go b/pkg/crane/image.go new file mode 100644 index 000000000..2d5d7f907 --- /dev/null +++ b/pkg/crane/image.go @@ -0,0 +1,123 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crane + +import ( + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// matchesPlatform checks if the given platform matches the required platforms. +// The given platform matches the required platform if +// - architecture and OS are identical. +// - OS version and variant are identical if provided. +// - features and OS features of the required platform are subsets of those of the given platform. +func matchesPlatform(given, required v1.Platform) bool { + // Required fields that must be identical. + if given.Architecture != required.Architecture || given.OS != required.OS { + return false + } + + // Optional fields that may be empty, but must be identical if provided. + if required.OSVersion != "" && given.OSVersion != required.OSVersion { + return false + } + if required.Variant != "" && given.Variant != required.Variant { + return false + } + + // Verify required platform's features are a subset of given platform's features. + if !isSubset(given.OSFeatures, required.OSFeatures) { + return false + } + if !isSubset(given.Features, required.Features) { + return false + } + + return true +} + +// isSubset checks if the required array of strings is a subset of the given lst. +func isSubset(lst, required []string) bool { + set := make(map[string]bool) + for _, value := range lst { + set[value] = true + } + + for _, value := range required { + if _, ok := set[value]; !ok { + return false + } + } + + return true +} + +func childByPlatform(idx v1.ImageIndex, platform v1.Platform) (partial.Artifact, error) { + index, err := idx.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if matchesPlatform(p, platform) { + if childDesc.MediaType.IsIndex() { + return idx.ImageIndex(childDesc.Digest) + } else if childDesc.MediaType.IsImage() { + return idx.Image(childDesc.Digest) + } else if childDesc.MediaType.IsSchema1() { + return idx.Image(childDesc.Digest) + } + } + } + return nil, fmt.Errorf("no child with platform %+v in index", platform) +} + +func GetImage(r string, opt ...Option) (v1.Image, error) { + o := makeOptions(opt...) + ar, err := Artifact(r, opt...) + if err != nil { + return nil, fmt.Errorf("reading image %q: %w", r, err) + } + + if img, ok := ar.(v1.Image); ok { + return img, nil + } else if idx, ok := ar.(v1.ImageIndex); ok && o.Platform != nil { + img, err := childByPlatform(idx, *o.Platform) + if err != nil { + return nil, err + } + if img, ok := img.(v1.Image); ok { + return img, nil + } + } + mt, err := ar.MediaType() + if err != nil { + return nil, err + } + return nil, fmt.Errorf("%s (%s) is not an image", r, mt) +} diff --git a/pkg/crane/manifest.go b/pkg/crane/manifest.go index a54926aef..083a6ce0e 100644 --- a/pkg/crane/manifest.go +++ b/pkg/crane/manifest.go @@ -14,19 +14,23 @@ package crane +import v1 "github.com/google/go-containerregistry/pkg/v1" + // Manifest returns the manifest for the remote image or index ref. func Manifest(ref string, opt ...Option) ([]byte, error) { - desc, err := getManifest(ref, opt...) + desc, err := getArtifact(ref, opt...) if err != nil { return nil, err } o := makeOptions(opt...) - if o.Platform != nil { - img, err := desc.Image() + if idx, ok := desc.(v1.ImageIndex); ok && o.Platform != nil { + img, err := childByPlatform(idx, *o.Platform) if err != nil { return nil, err } - return img.RawManifest() + if img, ok := img.(v1.Image); ok { + return img.RawManifest() + } } - return desc.Manifest, nil + return desc.RawManifest() } diff --git a/pkg/crane/options.go b/pkg/crane/options.go index d9d441761..525719a38 100644 --- a/pkg/crane/options.go +++ b/pkg/crane/options.go @@ -23,6 +23,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/sourcesink" ) // Options hold the options that crane uses when calling other packages. @@ -38,6 +39,9 @@ type Options struct { jobs int noclobber bool ctx context.Context + + sink sourcesink.Sink + source sourcesink.Source } // GetOptions exposes the underlying []remote.Option, []name.Option, and @@ -62,6 +66,14 @@ func makeOptions(opts ...Option) Options { o(&opt) } + // By default use remote source and sink + if opt.sink == nil { + opt.sink, _ = remote.NewPusher(opt.Remote...) + } + if opt.source == nil { + opt.source, _ = remote.NewPuller(opt.Remote...) + } + // Allow for untrusted certificates if the user // passed Insecure but no custom transport. if opt.insecure && opt.Transport == nil { @@ -176,3 +188,17 @@ func WithNoClobber(noclobber bool) Option { o.noclobber = noclobber } } + +// WithSink sets the sink +func WithSink(sink sourcesink.Sink) Option { + return func(o *Options) { + o.sink = sink + } +} + +// WithSource sets the source +func WithSource(source sourcesink.Source) Option { + return func(o *Options) { + o.source = source + } +} diff --git a/pkg/crane/push.go b/pkg/crane/push.go index 90a058502..c83edbaf5 100644 --- a/pkg/crane/push.go +++ b/pkg/crane/push.go @@ -19,7 +19,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" ) @@ -50,7 +49,7 @@ func Push(img v1.Image, dst string, opt ...Option) error { if err != nil { return fmt.Errorf("parsing reference %q: %w", dst, err) } - return remote.Write(tag, img, o.Remote...) + return o.sink.Push(o.ctx, tag, img) } // Upload pushes the v1.Layer to a given repo. @@ -60,6 +59,5 @@ func Upload(layer v1.Layer, repo string, opt ...Option) error { if err != nil { return fmt.Errorf("parsing repo %q: %w", repo, err) } - - return remote.WriteLayer(ref, layer, o.Remote...) + return o.sink.Upload(o.ctx, ref, layer) } diff --git a/pkg/crane/tag.go b/pkg/crane/tag.go index 13bc39587..480bc7668 100644 --- a/pkg/crane/tag.go +++ b/pkg/crane/tag.go @@ -28,7 +28,7 @@ func Tag(img, tag string, opt ...Option) error { if err != nil { return fmt.Errorf("parsing reference %q: %w", img, err) } - desc, err := remote.Get(ref, o.Remote...) + desc, err := remote.Artifact(ref, o.Remote...) if err != nil { return fmt.Errorf("fetching %q: %w", img, err) } diff --git a/pkg/v1/layout/layer.go b/pkg/v1/layout/layer.go new file mode 100644 index 000000000..7737c5e1a --- /dev/null +++ b/pkg/v1/layout/layer.go @@ -0,0 +1,57 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// remoteImagelayer implements partial.CompressedLayer +type localLayer struct { + path Path + digest v1.Hash +} + +// Compressed implements partial.CompressedLayer. +func (ll *localLayer) Compressed() (io.ReadCloser, error) { + return ll.path.Blob(ll.digest) +} + +// Digest implements partial.CompressedLayer. +func (ll *localLayer) Digest() (v1.Hash, error) { + return ll.digest, nil +} + +// MediaType implements partial.CompressedLayer. +func (ll *localLayer) MediaType() (types.MediaType, error) { + // TODO + return types.DockerLayer, nil +} + +// Size implements partial.CompressedLayer. +func (ll *localLayer) Size() (int64, error) { + return ll.path.BlobSize(ll.digest) +} + +// See partial.Exists. +func (ll *localLayer) Exists() (bool, error) { + return ll.path.BlobExists(ll.digest), nil +} + +var _ partial.CompressedLayer = (*localLayer)(nil) diff --git a/pkg/v1/layout/sink.go b/pkg/v1/layout/sink.go new file mode 100644 index 000000000..a6959ffc2 --- /dev/null +++ b/pkg/v1/layout/sink.go @@ -0,0 +1,283 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/sourcesink" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + specsv1 "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" +) + +func taggableToManifest(t partial.WithRawManifest) (partial.Artifact, error) { + if a, ok := t.(partial.Artifact); ok { + return a, nil + } + + desc := v1.Descriptor{ + // A reasonable default if Taggable doesn't implement MediaType. + MediaType: types.DockerManifestSchema2, + } + + b, err := t.RawManifest() + if err != nil { + return nil, err + } + + if wmt, ok := t.(partial.WithMediaType); ok { + desc.MediaType, err = wmt.MediaType() + if err != nil { + return nil, err + } + } + + desc.Digest, desc.Size, err = v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("unknown taggable type") +} + +func unpackTaggable(t partial.WithRawManifest) ([]byte, *v1.Descriptor, error) { + b, err := t.RawManifest() + if err != nil { + return nil, nil, err + } + + // A reasonable default if Taggable doesn't implement MediaType. + mt := types.DockerManifestSchema2 + + if wmt, ok := t.(partial.WithMediaType); ok { + m, err := wmt.MediaType() + if err != nil { + return nil, nil, err + } + mt = m + } + + h, sz, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, nil, err + } + + return b, &v1.Descriptor{ + MediaType: mt, + Size: sz, + Digest: h, + }, nil +} + +type sink struct { + path Path +} + +// Delete implements remote.Pusher. +func (lp *sink) Delete(_ context.Context, _ name.Reference) error { + return errors.New("unsupported operation") +} + +func (lp *sink) writeLayer(l v1.Layer) error { + dg, err := l.Digest() + if err != nil { + return err + } + rc, err := l.Compressed() + if err != nil { + return err + } + err = lp.path.WriteBlob(dg, rc) + if err != nil { + return err + } + return nil +} + +func (lp *sink) writeLayers(pctx context.Context, img v1.Image) error { + ls, err := img.Layers() + if err != nil { + return err + } + + g, _ := errgroup.WithContext(pctx) + + for _, l := range ls { + l := l + + g.Go(func() error { + return lp.writeLayer(l) + }) + } + + cl, err := partial.ConfigLayer(img) + if errors.Is(err, stream.ErrNotComputed) { + if err := g.Wait(); err != nil { + return err + } + + cl, err := partial.ConfigLayer(img) + if err != nil { + return err + } + + return lp.writeLayer(cl) + } else if err != nil { + return err + } + + g.Go(func() error { + return lp.writeLayer(cl) + }) + + return g.Wait() +} + +func (lp *sink) writeChildren(pctx context.Context, idx v1.ImageIndex) error { + children, err := partial.Manifests(idx) + if err != nil { + return err + } + + g, ctx := errgroup.WithContext(pctx) + + for _, child := range children { + child := child + if err := lp.writeChild(ctx, child, g); err != nil { + return err + } + } + + return g.Wait() +} + +func (lp *sink) writeDeps(ctx context.Context, m partial.Artifact) error { + if img, ok := m.(v1.Image); ok { + return lp.writeLayers(ctx, img) + } + + if idx, ok := m.(v1.ImageIndex); ok { + return lp.writeChildren(ctx, idx) + } + + // This has no deps, not an error (e.g. something you want to just PUT). + return nil +} + +func (lp *sink) writeManifest(ctx context.Context, t partial.WithRawManifest) error { + m, err := taggableToManifest(t) + if err != nil { + return err + } + + needDeps := true + + if errors.Is(err, stream.ErrNotComputed) { + if err := lp.writeDeps(ctx, m); err != nil { + return err + } + needDeps = false + } else if err != nil { + return err + } + + if needDeps { + if err := lp.writeDeps(ctx, m); err != nil { + return err + } + } + + b, desc, err := unpackTaggable(t) + if err != nil { + return err + } + if err := lp.path.WriteBlob(desc.Digest, io.NopCloser(bytes.NewBuffer(b))); err != nil { + return err + } + + return nil +} + +func (lp *sink) writeChild(ctx context.Context, child partial.Describable, g *errgroup.Group) error { + switch child := child.(type) { + case v1.ImageIndex: + // For recursive index, we want to do a depth-first launching of goroutines + // to avoid deadlocking. + // + // Note that this is rare, so the impact of this should be really small. + return lp.writeManifest(ctx, child) + case v1.Image: + g.Go(func() error { + return lp.writeManifest(ctx, child) + }) + case v1.Layer: + g.Go(func() error { + return lp.writeLayer(child) + }) + default: + // This can't happen. + return fmt.Errorf("encountered unknown child: %T", child) + } + return nil +} + +// Push implements remote.Pusher. +func (lp *sink) Push(ctx context.Context, ref name.Reference, t partial.WithRawManifest) error { + err := lp.writeManifest(ctx, t) + if err != nil { + return err + } + _, desc, err := unpackTaggable(t) + if err != nil { + return err + } + repo := ref.Context().String() + if tag, ok := ref.(name.Tag); ok { + repo = fmt.Sprintf("%s:%s", repo, tag.TagStr()) + } + desc.Annotations = map[string]string{ + specsv1.AnnotationRefName: repo, + } + return lp.path.AppendDescriptor(*desc) +} + +// Upload implements remote.Pusher. +func (lp *sink) Upload(_ context.Context, _ name.Repository, l v1.Layer) error { + digest, err := l.Digest() + if err != nil { + return err + } + rc, err := l.Compressed() + if err != nil { + return err + } + return lp.path.WriteBlob(digest, rc) +} + +func NewSink(path Path) sourcesink.Sink { + return &sink{ + path, + } +} + +var _ sourcesink.Sink = (*sink)(nil) diff --git a/pkg/v1/layout/sink_test.go b/pkg/v1/layout/sink_test.go new file mode 100644 index 000000000..61cd874b4 --- /dev/null +++ b/pkg/v1/layout/sink_test.go @@ -0,0 +1,202 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func mustOCILayout(t *testing.T) Path { + tmp := t.TempDir() + os.WriteFile(fmt.Sprintf("%s/index.json", tmp), []byte("{}"), os.ModePerm) + path, err := FromPath(tmp) + if err != nil { + t.Errorf("FromPath() = %v", err) + } + return path +} + +func mustHaveManifest(t *testing.T, p Path, ref string) { + idx, err := p.ImageIndex() + if err != nil { + t.Errorf("path.ImageIndex() = %v", err) + } + manifest, err := idx.IndexManifest() + if err != nil { + t.Errorf("idx.IndexManifest() = %v", err) + } + if manifest.Manifests[0].Annotations["org.opencontainers.image.ref.name"] != ref { + t.Errorf("image does not contain %s", ref) + } +} + +func enumerateImageBlobs(t *testing.T, img v1.Image) []v1.Hash { + mgd, err := img.Digest() + if err != nil { + t.Errorf("img.Digest() = %v", err) + } + cdg, err := img.ConfigName() + if err != nil { + t.Errorf("img.ConfigName() = %v", err) + } + blobs := []v1.Hash{ + mgd, + cdg, + } + layers, err := img.Layers() + if err != nil { + t.Errorf("img.Layers() = %v", err) + } + for _, layer := range layers { + ldg, err := layer.Digest() + if err != nil { + t.Errorf("layer.Digest() = %v", err) + } + blobs = append(blobs, ldg) + } + return blobs +} + +func enumerateImageIndexBlobs(t *testing.T, idx v1.ImageIndex) []v1.Hash { + mgd, err := idx.Digest() + if err != nil { + t.Errorf("idx.Digest() = %v", err) + } + blobs := []v1.Hash{mgd} + imf, err := idx.IndexManifest() + if err != nil { + t.Errorf("img.Layers() = %v", err) + } + for _, manifest := range imf.Manifests { + im, err := idx.Image(manifest.Digest) + if err != nil { + t.Errorf("idx.Image = %v", err) + } + blobs = append(blobs, enumerateImageBlobs(t, im)...) + } + return blobs +} + +func mustHaveBlobs(t *testing.T, p Path, blobs []v1.Hash) { + for _, blob := range blobs { + if !p.BlobExists(blob) { + t.Fatalf("blob %s/%s is missing", blob.Algorithm, blob.Hex) + } + } +} + +func TestCanPushRandomImage(t *testing.T) { + img, err := random.Image(1024, 10) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + path := mustOCILayout(t) + sink := NewSink(path) + ref := name.MustParseReference("local.random/image:latest") + err = sink.Push(context.TODO(), ref, img) + if err != nil { + t.Errorf("sink.Push() = %v", err) + } + mustHaveManifest(t, path, "local.random/image:latest") + mustHaveBlobs(t, path, enumerateImageBlobs(t, img)) +} + +func TestCanPushRandomImageIndex(t *testing.T) { + idx, err := random.Index(1024, 3, 3) + if err != nil { + t.Fatalf("random.Index() = %v", err) + } + path := mustOCILayout(t) + sink := NewSink(path) + ref := name.MustParseReference("local.random/index:latest") + err = sink.Push(context.TODO(), ref, idx) + if err != nil { + t.Errorf("sink.Push() = %v", err) + } + mustHaveManifest(t, path, "local.random/index:latest") + mustHaveBlobs(t, path, enumerateImageIndexBlobs(t, idx)) +} + +func TestCanPushImageIndex(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(manifestDigest) + if err != nil { + t.Fatalf("Image() = %v", err) + } + + path := mustOCILayout(t) + sink := NewSink(path) + ref := name.MustParseReference("local.repo/index:latest") + + err = sink.Push(context.TODO(), ref, img) + if err != nil { + t.Errorf("sink.Push() = %v", err) + } + mustHaveManifest(t, path, "local.repo/index:latest") + mustHaveBlobs(t, path, enumerateImageBlobs(t, img)) +} + +func TestCanPushImage(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(manifestDigest) + if err != nil { + t.Fatalf("Image() = %v", err) + } + + path := mustOCILayout(t) + sink := NewSink(path) + ref := name.MustParseReference("local.repo/index:latest") + + err = sink.Push(context.TODO(), ref, img) + if err != nil { + t.Errorf("sink.Push() = %v", err) + } + mustHaveManifest(t, path, "local.repo/index:latest") + mustHaveBlobs(t, path, enumerateImageBlobs(t, img)) +} + +func TestCanPushImageWithLatestTag(t *testing.T) { + lp, err := FromPath(testPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + img, err := lp.Image(manifestDigest) + if err != nil { + t.Fatalf("Image() = %v", err) + } + + path := mustOCILayout(t) + sink := NewSink(path) + + err = sink.Push(context.TODO(), name.MustParseReference("reg.local.repo/index"), img) + if err != nil { + t.Errorf("sink.Push() = %v", err) + } + mustHaveManifest(t, path, "reg.local.repo/index:latest") + mustHaveBlobs(t, path, enumerateImageBlobs(t, img)) +} diff --git a/pkg/v1/layout/source.go b/pkg/v1/layout/source.go new file mode 100644 index 000000000..77abac7a7 --- /dev/null +++ b/pkg/v1/layout/source.go @@ -0,0 +1,133 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "context" + "fmt" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/sourcesink" + specsv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +type source struct { + path Path +} + +func NewSource(path Path) sourcesink.Source { + return &source{ + path, + } +} + +var _ sourcesink.Source = (*source)(nil) + +// Artifact implements remote.Puller. +func (p *source) getDescriptor(ref name.Reference) (*v1.Descriptor, error) { + idx, err := p.path.ImageIndex() + if err != nil { + return nil, err + } + im, err := idx.IndexManifest() + if err != nil { + return nil, err + } + // Search for descriptors in reverse order to match most recent descriptors first + for i := len(im.Manifests) - 1; i >= 0; i-- { + manifest := im.Manifests[i] + if rref, ok := manifest.Annotations[specsv1.AnnotationRefName]; ok { + if ref.String() == rref { + return &manifest, nil + } + } + } + return nil, fmt.Errorf("unknown image: %s", ref.String()) +} + +// Artifact implements remote.Puller. +func (p *source) Artifact(_ context.Context, ref name.Reference) (partial.Artifact, error) { + desc, err := p.getDescriptor(ref) + if err != nil { + return nil, err + } + if desc.MediaType.IsImage() { + return p.path.Image(desc.Digest) + } else if desc.MediaType.IsIndex() { + reg, err := p.path.ImageIndex() + if err != nil { + return nil, err + } + return reg.ImageIndex(desc.Digest) + } else if desc.MediaType.IsSchema1() { + return nil, fmt.Errorf("layout puller does not support schema1 images") + } + return nil, fmt.Errorf("unknown media type: %s", desc.MediaType) +} + +// Head implements remote.Puller. +func (p *source) Head(_ context.Context, ref name.Reference) (*v1.Descriptor, error) { + return p.getDescriptor(ref) +} + +// Layer implements remote.Puller. +func (p *source) Layer(_ context.Context, ref name.Digest) (v1.Layer, error) { + h, err := v1.NewHash(ref.Identifier()) + if err != nil { + return nil, err + } + l, err := partial.CompressedToLayer(&localLayer{ + path: p.path, + digest: h, + }) + if err != nil { + return nil, err + } + return l, nil +} + +// List implements remote.Puller. +func (*source) List(_ context.Context, _ name.Repository) ([]string, error) { + return nil, fmt.Errorf("unsupported operation") +} + +// Get implements remote.Puller. +func (*source) Get(_ context.Context, _ name.Reference) (*remote.Descriptor, error) { + return nil, fmt.Errorf("unsupported operation") +} + +// Lister implements remote.Puller. +func (*source) Lister(_ context.Context, _ name.Repository) (*remote.Lister, error) { + return nil, fmt.Errorf("unsupported operation") +} + +// Catalogger implements remote.Puller. +func (*source) Catalogger(_ context.Context, _ name.Registry) (*remote.Catalogger, error) { + return nil, fmt.Errorf("unsupported operation") +} + +// Catalog implements remote.Puller. +func (*source) Catalog(_ context.Context, _ name.Registry) ([]string, error) { + return nil, fmt.Errorf("unsupported operation") +} + +// Referrers implements remote.Puller. +func (*source) Referrers(_ context.Context, _ name.Digest, _ map[string]string) (v1.ImageIndex, error) { + return nil, fmt.Errorf("unsupported operation") +} diff --git a/pkg/v1/layout/source_test.go b/pkg/v1/layout/source_test.go new file mode 100644 index 000000000..14e13cb05 --- /dev/null +++ b/pkg/v1/layout/source_test.go @@ -0,0 +1,125 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "context" + "path/filepath" + "testing" + + "github.com/google/go-containerregistry/pkg/name" +) + +var ( + testRefPath = filepath.Join("testdata", "test_with_ref") +) + +func TestPullerHeadWithDigest(t *testing.T) { + path, err := FromPath(testRefPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + digest := "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720" + source := NewSource(path) + desc, err := source.Head(context.TODO(), name.MustParseReference("reg.local/repo2@sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720")) + if err != nil { + t.Fatalf("source.Head() = %v", err) + } + + if desc.Digest.String() != digest { + t.Fatalf("wrong descriptor returned, expected %s but got %s ", digest, desc.Digest) + } +} + +func TestPullerHeadWithTag(t *testing.T) { + path, err := FromPath(testRefPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + digest := "sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5" + source := NewSource(path) + desc, err := source.Head(context.TODO(), name.MustParseReference("reg.local/repo4:latest")) + if err != nil { + t.Fatalf("source.Head() = %v", err) + } + if desc.Digest.String() != digest { + t.Fatalf("wrong descriptor returned, expected %s but got %s ", digest, desc.Digest) + } +} + +func TestPullerArtifact(t *testing.T) { + path, err := FromPath(testRefPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + expectedDigest := "sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5" + + source := NewSource(path) + desc, err := source.Artifact(context.TODO(), name.MustParseReference("reg.local/repo4:latest")) + if err != nil { + t.Fatalf("source.Artifact() = %v", err) + } + + digest, err := desc.Digest() + if err != nil { + t.Fatalf("desc.Digest() = %v", err) + } + + mt, err := desc.MediaType() + if err != nil { + t.Fatalf("desc.MediaType() = %v", err) + } + + if digest.String() != expectedDigest { + t.Fatalf("wrong descriptor returned, expected %s but got %s ", expectedDigest, digest.String()) + } + + if !mt.IsIndex() { + t.Fatalf("expcted an image index but got %s", mt) + } +} + +func TestPullerLayer(t *testing.T) { + path, err := FromPath(testRefPath) + if err != nil { + t.Fatalf("FromPath() = %v", err) + } + + expectedDigest := "sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e" + source := NewSource(path) + + layer, err := source.Layer(context.TODO(), name.MustParseReference("reg.local/repo4@sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e").(name.Digest)) + if err != nil { + t.Fatalf("source.Layer() = %v", err) + } + + digest, err := layer.Digest() + if err != nil { + t.Fatalf("layer.Digest() = %v", err) + } + + if digest.String() != expectedDigest { + t.Fatalf("wrong descriptor returned, expected %s but got %s ", expectedDigest, digest.String()) + } + + size, err := layer.Size() + if err != nil { + t.Fatalf("layer.Size() = %v", err) + } + + if size != 330 { + t.Fatalf("wrong size returned, expected 330 but got %d", size) + } +} diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 new file mode 100644 index 000000000..1597d0721 --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5 @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "1" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb new file mode 100644 index 000000000..e6587e23e --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "4" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 new file mode 100644 index 000000000..096f21fb7 Binary files /dev/null and b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3 differ diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 new file mode 100644 index 000000000..48609c6c6 --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":165,"digest":"sha256:321460fa87fd42433950b42d04b7aff249f4ed960d43404a9f699886906cc9d3"}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e new file mode 100644 index 000000000..4228c8902 --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e @@ -0,0 +1 @@ +{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17"], "type": "layers"}} diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 new file mode 100644 index 000000000..425c2d0b2 --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/930705ce23e3b6ed4c08746b6fe880089c864fbaf62482702ae3fdd66b8c7fe9 @@ -0,0 +1 @@ +{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:3610aa5267a210147ba6ca02cdd87610dfc08522de9c5f5015edd8ee14853fd8"], "type": "layers"}} diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b new file mode 100644 index 000000000..05c63217b Binary files /dev/null and b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b differ diff --git a/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 new file mode 100644 index 000000000..21dc412c3 --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":167,"digest":"sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b"}]} \ No newline at end of file diff --git a/pkg/v1/layout/testdata/test_with_ref/index.json b/pkg/v1/layout/testdata/test_with_ref/index.json new file mode 100644 index 000000000..2a7261ca2 --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/index.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 423, + "digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", + "annotations": { + "org.opencontainers.image.ref.name": "reg.local/repo1@sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650" + } + }, + { + "mediaType": "application/vnd.oci.descriptor.v1+json", + "size": 423, + "digest": "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720", + "annotations": { + "org.opencontainers.image.ref.name": "reg.local/repo2@sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720" + } + }, + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "size": 314, + "digest": "sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5", + "annotations": { + "org.opencontainers.image.ref.name": "reg.local/repo3@sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "size": 314, + "digest": "sha256:2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb", + "annotations": { + "org.opencontainers.image.ref.name": "reg.local/repo4@sha256:2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "size": 314, + "digest": "sha256:05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5", + "annotations": { + "org.opencontainers.image.ref.name": "reg.local/repo4:latest" + } + } + ] +} diff --git a/pkg/v1/layout/testdata/test_with_ref/oci-layout b/pkg/v1/layout/testdata/test_with_ref/oci-layout new file mode 100644 index 000000000..10ff2f3ce --- /dev/null +++ b/pkg/v1/layout/testdata/test_with_ref/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} diff --git a/pkg/v1/layout/write.go b/pkg/v1/layout/write.go index d6e35c391..4d8a1877a 100644 --- a/pkg/v1/layout/write.go +++ b/pkg/v1/layout/write.go @@ -318,6 +318,23 @@ func (l Path) RemoveBlob(hash v1.Hash) error { return nil } +// BlobExists checks a blob exists at blobs/{hash.Algorithm}/{hash.Hex} +func (l Path) BlobExists(hash v1.Hash) bool { + dir := l.path("blobs", hash.Algorithm, hash.Hex) + _, err := os.Stat(dir) + return !errors.Is(err, os.ErrNotExist) +} + +// BlobExists checks a blob exists at blobs/{hash.Algorithm}/{hash.Hex} +func (l Path) BlobSize(hash v1.Hash) (int64, error) { + dir := l.path("blobs", hash.Algorithm, hash.Hex) + stat, err := os.Stat(dir) + if err != nil { + return 0, err + } + return stat.Size(), nil +} + // WriteImage writes an image, including its manifest, config and all of its // layers, to the blobs directory. If any blob already exists, as determined by // the hash filename, does not write it. diff --git a/pkg/v1/layout/write_test.go b/pkg/v1/layout/write_test.go index 530e0e8ea..5fd150ebb 100644 --- a/pkg/v1/layout/write_test.go +++ b/pkg/v1/layout/write_test.go @@ -443,6 +443,52 @@ func TestRemoveBlob(t *testing.T) { } } +func TestBlobExistsAndSize(t *testing.T) { + // need to set up a basic path + tmp := t.TempDir() + + var ii v1.ImageIndex = empty.Index + l, err := Write(tmp, ii) + if err != nil { + t.Fatal(err) + } + + // create a random blob + b := []byte("abcdefghijklmnop") + hash, _, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + + if err := l.WriteBlob(hash, io.NopCloser(bytes.NewReader(b))); err != nil { + t.Fatal(err) + } + // make sure it exists + b2, err := l.Bytes(hash) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, b2) { + t.Fatal("mismatched bytes") + } + if !l.BlobExists(hash) { + t.Fatal("blob should exist") + } + size, err := l.BlobSize(hash) + if err != nil { + t.Fatalf("BlobSize = %v", err) + } + if size != int64(len(b)) { + t.Fatalf("invalid size reported by BlobSize, expected %d, but got %d", len(b), size) + } + if err := l.RemoveBlob(hash); err != nil { + t.Fatal(err) + } + if l.BlobExists(hash) { + t.Fatal("blob should not exist") + } +} + func TestStreamingWriteLayer(t *testing.T) { // need to set up a basic path tmp := t.TempDir() diff --git a/pkg/v1/partial/artifact.go b/pkg/v1/partial/artifact.go new file mode 100644 index 000000000..6ed48a540 --- /dev/null +++ b/pkg/v1/partial/artifact.go @@ -0,0 +1,28 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partial + +import ( + types "github.com/google/go-containerregistry/pkg/v1/types" +) + +type Artifact interface { + Describable + WithRawManifest +} + +type WithMediaType interface { + MediaType() (types.MediaType, error) +} diff --git a/pkg/v1/remote/artifact.go b/pkg/v1/remote/artifact.go new file mode 100644 index 000000000..9d3fa12f0 --- /dev/null +++ b/pkg/v1/remote/artifact.go @@ -0,0 +1,31 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +// Get returns a partial.Artifact for the given reference. +// +// See Head if you don't need the response body. +func Artifact(ref name.Reference, options ...Option) (partial.Artifact, error) { + o, err := makeOptions(options...) + if err != nil { + return nil, err + } + return newPuller(o).Artifact(o.context, ref) +} diff --git a/pkg/v1/remote/index_test.go b/pkg/v1/remote/index_test.go index 1b4b5cf1e..87fa519aa 100644 --- a/pkg/v1/remote/index_test.go +++ b/pkg/v1/remote/index_test.go @@ -25,6 +25,7 @@ import ( "github.com/google/go-cmp/cmp" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -53,7 +54,7 @@ func mustChild(t *testing.T, idx v1.ImageIndex, h v1.Hash) v1.Image { return img } -func mustMediaType(t *testing.T, tag withMediaType) types.MediaType { +func mustMediaType(t *testing.T, tag partial.WithMediaType) types.MediaType { mt, err := tag.MediaType() if err != nil { t.Fatalf("MediaType() = %v", err) diff --git a/pkg/v1/remote/puller.go b/pkg/v1/remote/puller.go index 7da8017ee..b5dab2bc0 100644 --- a/pkg/v1/remote/puller.go +++ b/pkg/v1/remote/puller.go @@ -16,11 +16,13 @@ package remote import ( "context" + "fmt" "sync" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/sourcesink" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -31,6 +33,8 @@ type Puller struct { readers sync.Map } +var _ sourcesink.Source = (*Puller)(nil) + func NewPuller(options ...Option) (*Puller, error) { o, err := makeOptions(options...) if err != nil { @@ -102,6 +106,25 @@ func (p *Puller) get(ctx context.Context, ref name.Reference, acceptable []types return f.get(ctx, ref, acceptable, platform) } +func (p *Puller) Artifact(ctx context.Context, ref name.Reference) (partial.Artifact, error) { + return p.artifact(ctx, ref, allManifestMediaTypes, p.o.platform) +} + +func (p *Puller) artifact(ctx context.Context, ref name.Reference, acceptable []types.MediaType, platform v1.Platform) (partial.Artifact, error) { + desc, err := p.get(ctx, ref, acceptable, platform) + if err != nil { + return nil, err + } + if desc.MediaType.IsImage() { + return desc.Image() + } else if desc.MediaType.IsIndex() { + return desc.ImageIndex() + } else if desc.MediaType.IsSchema1() { + return desc.Schema1() + } + return nil, fmt.Errorf("unknown media type: %s", desc.MediaType) +} + // Layer is like remote.Layer, but avoids re-authenticating when possible. func (p *Puller) Layer(ctx context.Context, ref name.Digest) (v1.Layer, error) { f, err := p.fetcher(ctx, ref.Context()) diff --git a/pkg/v1/remote/pusher.go b/pkg/v1/remote/pusher.go index 1c07bd475..79f24a4d0 100644 --- a/pkg/v1/remote/pusher.go +++ b/pkg/v1/remote/pusher.go @@ -28,14 +28,14 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/sourcesink" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" ) type manifest interface { - Taggable - partial.Describable + partial.Artifact } // key is either v1.Hash or v1.Layer (for stream.Layer) @@ -91,6 +91,8 @@ func (w *workers) Stream(layer v1.Layer, f func() error) error { return v.(error) } +var _ sourcesink.Sink = (*Pusher)(nil) + type Pusher struct { o *options @@ -103,7 +105,6 @@ func NewPusher(options ...Option) (*Pusher, error) { if err != nil { return nil, err } - return newPusher(o), nil } @@ -125,7 +126,7 @@ func (p *Pusher) writer(ctx context.Context, repo name.Repository, o *options) ( return rw, rw.init(ctx) } -func (p *Pusher) Push(ctx context.Context, ref name.Reference, t Taggable) error { +func (p *Pusher) Push(ctx context.Context, ref name.Reference, t partial.WithRawManifest) error { w, err := p.writer(ctx, ref.Context(), p.o) if err != nil { return err @@ -220,6 +221,7 @@ func (d describable) MediaType() (types.MediaType, error) { return d.desc.MediaType, nil } +// This is basically partia type tagManifest struct { Taggable partial.Describable @@ -256,7 +258,7 @@ func taggableToManifest(t Taggable) (manifest, error) { return nil, err } - if wmt, ok := t.(withMediaType); ok { + if wmt, ok := t.(partial.WithMediaType); ok { desc.MediaType, err = wmt.MediaType() if err != nil { return nil, err diff --git a/pkg/v1/remote/write.go b/pkg/v1/remote/write.go index 04a3989a6..cb496e060 100644 --- a/pkg/v1/remote/write.go +++ b/pkg/v1/remote/write.go @@ -33,6 +33,7 @@ import ( "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" @@ -40,7 +41,7 @@ import ( // Taggable is an interface that enables a manifest PUT (e.g. for tagging). type Taggable interface { - RawManifest() ([]byte, error) + partial.WithRawManifest } // Write pushes the provided img to the specified image reference. @@ -426,10 +427,6 @@ func (w *writer) uploadOne(ctx context.Context, l v1.Layer) error { return retry.Retry(tryUpload, w.predicate, w.backoff) } -type withMediaType interface { - MediaType() (types.MediaType, error) -} - // This is really silly, but go interfaces don't let me satisfy remote.Taggable // with remote.Descriptor because of name collisions between method names and // struct fields. @@ -448,7 +445,7 @@ func unpackTaggable(t Taggable) ([]byte, *v1.Descriptor, error) { // A reasonable default if Taggable doesn't implement MediaType. mt := types.DockerManifestSchema2 - if wmt, ok := t.(withMediaType); ok { + if wmt, ok := t.(partial.WithMediaType); ok { m, err := wmt.MediaType() if err != nil { return nil, nil, err diff --git a/pkg/v1/sourcesink/sink.go b/pkg/v1/sourcesink/sink.go new file mode 100644 index 000000000..3bd9df7aa --- /dev/null +++ b/pkg/v1/sourcesink/sink.go @@ -0,0 +1,29 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sourcesink + +import ( + "context" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +type Sink interface { + Delete(ctx context.Context, ref name.Reference) error + Push(ctx context.Context, ref name.Reference, t partial.WithRawManifest) error + Upload(ctx context.Context, repo name.Repository, l v1.Layer) error +} diff --git a/pkg/v1/sourcesink/source.go b/pkg/v1/sourcesink/source.go new file mode 100644 index 000000000..7a2b93623 --- /dev/null +++ b/pkg/v1/sourcesink/source.go @@ -0,0 +1,29 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sourcesink + +import ( + "context" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +type Source interface { + Layer(ctx context.Context, ref name.Digest) (v1.Layer, error) + Head(ctx context.Context, ref name.Reference) (*v1.Descriptor, error) + Artifact(ctx context.Context, ref name.Reference) (partial.Artifact, error) +}