Skip to content

Commit

Permalink
feat: implement gc command (#1811)
Browse files Browse the repository at this point in the history
* feat: implement prune flag

* address changes

* revert

* boilerplate

* rename

* boilerplate

* Update .gitattributes

---------

Co-authored-by: Jason Hall <[email protected]>
  • Loading branch information
thesayyn and imjasonh authored Nov 29, 2023
1 parent 5a53a12 commit ceb0580
Show file tree
Hide file tree
Showing 16 changed files with 376 additions and 2 deletions.
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@
**/zz_deepcopy_generated.go linguist-generated=true
cmd/crane/doc/crane*.md linguist-generated=true
go.sum linguist-generated=true
**/testdata/** ignore-lint=true
66 changes: 66 additions & 0 deletions cmd/crane/cmd/gc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
"fmt"
"os"

"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/spf13/cobra"
)

func NewCmdLayout() *cobra.Command {
cmd := &cobra.Command{
Use: "layout",
}
cmd.AddCommand(newCmdGc())
return cmd
}

// NewCmdGc creates a new cobra.Command for the pull subcommand.
func newCmdGc() *cobra.Command {
cmd := &cobra.Command{
Use: "gc OCI-LAYOUT",
Short: "Garbage collect unreferenced blobs in a local oci-layout",
Args: cobra.ExactArgs(1),
Hidden: true, // TODO: promote to public once theres some milage
RunE: func(_ *cobra.Command, args []string) error {
path := args[0]

p, err := layout.FromPath(path)

if err != nil {
return err
}

blobs, err := p.GarbageCollect()
if err != nil {
return err
}

for _, blob := range blobs {
if err := p.RemoveBlob(blob); err != nil {
return err
}
fmt.Fprintf(os.Stderr, "garbage collecting: %s\n", blob.String())
}

return nil
},
}

return cmd
}
3 changes: 2 additions & 1 deletion cmd/crane/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ func New(use, short string, options []crane.Option) *cobra.Command {
NewCmdTag(&options),
NewCmdValidate(&options),
NewCmdVersion(),
newCmdRegistry(),
NewCmdRegistry(),
NewCmdLayout(),
)

root.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable debug logs")
Expand Down
2 changes: 1 addition & 1 deletion cmd/crane/cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"github.com/google/go-containerregistry/pkg/registry"
)

func newCmdRegistry() *cobra.Command {
func NewCmdRegistry() *cobra.Command {
cmd := &cobra.Command{
Use: "registry",
}
Expand Down
137 changes: 137 additions & 0 deletions pkg/v1/layout/gc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This is an EXPERIMENTAL package, and may change in arbitrary ways without notice.
package layout

import (
"fmt"
"io/fs"
"path/filepath"
"strings"

v1 "github.com/google/go-containerregistry/pkg/v1"
)

// GarbageCollect removes unreferenced blobs from the oci-layout
//
// This is an experimental api, and not subject to any stability guarantees
// We may abandon it at any time, without prior notice.
// Deprecated: Use it at your own risk!
func (l Path) GarbageCollect() ([]v1.Hash, error) {
idx, err := l.ImageIndex()
if err != nil {
return nil, err
}
blobsToKeep := map[string]bool{}
if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil {
return nil, err
}
blobsDir := l.path("blobs")
removedBlobs := []v1.Hash{}

err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

if d.IsDir() {
return nil
}

rel, err := filepath.Rel(blobsDir, path)
if err != nil {
return err
}
hashString := strings.Replace(rel, "/", ":", 1)
if present := blobsToKeep[hashString]; !present {
h, err := v1.NewHash(hashString)
if err != nil {
return err
}
removedBlobs = append(removedBlobs, h)
}
return nil
})

if err != nil {
return nil, err
}

return removedBlobs, nil
}

func (l Path) garbageCollectImageIndex(index v1.ImageIndex, blobsToKeep map[string]bool) error {
idxm, err := index.IndexManifest()
if err != nil {
return err
}

h, err := index.Digest()
if err != nil {
return err
}

blobsToKeep[h.String()] = true

for _, descriptor := range idxm.Manifests {
if descriptor.MediaType.IsImage() {
img, err := index.Image(descriptor.Digest)
if err != nil {
return err
}
if err := l.garbageCollectImage(img, blobsToKeep); err != nil {
return err
}
} else if descriptor.MediaType.IsIndex() {
idx, err := index.ImageIndex(descriptor.Digest)
if err != nil {
return err
}
if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil {
return err
}
} else {
return fmt.Errorf("gc: unknown media type: %s", descriptor.MediaType)
}
}
return nil
}

func (l Path) garbageCollectImage(image v1.Image, blobsToKeep map[string]bool) error {
h, err := image.Digest()
if err != nil {
return err
}
blobsToKeep[h.String()] = true

h, err = image.ConfigName()
if err != nil {
return err
}
blobsToKeep[h.String()] = true

ls, err := image.Layers()
if err != nil {
return err
}
for _, l := range ls {
h, err := l.Digest()
if err != nil {
return err
}
blobsToKeep[h.String()] = true
}
return nil
}
96 changes: 96 additions & 0 deletions pkg/v1/layout/gc_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package layout

import (
"path/filepath"
"testing"
)

var (
gcIndexPath = filepath.Join("testdata", "test_gc_index")
gcIndexBlobHash = "sha256:492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0"
gcUnknownMediaTypePath = filepath.Join("testdata", "test_gc_image_unknown_mediatype")
gcUnknownMediaTypeErr = "gc: unknown media type: application/vnd.oci.descriptor.v1+json"
gcTestOneImagePath = filepath.Join("testdata", "test_index_one_image")
gcTestIndexMediaTypePath = filepath.Join("testdata", "test_index_media_type")
)

func TestGcIndex(t *testing.T) {
lp, err := FromPath(gcIndexPath)
if err != nil {
t.Fatalf("FromPath() = %v", err)
}

removed, err := lp.GarbageCollect()
if err != nil {
t.Fatalf("GarbageCollect() = %v", err)
}

if len(removed) != 1 {
t.Fatalf("expected to have only one gc-able blob")
}
if removed[0].String() != gcIndexBlobHash {
t.Fatalf("wrong blob is gc-ed: expected '%s', got '%s'", gcIndexBlobHash, removed[0].String())
}
}

func TestGcOneImage(t *testing.T) {
lp, err := FromPath(gcTestOneImagePath)
if err != nil {
t.Fatalf("FromPath() = %v", err)
}

removed, err := lp.GarbageCollect()
if err != nil {
t.Fatalf("GarbageCollect() = %v", err)
}

if len(removed) != 0 {
t.Fatalf("expected to have to gc-able blobs")
}
}

func TestGcIndexMediaType(t *testing.T) {
lp, err := FromPath(gcTestIndexMediaTypePath)
if err != nil {
t.Fatalf("FromPath() = %v", err)
}

removed, err := lp.GarbageCollect()
if err != nil {
t.Fatalf("GarbageCollect() = %v", err)
}

if len(removed) != 0 {
t.Fatalf("expected to have to gc-able blobs")
}
}

func TestGcUnknownMediaType(t *testing.T) {
lp, err := FromPath(gcUnknownMediaTypePath)
if err != nil {
t.Fatalf("FromPath() = %v", err)
}

_, err = lp.GarbageCollect()
if err == nil {
t.Fatalf("expected GarbageCollect to return err but did not")
}

if err.Error() != gcUnknownMediaTypeErr {
t.Fatalf("expected error '%s', got '%s'", gcUnknownMediaTypeErr, err.Error())
}
}
10 changes: 10 additions & 0 deletions pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/index.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"schemaVersion": 2,
"manifests": [
{
"mediaType": "application/vnd.oci.descriptor.v1+json",
"size": 423,
"digest": "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"imageLayoutVersion": "1.0.0"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"schemaVersion": 2,
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 423,
"digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650",
"annotations": {
"org.opencontainers.image.ref.name": "1"
}
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"schemaVersion": 2,
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 423,
"digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650",
"annotations": {
"org.opencontainers.image.ref.name": "4"
}
}
]
}
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17"], "type": "layers"}}
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":167,"digest":"sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b"}]}
Loading

0 comments on commit ceb0580

Please sign in to comment.