Skip to content

Add garbage collection for docker images #2435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ require (
github.com/shirou/gopsutil/v3 v3.24.5
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
golang.org/x/sys v0.30.0
golang.org/x/tools v0.30.0
gopkg.in/dnaeon/go-vcr.v3 v3.2.0
gopkg.in/yaml.v3 v3.0.1
Expand Down Expand Up @@ -153,7 +154,6 @@ require (
golang.org/x/net v0.35.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/term v0.29.0 // indirect
golang.org/x/text v0.22.0 // indirect
golang.org/x/time v0.7.0 // indirect
Expand Down
168 changes: 168 additions & 0 deletions internal/common/bytesize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package common

import (
"encoding/json"
"fmt"
"math"
"regexp"
"strconv"

"gopkg.in/yaml.v3"
)

// Common units for sizes in bytes.
const (
Byte = ByteSize(1)
KiloByte = 1024 * Byte
MegaByte = 1024 * KiloByte
GigaByte = 1024 * MegaByte
)

const (
byteString = "B"
kiloByteString = "KB"
megaByteString = "MB"
gigaByteString = "GB"
)

// ByteSize represents the size of a file.
type ByteSize uint64
Copy link
Member Author

@jsoriano jsoriano Feb 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Type mostly copied from https://github.com/elastic/package-spec/blob/964c4a69e024cc464c4808720ba0db9f001a82a7/code/go/internal/spectypes/filesize.go, adding support for GB and floats, to support sizes reported by docker system df.


// Ensure FileSize implements these interfaces.
var (
_ json.Marshaler = new(ByteSize)
_ json.Unmarshaler = new(ByteSize)
_ yaml.Marshaler = new(ByteSize)
_ yaml.Unmarshaler = new(ByteSize)
)

func parseFileSizeInt(s string) (uint64, error) {
// os.FileInfo reports size as int64, don't support bigger numbers.
maxBitSize := 63
return strconv.ParseUint(s, 10, maxBitSize)
}

// MarshalJSON implements the json.Marshaler interface for FileSize, it returns
// the string representation in a format that can be unmarshaled back to an
// equivalent value.
func (s ByteSize) MarshalJSON() ([]byte, error) {
return json.Marshal(s.String())
}

// MarshalYAML implements the yaml.Marshaler interface for FileSize, it returns
// the string representation in a format that can be unmarshaled back to an
// equivalent value.
func (s ByteSize) MarshalYAML() (interface{}, error) {
return s.String(), nil
}

// UnmarshalJSON implements the json.Unmarshaler interface for FileSize.
func (s *ByteSize) UnmarshalJSON(d []byte) error {
// Support unquoted plain numbers.
bytes, err := parseFileSizeInt(string(d))
if err == nil {
*s = ByteSize(bytes)
return nil
}

var text string
err = json.Unmarshal(d, &text)
if err != nil {
return err
}

return s.unmarshalString(text)
}

// UnmarshalYAML implements the yaml.Unmarshaler interface for FileSize.
func (s *ByteSize) UnmarshalYAML(value *yaml.Node) error {
// Support unquoted plain numbers.
bytes, err := parseFileSizeInt(value.Value)
if err == nil {
*s = ByteSize(bytes)
return nil
}

return s.unmarshalString(value.Value)
}

var bytesPattern = regexp.MustCompile(fmt.Sprintf(`^(\d+(\.\d+)?)(%s|%s|%s|%s|)$`, byteString, kiloByteString, megaByteString, gigaByteString))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
var bytesPattern = regexp.MustCompile(fmt.Sprintf(`^(\d+(\.\d+)?)(%s|%s|%s|%s|)$`, byteString, kiloByteString, megaByteString, gigaByteString))
var bytesPattern = regexp.MustCompile(`^(\d+(\.\d+)?)(` + byteString + `|` + kiloByteString + `|` + megaByteString + `|` + gigaByteString + `|)$`)

(compile-time string construction instead of init-time)

but for a more efficient pattern

Suggested change
var bytesPattern = regexp.MustCompile(fmt.Sprintf(`^(\d+(\.\d+)?)(%s|%s|%s|%s|)$`, byteString, kiloByteString, megaByteString, gigaByteString))
var bytesPattern = regexp.MustCompile(`^(\d+(\.\d+)?)(B|[KMG]B|)$`)


func (s *ByteSize) unmarshalString(text string) error {
match := bytesPattern.FindStringSubmatch(text)
if len(match) < 3 {
return fmt.Errorf("invalid format for size in bytes (%s)", text)
}

if match[2] == "" {
q, err := parseFileSizeInt(match[1])
if err != nil {
return fmt.Errorf("invalid format for size in bytes (%s): %w", text, err)
}

unit := match[3]
switch unit {
case gigaByteString:
*s = ByteSize(q) * GigaByte
case megaByteString:
*s = ByteSize(q) * MegaByte
case kiloByteString:
*s = ByteSize(q) * KiloByte
case byteString, "":
*s = ByteSize(q) * Byte
default:
return fmt.Errorf("invalid unit for filesize (%s): %s", text, unit)
}
} else {
q, err := strconv.ParseFloat(match[1], 64)
if err != nil {
return fmt.Errorf("invalid format for size in bytes (%s): %w", text, err)
}

unit := match[3]
switch unit {
case gigaByteString:
*s = approxFloat(q, GigaByte)
case megaByteString:
*s = approxFloat(q, MegaByte)
case kiloByteString:
*s = approxFloat(q, KiloByte)
case byteString, "":
*s = approxFloat(q, Byte)
default:
return fmt.Errorf("invalid unit for filesize (%s): %s", text, unit)
}
}

return nil
}

func approxFloat(n float64, unit ByteSize) ByteSize {
approx := n * float64(unit)
return ByteSize(math.Round(approx))
}

// String returns the string representation of the FileSize.
func (s ByteSize) String() string {
format := func(q ByteSize, unit string) string {
return fmt.Sprintf("%d%s", q, unit)
}

if s >= GigaByte && (s%GigaByte == 0) {
return format(s/GigaByte, gigaByteString)
}

if s >= MegaByte && (s%MegaByte == 0) {
return format(s/MegaByte, megaByteString)
}

if s >= KiloByte && (s%KiloByte == 0) {
return format(s/KiloByte, kiloByteString)
}

return format(s, byteString)
}
103 changes: 103 additions & 0 deletions internal/common/bytesize_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package common

import (
"encoding/json"
"testing"

"gopkg.in/yaml.v3"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestFileSizeMarshallJSON(t *testing.T) {
cases := []struct {
fileSize ByteSize
expected string
}{
{ByteSize(0), `"0B"`},
{ByteSize(1024), `"1KB"`},
{ByteSize(1025), `"1025B"`},
{5 * MegaByte, `"5MB"`},
{5 * GigaByte, `"5GB"`},
}

for _, c := range cases {
t.Run(c.expected, func(t *testing.T) {
d, err := json.Marshal(c.fileSize)
require.NoError(t, err)
assert.Equal(t, c.expected, string(d))
})
}
}

func TestFileSizeMarshallYAML(t *testing.T) {
cases := []struct {
fileSize ByteSize
expected string
}{
{ByteSize(0), "0B\n"},
{ByteSize(1024), "1KB\n"},
{ByteSize(1025), "1025B\n"},
{5 * MegaByte, "5MB\n"},
{5 * GigaByte, "5GB\n"},
}

for _, c := range cases {
t.Run(c.expected, func(t *testing.T) {
d, err := yaml.Marshal(c.fileSize)
require.NoError(t, err)
assert.Equal(t, c.expected, string(d))
})
}
}

func TestFileSizeUnmarshal(t *testing.T) {
t.Run("json", func(t *testing.T) {
testFileSizeUnmarshalFormat(t, json.Unmarshal)
})
t.Run("yaml", func(t *testing.T) {
testFileSizeUnmarshalFormat(t, yaml.Unmarshal)
})
}

func testFileSizeUnmarshalFormat(t *testing.T, unmarshaler func([]byte, interface{}) error) {
cases := []struct {
json string
expected ByteSize
valid bool
}{
{"0", 0, true},
{"1024", 1024 * Byte, true},
{`"1024"`, 1024 * Byte, true},
{`"1024B"`, 1024 * Byte, true},
{`"10MB"`, 10 * MegaByte, true},
{`"40GB"`, 40 * GigaByte, true},
{`"56.21GB"`, approxFloat(56.21, GigaByte), true},
{`"2KB"`, 2 * KiloByte, true},
{`"KB"`, 0, false},
{`"1s"`, 0, false},
{`""`, 0, false},
{`"B"`, 0, false},
{`"-200MB"`, 0, false},
{`"-1"`, 0, false},
{`"10000000000000000000MB"`, 0, false},
}

for _, c := range cases {
t.Run(c.json, func(t *testing.T) {
var found ByteSize
err := unmarshaler([]byte(c.json), &found)
if c.valid {
require.NoError(t, err)
assert.Equal(t, c.expected, found)
} else {
require.Error(t, err)
}
})
}
}
17 changes: 17 additions & 0 deletions internal/compose/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"os"
"os/exec"
"regexp"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -60,7 +61,23 @@ type Config struct {
Services map[string]service
}

// Images lists the images found in the configuration.
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This only lists images found directly in the docker compose configuration, it won't find images found in Dockerfiles. Maybe we can think in other approaches to find the images to track. Ideas welcome.

func (c *Config) Images() []string {
var images []string
for _, service := range c.Services {
if service.Image == "" {
continue
}
if slices.Contains(images, service.Image) {
continue
}
images = append(images, service.Image)
}
return images
}

type service struct {
Image string
Ports []portMapping
Environment map[string]string
}
Expand Down
Loading