-
Notifications
You must be signed in to change notification settings - Fork 124
Add garbage collection for docker images #2435
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||
---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,168 @@ | ||||||||||
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||||||||||
// or more contributor license agreements. Licensed under the Elastic License; | ||||||||||
// you may not use this file except in compliance with the Elastic License. | ||||||||||
|
||||||||||
package common | ||||||||||
|
||||||||||
import ( | ||||||||||
"encoding/json" | ||||||||||
"fmt" | ||||||||||
"math" | ||||||||||
"regexp" | ||||||||||
"strconv" | ||||||||||
|
||||||||||
"gopkg.in/yaml.v3" | ||||||||||
) | ||||||||||
|
||||||||||
// Common units for sizes in bytes. | ||||||||||
const ( | ||||||||||
Byte = ByteSize(1) | ||||||||||
KiloByte = 1024 * Byte | ||||||||||
MegaByte = 1024 * KiloByte | ||||||||||
GigaByte = 1024 * MegaByte | ||||||||||
) | ||||||||||
|
||||||||||
const ( | ||||||||||
byteString = "B" | ||||||||||
kiloByteString = "KB" | ||||||||||
megaByteString = "MB" | ||||||||||
gigaByteString = "GB" | ||||||||||
) | ||||||||||
|
||||||||||
// ByteSize represents the size of a file. | ||||||||||
type ByteSize uint64 | ||||||||||
|
||||||||||
// Ensure FileSize implements these interfaces. | ||||||||||
var ( | ||||||||||
_ json.Marshaler = new(ByteSize) | ||||||||||
_ json.Unmarshaler = new(ByteSize) | ||||||||||
_ yaml.Marshaler = new(ByteSize) | ||||||||||
_ yaml.Unmarshaler = new(ByteSize) | ||||||||||
) | ||||||||||
|
||||||||||
func parseFileSizeInt(s string) (uint64, error) { | ||||||||||
// os.FileInfo reports size as int64, don't support bigger numbers. | ||||||||||
maxBitSize := 63 | ||||||||||
return strconv.ParseUint(s, 10, maxBitSize) | ||||||||||
} | ||||||||||
|
||||||||||
// MarshalJSON implements the json.Marshaler interface for FileSize, it returns | ||||||||||
// the string representation in a format that can be unmarshaled back to an | ||||||||||
// equivalent value. | ||||||||||
func (s ByteSize) MarshalJSON() ([]byte, error) { | ||||||||||
return json.Marshal(s.String()) | ||||||||||
} | ||||||||||
|
||||||||||
// MarshalYAML implements the yaml.Marshaler interface for FileSize, it returns | ||||||||||
// the string representation in a format that can be unmarshaled back to an | ||||||||||
// equivalent value. | ||||||||||
func (s ByteSize) MarshalYAML() (interface{}, error) { | ||||||||||
return s.String(), nil | ||||||||||
} | ||||||||||
|
||||||||||
// UnmarshalJSON implements the json.Unmarshaler interface for FileSize. | ||||||||||
func (s *ByteSize) UnmarshalJSON(d []byte) error { | ||||||||||
// Support unquoted plain numbers. | ||||||||||
bytes, err := parseFileSizeInt(string(d)) | ||||||||||
if err == nil { | ||||||||||
*s = ByteSize(bytes) | ||||||||||
return nil | ||||||||||
} | ||||||||||
|
||||||||||
var text string | ||||||||||
err = json.Unmarshal(d, &text) | ||||||||||
if err != nil { | ||||||||||
return err | ||||||||||
} | ||||||||||
|
||||||||||
return s.unmarshalString(text) | ||||||||||
} | ||||||||||
|
||||||||||
// UnmarshalYAML implements the yaml.Unmarshaler interface for FileSize. | ||||||||||
func (s *ByteSize) UnmarshalYAML(value *yaml.Node) error { | ||||||||||
// Support unquoted plain numbers. | ||||||||||
bytes, err := parseFileSizeInt(value.Value) | ||||||||||
if err == nil { | ||||||||||
*s = ByteSize(bytes) | ||||||||||
return nil | ||||||||||
} | ||||||||||
|
||||||||||
return s.unmarshalString(value.Value) | ||||||||||
} | ||||||||||
|
||||||||||
var bytesPattern = regexp.MustCompile(fmt.Sprintf(`^(\d+(\.\d+)?)(%s|%s|%s|%s|)$`, byteString, kiloByteString, megaByteString, gigaByteString)) | ||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
(compile-time string construction instead of init-time) but for a more efficient pattern
Suggested change
|
||||||||||
|
||||||||||
func (s *ByteSize) unmarshalString(text string) error { | ||||||||||
match := bytesPattern.FindStringSubmatch(text) | ||||||||||
if len(match) < 3 { | ||||||||||
return fmt.Errorf("invalid format for size in bytes (%s)", text) | ||||||||||
} | ||||||||||
|
||||||||||
if match[2] == "" { | ||||||||||
q, err := parseFileSizeInt(match[1]) | ||||||||||
if err != nil { | ||||||||||
return fmt.Errorf("invalid format for size in bytes (%s): %w", text, err) | ||||||||||
} | ||||||||||
|
||||||||||
unit := match[3] | ||||||||||
switch unit { | ||||||||||
case gigaByteString: | ||||||||||
*s = ByteSize(q) * GigaByte | ||||||||||
case megaByteString: | ||||||||||
*s = ByteSize(q) * MegaByte | ||||||||||
case kiloByteString: | ||||||||||
*s = ByteSize(q) * KiloByte | ||||||||||
case byteString, "": | ||||||||||
*s = ByteSize(q) * Byte | ||||||||||
default: | ||||||||||
return fmt.Errorf("invalid unit for filesize (%s): %s", text, unit) | ||||||||||
} | ||||||||||
} else { | ||||||||||
q, err := strconv.ParseFloat(match[1], 64) | ||||||||||
if err != nil { | ||||||||||
return fmt.Errorf("invalid format for size in bytes (%s): %w", text, err) | ||||||||||
} | ||||||||||
|
||||||||||
unit := match[3] | ||||||||||
switch unit { | ||||||||||
case gigaByteString: | ||||||||||
*s = approxFloat(q, GigaByte) | ||||||||||
case megaByteString: | ||||||||||
*s = approxFloat(q, MegaByte) | ||||||||||
case kiloByteString: | ||||||||||
*s = approxFloat(q, KiloByte) | ||||||||||
case byteString, "": | ||||||||||
*s = approxFloat(q, Byte) | ||||||||||
default: | ||||||||||
return fmt.Errorf("invalid unit for filesize (%s): %s", text, unit) | ||||||||||
} | ||||||||||
} | ||||||||||
|
||||||||||
return nil | ||||||||||
} | ||||||||||
|
||||||||||
func approxFloat(n float64, unit ByteSize) ByteSize { | ||||||||||
approx := n * float64(unit) | ||||||||||
return ByteSize(math.Round(approx)) | ||||||||||
} | ||||||||||
|
||||||||||
// String returns the string representation of the FileSize. | ||||||||||
func (s ByteSize) String() string { | ||||||||||
format := func(q ByteSize, unit string) string { | ||||||||||
return fmt.Sprintf("%d%s", q, unit) | ||||||||||
} | ||||||||||
|
||||||||||
if s >= GigaByte && (s%GigaByte == 0) { | ||||||||||
return format(s/GigaByte, gigaByteString) | ||||||||||
} | ||||||||||
|
||||||||||
if s >= MegaByte && (s%MegaByte == 0) { | ||||||||||
return format(s/MegaByte, megaByteString) | ||||||||||
} | ||||||||||
|
||||||||||
if s >= KiloByte && (s%KiloByte == 0) { | ||||||||||
return format(s/KiloByte, kiloByteString) | ||||||||||
} | ||||||||||
|
||||||||||
return format(s, byteString) | ||||||||||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
// or more contributor license agreements. Licensed under the Elastic License; | ||
// you may not use this file except in compliance with the Elastic License. | ||
|
||
package common | ||
|
||
import ( | ||
"encoding/json" | ||
"testing" | ||
|
||
"gopkg.in/yaml.v3" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
func TestFileSizeMarshallJSON(t *testing.T) { | ||
cases := []struct { | ||
fileSize ByteSize | ||
expected string | ||
}{ | ||
{ByteSize(0), `"0B"`}, | ||
{ByteSize(1024), `"1KB"`}, | ||
{ByteSize(1025), `"1025B"`}, | ||
{5 * MegaByte, `"5MB"`}, | ||
{5 * GigaByte, `"5GB"`}, | ||
} | ||
|
||
for _, c := range cases { | ||
t.Run(c.expected, func(t *testing.T) { | ||
d, err := json.Marshal(c.fileSize) | ||
require.NoError(t, err) | ||
assert.Equal(t, c.expected, string(d)) | ||
}) | ||
} | ||
} | ||
|
||
func TestFileSizeMarshallYAML(t *testing.T) { | ||
cases := []struct { | ||
fileSize ByteSize | ||
expected string | ||
}{ | ||
{ByteSize(0), "0B\n"}, | ||
{ByteSize(1024), "1KB\n"}, | ||
{ByteSize(1025), "1025B\n"}, | ||
{5 * MegaByte, "5MB\n"}, | ||
{5 * GigaByte, "5GB\n"}, | ||
} | ||
|
||
for _, c := range cases { | ||
t.Run(c.expected, func(t *testing.T) { | ||
d, err := yaml.Marshal(c.fileSize) | ||
require.NoError(t, err) | ||
assert.Equal(t, c.expected, string(d)) | ||
}) | ||
} | ||
} | ||
|
||
func TestFileSizeUnmarshal(t *testing.T) { | ||
t.Run("json", func(t *testing.T) { | ||
testFileSizeUnmarshalFormat(t, json.Unmarshal) | ||
}) | ||
t.Run("yaml", func(t *testing.T) { | ||
testFileSizeUnmarshalFormat(t, yaml.Unmarshal) | ||
}) | ||
} | ||
|
||
func testFileSizeUnmarshalFormat(t *testing.T, unmarshaler func([]byte, interface{}) error) { | ||
cases := []struct { | ||
json string | ||
expected ByteSize | ||
valid bool | ||
}{ | ||
{"0", 0, true}, | ||
{"1024", 1024 * Byte, true}, | ||
{`"1024"`, 1024 * Byte, true}, | ||
{`"1024B"`, 1024 * Byte, true}, | ||
{`"10MB"`, 10 * MegaByte, true}, | ||
{`"40GB"`, 40 * GigaByte, true}, | ||
{`"56.21GB"`, approxFloat(56.21, GigaByte), true}, | ||
{`"2KB"`, 2 * KiloByte, true}, | ||
{`"KB"`, 0, false}, | ||
{`"1s"`, 0, false}, | ||
{`""`, 0, false}, | ||
{`"B"`, 0, false}, | ||
{`"-200MB"`, 0, false}, | ||
{`"-1"`, 0, false}, | ||
{`"10000000000000000000MB"`, 0, false}, | ||
} | ||
|
||
for _, c := range cases { | ||
t.Run(c.json, func(t *testing.T) { | ||
var found ByteSize | ||
err := unmarshaler([]byte(c.json), &found) | ||
if c.valid { | ||
require.NoError(t, err) | ||
assert.Equal(t, c.expected, found) | ||
} else { | ||
require.Error(t, err) | ||
} | ||
}) | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,6 +14,7 @@ import ( | |
"os" | ||
"os/exec" | ||
"regexp" | ||
"slices" | ||
"strconv" | ||
"strings" | ||
"time" | ||
|
@@ -60,7 +61,23 @@ type Config struct { | |
Services map[string]service | ||
} | ||
|
||
// Images lists the images found in the configuration. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This only lists images found directly in the docker compose configuration, it won't find images found in Dockerfiles. Maybe we can think in other approaches to find the images to track. Ideas welcome. |
||
func (c *Config) Images() []string { | ||
var images []string | ||
for _, service := range c.Services { | ||
if service.Image == "" { | ||
continue | ||
} | ||
if slices.Contains(images, service.Image) { | ||
continue | ||
} | ||
images = append(images, service.Image) | ||
} | ||
return images | ||
} | ||
|
||
type service struct { | ||
Image string | ||
Ports []portMapping | ||
Environment map[string]string | ||
} | ||
|
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Type mostly copied from https://github.com/elastic/package-spec/blob/964c4a69e024cc464c4808720ba0db9f001a82a7/code/go/internal/spectypes/filesize.go, adding support for GB and floats, to support sizes reported by
docker system df
.