From 243de23dbf5cdf273c8a9e3d73eef67186be2b16 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Wed, 12 Jun 2024 13:00:15 +0300 Subject: [PATCH] Add optional pebble support (#478) (#484) * refactor snap.NewSnap to use a struct for options * add pebble implementation * pick runtime through environment values * add pebble component (do not include in snap) * consolidate pebble configs * fix issues with installing tzdata deb package --- build-scripts/components/pebble/build.sh | 9 +++ build-scripts/components/pebble/repository | 1 + build-scripts/components/pebble/version | 1 + k8s/pebble/000-k8s.yaml | 53 +++++++++++++ k8s/pebble/pebble.service | 27 +++++++ src/k8s/cmd/util/environ.go | 20 ++++- src/k8s/hack/deps.sh | 2 +- src/k8s/pkg/snap/options.go | 13 --- src/k8s/pkg/snap/pebble.go | 71 +++++++++++++++++ src/k8s/pkg/snap/pebble_test.go | 79 +++++++++++++++++++ src/k8s/pkg/snap/service.go | 15 ++++ ...ernal_test.go => service_internal_test.go} | 2 +- src/k8s/pkg/snap/snap.go | 41 ++++------ src/k8s/pkg/snap/snap_test.go | 21 +++-- 14 files changed, 308 insertions(+), 47 deletions(-) create mode 100755 build-scripts/components/pebble/build.sh create mode 100644 build-scripts/components/pebble/repository create mode 100644 build-scripts/components/pebble/version create mode 100644 k8s/pebble/000-k8s.yaml create mode 100644 k8s/pebble/pebble.service delete mode 100644 src/k8s/pkg/snap/options.go create mode 100644 src/k8s/pkg/snap/pebble.go create mode 100644 src/k8s/pkg/snap/pebble_test.go create mode 100644 src/k8s/pkg/snap/service.go rename src/k8s/pkg/snap/{snap_internal_test.go => service_internal_test.go} (92%) diff --git a/build-scripts/components/pebble/build.sh b/build-scripts/components/pebble/build.sh new file mode 100755 index 000000000..2919d49c2 --- /dev/null +++ b/build-scripts/components/pebble/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +VERSION="${2}" +INSTALL="${1}/bin" + +mkdir -p "${INSTALL}" + +go build -ldflags '-s -w' ./cmd/pebble +cp pebble "${INSTALL}/pebble" diff --git a/build-scripts/components/pebble/repository b/build-scripts/components/pebble/repository new file mode 100644 index 000000000..f9006ccb5 --- /dev/null +++ b/build-scripts/components/pebble/repository @@ -0,0 +1 @@ +https://github.com/canonical/pebble diff --git a/build-scripts/components/pebble/version b/build-scripts/components/pebble/version new file mode 100644 index 000000000..cd74ac3b5 --- /dev/null +++ b/build-scripts/components/pebble/version @@ -0,0 +1 @@ +v1.11.0 diff --git a/k8s/pebble/000-k8s.yaml b/k8s/pebble/000-k8s.yaml new file mode 100644 index 000000000..f5c92e8e5 --- /dev/null +++ b/k8s/pebble/000-k8s.yaml @@ -0,0 +1,53 @@ +services: + k8sd: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/k8sd" + startup: enabled + + containerd: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/containerd" + startup: disabled + before: [kubelet] + + k8s-dqlite: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/k8s-dqlite" + startup: disabled + before: [kube-apiserver] + + kube-apiserver: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/kube-apiserver" + startup: disabled + before: [kubelet, kube-controller-manager, kube-proxy, kube-scheduler] + + kubelet: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/kubelet" + startup: disabled + after: [containerd] + + kube-controller-manager: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/kube-controller-manager" + startup: disabled + after: [kube-apiserver] + + kube-scheduler: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/kube-scheduler" + startup: disabled + after: [kube-apiserver] + + kube-proxy: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/kube-proxy" + startup: disabled + after: [kube-apiserver] + + k8s-apiserver-proxy: + override: replace + command: bash -c "$SNAP/k8s/wrappers/services/k8s-apiserver-proxy" + startup: disabled + before: [kube-proxy, kubelet] diff --git a/k8s/pebble/pebble.service b/k8s/pebble/pebble.service new file mode 100644 index 000000000..dfd7afc3a --- /dev/null +++ b/k8s/pebble/pebble.service @@ -0,0 +1,27 @@ +[Unit] +Description=pebble +After=network.target + +[Service] +ExecStart=/snap/k8s/current/bin/pebble run +Restart=always +RestartSec=2 + +# Must set environment here as well, since Docker ENV is not propagated to the service +Environment=SNAP=/snap/k8s/current +Environment=SNAP_REVISION=current +Environment=SNAP_COMMON=/var/snap/k8s/common +Environment=REAL_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/k8s/current/bin +Environment=K8SD_RUNTIME_ENVIRONMENT=pebble + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=infinity +TasksMax=infinity +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target diff --git a/src/k8s/cmd/util/environ.go b/src/k8s/cmd/util/environ.go index 0c95a9c78..73e6f894f 100644 --- a/src/k8s/cmd/util/environ.go +++ b/src/k8s/cmd/util/environ.go @@ -33,7 +33,21 @@ type ExecutionEnvironment struct { // DefaultExecutionEnvironment is used to run the CLI. func DefaultExecutionEnvironment() ExecutionEnvironment { - snap := snap.NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_COMMON")) + var s snap.Snap + switch os.Getenv("K8SD_RUNTIME_ENVIRONMENT") { + case "", "snap": + s = snap.NewSnap(snap.SnapOpts{ + SnapDir: os.Getenv("SNAP"), + SnapCommonDir: os.Getenv("SNAP_COMMON"), + }) + case "pebble": + s = snap.NewPebble(snap.PebbleOpts{ + SnapDir: os.Getenv("SNAP"), + SnapCommonDir: os.Getenv("SNAP_COMMON"), + }) + default: + panic(fmt.Sprintf("invalid runtime environment %q", os.Getenv("K8SD_RUNTIME_ENVIRONMENT"))) + } return ExecutionEnvironment{ Stdin: os.Stdin, @@ -42,9 +56,9 @@ func DefaultExecutionEnvironment() ExecutionEnvironment { Exit: os.Exit, Environ: os.Environ(), Getuid: os.Getuid, - Snap: snap, + Snap: s, Client: func(ctx context.Context) (client.Client, error) { - return client.New(ctx, snap) + return client.New(ctx, s) }, } } diff --git a/src/k8s/hack/deps.sh b/src/k8s/hack/deps.sh index 0d94586ee..6ae4ad2df 100755 --- a/src/k8s/hack/deps.sh +++ b/src/k8s/hack/deps.sh @@ -1,3 +1,3 @@ #!/bin/bash -eu -sudo apt install -y build-essential automake libtool gettext autopoint tclsh tcl libsqlite3-dev pkg-config git > /dev/null +sudo DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y --no-install-recommends build-essential automake libtool gettext autopoint tclsh tcl libsqlite3-dev pkg-config git > /dev/null diff --git a/src/k8s/pkg/snap/options.go b/src/k8s/pkg/snap/options.go deleted file mode 100644 index 40fbb00d8..000000000 --- a/src/k8s/pkg/snap/options.go +++ /dev/null @@ -1,13 +0,0 @@ -package snap - -import ( - "context" - "os/exec" -) - -// WithCommandRunner configures how shell commands are executed. -func WithCommandRunner(f func(context.Context, []string, ...func(*exec.Cmd)) error) func(s *snap) { - return func(s *snap) { - s.runCommand = f - } -} diff --git a/src/k8s/pkg/snap/pebble.go b/src/k8s/pkg/snap/pebble.go new file mode 100644 index 000000000..27bfa4fca --- /dev/null +++ b/src/k8s/pkg/snap/pebble.go @@ -0,0 +1,71 @@ +package snap + +import ( + "context" + "os/exec" + "path/filepath" + + "github.com/canonical/k8s/pkg/utils" +) + +type PebbleOpts struct { + SnapDir string + SnapCommonDir string + RunCommand func(ctx context.Context, command []string, opts ...func(c *exec.Cmd)) error +} + +// pebble implements the Snap interface. +// pebble is the same as snap, but uses pebble for managing services, and disables snapctl. +type pebble struct { + snap +} + +// NewPebble creates a new interface with the K8s snap. +func NewPebble(opts PebbleOpts) *pebble { + runCommand := utils.RunCommand + if opts.RunCommand != nil { + runCommand = opts.RunCommand + } + s := &pebble{ + snap: snap{ + snapDir: opts.SnapDir, + snapCommonDir: opts.SnapCommonDir, + runCommand: runCommand, + }, + } + + return s +} + +// StartService starts a k8s service. The name can be either prefixed or not. +func (s *pebble) StartService(ctx context.Context, name string) error { + return s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", "pebble"), "start", name}) +} + +// StopService stops a k8s service. The name can be either prefixed or not. +func (s *pebble) StopService(ctx context.Context, name string) error { + return s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", "pebble"), "stop", name}) +} + +// RestartService restarts a k8s service. The name can be either prefixed or not. +func (s *pebble) RestartService(ctx context.Context, name string) error { + return s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", "pebble"), "restart", name}) +} + +func (s *pebble) Strict() bool { + return false +} + +func (s *pebble) OnLXD(ctx context.Context) (bool, error) { + return true, nil +} + +func (s *pebble) SnapctlGet(ctx context.Context, args ...string) ([]byte, error) { + return []byte(`{"meta": {"apiVersion": "1.30", "orb": "none"}`), nil +} + +func (s *pebble) SnapctlSet(ctx context.Context, args ...string) error { + return nil +} + +var _ Snap = &pebble{} diff --git a/src/k8s/pkg/snap/pebble_test.go b/src/k8s/pkg/snap/pebble_test.go new file mode 100644 index 000000000..f85491140 --- /dev/null +++ b/src/k8s/pkg/snap/pebble_test.go @@ -0,0 +1,79 @@ +package snap_test + +import ( + "context" + "fmt" + "testing" + + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/snap/mock" + + . "github.com/onsi/gomega" +) + +func TestPebble(t *testing.T) { + t.Run("Start", func(t *testing.T) { + g := NewWithT(t) + mockRunner := &mock.Runner{} + snap := snap.NewPebble(snap.PebbleOpts{ + SnapDir: "testdir", + SnapCommonDir: "testdir", + RunCommand: mockRunner.Run, + }) + + err := snap.StartService(context.Background(), "test-service") + g.Expect(err).To(BeNil()) + g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("testdir/bin/pebble start test-service")) + + t.Run("Fail", func(t *testing.T) { + g := NewWithT(t) + mockRunner.Err = fmt.Errorf("some error") + + err := snap.StartService(context.Background(), "test-service") + g.Expect(err).NotTo(BeNil()) + }) + }) + + t.Run("Stop", func(t *testing.T) { + g := NewWithT(t) + mockRunner := &mock.Runner{} + snap := snap.NewPebble(snap.PebbleOpts{ + SnapDir: "testdir", + SnapCommonDir: "testdir", + RunCommand: mockRunner.Run, + }) + err := snap.StopService(context.Background(), "test-service") + g.Expect(err).To(BeNil()) + g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("testdir/bin/pebble stop test-service")) + + t.Run("Fail", func(t *testing.T) { + g := NewWithT(t) + mockRunner.Err = fmt.Errorf("some error") + + err := snap.StartService(context.Background(), "test-service") + g.Expect(err).NotTo(BeNil()) + }) + }) + + t.Run("Restart", func(t *testing.T) { + g := NewWithT(t) + mockRunner := &mock.Runner{} + snap := snap.NewPebble(snap.PebbleOpts{ + SnapDir: "testdir", + SnapCommonDir: "testdir", + RunCommand: mockRunner.Run, + }) + + err := snap.RestartService(context.Background(), "test-service") + g.Expect(err).To(BeNil()) + g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("testdir/bin/pebble restart test-service")) + + t.Run("Fail", func(t *testing.T) { + g := NewWithT(t) + mockRunner.Err = fmt.Errorf("some error") + + err := snap.StartService(context.Background(), "service") + g.Expect(err).NotTo(BeNil()) + }) + }) +} diff --git a/src/k8s/pkg/snap/service.go b/src/k8s/pkg/snap/service.go new file mode 100644 index 000000000..2bf0b9c5d --- /dev/null +++ b/src/k8s/pkg/snap/service.go @@ -0,0 +1,15 @@ +package snap + +import ( + "fmt" + "strings" +) + +// serviceName infers the name of the snapctl daemon from the service name. +// if the serviceName is the snap name `k8s` (=referes to all services) it will return it as is. +func serviceName(serviceName string) string { + if strings.HasPrefix(serviceName, "k8s.") || serviceName == "k8s" { + return serviceName + } + return fmt.Sprintf("k8s.%s", serviceName) +} diff --git a/src/k8s/pkg/snap/snap_internal_test.go b/src/k8s/pkg/snap/service_internal_test.go similarity index 92% rename from src/k8s/pkg/snap/snap_internal_test.go rename to src/k8s/pkg/snap/service_internal_test.go index 797aab840..325b6da46 100644 --- a/src/k8s/pkg/snap/snap_internal_test.go +++ b/src/k8s/pkg/snap/service_internal_test.go @@ -6,7 +6,7 @@ import ( . "github.com/onsi/gomega" ) -func TestServiceName(t *testing.T) { +func Test_serviceName(t *testing.T) { tests := []struct { name string input string diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go index 4ec4a6513..67422969b 100644 --- a/src/k8s/pkg/snap/snap.go +++ b/src/k8s/pkg/snap/snap.go @@ -7,7 +7,7 @@ import ( "os" "os/exec" "path" - "strings" + "path/filepath" "github.com/canonical/k8s/pkg/client/dqlite" "github.com/canonical/k8s/pkg/client/helm" @@ -18,6 +18,12 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" ) +type SnapOpts struct { + SnapDir string + SnapCommonDir string + RunCommand func(ctx context.Context, command []string, opts ...func(c *exec.Cmd)) error +} + // snap implements the Snap interface. type snap struct { snapDir string @@ -27,33 +33,20 @@ type snap struct { // NewSnap creates a new interface with the K8s snap. // NewSnap accepts the $SNAP, $SNAP_COMMON, directories, and a number of options. -func NewSnap(snapDir, snapCommonDir string, options ...func(s *snap)) *snap { - s := &snap{ - snapDir: snapDir, - snapCommonDir: snapCommonDir, - runCommand: utils.RunCommand, +func NewSnap(opts SnapOpts) *snap { + runCommand := utils.RunCommand + if opts.RunCommand != nil { + runCommand = opts.RunCommand } - - for _, option := range options { - option(s) // Apply each passed option to the snap instance + s := &snap{ + snapDir: opts.SnapDir, + snapCommonDir: opts.SnapCommonDir, + runCommand: runCommand, } return s } -func (s *snap) path(parts ...string) string { - return path.Join(append([]string{s.snapDir}, parts...)...) -} - -// serviceName infers the name of the snapctl daemon from the service name. -// if the serviceName is the snap name `k8s` (=referes to all services) it will return it as is. -func serviceName(serviceName string) string { - if strings.HasPrefix(serviceName, "k8s.") || serviceName == "k8s" { - return serviceName - } - return fmt.Sprintf("k8s.%s", serviceName) -} - // StartService starts a k8s service. The name can be either prefixed or not. func (s *snap) StartService(ctx context.Context, name string) error { return s.runCommand(ctx, []string{"snapctl", "start", "--enable", serviceName(name)}) @@ -75,7 +68,7 @@ type snapcraftYml struct { func (s *snap) Strict() bool { var meta snapcraftYml - contents, err := os.ReadFile(s.path("meta", "snap.yaml")) + contents, err := os.ReadFile(filepath.Join(s.snapDir, "meta", "snap.yaml")) if err != nil { return false } @@ -214,7 +207,7 @@ func (s *snap) KubernetesNodeClient(namespace string) (*kubernetes.Client, error func (s *snap) HelmClient() helm.Client { return helm.NewClient( - path.Join(s.snapDir, "k8s", "manifests"), + filepath.Join(s.snapDir, "k8s", "manifests"), func(namespace string) genericclioptions.RESTClientGetter { return s.restClientGetter(path.Join(s.KubernetesConfigDir(), "admin.conf"), namespace) }, diff --git a/src/k8s/pkg/snap/snap_test.go b/src/k8s/pkg/snap/snap_test.go index 1087074f0..f694a0ef0 100644 --- a/src/k8s/pkg/snap/snap_test.go +++ b/src/k8s/pkg/snap/snap_test.go @@ -11,11 +11,15 @@ import ( . "github.com/onsi/gomega" ) -func TestServices(t *testing.T) { +func TestSnap(t *testing.T) { t.Run("Start", func(t *testing.T) { g := NewWithT(t) mockRunner := &mock.Runner{} - snap := snap.NewSnap("testdir", "testdir", snap.WithCommandRunner(mockRunner.Run)) + snap := snap.NewSnap(snap.SnapOpts{ + SnapDir: "testdir", + SnapCommonDir: "testdir", + RunCommand: mockRunner.Run, + }) err := snap.StartService(context.Background(), "test-service") g.Expect(err).To(BeNil()) @@ -33,8 +37,11 @@ func TestServices(t *testing.T) { t.Run("Stop", func(t *testing.T) { g := NewWithT(t) mockRunner := &mock.Runner{} - snap := snap.NewSnap("testdir", "testdir", snap.WithCommandRunner(mockRunner.Run)) - + snap := snap.NewSnap(snap.SnapOpts{ + SnapDir: "testdir", + SnapCommonDir: "testdir", + RunCommand: mockRunner.Run, + }) err := snap.StopService(context.Background(), "test-service") g.Expect(err).To(BeNil()) g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("snapctl stop --disable k8s.test-service")) @@ -51,7 +58,11 @@ func TestServices(t *testing.T) { t.Run("Restart", func(t *testing.T) { g := NewWithT(t) mockRunner := &mock.Runner{} - snap := snap.NewSnap("testdir", "testdir", snap.WithCommandRunner(mockRunner.Run)) + snap := snap.NewSnap(snap.SnapOpts{ + SnapDir: "testdir", + SnapCommonDir: "testdir", + RunCommand: mockRunner.Run, + }) err := snap.RestartService(context.Background(), "test-service") g.Expect(err).To(BeNil())