Add optional pebble support (#478) (#484)

* refactor snap.NewSnap to use a struct for options

* add pebble implementation

* pick runtime through environment values

* add pebble component (do not include in snap)

* consolidate pebble configs

* fix issues with installing tzdata deb package

build-scripts/components/pebble/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+VERSION="${2}"
+INSTALL="${1}/bin"
+
+mkdir -p "${INSTALL}"
+
+go build -ldflags '-s -w' ./cmd/pebble
+cp pebble "${INSTALL}/pebble"

build-scripts/components/pebble/repository

@@ -0,0 +1 @@
+https://github.com/canonical/pebble

build-scripts/components/pebble/version

@@ -0,0 +1 @@
+v1.11.0

k8s/pebble/000-k8s.yaml

@@ -0,0 +1,53 @@
+services:
+  k8sd:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/k8sd"
+    startup: enabled
+
+  containerd:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/containerd"
+    startup: disabled
+    before: [kubelet]
+
+  k8s-dqlite:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/k8s-dqlite"
+    startup: disabled
+    before: [kube-apiserver]
+
+  kube-apiserver:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/kube-apiserver"
+    startup: disabled
+    before: [kubelet, kube-controller-manager, kube-proxy, kube-scheduler]
+
+  kubelet:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/kubelet"
+    startup: disabled
+    after: [containerd]
+
+  kube-controller-manager:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/kube-controller-manager"
+    startup: disabled
+    after: [kube-apiserver]
+
+  kube-scheduler:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/kube-scheduler"
+    startup: disabled
+    after: [kube-apiserver]
+
+  kube-proxy:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/kube-proxy"
+    startup: disabled
+    after: [kube-apiserver]
+
+  k8s-apiserver-proxy:
+    override: replace
+    command: bash -c "$SNAP/k8s/wrappers/services/k8s-apiserver-proxy"
+    startup: disabled
+    before: [kube-proxy, kubelet]

k8s/pebble/pebble.service

@@ -0,0 +1,27 @@
+[Unit]
+Description=pebble
+After=network.target
+
+[Service]
+ExecStart=/snap/k8s/current/bin/pebble run
+Restart=always
+RestartSec=2
+
+# Must set environment here as well, since Docker ENV is not propagated to the service
+Environment=SNAP=/snap/k8s/current
+Environment=SNAP_REVISION=current
+Environment=SNAP_COMMON=/var/snap/k8s/common
+Environment=REAL_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/k8s/current/bin
+Environment=K8SD_RUNTIME_ENVIRONMENT=pebble
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+LimitNOFILE=infinity
+TasksMax=infinity
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

src/k8s/cmd/util/environ.go

@@ -33,7 +33,21 @@ type ExecutionEnvironment struct {
 
 // DefaultExecutionEnvironment is used to run the CLI.
 func DefaultExecutionEnvironment() ExecutionEnvironment {
-	snap := snap.NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_COMMON"))
+	var s snap.Snap
+	switch os.Getenv("K8SD_RUNTIME_ENVIRONMENT") {
+	case "", "snap":
+		s = snap.NewSnap(snap.SnapOpts{
+			SnapDir:       os.Getenv("SNAP"),
+			SnapCommonDir: os.Getenv("SNAP_COMMON"),
+		})
+	case "pebble":
+		s = snap.NewPebble(snap.PebbleOpts{
+			SnapDir:       os.Getenv("SNAP"),
+			SnapCommonDir: os.Getenv("SNAP_COMMON"),
+		})
+	default:
+		panic(fmt.Sprintf("invalid runtime environment %q", os.Getenv("K8SD_RUNTIME_ENVIRONMENT")))
+	}
 
 	return ExecutionEnvironment{
 		Stdin:   os.Stdin,
@@ -42,9 +56,9 @@ func DefaultExecutionEnvironment() ExecutionEnvironment {
 		Exit:    os.Exit,
 		Environ: os.Environ(),
 		Getuid:  os.Getuid,
-		Snap:    snap,
+		Snap:    s,
 		Client: func(ctx context.Context) (client.Client, error) {
-			return client.New(ctx, snap)
+			return client.New(ctx, s)
 		},
 	}
 }

src/k8s/hack/deps.sh

@@ -1,3 +1,3 @@
 #!/bin/bash -eu
 
-sudo apt install -y build-essential automake libtool gettext autopoint tclsh tcl libsqlite3-dev pkg-config git > /dev/null
+sudo DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y --no-install-recommends build-essential automake libtool gettext autopoint tclsh tcl libsqlite3-dev pkg-config git > /dev/null

src/k8s/pkg/snap/pebble.go

@@ -0,0 +1,71 @@
+package snap
+
+import (
+	"context"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/canonical/k8s/pkg/utils"
+)
+
+type PebbleOpts struct {
+	SnapDir       string
+	SnapCommonDir string
+	RunCommand    func(ctx context.Context, command []string, opts ...func(c *exec.Cmd)) error
+}
+
+// pebble implements the Snap interface.
+// pebble is the same as snap, but uses pebble for managing services, and disables snapctl.
+type pebble struct {
+	snap
+}
+
+// NewPebble creates a new interface with the K8s snap.
+func NewPebble(opts PebbleOpts) *pebble {
+	runCommand := utils.RunCommand
+	if opts.RunCommand != nil {
+		runCommand = opts.RunCommand
+	}
+	s := &pebble{
+		snap: snap{
+			snapDir:       opts.SnapDir,
+			snapCommonDir: opts.SnapCommonDir,
+			runCommand:    runCommand,
+		},
+	}
+
+	return s
+}
+
+// StartService starts a k8s service. The name can be either prefixed or not.
+func (s *pebble) StartService(ctx context.Context, name string) error {
+	return s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", "pebble"), "start", name})
+}
+
+// StopService stops a k8s service. The name can be either prefixed or not.
+func (s *pebble) StopService(ctx context.Context, name string) error {
+	return s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", "pebble"), "stop", name})
+}
+
+// RestartService restarts a k8s service. The name can be either prefixed or not.
+func (s *pebble) RestartService(ctx context.Context, name string) error {
+	return s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", "pebble"), "restart", name})
+}
+
+func (s *pebble) Strict() bool {
+	return false
+}
+
+func (s *pebble) OnLXD(ctx context.Context) (bool, error) {
+	return true, nil
+}
+
+func (s *pebble) SnapctlGet(ctx context.Context, args ...string) ([]byte, error) {
+	return []byte(`{"meta": {"apiVersion": "1.30", "orb": "none"}`), nil
+}
+
+func (s *pebble) SnapctlSet(ctx context.Context, args ...string) error {
+	return nil
+}
+
+var _ Snap = &pebble{}

src/k8s/pkg/snap/pebble_test.go

@@ -0,0 +1,79 @@
+package snap_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/canonical/k8s/pkg/snap"
+	"github.com/canonical/k8s/pkg/snap/mock"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestPebble(t *testing.T) {
+	t.Run("Start", func(t *testing.T) {
+		g := NewWithT(t)
+		mockRunner := &mock.Runner{}
+		snap := snap.NewPebble(snap.PebbleOpts{
+			SnapDir:       "testdir",
+			SnapCommonDir: "testdir",
+			RunCommand:    mockRunner.Run,
+		})
+
+		err := snap.StartService(context.Background(), "test-service")
+		g.Expect(err).To(BeNil())
+		g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("testdir/bin/pebble start test-service"))
+
+		t.Run("Fail", func(t *testing.T) {
+			g := NewWithT(t)
+			mockRunner.Err = fmt.Errorf("some error")
+
+			err := snap.StartService(context.Background(), "test-service")
+			g.Expect(err).NotTo(BeNil())
+		})
+	})
+
+	t.Run("Stop", func(t *testing.T) {
+		g := NewWithT(t)
+		mockRunner := &mock.Runner{}
+		snap := snap.NewPebble(snap.PebbleOpts{
+			SnapDir:       "testdir",
+			SnapCommonDir: "testdir",
+			RunCommand:    mockRunner.Run,
+		})
+		err := snap.StopService(context.Background(), "test-service")
+		g.Expect(err).To(BeNil())
+		g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("testdir/bin/pebble stop test-service"))
+
+		t.Run("Fail", func(t *testing.T) {
+			g := NewWithT(t)
+			mockRunner.Err = fmt.Errorf("some error")
+
+			err := snap.StartService(context.Background(), "test-service")
+			g.Expect(err).NotTo(BeNil())
+		})
+	})
+
+	t.Run("Restart", func(t *testing.T) {
+		g := NewWithT(t)
+		mockRunner := &mock.Runner{}
+		snap := snap.NewPebble(snap.PebbleOpts{
+			SnapDir:       "testdir",
+			SnapCommonDir: "testdir",
+			RunCommand:    mockRunner.Run,
+		})
+
+		err := snap.RestartService(context.Background(), "test-service")
+		g.Expect(err).To(BeNil())
+		g.Expect(mockRunner.CalledWithCommand).To(ConsistOf("testdir/bin/pebble restart test-service"))
+
+		t.Run("Fail", func(t *testing.T) {
+			g := NewWithT(t)
+			mockRunner.Err = fmt.Errorf("some error")
+
+			err := snap.StartService(context.Background(), "service")
+			g.Expect(err).NotTo(BeNil())
+		})
+	})
+}

src/k8s/pkg/snap/service.go

@@ -0,0 +1,15 @@
+package snap
+
+import (
+	"fmt"
+	"strings"
+)
+
+// serviceName infers the name of the snapctl daemon from the service name.
+// if the serviceName is the snap name `k8s` (=referes to all services) it will return it as is.
+func serviceName(serviceName string) string {
+	if strings.HasPrefix(serviceName, "k8s.") || serviceName == "k8s" {
+		return serviceName
+	}
+	return fmt.Sprintf("k8s.%s", serviceName)
+}

src/k8s/pkg/snap/snap_internal_test.go → src/k8s/pkg/snap/service_internal_test.go

@@ -6,7 +6,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestServiceName(t *testing.T) {
+func Test_serviceName(t *testing.T) {
 	tests := []struct {
 		name     string
 		input    string