Allow changing cluster configuration through snap set k8s (#399)

snap/hooks/configure

@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+. $SNAP/k8s/lib.sh
+
+k8s::common::setup_env
+
+k8s::cmd::k8s x-snapd-config reconcile

snap/hooks/install

@@ -4,6 +4,9 @@
 
 k8s::common::setup_env
 
+# disable snap set/get by default
+k8s::cmd::k8s x-snapd-config disable
+
 # k8s has a REST interface to initialize a cluster.
 # In order to interact with the REST API the k8sd service
 # needs to be started and configured in the installation step.

src/k8s/cmd/k8s/k8s.go

@@ -109,8 +109,9 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 		newLocalNodeStatusCommand(env),
 		newRevokeAuthTokenCmd(env),
 		newGenerateDocsCmd(env),
-		xPrintShimPidsCmd,
 		newHelmCmd(env),
+		xPrintShimPidsCmd,
+		newXSnapdConfigCmd(env),
 	)
 
 	cmd.DisableAutoGenTag = true

src/k8s/cmd/k8s/k8s_x_snapd_config.go

@@ -0,0 +1,99 @@
+package k8s
+
+import (
+	apiv1 "github.com/canonical/k8s/api/v1"
+	cmdutil "github.com/canonical/k8s/cmd/util"
+	"github.com/canonical/k8s/pkg/utils/experimental/snapdconfig"
+	"github.com/spf13/cobra"
+)
+
+func newXSnapdConfigCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
+	disableCmd := &cobra.Command{
+		Use:   "disable",
+		Short: "Disable the use of snap get/set to manage the cluster configuration",
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := snapdconfig.Disable(cmd.Context(), env.Snap); err != nil {
+				cmd.PrintErrf("Error: failed to disable snapd configuration: %v\n", err)
+				env.Exit(1)
+			}
+		},
+	}
+	reconcileCmd := &cobra.Command{
+		Use:   "reconcile",
+		Short: "Reconcile the cluster configuration changes from k8s {set,get} <-> snap {set,get} k8s",
+		Run: func(cmd *cobra.Command, args []string) {
+			mode, empty, err := snapdconfig.ParseMeta(cmd.Context(), env.Snap)
+			if err != nil {
+				if !empty {
+					cmd.PrintErrf("Error: failed to parse meta configuration: %v\n", err)
+					env.Exit(1)
+					return
+				}
+
+				cmd.PrintErrf("Warning: failed to parse meta configuration: %v\n", err)
+				cmd.PrintErrf("Warning: ignoring further errors to prevent infinite loop\n")
+				if setErr := snapdconfig.SetMeta(cmd.Context(), env.Snap, snapdconfig.Meta{
+					APIVersion: "1.30",
+					Orb:        "none",
+					Error:      err.Error(),
+				}); setErr != nil {
+					cmd.PrintErrf("Warning: failed to set meta configuration to safe defaults: %v\n", setErr)
+				}
+				return
+			}
+
+			switch mode.Orb {
+			case "none":
+				cmd.PrintErrln("Warning: meta.orb is none, skipping reconcile actions")
+				return
+			case "k8sd":
+				client, err := env.Client(cmd.Context())
+				if err != nil {
+					cmd.PrintErrf("Error: failed to create k8sd client: %v\n", err)
+					env.Exit(1)
+					return
+				}
+				config, err := client.GetClusterConfig(cmd.Context(), apiv1.GetClusterConfigRequest{})
+				if err != nil {
+					cmd.PrintErrf("Error: failed to retrieve cluster configuration: %v\n", err)
+					env.Exit(1)
+					return
+				}
+				if err := snapdconfig.SetSnapdFromK8sd(cmd.Context(), config, env.Snap); err != nil {
+					cmd.PrintErrf("Error: failed to update snapd state: %v\n", err)
+					env.Exit(1)
+					return
+				}
+			case "snapd":
+				client, err := env.Client(cmd.Context())
+				if err != nil {
+					cmd.PrintErrf("Error: failed to create k8sd client: %v\n", err)
+					env.Exit(1)
+					return
+				}
+				if err := snapdconfig.SetK8sdFromSnapd(cmd.Context(), client, env.Snap); err != nil {
+					cmd.PrintErrf("Error: failed to update k8sd state: %v\n", err)
+					env.Exit(1)
+					return
+				}
+			}
+
+			mode.Orb = "snapd"
+			if err := snapdconfig.SetMeta(cmd.Context(), env.Snap, mode); err != nil {
+				cmd.PrintErrf("Error: failed to set snapd configuration: %v\n", err)
+				env.Exit(1)
+				return
+			}
+		},
+	}
+	cmd := &cobra.Command{
+		Use:    "x-snapd-config",
+		Short:  "Manage snapd configuration",
+		Hidden: true,
+	}
+
+	cmd.AddCommand(reconcileCmd)
+	cmd.AddCommand(disableCmd)
+
+	return cmd
+}

src/k8s/pkg/k8sd/controllers/control_plane_configuration.go

@@ -11,6 +11,7 @@ import (
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
+	"github.com/canonical/k8s/pkg/utils/experimental/snapdconfig"
 )
 
 // ControlPlaneConfigurationController watches for changes in the cluster configuration
@@ -107,5 +108,12 @@ func (c *ControlPlaneConfigurationController) reconcile(ctx context.Context, con
 		}
 	}
 
+	// snapd
+	if meta, _, err := snapdconfig.ParseMeta(ctx, c.snap); err == nil && meta.Orb != "none" {
+		if err := snapdconfig.SetSnapdFromK8sd(ctx, config.ToUserFacing(), c.snap); err != nil {
+			log.Printf("Warning: failed to update snapd configuration: %v", err)
+		}
+	}
+
 	return nil
 }

src/k8s/pkg/snap/interface.go

@@ -20,6 +20,9 @@ type Snap interface {
 	StopService(ctx context.Context, serviceName string) error    // snapctl stop $service
 	RestartService(ctx context.Context, serviceName string) error // snapctl restart $service
 
+	SnapctlGet(ctx context.Context, args ...string) ([]byte, error) // snapctl get $args...
+	SnapctlSet(ctx context.Context, args ...string) error           // snapctl set $args...
+
 	CNIConfDir() string       // /etc/cni/net.d
 	CNIBinDir() string        // /opt/cni/bin
 	CNIPluginsBinary() string // /snap/k8s/current/bin/cni

src/k8s/pkg/snap/mock/mock.go

@@ -2,6 +2,7 @@ package mock
 
 import (
 	"context"
+	"strings"
 
 	"github.com/canonical/k8s/pkg/client/dqlite"
 	"github.com/canonical/k8s/pkg/client/helm"
@@ -38,6 +39,7 @@ type Mock struct {
 	KubernetesNodeClient        *kubernetes.Client
 	HelmClient                  helm.Client
 	K8sDqliteClient             *dqlite.Client
+	SnapctlGet                  map[string][]byte
 }
 
 // Snap is a mock implementation for snap.Snap.
@@ -49,6 +51,11 @@ type Snap struct {
 	RestartServiceCalledWith []string
 	RestartServiceErr        error
 
+	SnapctlSetCalledWith [][]string
+	SnapctlSetErr        error
+	SnapctlGetCalledWith [][]string
+	SnapctlGetErr        error
+
 	Mock Mock
 }
 
@@ -158,5 +165,13 @@ func (s *Snap) HelmClient() helm.Client {
 func (s *Snap) K8sDqliteClient(context.Context) (*dqlite.Client, error) {
 	return s.Mock.K8sDqliteClient, nil
 }
+func (s *Snap) SnapctlGet(ctx context.Context, args ...string) ([]byte, error) {
+	s.SnapctlGetCalledWith = append(s.SnapctlGetCalledWith, args)
+	return s.Mock.SnapctlGet[strings.Join(args, " ")], s.SnapctlGetErr
+}
+func (s *Snap) SnapctlSet(ctx context.Context, args ...string) error {
+	s.SnapctlSetCalledWith = append(s.SnapctlGetCalledWith, args)
+	return s.SnapctlSetErr
+}
 
 var _ snap.Snap = &Snap{}

src/k8s/pkg/snap/mock/runner.go

@@ -3,6 +3,7 @@ package mock
 import (
 	"context"
 	"log"
+	"os/exec"
 	"strings"
 )
 
@@ -14,7 +15,7 @@ type Runner struct {
 }
 
 // Run is a mock implementation of CommandRunner.
-func (m *Runner) Run(ctx context.Context, command ...string) error {
+func (m *Runner) Run(ctx context.Context, command []string, opts ...func(*exec.Cmd)) error {
 	if m.Log {
 		log.Printf("mock execute %#v", command)
 	}

src/k8s/pkg/snap/options.go

@@ -2,10 +2,11 @@ package snap
 
 import (
 	"context"
+	"os/exec"
 )
 
 // WithCommandRunner configures how shell commands are executed.
-func WithCommandRunner(f func(context.Context, ...string) error) func(s *snap) {
+func WithCommandRunner(f func(context.Context, []string, ...func(*exec.Cmd)) error) func(s *snap) {
 	return func(s *snap) {
 		s.runCommand = f
 	}

src/k8s/pkg/snap/snap.go

@@ -1,9 +1,11 @@
 package snap
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path"
 	"strings"
 
@@ -20,7 +22,7 @@ import (
 type snap struct {
 	snapDir       string
 	snapCommonDir string
-	runCommand    func(ctx context.Context, command ...string) error
+	runCommand    func(ctx context.Context, command []string, opts ...func(c *exec.Cmd)) error
 }
 
 // NewSnap creates a new interface with the K8s snap.
@@ -54,17 +56,17 @@ func serviceName(serviceName string) string {
 
 // StartService starts a k8s service. The name can be either prefixed or not.
 func (s *snap) StartService(ctx context.Context, name string) error {
-	return s.runCommand(ctx, "snapctl", "start", "--enable", serviceName(name))
+	return s.runCommand(ctx, []string{"snapctl", "start", "--enable", serviceName(name)})
 }
 
 // StopService stops a k8s service. The name can be either prefixed or not.
 func (s *snap) StopService(ctx context.Context, name string) error {
-	return s.runCommand(ctx, "snapctl", "stop", "--disable", serviceName(name))
+	return s.runCommand(ctx, []string{"snapctl", "stop", "--disable", serviceName(name)})
 }
 
 // RestartService restarts a k8s service. The name can be either prefixed or not.
 func (s *snap) RestartService(ctx context.Context, name string) error {
-	return s.runCommand(ctx, "snapctl", "restart", serviceName(name))
+	return s.runCommand(ctx, []string{"snapctl", "restart", serviceName(name)})
 }
 
 type snapcraftYml struct {
@@ -231,4 +233,16 @@ func (s *snap) K8sDqliteClient(ctx context.Context) (*dqlite.Client, error) {
 	return client, nil
 }
 
+func (s *snap) SnapctlGet(ctx context.Context, args ...string) ([]byte, error) {
+	var b bytes.Buffer
+	if err := s.runCommand(ctx, append([]string{"snapctl", "get"}, args...), func(c *exec.Cmd) { c.Stdout = &b }); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+func (s *snap) SnapctlSet(ctx context.Context, args ...string) error {
+	return s.runCommand(ctx, append([]string{"snapctl", "set"}, args...))
+}
+
 var _ Snap = &snap{}

src/k8s/pkg/utils/experimental/snapdconfig/disable.go

@@ -0,0 +1,20 @@
+package snapdconfig
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/canonical/k8s/pkg/snap"
+)
+
+func Disable(ctx context.Context, s snap.Snap) error {
+	b, err := json.Marshal(Meta{Orb: "none", APIVersion: "1.30"})
+	if err != nil {
+		return fmt.Errorf("failed to marshal config: %w", err)
+	}
+	if err := s.SnapctlSet(ctx, fmt.Sprintf("meta=%s", string(b)), "dns!", "network!", "gateway!", "ingress!", "load-balancer!", "local-storage!"); err != nil {
+		return fmt.Errorf("failed to snapctl set: %w", err)
+	}
+	return nil
+}

src/k8s/pkg/utils/experimental/snapdconfig/k8s_to_snapd.go

@@ -0,0 +1,36 @@
+package snapdconfig
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	apiv1 "github.com/canonical/k8s/api/v1"
+	"github.com/canonical/k8s/pkg/snap"
+)
+
+// SetSnapdFromK8sd uses snapctl to update the local snapd configuration with the new k8sd cluster configuration.
+func SetSnapdFromK8sd(ctx context.Context, config apiv1.UserFacingClusterConfig, snap snap.Snap) error {
+	var sets []string
+	for key, cfg := range map[string]any{
+		"meta":          Meta{Orb: "snapd", APIVersion: "1.30"},
+		"dns":           config.DNS,
+		"network":       config.Network,
+		"local-storage": config.LocalStorage,
+		"load-balancer": config.LoadBalancer,
+		"ingress":       config.Ingress,
+		"gateway":       config.Gateway,
+	} {
+		b, err := json.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("failed to marshal %s config: %w", key, err)
+		}
+		sets = append(sets, fmt.Sprintf("%s=%s", key, string(b)))
+	}
+
+	if err := snap.SnapctlSet(ctx, sets...); err != nil {
+		return fmt.Errorf("failed to set snapd configuration: %w", err)
+	}
+
+	return nil
+}