From 1ee153714da2ab29b40583af43485facb67eae9d Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Sat, 4 May 2024 17:45:15 +0300 Subject: [PATCH] Allow changing cluster configuration through `snap set k8s` (#399) --- snap/hooks/configure | 7 ++ snap/hooks/install | 3 + src/k8s/cmd/k8s/k8s.go | 3 +- src/k8s/cmd/k8s/k8s_x_snapd_config.go | 99 +++++++++++++++++++ src/k8s/docs/snapd-config.md | 0 .../control_plane_configuration.go | 8 ++ src/k8s/pkg/snap/interface.go | 3 + src/k8s/pkg/snap/mock/mock.go | 15 +++ src/k8s/pkg/snap/mock/runner.go | 3 +- src/k8s/pkg/snap/options.go | 3 +- src/k8s/pkg/snap/snap.go | 22 ++++- .../utils/experimental/snapdconfig/disable.go | 20 ++++ .../experimental/snapdconfig/k8s_to_snapd.go | 36 +++++++ .../utils/experimental/snapdconfig/meta.go | 69 +++++++++++++ .../experimental/snapdconfig/snapd_to_k8s.go | 30 ++++++ src/k8s/pkg/utils/sys.go | 7 +- 16 files changed, 320 insertions(+), 8 deletions(-) create mode 100755 snap/hooks/configure create mode 100644 src/k8s/cmd/k8s/k8s_x_snapd_config.go create mode 100644 src/k8s/docs/snapd-config.md create mode 100644 src/k8s/pkg/utils/experimental/snapdconfig/disable.go create mode 100644 src/k8s/pkg/utils/experimental/snapdconfig/k8s_to_snapd.go create mode 100644 src/k8s/pkg/utils/experimental/snapdconfig/meta.go create mode 100644 src/k8s/pkg/utils/experimental/snapdconfig/snapd_to_k8s.go diff --git a/snap/hooks/configure b/snap/hooks/configure new file mode 100755 index 000000000..e428bddc6 --- /dev/null +++ b/snap/hooks/configure @@ -0,0 +1,7 @@ +#!/bin/bash -e + +. $SNAP/k8s/lib.sh + +k8s::common::setup_env + +k8s::cmd::k8s x-snapd-config reconcile diff --git a/snap/hooks/install b/snap/hooks/install index 0175b3942..24dc9c6d5 100755 --- a/snap/hooks/install +++ b/snap/hooks/install @@ -4,6 +4,9 @@ k8s::common::setup_env +# disable snap set/get by default +k8s::cmd::k8s x-snapd-config disable + # k8s has a REST interface to initialize a cluster. # In order to interact with the REST API the k8sd service # needs to be started and configured in the installation step. diff --git a/src/k8s/cmd/k8s/k8s.go b/src/k8s/cmd/k8s/k8s.go index ea1ee815e..297d9bd1c 100644 --- a/src/k8s/cmd/k8s/k8s.go +++ b/src/k8s/cmd/k8s/k8s.go @@ -109,8 +109,9 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { newLocalNodeStatusCommand(env), newRevokeAuthTokenCmd(env), newGenerateDocsCmd(env), - xPrintShimPidsCmd, newHelmCmd(env), + xPrintShimPidsCmd, + newXSnapdConfigCmd(env), ) cmd.DisableAutoGenTag = true diff --git a/src/k8s/cmd/k8s/k8s_x_snapd_config.go b/src/k8s/cmd/k8s/k8s_x_snapd_config.go new file mode 100644 index 000000000..c6ee24b8c --- /dev/null +++ b/src/k8s/cmd/k8s/k8s_x_snapd_config.go @@ -0,0 +1,99 @@ +package k8s + +import ( + apiv1 "github.com/canonical/k8s/api/v1" + cmdutil "github.com/canonical/k8s/cmd/util" + "github.com/canonical/k8s/pkg/utils/experimental/snapdconfig" + "github.com/spf13/cobra" +) + +func newXSnapdConfigCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { + disableCmd := &cobra.Command{ + Use: "disable", + Short: "Disable the use of snap get/set to manage the cluster configuration", + Run: func(cmd *cobra.Command, args []string) { + if err := snapdconfig.Disable(cmd.Context(), env.Snap); err != nil { + cmd.PrintErrf("Error: failed to disable snapd configuration: %v\n", err) + env.Exit(1) + } + }, + } + reconcileCmd := &cobra.Command{ + Use: "reconcile", + Short: "Reconcile the cluster configuration changes from k8s {set,get} <-> snap {set,get} k8s", + Run: func(cmd *cobra.Command, args []string) { + mode, empty, err := snapdconfig.ParseMeta(cmd.Context(), env.Snap) + if err != nil { + if !empty { + cmd.PrintErrf("Error: failed to parse meta configuration: %v\n", err) + env.Exit(1) + return + } + + cmd.PrintErrf("Warning: failed to parse meta configuration: %v\n", err) + cmd.PrintErrf("Warning: ignoring further errors to prevent infinite loop\n") + if setErr := snapdconfig.SetMeta(cmd.Context(), env.Snap, snapdconfig.Meta{ + APIVersion: "1.30", + Orb: "none", + Error: err.Error(), + }); setErr != nil { + cmd.PrintErrf("Warning: failed to set meta configuration to safe defaults: %v\n", setErr) + } + return + } + + switch mode.Orb { + case "none": + cmd.PrintErrln("Warning: meta.orb is none, skipping reconcile actions") + return + case "k8sd": + client, err := env.Client(cmd.Context()) + if err != nil { + cmd.PrintErrf("Error: failed to create k8sd client: %v\n", err) + env.Exit(1) + return + } + config, err := client.GetClusterConfig(cmd.Context(), apiv1.GetClusterConfigRequest{}) + if err != nil { + cmd.PrintErrf("Error: failed to retrieve cluster configuration: %v\n", err) + env.Exit(1) + return + } + if err := snapdconfig.SetSnapdFromK8sd(cmd.Context(), config, env.Snap); err != nil { + cmd.PrintErrf("Error: failed to update snapd state: %v\n", err) + env.Exit(1) + return + } + case "snapd": + client, err := env.Client(cmd.Context()) + if err != nil { + cmd.PrintErrf("Error: failed to create k8sd client: %v\n", err) + env.Exit(1) + return + } + if err := snapdconfig.SetK8sdFromSnapd(cmd.Context(), client, env.Snap); err != nil { + cmd.PrintErrf("Error: failed to update k8sd state: %v\n", err) + env.Exit(1) + return + } + } + + mode.Orb = "snapd" + if err := snapdconfig.SetMeta(cmd.Context(), env.Snap, mode); err != nil { + cmd.PrintErrf("Error: failed to set snapd configuration: %v\n", err) + env.Exit(1) + return + } + }, + } + cmd := &cobra.Command{ + Use: "x-snapd-config", + Short: "Manage snapd configuration", + Hidden: true, + } + + cmd.AddCommand(reconcileCmd) + cmd.AddCommand(disableCmd) + + return cmd +} diff --git a/src/k8s/docs/snapd-config.md b/src/k8s/docs/snapd-config.md new file mode 100644 index 000000000..e69de29bb diff --git a/src/k8s/pkg/k8sd/controllers/control_plane_configuration.go b/src/k8s/pkg/k8sd/controllers/control_plane_configuration.go index 52d4a4dc4..0ff72c5c2 100644 --- a/src/k8s/pkg/k8sd/controllers/control_plane_configuration.go +++ b/src/k8s/pkg/k8sd/controllers/control_plane_configuration.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils/experimental/snapdconfig" ) // ControlPlaneConfigurationController watches for changes in the cluster configuration @@ -107,5 +108,12 @@ func (c *ControlPlaneConfigurationController) reconcile(ctx context.Context, con } } + // snapd + if meta, _, err := snapdconfig.ParseMeta(ctx, c.snap); err == nil && meta.Orb != "none" { + if err := snapdconfig.SetSnapdFromK8sd(ctx, config.ToUserFacing(), c.snap); err != nil { + log.Printf("Warning: failed to update snapd configuration: %v", err) + } + } + return nil } diff --git a/src/k8s/pkg/snap/interface.go b/src/k8s/pkg/snap/interface.go index 0fa9fb610..fd252392e 100644 --- a/src/k8s/pkg/snap/interface.go +++ b/src/k8s/pkg/snap/interface.go @@ -20,6 +20,9 @@ type Snap interface { StopService(ctx context.Context, serviceName string) error // snapctl stop $service RestartService(ctx context.Context, serviceName string) error // snapctl restart $service + SnapctlGet(ctx context.Context, args ...string) ([]byte, error) // snapctl get $args... + SnapctlSet(ctx context.Context, args ...string) error // snapctl set $args... + CNIConfDir() string // /etc/cni/net.d CNIBinDir() string // /opt/cni/bin CNIPluginsBinary() string // /snap/k8s/current/bin/cni diff --git a/src/k8s/pkg/snap/mock/mock.go b/src/k8s/pkg/snap/mock/mock.go index 0cd9e13e8..ad117cba4 100644 --- a/src/k8s/pkg/snap/mock/mock.go +++ b/src/k8s/pkg/snap/mock/mock.go @@ -2,6 +2,7 @@ package mock import ( "context" + "strings" "github.com/canonical/k8s/pkg/client/dqlite" "github.com/canonical/k8s/pkg/client/helm" @@ -38,6 +39,7 @@ type Mock struct { KubernetesNodeClient *kubernetes.Client HelmClient helm.Client K8sDqliteClient *dqlite.Client + SnapctlGet map[string][]byte } // Snap is a mock implementation for snap.Snap. @@ -49,6 +51,11 @@ type Snap struct { RestartServiceCalledWith []string RestartServiceErr error + SnapctlSetCalledWith [][]string + SnapctlSetErr error + SnapctlGetCalledWith [][]string + SnapctlGetErr error + Mock Mock } @@ -158,5 +165,13 @@ func (s *Snap) HelmClient() helm.Client { func (s *Snap) K8sDqliteClient(context.Context) (*dqlite.Client, error) { return s.Mock.K8sDqliteClient, nil } +func (s *Snap) SnapctlGet(ctx context.Context, args ...string) ([]byte, error) { + s.SnapctlGetCalledWith = append(s.SnapctlGetCalledWith, args) + return s.Mock.SnapctlGet[strings.Join(args, " ")], s.SnapctlGetErr +} +func (s *Snap) SnapctlSet(ctx context.Context, args ...string) error { + s.SnapctlSetCalledWith = append(s.SnapctlGetCalledWith, args) + return s.SnapctlSetErr +} var _ snap.Snap = &Snap{} diff --git a/src/k8s/pkg/snap/mock/runner.go b/src/k8s/pkg/snap/mock/runner.go index a84a65e67..ec69bdd60 100644 --- a/src/k8s/pkg/snap/mock/runner.go +++ b/src/k8s/pkg/snap/mock/runner.go @@ -3,6 +3,7 @@ package mock import ( "context" "log" + "os/exec" "strings" ) @@ -14,7 +15,7 @@ type Runner struct { } // Run is a mock implementation of CommandRunner. -func (m *Runner) Run(ctx context.Context, command ...string) error { +func (m *Runner) Run(ctx context.Context, command []string, opts ...func(*exec.Cmd)) error { if m.Log { log.Printf("mock execute %#v", command) } diff --git a/src/k8s/pkg/snap/options.go b/src/k8s/pkg/snap/options.go index 7e150f097..40fbb00d8 100644 --- a/src/k8s/pkg/snap/options.go +++ b/src/k8s/pkg/snap/options.go @@ -2,10 +2,11 @@ package snap import ( "context" + "os/exec" ) // WithCommandRunner configures how shell commands are executed. -func WithCommandRunner(f func(context.Context, ...string) error) func(s *snap) { +func WithCommandRunner(f func(context.Context, []string, ...func(*exec.Cmd)) error) func(s *snap) { return func(s *snap) { s.runCommand = f } diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go index 6bdbb51c4..4ec4a6513 100644 --- a/src/k8s/pkg/snap/snap.go +++ b/src/k8s/pkg/snap/snap.go @@ -1,9 +1,11 @@ package snap import ( + "bytes" "context" "fmt" "os" + "os/exec" "path" "strings" @@ -20,7 +22,7 @@ import ( type snap struct { snapDir string snapCommonDir string - runCommand func(ctx context.Context, command ...string) error + runCommand func(ctx context.Context, command []string, opts ...func(c *exec.Cmd)) error } // NewSnap creates a new interface with the K8s snap. @@ -54,17 +56,17 @@ func serviceName(serviceName string) string { // StartService starts a k8s service. The name can be either prefixed or not. func (s *snap) StartService(ctx context.Context, name string) error { - return s.runCommand(ctx, "snapctl", "start", "--enable", serviceName(name)) + return s.runCommand(ctx, []string{"snapctl", "start", "--enable", serviceName(name)}) } // StopService stops a k8s service. The name can be either prefixed or not. func (s *snap) StopService(ctx context.Context, name string) error { - return s.runCommand(ctx, "snapctl", "stop", "--disable", serviceName(name)) + return s.runCommand(ctx, []string{"snapctl", "stop", "--disable", serviceName(name)}) } // RestartService restarts a k8s service. The name can be either prefixed or not. func (s *snap) RestartService(ctx context.Context, name string) error { - return s.runCommand(ctx, "snapctl", "restart", serviceName(name)) + return s.runCommand(ctx, []string{"snapctl", "restart", serviceName(name)}) } type snapcraftYml struct { @@ -231,4 +233,16 @@ func (s *snap) K8sDqliteClient(ctx context.Context) (*dqlite.Client, error) { return client, nil } +func (s *snap) SnapctlGet(ctx context.Context, args ...string) ([]byte, error) { + var b bytes.Buffer + if err := s.runCommand(ctx, append([]string{"snapctl", "get"}, args...), func(c *exec.Cmd) { c.Stdout = &b }); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func (s *snap) SnapctlSet(ctx context.Context, args ...string) error { + return s.runCommand(ctx, append([]string{"snapctl", "set"}, args...)) +} + var _ Snap = &snap{} diff --git a/src/k8s/pkg/utils/experimental/snapdconfig/disable.go b/src/k8s/pkg/utils/experimental/snapdconfig/disable.go new file mode 100644 index 000000000..154c51251 --- /dev/null +++ b/src/k8s/pkg/utils/experimental/snapdconfig/disable.go @@ -0,0 +1,20 @@ +package snapdconfig + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/canonical/k8s/pkg/snap" +) + +func Disable(ctx context.Context, s snap.Snap) error { + b, err := json.Marshal(Meta{Orb: "none", APIVersion: "1.30"}) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + if err := s.SnapctlSet(ctx, fmt.Sprintf("meta=%s", string(b)), "dns!", "network!", "gateway!", "ingress!", "load-balancer!", "local-storage!"); err != nil { + return fmt.Errorf("failed to snapctl set: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/utils/experimental/snapdconfig/k8s_to_snapd.go b/src/k8s/pkg/utils/experimental/snapdconfig/k8s_to_snapd.go new file mode 100644 index 000000000..b871b5e4b --- /dev/null +++ b/src/k8s/pkg/utils/experimental/snapdconfig/k8s_to_snapd.go @@ -0,0 +1,36 @@ +package snapdconfig + +import ( + "context" + "encoding/json" + "fmt" + + apiv1 "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/snap" +) + +// SetSnapdFromK8sd uses snapctl to update the local snapd configuration with the new k8sd cluster configuration. +func SetSnapdFromK8sd(ctx context.Context, config apiv1.UserFacingClusterConfig, snap snap.Snap) error { + var sets []string + for key, cfg := range map[string]any{ + "meta": Meta{Orb: "snapd", APIVersion: "1.30"}, + "dns": config.DNS, + "network": config.Network, + "local-storage": config.LocalStorage, + "load-balancer": config.LoadBalancer, + "ingress": config.Ingress, + "gateway": config.Gateway, + } { + b, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("failed to marshal %s config: %w", key, err) + } + sets = append(sets, fmt.Sprintf("%s=%s", key, string(b))) + } + + if err := snap.SnapctlSet(ctx, sets...); err != nil { + return fmt.Errorf("failed to set snapd configuration: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/utils/experimental/snapdconfig/meta.go b/src/k8s/pkg/utils/experimental/snapdconfig/meta.go new file mode 100644 index 000000000..74f20d8f4 --- /dev/null +++ b/src/k8s/pkg/utils/experimental/snapdconfig/meta.go @@ -0,0 +1,69 @@ +package snapdconfig + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/canonical/k8s/pkg/snap" +) + +// Meta represents meta configuration that describes how to parse the snapd configuration values. +type Meta struct { + // Orb is one of "", "k8sd", "snapd", "none". + Orb string `json:"orb"` + // APIVersion is one of "", "1.30". + APIVersion string `json:"apiVersion"` + // Error is an error message that is set if the config mode cannot be parsed. + Error string `json:"error,omitempty"` +} + +// ParseMeta parses the output of "snapctl get -d meta" and returns the active snapd configuration mode. +// ParseMeta returns an error and true if the error is because of missing/empty config, rather than an operational error. +func ParseMeta(ctx context.Context, s snap.Snap) (Meta, bool, error) { + var parse struct { + Meta Meta `json:"meta"` + } + + b, err := s.SnapctlGet(ctx, "-d", "meta") + if err != nil { + return Meta{}, false, fmt.Errorf("failed to get snapd config mode: %w", err) + } + if err := json.Unmarshal(b, &parse); err != nil { + return Meta{}, false, fmt.Errorf("failed to parse snap config mode: %w", err) + } + + // default meta.orb is none + if parse.Meta.Orb == "" { + parse.Meta.Orb = "none" + } + switch parse.Meta.Orb { + case "k8sd", "snapd", "none": + default: + return Meta{}, false, fmt.Errorf("invalid meta.orb value %q", parse.Meta.Orb) + } + + // default meta.version is 1.30 + if parse.Meta.APIVersion == "" { + parse.Meta.APIVersion = "1.30" + } + switch parse.Meta.APIVersion { + case "1.30": + default: + return Meta{}, false, fmt.Errorf("invalid meta.apiVersion value %q", parse.Meta.APIVersion) + } + + return parse.Meta, true, nil +} + +// SetMeta sets the active snapd configuration mode. +func SetMeta(ctx context.Context, s snap.Snap, meta Meta) error { + b, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + if err := s.SnapctlSet(ctx, fmt.Sprintf("meta=%s", string(b))); err != nil { + return fmt.Errorf("failed to snapctl set meta: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/utils/experimental/snapdconfig/snapd_to_k8s.go b/src/k8s/pkg/utils/experimental/snapdconfig/snapd_to_k8s.go new file mode 100644 index 000000000..486e14cd2 --- /dev/null +++ b/src/k8s/pkg/utils/experimental/snapdconfig/snapd_to_k8s.go @@ -0,0 +1,30 @@ +package snapdconfig + +import ( + "context" + "encoding/json" + "fmt" + + apiv1 "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/k8s/client" + "github.com/canonical/k8s/pkg/snap" +) + +// SetK8sdFromSnapd updates the k8sd cluster configuration from the current local snapd configuration. +func SetK8sdFromSnapd(ctx context.Context, client client.Client, snap snap.Snap) error { + b, err := snap.SnapctlGet(ctx, "-d", "dns", "network", "local-storage", "load-balancer", "ingress", "gateway") + if err != nil { + return fmt.Errorf("failed to retrieve snapd configuration: %w", err) + } + + var config apiv1.UserFacingClusterConfig + if err := json.Unmarshal(b, &config); err != nil { + return fmt.Errorf("failed to parse snapd configuration: %w", err) + } + + if err := client.UpdateClusterConfig(ctx, apiv1.UpdateClusterConfigRequest{Config: config}); err != nil { + return fmt.Errorf("failed to update k8s configuration: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/utils/sys.go b/src/k8s/pkg/utils/sys.go index a441688cc..eae02b7f3 100644 --- a/src/k8s/pkg/utils/sys.go +++ b/src/k8s/pkg/utils/sys.go @@ -9,7 +9,7 @@ import ( // runCommand executes a command with a given context. // runCommand returns nil if the command completes successfully and the exit code is 0. -func RunCommand(ctx context.Context, command ...string) error { +func RunCommand(ctx context.Context, command []string, opts ...func(*exec.Cmd)) error { var args []string if len(command) > 1 { args = command[1:] @@ -17,6 +17,11 @@ func RunCommand(ctx context.Context, command ...string) error { cmd := exec.CommandContext(ctx, command[0], args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + + for _, o := range opts { + o(cmd) + } + if err := cmd.Run(); err != nil { return fmt.Errorf("command %v failed with exit code %d: %w", command, cmd.ProcessState.ExitCode(), err) }