Cluster config annotations (#459)

* add support for config annotations

* set/get annotations through the API

* pass annotations to feature controllers

* configurable metrics-server image through private annotations

src/k8s/api/v1/cluster_config.go

@@ -29,6 +29,31 @@ type UserFacingClusterConfig struct {
 	Gateway       GatewayConfig       `json:"gateway,omitempty" yaml:"gateway,omitempty"`
 	MetricsServer MetricsServerConfig `json:"metrics-server,omitempty" yaml:"metrics-server,omitempty"`
 	CloudProvider *string             `json:"cloud-provider,omitempty" yaml:"cloud-provider,omitempty"`
+	Annotations   map[string]string   `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+}
+
+func (c UserFacingClusterConfig) Empty() bool {
+	switch {
+	case c.Network != NetworkConfig{}:
+		return false
+	case c.DNS != DNSConfig{}:
+		return false
+	case c.Ingress != IngressConfig{}:
+		return false
+	case c.LoadBalancer != LoadBalancerConfig{}:
+		return false
+	case c.LocalStorage != LocalStorageConfig{}:
+		return false
+	case c.Gateway != GatewayConfig{}:
+		return false
+	case c.MetricsServer != MetricsServerConfig{}:
+		return false
+	case getField(c.CloudProvider) != "":
+		return false
+	case len(c.Annotations) > 0:
+		return false
+	}
+	return true
 }
 
 type DNSConfig struct {

src/k8s/api/v1/types.go

@@ -153,8 +153,7 @@ func (c ClusterStatus) String() string {
 	result.WriteString(c.datastoreToString())
 
 	// Config
-	var emptyConfig UserFacingClusterConfig
-	if c.Config != emptyConfig {
+	if !c.Config.Empty() {
 		b, _ := yaml.Marshal(c.Config)
 		result.WriteString(string(b))
 	}

src/k8s/cmd/k8s/k8s_get.go

@@ -47,6 +47,7 @@ func newGetCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 
 			config.MetricsServer = apiv1.MetricsServerConfig{}
 			config.CloudProvider = nil
+			config.Annotations = nil
 
 			var key string
 			if len(args) == 1 {

src/k8s/cmd/k8s/k8s_set.go

@@ -73,6 +73,7 @@ func newSetCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 }
 
 var knownSetKeys = map[string]struct{}{
+	"annotations":                    {},
 	"cloud-provider":                 {},
 	"dns.cluster-domain":             {},
 	"dns.enabled":                    {},
@@ -108,6 +109,8 @@ func updateConfigMapstructure(config *apiv1.UserFacingClusterConfig, arg string)
 		DecodeHook: mapstructure.ComposeDecodeHookFunc(
 			utils.YAMLToStringSliceHookFunc,
 			utils.StringToFieldsSliceHookFunc(','),
+			utils.YAMLToStringMapHookFunc,
+			utils.StringToStringMapHookFunc,
 		),
 	})
 	if err != nil {

src/k8s/cmd/k8s/k8s_set_test.go

@@ -79,6 +79,47 @@ func generateMapstructureTestCasesStringSlice(keyName string, fieldName string)
 	}
 }
 
+func generateMapstructureTestCasesMap(keyName string, fieldName string) []mapstructureTestCase {
+	return []mapstructureTestCase{
+		{
+			val:        fmt.Sprintf("%s=", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{})},
+		},
+		{
+			val:        fmt.Sprintf("%s={}", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{})},
+		},
+		{
+			val:        fmt.Sprintf("%s=k1=", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": ""})},
+		},
+		{
+			val:        fmt.Sprintf("%s=k1=,k2=test", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "", "k2": "test"})},
+		},
+		{
+			val:        fmt.Sprintf("%s=k1=v1", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1"})},
+		},
+		{
+			val:        fmt.Sprintf("%s=k1=v1,k2=v2", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1", "k2": "v2"})},
+		},
+		{
+			val:        fmt.Sprintf("%s={k1: v1}", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1"})},
+		},
+		{
+			val:        fmt.Sprintf("%s={k1: v1, k2: v2}", keyName),
+			assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1", "k2": "v2"})},
+		},
+		{
+			val:       fmt.Sprintf("%s=k1,k2", keyName),
+			expectErr: true,
+		},
+	}
+}
+
 func generateMapstructureTestCasesString(keyName string, fieldName string) []mapstructureTestCase {
 	return []mapstructureTestCase{
 		{
@@ -139,6 +180,8 @@ func Test_updateConfigMapstructure(t *testing.T) {
 		generateMapstructureTestCasesInt("load-balancer.bgp-local-asn", "LoadBalancer.BGPLocalASN"),
 		generateMapstructureTestCasesInt("load-balancer.bgp-peer-asn", "LoadBalancer.BGPPeerASN"),
 		generateMapstructureTestCasesInt("load-balancer.bgp-peer-port", "LoadBalancer.BGPPeerPort"),
+
+		generateMapstructureTestCasesMap("annotations", "Annotations"),
 	} {
 		for _, tc := range tcs {
 			t.Run(tc.val, func(t *testing.T) {

src/k8s/pkg/k8sd/controllers/feature.go

@@ -73,31 +73,31 @@ func (c *FeatureController) Run(ctx context.Context, getClusterConfig func(conte
 	c.waitReady()
 
 	go c.reconcileLoop(ctx, getClusterConfig, "network", c.triggerNetworkCh, c.reconciledNetworkCh, func(cfg types.ClusterConfig) error {
-		return features.Implementation.ApplyNetwork(ctx, c.snap, cfg.Network)
+		return features.Implementation.ApplyNetwork(ctx, c.snap, cfg.Network, cfg.Annotations)
 	})
 
 	go c.reconcileLoop(ctx, getClusterConfig, "gateway", c.triggerGatewayCh, c.reconciledGatewayCh, func(cfg types.ClusterConfig) error {
-		return features.Implementation.ApplyGateway(ctx, c.snap, cfg.Gateway, cfg.Network)
+		return features.Implementation.ApplyGateway(ctx, c.snap, cfg.Gateway, cfg.Network, cfg.Annotations)
 	})
 
 	go c.reconcileLoop(ctx, getClusterConfig, "ingress", c.triggerIngressCh, c.reconciledIngressCh, func(cfg types.ClusterConfig) error {
-		return features.Implementation.ApplyIngress(ctx, c.snap, cfg.Ingress, cfg.Network)
+		return features.Implementation.ApplyIngress(ctx, c.snap, cfg.Ingress, cfg.Network, cfg.Annotations)
 	})
 
 	go c.reconcileLoop(ctx, getClusterConfig, "load balancer", c.triggerLoadBalancerCh, c.reconciledLoadBalancerCh, func(cfg types.ClusterConfig) error {
-		return features.Implementation.ApplyLoadBalancer(ctx, c.snap, cfg.LoadBalancer, cfg.Network)
+		return features.Implementation.ApplyLoadBalancer(ctx, c.snap, cfg.LoadBalancer, cfg.Network, cfg.Annotations)
 	})
 
 	go c.reconcileLoop(ctx, getClusterConfig, "local storage", c.triggerLocalStorageCh, c.reconciledLocalStorageCh, func(cfg types.ClusterConfig) error {
-		return features.Implementation.ApplyLocalStorage(ctx, c.snap, cfg.LocalStorage)
+		return features.Implementation.ApplyLocalStorage(ctx, c.snap, cfg.LocalStorage, cfg.Annotations)
 	})
 
 	go c.reconcileLoop(ctx, getClusterConfig, "metrics server", c.triggerMetricsServerCh, c.reconciledMetricsServerCh, func(cfg types.ClusterConfig) error {
-		return features.Implementation.ApplyMetricsServer(ctx, c.snap, cfg.MetricsServer)
+		return features.Implementation.ApplyMetricsServer(ctx, c.snap, cfg.MetricsServer, cfg.Annotations)
 	})
 
 	go c.reconcileLoop(ctx, getClusterConfig, "DNS", c.triggerDNSCh, c.reconciledDNSCh, func(cfg types.ClusterConfig) error {
-		if dnsIP, err := features.Implementation.ApplyDNS(ctx, c.snap, cfg.DNS, cfg.Kubelet); err != nil {
+		if dnsIP, err := features.Implementation.ApplyDNS(ctx, c.snap, cfg.DNS, cfg.Kubelet, cfg.Annotations); err != nil {
 			return fmt.Errorf("failed to apply DNS configuration: %w", err)
 		} else if dnsIP != "" {
 			if err := notifyDNSChangedIP(ctx, dnsIP); err != nil {

src/k8s/pkg/k8sd/features/cilium/gateway.go

@@ -14,7 +14,7 @@ import (
 // ApplyGateway will remove the Gateway API CRDs from the cluster and disable the GatewayAPI controllers on Cilium, when gateway.Enabled is false.
 // ApplyGateway will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyGateway returns an error if anything fails.
-func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network) error {
+func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network, _ types.Annotations) error {
 	m := snap.HelmClient()
 
 	if _, err := m.Apply(ctx, chartGateway, helm.StatePresentOrDeleted(gateway.GetEnabled()), nil); err != nil {

src/k8s/pkg/k8sd/features/cilium/ingress.go

@@ -14,7 +14,7 @@ import (
 // ApplyIngress will disable Cilium's ingress controller when ingress.Disabled is false.
 // ApplyIngress will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyIngress returns an error if anything fails.
-func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network) error {
+func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network, _ types.Annotations) error {
 	m := snap.HelmClient()
 
 	var values map[string]any

src/k8s/pkg/k8sd/features/cilium/loadbalancer.go

@@ -15,7 +15,7 @@ import (
 // ApplyLoadBalancer will disable L2 and BGP on Cilium, and remove any previously created CRs when loadbalancer.Enabled is false.
 // ApplyLoadBalancer will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyLoadBalancer returns an error if anything fails.
-func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error {
+func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, _ types.Annotations) error {
 	if !loadbalancer.GetEnabled() {
 		if err := disableLoadBalancer(ctx, snap, network); err != nil {
 			return fmt.Errorf("failed to disable LoadBalancer: %w", err)

src/k8s/pkg/k8sd/features/cilium/network.go

@@ -19,7 +19,7 @@ import (
 // ApplyNetwork requires that bpf and cgroups2 are already mounted and available when running under strict snap confinement. If they are not, it will fail (since Cilium will not have the required permissions to mount them).
 // ApplyNetwork requires that `/sys` is mounted as a shared mount when running under classic snap confinement. This is to ensure that Cilium will be able to automatically mount bpf and cgroups2 on the pods.
 // ApplyNetwork returns an error if anything fails.
-func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error {
+func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ types.Annotations) error {
 	m := snap.HelmClient()
 
 	if !cfg.GetEnabled() {

src/k8s/pkg/k8sd/features/coredns/coredns.go

@@ -15,7 +15,7 @@ import (
 // ApplyDNS will install or refresh CoreDNS if dns.Enabled is true.
 // ApplyDNS will return the ClusterIP address of the coredns service, if successful.
 // ApplyDNS returns an error if anything fails.
-func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet) (string, error) {
+func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet, _ types.Annotations) (string, error) {
 	m := snap.HelmClient()
 
 	if !dns.GetEnabled() {

src/k8s/pkg/k8sd/features/interface.go

@@ -10,56 +10,56 @@ import (
 // Interface abstracts the management of built-in Canonical Kubernetes features.
 type Interface interface {
 	// ApplyDNS is used to configure the DNS feature on Canonical Kubernetes.
-	ApplyDNS(context.Context, snap.Snap, types.DNS, types.Kubelet) (string, error)
+	ApplyDNS(context.Context, snap.Snap, types.DNS, types.Kubelet, types.Annotations) (string, error)
 	// ApplyNetwork is used to configure the network feature on Canonical Kubernetes.
-	ApplyNetwork(context.Context, snap.Snap, types.Network) error
+	ApplyNetwork(context.Context, snap.Snap, types.Network, types.Annotations) error
 	// ApplyLoadBalancer is used to configure the load-balancer feature on Canonical Kubernetes.
-	ApplyLoadBalancer(context.Context, snap.Snap, types.LoadBalancer, types.Network) error
+	ApplyLoadBalancer(context.Context, snap.Snap, types.LoadBalancer, types.Network, types.Annotations) error
 	// ApplyIngress is used to configure the ingress controller feature on Canonical Kubernetes.
-	ApplyIngress(context.Context, snap.Snap, types.Ingress, types.Network) error
+	ApplyIngress(context.Context, snap.Snap, types.Ingress, types.Network, types.Annotations) error
 	// ApplyGateway is used to configure the gateway feature on Canonical Kubernetes.
-	ApplyGateway(context.Context, snap.Snap, types.Gateway, types.Network) error
+	ApplyGateway(context.Context, snap.Snap, types.Gateway, types.Network, types.Annotations) error
 	// ApplyMetricsServer is used to configure the metrics-server feature on Canonical Kubernetes.
-	ApplyMetricsServer(context.Context, snap.Snap, types.MetricsServer) error
+	ApplyMetricsServer(context.Context, snap.Snap, types.MetricsServer, types.Annotations) error
 	// ApplyLocalStorage is used to configure the Local Storage feature on Canonical Kubernetes.
-	ApplyLocalStorage(context.Context, snap.Snap, types.LocalStorage) error
+	ApplyLocalStorage(context.Context, snap.Snap, types.LocalStorage, types.Annotations) error
 }
 
 // implementation implements Interface.
 type implementation struct {
-	applyDNS           func(context.Context, snap.Snap, types.DNS, types.Kubelet) (string, error)
-	applyNetwork       func(context.Context, snap.Snap, types.Network) error
-	applyLoadBalancer  func(context.Context, snap.Snap, types.LoadBalancer, types.Network) error
-	applyIngress       func(context.Context, snap.Snap, types.Ingress, types.Network) error
-	applyGateway       func(context.Context, snap.Snap, types.Gateway, types.Network) error
-	applyMetricsServer func(context.Context, snap.Snap, types.MetricsServer) error
-	applyLocalStorage  func(context.Context, snap.Snap, types.LocalStorage) error
+	applyDNS           func(context.Context, snap.Snap, types.DNS, types.Kubelet, types.Annotations) (string, error)
+	applyNetwork       func(context.Context, snap.Snap, types.Network, types.Annotations) error
+	applyLoadBalancer  func(context.Context, snap.Snap, types.LoadBalancer, types.Network, types.Annotations) error
+	applyIngress       func(context.Context, snap.Snap, types.Ingress, types.Network, types.Annotations) error
+	applyGateway       func(context.Context, snap.Snap, types.Gateway, types.Network, types.Annotations) error
+	applyMetricsServer func(context.Context, snap.Snap, types.MetricsServer, types.Annotations) error
+	applyLocalStorage  func(context.Context, snap.Snap, types.LocalStorage, types.Annotations) error
 }
 
-func (i *implementation) ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet) (string, error) {
-	return i.applyDNS(ctx, snap, dns, kubelet)
+func (i *implementation) ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet, annotations types.Annotations) (string, error) {
+	return i.applyDNS(ctx, snap, dns, kubelet, annotations)
 }
 
-func (i *implementation) ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error {
-	return i.applyNetwork(ctx, snap, cfg)
+func (i *implementation) ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annotations types.Annotations) error {
+	return i.applyNetwork(ctx, snap, cfg, annotations)
 }
 
-func (i *implementation) ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error {
-	return i.applyLoadBalancer(ctx, snap, loadbalancer, network)
+func (i *implementation) ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, annotations types.Annotations) error {
+	return i.applyLoadBalancer(ctx, snap, loadbalancer, network, annotations)
 }
 
-func (i *implementation) ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network) error {
-	return i.applyIngress(ctx, snap, ingress, network)
+func (i *implementation) ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network, annotations types.Annotations) error {
+	return i.applyIngress(ctx, snap, ingress, network, annotations)
 }
 
-func (i *implementation) ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network) error {
-	return i.applyGateway(ctx, snap, gateway, network)
+func (i *implementation) ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network, annotations types.Annotations) error {
+	return i.applyGateway(ctx, snap, gateway, network, annotations)
 }
 
-func (i *implementation) ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer) error {
-	return i.applyMetricsServer(ctx, snap, cfg)
+func (i *implementation) ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer, annotations types.Annotations) error {
+	return i.applyMetricsServer(ctx, snap, cfg, annotations)
 }
 
-func (i *implementation) ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage) error {
-	return i.applyLocalStorage(ctx, snap, cfg)
+func (i *implementation) ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage, annotations types.Annotations) error {
+	return i.applyLocalStorage(ctx, snap, cfg, annotations)
 }

src/k8s/pkg/k8sd/features/localpv/localpv.go

@@ -11,7 +11,7 @@ import (
 // ApplyLocalStorage deploys the rawfile-localpv CSI driver on the cluster based on the given configuration, when cfg.Enabled is true.
 // ApplyLocalStorage removes the rawfile-localpv when cfg.Enabled is false.
 // ApplyLocalStorage returns an error if anything fails.
-func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage) error {
+func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage, _ types.Annotations) error {
 	m := snap.HelmClient()
 
 	values := map[string]any{

src/k8s/pkg/k8sd/features/metrics-server/internal.go

@@ -0,0 +1,29 @@
+package metrics_server
+
+import "github.com/canonical/k8s/pkg/k8sd/types"
+
+const (
+	annotationImageRepo = "k8sd/v1alpha1/metrics-server/image-repo"
+	annotationImageTag  = "k8sd/v1alpha1/metrics-server/image-tag"
+)
+
+type config struct {
+	imageRepo string
+	imageTag  string
+}
+
+func internalConfig(annotations types.Annotations) config {
+	config := config{
+		imageRepo: imageRepo,
+		imageTag:  imageTag,
+	}
+
+	if v, ok := annotations.Get(annotationImageRepo); ok {
+		config.imageRepo = v
+	}
+	if v, ok := annotations.Get(annotationImageTag); ok {
+		config.imageTag = v
+	}
+
+	return config
+}

src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go

@@ -11,13 +11,15 @@ import (
 // ApplyMetricsServer deploys metrics-server when cfg.Enabled is true.
 // ApplyMetricsServer removes metrics-server when cfg.Enabled is false.
 // ApplyMetricsServer returns an error if anything fails.
-func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer) error {
+func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer, annotations types.Annotations) error {
 	m := snap.HelmClient()
 
+	config := internalConfig(annotations)
+
 	values := map[string]any{
 		"image": map[string]any{
-			"repository": imageRepo,
-			"tag":        imageTag,
+			"repository": config.imageRepo,
+			"tag":        config.imageTag,
 		},
 		"securityContext": map[string]any{
 			// ROCKs with Pebble as the entrypoint do not work with this option.

src/k8s/pkg/k8sd/features/metrics-server/metrics_server_test.go

@@ -43,7 +43,7 @@ func TestApplyMetricsServer(t *testing.T) {
 				},
 			}
 
-			err := metrics_server.ApplyMetricsServer(context.Background(), s, tc.config)
+			err := metrics_server.ApplyMetricsServer(context.Background(), s, tc.config, nil)
 			g.Expect(err).ToNot(HaveOccurred())
 
 			g.Expect(h.ApplyCalledWith).To(ConsistOf(SatisfyAll(
@@ -53,4 +53,29 @@ func TestApplyMetricsServer(t *testing.T) {
 			)))
 		})
 	}
+
+	t.Run("Annotations", func(t *testing.T) {
+		g := NewWithT(t)
+		h := &helmmock.Mock{}
+		s := &snapmock.Snap{
+			Mock: snapmock.Mock{
+				HelmClient: h,
+			},
+		}
+
+		cfg := types.MetricsServer{
+			Enabled: utils.Pointer(true),
+		}
+		annotations := types.Annotations{
+			"k8sd/v1alpha1/metrics-server/image-repo": "custom-image",
+			"k8sd/v1alpha1/metrics-server/image-tag":  "custom-tag",
+		}
+
+		err := metrics_server.ApplyMetricsServer(context.Background(), s, cfg, annotations)
+		g.Expect(err).To(BeNil())
+		g.Expect(h.ApplyCalledWith).To(ConsistOf(HaveField("Values", HaveKeyWithValue("image", SatisfyAll(
+			HaveKeyWithValue("repository", "custom-image"),
+			HaveKeyWithValue("tag", "custom-tag"),
+		)))))
+	})
 }

src/k8s/pkg/k8sd/types/cluster_config.go

@@ -14,6 +14,6 @@ type ClusterConfig struct {
 	Gateway       Gateway       `json:"gateway,omitempty"`
 	LocalStorage  LocalStorage  `json:"local-storage,omitempty"`
 	MetricsServer MetricsServer `json:"metrics-server,omitempty"`
-}
 
-func (c ClusterConfig) Empty() bool { return c == ClusterConfig{} }
+	Annotations Annotations `json:"annotations,omitempty"`
+}