From f026b18c18a4e80d5dd0b6147ef6a638127d415d Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Mon, 3 Jun 2024 20:33:52 +0300 Subject: [PATCH] Cluster config annotations (#459) * add support for config annotations * set/get annotations through the API * pass annotations to feature controllers * configurable metrics-server image through private annotations --- src/k8s/api/v1/cluster_config.go | 25 +++++++++ src/k8s/api/v1/types.go | 3 +- src/k8s/cmd/k8s/k8s_get.go | 1 + src/k8s/cmd/k8s/k8s_set.go | 3 + src/k8s/cmd/k8s/k8s_set_test.go | 43 ++++++++++++++ src/k8s/pkg/k8sd/controllers/feature.go | 14 ++--- src/k8s/pkg/k8sd/features/cilium/gateway.go | 2 +- src/k8s/pkg/k8sd/features/cilium/ingress.go | 2 +- .../pkg/k8sd/features/cilium/loadbalancer.go | 2 +- src/k8s/pkg/k8sd/features/cilium/network.go | 2 +- src/k8s/pkg/k8sd/features/coredns/coredns.go | 2 +- src/k8s/pkg/k8sd/features/interface.go | 56 +++++++++---------- src/k8s/pkg/k8sd/features/localpv/localpv.go | 2 +- .../k8sd/features/metrics-server/internal.go | 29 ++++++++++ .../features/metrics-server/metrics_server.go | 8 ++- .../metrics-server/metrics_server_test.go | 27 ++++++++- src/k8s/pkg/k8sd/types/cluster_config.go | 4 +- .../k8sd/types/cluster_config_annotations.go | 12 ++++ .../pkg/k8sd/types/cluster_config_convert.go | 3 + .../k8sd/types/cluster_config_convert_test.go | 8 ++- .../pkg/k8sd/types/cluster_config_merge.go | 3 + .../k8sd/types/cluster_config_merge_util.go | 27 +++++++++ .../types/cluster_config_merge_util_test.go | 31 +++++++++- src/k8s/pkg/k8sd/types/cluster_config_test.go | 14 ----- src/k8s/pkg/utils/mapstructure.go | 42 ++++++++++++++ 25 files changed, 300 insertions(+), 65 deletions(-) create mode 100644 src/k8s/pkg/k8sd/features/metrics-server/internal.go create mode 100644 src/k8s/pkg/k8sd/types/cluster_config_annotations.go delete mode 100644 src/k8s/pkg/k8sd/types/cluster_config_test.go diff --git a/src/k8s/api/v1/cluster_config.go b/src/k8s/api/v1/cluster_config.go index f904af487..76d97b7b7 100644 --- a/src/k8s/api/v1/cluster_config.go +++ b/src/k8s/api/v1/cluster_config.go @@ -29,6 +29,31 @@ type UserFacingClusterConfig struct { Gateway GatewayConfig `json:"gateway,omitempty" yaml:"gateway,omitempty"` MetricsServer MetricsServerConfig `json:"metrics-server,omitempty" yaml:"metrics-server,omitempty"` CloudProvider *string `json:"cloud-provider,omitempty" yaml:"cloud-provider,omitempty"` + Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"` +} + +func (c UserFacingClusterConfig) Empty() bool { + switch { + case c.Network != NetworkConfig{}: + return false + case c.DNS != DNSConfig{}: + return false + case c.Ingress != IngressConfig{}: + return false + case c.LoadBalancer != LoadBalancerConfig{}: + return false + case c.LocalStorage != LocalStorageConfig{}: + return false + case c.Gateway != GatewayConfig{}: + return false + case c.MetricsServer != MetricsServerConfig{}: + return false + case getField(c.CloudProvider) != "": + return false + case len(c.Annotations) > 0: + return false + } + return true } type DNSConfig struct { diff --git a/src/k8s/api/v1/types.go b/src/k8s/api/v1/types.go index 37d520d5d..1826256c2 100644 --- a/src/k8s/api/v1/types.go +++ b/src/k8s/api/v1/types.go @@ -153,8 +153,7 @@ func (c ClusterStatus) String() string { result.WriteString(c.datastoreToString()) // Config - var emptyConfig UserFacingClusterConfig - if c.Config != emptyConfig { + if !c.Config.Empty() { b, _ := yaml.Marshal(c.Config) result.WriteString(string(b)) } diff --git a/src/k8s/cmd/k8s/k8s_get.go b/src/k8s/cmd/k8s/k8s_get.go index 374f4cc95..3c2114948 100644 --- a/src/k8s/cmd/k8s/k8s_get.go +++ b/src/k8s/cmd/k8s/k8s_get.go @@ -47,6 +47,7 @@ func newGetCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { config.MetricsServer = apiv1.MetricsServerConfig{} config.CloudProvider = nil + config.Annotations = nil var key string if len(args) == 1 { diff --git a/src/k8s/cmd/k8s/k8s_set.go b/src/k8s/cmd/k8s/k8s_set.go index 5c2fc059b..496ea6e50 100644 --- a/src/k8s/cmd/k8s/k8s_set.go +++ b/src/k8s/cmd/k8s/k8s_set.go @@ -73,6 +73,7 @@ func newSetCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { } var knownSetKeys = map[string]struct{}{ + "annotations": {}, "cloud-provider": {}, "dns.cluster-domain": {}, "dns.enabled": {}, @@ -108,6 +109,8 @@ func updateConfigMapstructure(config *apiv1.UserFacingClusterConfig, arg string) DecodeHook: mapstructure.ComposeDecodeHookFunc( utils.YAMLToStringSliceHookFunc, utils.StringToFieldsSliceHookFunc(','), + utils.YAMLToStringMapHookFunc, + utils.StringToStringMapHookFunc, ), }) if err != nil { diff --git a/src/k8s/cmd/k8s/k8s_set_test.go b/src/k8s/cmd/k8s/k8s_set_test.go index 52c624840..361fd8fe8 100644 --- a/src/k8s/cmd/k8s/k8s_set_test.go +++ b/src/k8s/cmd/k8s/k8s_set_test.go @@ -79,6 +79,47 @@ func generateMapstructureTestCasesStringSlice(keyName string, fieldName string) } } +func generateMapstructureTestCasesMap(keyName string, fieldName string) []mapstructureTestCase { + return []mapstructureTestCase{ + { + val: fmt.Sprintf("%s=", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{})}, + }, + { + val: fmt.Sprintf("%s={}", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{})}, + }, + { + val: fmt.Sprintf("%s=k1=", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": ""})}, + }, + { + val: fmt.Sprintf("%s=k1=,k2=test", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "", "k2": "test"})}, + }, + { + val: fmt.Sprintf("%s=k1=v1", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1"})}, + }, + { + val: fmt.Sprintf("%s=k1=v1,k2=v2", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1", "k2": "v2"})}, + }, + { + val: fmt.Sprintf("%s={k1: v1}", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1"})}, + }, + { + val: fmt.Sprintf("%s={k1: v1, k2: v2}", keyName), + assertions: []types.GomegaMatcher{HaveField(fieldName, map[string]string{"k1": "v1", "k2": "v2"})}, + }, + { + val: fmt.Sprintf("%s=k1,k2", keyName), + expectErr: true, + }, + } +} + func generateMapstructureTestCasesString(keyName string, fieldName string) []mapstructureTestCase { return []mapstructureTestCase{ { @@ -139,6 +180,8 @@ func Test_updateConfigMapstructure(t *testing.T) { generateMapstructureTestCasesInt("load-balancer.bgp-local-asn", "LoadBalancer.BGPLocalASN"), generateMapstructureTestCasesInt("load-balancer.bgp-peer-asn", "LoadBalancer.BGPPeerASN"), generateMapstructureTestCasesInt("load-balancer.bgp-peer-port", "LoadBalancer.BGPPeerPort"), + + generateMapstructureTestCasesMap("annotations", "Annotations"), } { for _, tc := range tcs { t.Run(tc.val, func(t *testing.T) { diff --git a/src/k8s/pkg/k8sd/controllers/feature.go b/src/k8s/pkg/k8sd/controllers/feature.go index 1bd6ea086..e3dadb4f1 100644 --- a/src/k8s/pkg/k8sd/controllers/feature.go +++ b/src/k8s/pkg/k8sd/controllers/feature.go @@ -73,31 +73,31 @@ func (c *FeatureController) Run(ctx context.Context, getClusterConfig func(conte c.waitReady() go c.reconcileLoop(ctx, getClusterConfig, "network", c.triggerNetworkCh, c.reconciledNetworkCh, func(cfg types.ClusterConfig) error { - return features.Implementation.ApplyNetwork(ctx, c.snap, cfg.Network) + return features.Implementation.ApplyNetwork(ctx, c.snap, cfg.Network, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, "gateway", c.triggerGatewayCh, c.reconciledGatewayCh, func(cfg types.ClusterConfig) error { - return features.Implementation.ApplyGateway(ctx, c.snap, cfg.Gateway, cfg.Network) + return features.Implementation.ApplyGateway(ctx, c.snap, cfg.Gateway, cfg.Network, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, "ingress", c.triggerIngressCh, c.reconciledIngressCh, func(cfg types.ClusterConfig) error { - return features.Implementation.ApplyIngress(ctx, c.snap, cfg.Ingress, cfg.Network) + return features.Implementation.ApplyIngress(ctx, c.snap, cfg.Ingress, cfg.Network, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, "load balancer", c.triggerLoadBalancerCh, c.reconciledLoadBalancerCh, func(cfg types.ClusterConfig) error { - return features.Implementation.ApplyLoadBalancer(ctx, c.snap, cfg.LoadBalancer, cfg.Network) + return features.Implementation.ApplyLoadBalancer(ctx, c.snap, cfg.LoadBalancer, cfg.Network, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, "local storage", c.triggerLocalStorageCh, c.reconciledLocalStorageCh, func(cfg types.ClusterConfig) error { - return features.Implementation.ApplyLocalStorage(ctx, c.snap, cfg.LocalStorage) + return features.Implementation.ApplyLocalStorage(ctx, c.snap, cfg.LocalStorage, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, "metrics server", c.triggerMetricsServerCh, c.reconciledMetricsServerCh, func(cfg types.ClusterConfig) error { - return features.Implementation.ApplyMetricsServer(ctx, c.snap, cfg.MetricsServer) + return features.Implementation.ApplyMetricsServer(ctx, c.snap, cfg.MetricsServer, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, "DNS", c.triggerDNSCh, c.reconciledDNSCh, func(cfg types.ClusterConfig) error { - if dnsIP, err := features.Implementation.ApplyDNS(ctx, c.snap, cfg.DNS, cfg.Kubelet); err != nil { + if dnsIP, err := features.Implementation.ApplyDNS(ctx, c.snap, cfg.DNS, cfg.Kubelet, cfg.Annotations); err != nil { return fmt.Errorf("failed to apply DNS configuration: %w", err) } else if dnsIP != "" { if err := notifyDNSChangedIP(ctx, dnsIP); err != nil { diff --git a/src/k8s/pkg/k8sd/features/cilium/gateway.go b/src/k8s/pkg/k8sd/features/cilium/gateway.go index 3c91a125d..e3a7d7891 100644 --- a/src/k8s/pkg/k8sd/features/cilium/gateway.go +++ b/src/k8s/pkg/k8sd/features/cilium/gateway.go @@ -14,7 +14,7 @@ import ( // ApplyGateway will remove the Gateway API CRDs from the cluster and disable the GatewayAPI controllers on Cilium, when gateway.Enabled is false. // ApplyGateway will rollout restart the Cilium pods in case any Cilium configuration was changed. // ApplyGateway returns an error if anything fails. -func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network) error { +func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network, _ types.Annotations) error { m := snap.HelmClient() if _, err := m.Apply(ctx, chartGateway, helm.StatePresentOrDeleted(gateway.GetEnabled()), nil); err != nil { diff --git a/src/k8s/pkg/k8sd/features/cilium/ingress.go b/src/k8s/pkg/k8sd/features/cilium/ingress.go index 604c22490..56fd4bcf8 100644 --- a/src/k8s/pkg/k8sd/features/cilium/ingress.go +++ b/src/k8s/pkg/k8sd/features/cilium/ingress.go @@ -14,7 +14,7 @@ import ( // ApplyIngress will disable Cilium's ingress controller when ingress.Disabled is false. // ApplyIngress will rollout restart the Cilium pods in case any Cilium configuration was changed. // ApplyIngress returns an error if anything fails. -func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network) error { +func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network, _ types.Annotations) error { m := snap.HelmClient() var values map[string]any diff --git a/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go b/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go index e3c7cfbbe..a2e347187 100644 --- a/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go +++ b/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go @@ -15,7 +15,7 @@ import ( // ApplyLoadBalancer will disable L2 and BGP on Cilium, and remove any previously created CRs when loadbalancer.Enabled is false. // ApplyLoadBalancer will rollout restart the Cilium pods in case any Cilium configuration was changed. // ApplyLoadBalancer returns an error if anything fails. -func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error { +func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, _ types.Annotations) error { if !loadbalancer.GetEnabled() { if err := disableLoadBalancer(ctx, snap, network); err != nil { return fmt.Errorf("failed to disable LoadBalancer: %w", err) diff --git a/src/k8s/pkg/k8sd/features/cilium/network.go b/src/k8s/pkg/k8sd/features/cilium/network.go index e7d8051e3..68b310505 100644 --- a/src/k8s/pkg/k8sd/features/cilium/network.go +++ b/src/k8s/pkg/k8sd/features/cilium/network.go @@ -19,7 +19,7 @@ import ( // ApplyNetwork requires that bpf and cgroups2 are already mounted and available when running under strict snap confinement. If they are not, it will fail (since Cilium will not have the required permissions to mount them). // ApplyNetwork requires that `/sys` is mounted as a shared mount when running under classic snap confinement. This is to ensure that Cilium will be able to automatically mount bpf and cgroups2 on the pods. // ApplyNetwork returns an error if anything fails. -func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error { +func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ types.Annotations) error { m := snap.HelmClient() if !cfg.GetEnabled() { diff --git a/src/k8s/pkg/k8sd/features/coredns/coredns.go b/src/k8s/pkg/k8sd/features/coredns/coredns.go index 33ce60d75..08698ccd2 100644 --- a/src/k8s/pkg/k8sd/features/coredns/coredns.go +++ b/src/k8s/pkg/k8sd/features/coredns/coredns.go @@ -15,7 +15,7 @@ import ( // ApplyDNS will install or refresh CoreDNS if dns.Enabled is true. // ApplyDNS will return the ClusterIP address of the coredns service, if successful. // ApplyDNS returns an error if anything fails. -func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet) (string, error) { +func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet, _ types.Annotations) (string, error) { m := snap.HelmClient() if !dns.GetEnabled() { diff --git a/src/k8s/pkg/k8sd/features/interface.go b/src/k8s/pkg/k8sd/features/interface.go index f72fb05f6..45439f928 100644 --- a/src/k8s/pkg/k8sd/features/interface.go +++ b/src/k8s/pkg/k8sd/features/interface.go @@ -10,56 +10,56 @@ import ( // Interface abstracts the management of built-in Canonical Kubernetes features. type Interface interface { // ApplyDNS is used to configure the DNS feature on Canonical Kubernetes. - ApplyDNS(context.Context, snap.Snap, types.DNS, types.Kubelet) (string, error) + ApplyDNS(context.Context, snap.Snap, types.DNS, types.Kubelet, types.Annotations) (string, error) // ApplyNetwork is used to configure the network feature on Canonical Kubernetes. - ApplyNetwork(context.Context, snap.Snap, types.Network) error + ApplyNetwork(context.Context, snap.Snap, types.Network, types.Annotations) error // ApplyLoadBalancer is used to configure the load-balancer feature on Canonical Kubernetes. - ApplyLoadBalancer(context.Context, snap.Snap, types.LoadBalancer, types.Network) error + ApplyLoadBalancer(context.Context, snap.Snap, types.LoadBalancer, types.Network, types.Annotations) error // ApplyIngress is used to configure the ingress controller feature on Canonical Kubernetes. - ApplyIngress(context.Context, snap.Snap, types.Ingress, types.Network) error + ApplyIngress(context.Context, snap.Snap, types.Ingress, types.Network, types.Annotations) error // ApplyGateway is used to configure the gateway feature on Canonical Kubernetes. - ApplyGateway(context.Context, snap.Snap, types.Gateway, types.Network) error + ApplyGateway(context.Context, snap.Snap, types.Gateway, types.Network, types.Annotations) error // ApplyMetricsServer is used to configure the metrics-server feature on Canonical Kubernetes. - ApplyMetricsServer(context.Context, snap.Snap, types.MetricsServer) error + ApplyMetricsServer(context.Context, snap.Snap, types.MetricsServer, types.Annotations) error // ApplyLocalStorage is used to configure the Local Storage feature on Canonical Kubernetes. - ApplyLocalStorage(context.Context, snap.Snap, types.LocalStorage) error + ApplyLocalStorage(context.Context, snap.Snap, types.LocalStorage, types.Annotations) error } // implementation implements Interface. type implementation struct { - applyDNS func(context.Context, snap.Snap, types.DNS, types.Kubelet) (string, error) - applyNetwork func(context.Context, snap.Snap, types.Network) error - applyLoadBalancer func(context.Context, snap.Snap, types.LoadBalancer, types.Network) error - applyIngress func(context.Context, snap.Snap, types.Ingress, types.Network) error - applyGateway func(context.Context, snap.Snap, types.Gateway, types.Network) error - applyMetricsServer func(context.Context, snap.Snap, types.MetricsServer) error - applyLocalStorage func(context.Context, snap.Snap, types.LocalStorage) error + applyDNS func(context.Context, snap.Snap, types.DNS, types.Kubelet, types.Annotations) (string, error) + applyNetwork func(context.Context, snap.Snap, types.Network, types.Annotations) error + applyLoadBalancer func(context.Context, snap.Snap, types.LoadBalancer, types.Network, types.Annotations) error + applyIngress func(context.Context, snap.Snap, types.Ingress, types.Network, types.Annotations) error + applyGateway func(context.Context, snap.Snap, types.Gateway, types.Network, types.Annotations) error + applyMetricsServer func(context.Context, snap.Snap, types.MetricsServer, types.Annotations) error + applyLocalStorage func(context.Context, snap.Snap, types.LocalStorage, types.Annotations) error } -func (i *implementation) ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet) (string, error) { - return i.applyDNS(ctx, snap, dns, kubelet) +func (i *implementation) ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet, annotations types.Annotations) (string, error) { + return i.applyDNS(ctx, snap, dns, kubelet, annotations) } -func (i *implementation) ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error { - return i.applyNetwork(ctx, snap, cfg) +func (i *implementation) ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annotations types.Annotations) error { + return i.applyNetwork(ctx, snap, cfg, annotations) } -func (i *implementation) ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error { - return i.applyLoadBalancer(ctx, snap, loadbalancer, network) +func (i *implementation) ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, annotations types.Annotations) error { + return i.applyLoadBalancer(ctx, snap, loadbalancer, network, annotations) } -func (i *implementation) ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network) error { - return i.applyIngress(ctx, snap, ingress, network) +func (i *implementation) ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network, annotations types.Annotations) error { + return i.applyIngress(ctx, snap, ingress, network, annotations) } -func (i *implementation) ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network) error { - return i.applyGateway(ctx, snap, gateway, network) +func (i *implementation) ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network, annotations types.Annotations) error { + return i.applyGateway(ctx, snap, gateway, network, annotations) } -func (i *implementation) ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer) error { - return i.applyMetricsServer(ctx, snap, cfg) +func (i *implementation) ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer, annotations types.Annotations) error { + return i.applyMetricsServer(ctx, snap, cfg, annotations) } -func (i *implementation) ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage) error { - return i.applyLocalStorage(ctx, snap, cfg) +func (i *implementation) ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage, annotations types.Annotations) error { + return i.applyLocalStorage(ctx, snap, cfg, annotations) } diff --git a/src/k8s/pkg/k8sd/features/localpv/localpv.go b/src/k8s/pkg/k8sd/features/localpv/localpv.go index a4a1538c2..171971e31 100644 --- a/src/k8s/pkg/k8sd/features/localpv/localpv.go +++ b/src/k8s/pkg/k8sd/features/localpv/localpv.go @@ -11,7 +11,7 @@ import ( // ApplyLocalStorage deploys the rawfile-localpv CSI driver on the cluster based on the given configuration, when cfg.Enabled is true. // ApplyLocalStorage removes the rawfile-localpv when cfg.Enabled is false. // ApplyLocalStorage returns an error if anything fails. -func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage) error { +func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage, _ types.Annotations) error { m := snap.HelmClient() values := map[string]any{ diff --git a/src/k8s/pkg/k8sd/features/metrics-server/internal.go b/src/k8s/pkg/k8sd/features/metrics-server/internal.go new file mode 100644 index 000000000..2300b7139 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metrics-server/internal.go @@ -0,0 +1,29 @@ +package metrics_server + +import "github.com/canonical/k8s/pkg/k8sd/types" + +const ( + annotationImageRepo = "k8sd/v1alpha1/metrics-server/image-repo" + annotationImageTag = "k8sd/v1alpha1/metrics-server/image-tag" +) + +type config struct { + imageRepo string + imageTag string +} + +func internalConfig(annotations types.Annotations) config { + config := config{ + imageRepo: imageRepo, + imageTag: imageTag, + } + + if v, ok := annotations.Get(annotationImageRepo); ok { + config.imageRepo = v + } + if v, ok := annotations.Get(annotationImageTag); ok { + config.imageTag = v + } + + return config +} diff --git a/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go b/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go index 96d6ff89f..224db024e 100644 --- a/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go +++ b/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go @@ -11,13 +11,15 @@ import ( // ApplyMetricsServer deploys metrics-server when cfg.Enabled is true. // ApplyMetricsServer removes metrics-server when cfg.Enabled is false. // ApplyMetricsServer returns an error if anything fails. -func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer) error { +func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer, annotations types.Annotations) error { m := snap.HelmClient() + config := internalConfig(annotations) + values := map[string]any{ "image": map[string]any{ - "repository": imageRepo, - "tag": imageTag, + "repository": config.imageRepo, + "tag": config.imageTag, }, "securityContext": map[string]any{ // ROCKs with Pebble as the entrypoint do not work with this option. diff --git a/src/k8s/pkg/k8sd/features/metrics-server/metrics_server_test.go b/src/k8s/pkg/k8sd/features/metrics-server/metrics_server_test.go index 855bdd7bb..c537211c4 100644 --- a/src/k8s/pkg/k8sd/features/metrics-server/metrics_server_test.go +++ b/src/k8s/pkg/k8sd/features/metrics-server/metrics_server_test.go @@ -43,7 +43,7 @@ func TestApplyMetricsServer(t *testing.T) { }, } - err := metrics_server.ApplyMetricsServer(context.Background(), s, tc.config) + err := metrics_server.ApplyMetricsServer(context.Background(), s, tc.config, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(h.ApplyCalledWith).To(ConsistOf(SatisfyAll( @@ -53,4 +53,29 @@ func TestApplyMetricsServer(t *testing.T) { ))) }) } + + t.Run("Annotations", func(t *testing.T) { + g := NewWithT(t) + h := &helmmock.Mock{} + s := &snapmock.Snap{ + Mock: snapmock.Mock{ + HelmClient: h, + }, + } + + cfg := types.MetricsServer{ + Enabled: utils.Pointer(true), + } + annotations := types.Annotations{ + "k8sd/v1alpha1/metrics-server/image-repo": "custom-image", + "k8sd/v1alpha1/metrics-server/image-tag": "custom-tag", + } + + err := metrics_server.ApplyMetricsServer(context.Background(), s, cfg, annotations) + g.Expect(err).To(BeNil()) + g.Expect(h.ApplyCalledWith).To(ConsistOf(HaveField("Values", HaveKeyWithValue("image", SatisfyAll( + HaveKeyWithValue("repository", "custom-image"), + HaveKeyWithValue("tag", "custom-tag"), + ))))) + }) } diff --git a/src/k8s/pkg/k8sd/types/cluster_config.go b/src/k8s/pkg/k8sd/types/cluster_config.go index 39af14a3c..51109bafa 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config.go +++ b/src/k8s/pkg/k8sd/types/cluster_config.go @@ -14,6 +14,6 @@ type ClusterConfig struct { Gateway Gateway `json:"gateway,omitempty"` LocalStorage LocalStorage `json:"local-storage,omitempty"` MetricsServer MetricsServer `json:"metrics-server,omitempty"` -} -func (c ClusterConfig) Empty() bool { return c == ClusterConfig{} } + Annotations Annotations `json:"annotations,omitempty"` +} diff --git a/src/k8s/pkg/k8sd/types/cluster_config_annotations.go b/src/k8s/pkg/k8sd/types/cluster_config_annotations.go new file mode 100644 index 000000000..76fcd11b2 --- /dev/null +++ b/src/k8s/pkg/k8sd/types/cluster_config_annotations.go @@ -0,0 +1,12 @@ +package types + +type Annotations map[string]string + +func (a Annotations) Get(key string) (value string, exists bool) { + if a == nil { + return "", false + } + + v, ok := a[key] + return v, ok +} diff --git a/src/k8s/pkg/k8sd/types/cluster_config_convert.go b/src/k8s/pkg/k8sd/types/cluster_config_convert.go index 8202228f7..050b15481 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_convert.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_convert.go @@ -2,6 +2,7 @@ package types import ( "fmt" + apiv1 "github.com/canonical/k8s/api/v1" "github.com/canonical/k8s/pkg/utils" ) @@ -80,6 +81,7 @@ func ClusterConfigFromUserFacing(u apiv1.UserFacingClusterConfig) (ClusterConfig } return ClusterConfig{ + Annotations: Annotations(u.Annotations), Kubelet: Kubelet{ ClusterDNS: u.DNS.ServiceIP, ClusterDomain: u.DNS.ClusterDomain, @@ -165,5 +167,6 @@ func (c ClusterConfig) ToUserFacing() apiv1.UserFacingClusterConfig { Enabled: c.Gateway.Enabled, }, CloudProvider: c.Kubelet.CloudProvider, + Annotations: map[string]string(c.Annotations), } } diff --git a/src/k8s/pkg/k8sd/types/cluster_config_convert_test.go b/src/k8s/pkg/k8sd/types/cluster_config_convert_test.go index 4d6abbe18..26bb7c2ae 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_convert_test.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_convert_test.go @@ -1,11 +1,11 @@ package types_test import ( - "github.com/canonical/k8s/pkg/utils" "testing" apiv1 "github.com/canonical/k8s/api/v1" "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -80,6 +80,9 @@ func TestClusterConfigFromBootstrapConfig(t *testing.T) { name: "Full", bootstrap: apiv1.BootstrapConfig{ ClusterConfig: apiv1.UserFacingClusterConfig{ + Annotations: map[string]string{ + "key": "value", + }, Network: apiv1.NetworkConfig{ Enabled: utils.Pointer(true), }, @@ -157,6 +160,9 @@ func TestClusterConfigFromBootstrapConfig(t *testing.T) { MetricsServer: types.MetricsServer{ Enabled: utils.Pointer(true), }, + Annotations: types.Annotations{ + "key": "value", + }, }, }, { diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge.go b/src/k8s/pkg/k8sd/types/cluster_config_merge.go index 208c57da7..fb1840d1a 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge.go @@ -143,6 +143,9 @@ func MergeClusterConfig(existing ClusterConfig, new ClusterConfig) (ClusterConfi } } + // merge annotations + config.Annotations = mergeAnnotationsField(existing.Annotations, new.Annotations) + if err := config.Validate(); err != nil { return ClusterConfig{}, fmt.Errorf("updated cluster configuration is not valid: %w", err) } diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge_util.go b/src/k8s/pkg/k8sd/types/cluster_config_merge_util.go index f4d839faf..ab5ace7c2 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge_util.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge_util.go @@ -39,6 +39,33 @@ func mergeSliceField[T comparable](old *[]T, new *[]T, allowChange bool) (*[]T, return new, nil } +func mergeAnnotationsField(old Annotations, new Annotations) Annotations { + // old value is not set, use new + if old == nil { + return new + } + // new value is not set, use old + if new == nil { + return old + } + + // merge fields, start from old and then add new + // if any field is set to "-", delete it from the final result + m := make(map[string]string, len(old)+len(new)) + for k, v := range old { + m[k] = v + } + for k, v := range new { + if v == "-" { + delete(m, k) + } else { + m[k] = v + } + } + + return Annotations(m) +} + func getField[T any](val *T) T { if val != nil { return *val diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge_util_test.go b/src/k8s/pkg/k8sd/types/cluster_config_merge_util_test.go index 7b1ee8909..251826d1c 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge_util_test.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge_util_test.go @@ -1,9 +1,9 @@ package types import ( - "github.com/canonical/k8s/pkg/utils" "testing" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) @@ -140,3 +140,32 @@ func Test_mergeSliceField(t *testing.T) { } }) } + +func Test_mergeAnnotationsField(t *testing.T) { + for _, tc := range []struct { + name string + old Annotations + new Annotations + expectErr bool + expectVal Annotations + }{ + {name: "keep-empty"}, + {name: "set-empty", new: Annotations{"k1": "v1"}, expectVal: Annotations{"k1": "v1"}}, + {name: "keep-old", old: Annotations{"k1": "v1"}, expectVal: Annotations{"k1": "v1"}}, + {name: "update", old: Annotations{"k1": "v1"}, new: Annotations{"k1": "v2"}, expectVal: Annotations{"k1": "v2"}}, + {name: "update-add-fields", old: Annotations{"k1": "v1"}, new: Annotations{"k1": "v2", "k2": "v2"}, expectVal: Annotations{"k1": "v2", "k2": "v2"}}, + {name: "delete-fields", old: Annotations{"k1": "v1", "k2": "v2"}, new: Annotations{"k1": "-"}, expectVal: Annotations{"k2": "v2"}}, + {name: "delete-last-field", old: Annotations{"k1": "v1"}, new: Annotations{"k1": "-"}, expectVal: Annotations{}}, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + result := mergeAnnotationsField(tc.old, tc.new) + if tc.expectVal != nil { + g.Expect(result).To(Equal(tc.expectVal)) + } else { + g.Expect(result).To(BeNil()) + } + }) + } +} diff --git a/src/k8s/pkg/k8sd/types/cluster_config_test.go b/src/k8s/pkg/k8sd/types/cluster_config_test.go deleted file mode 100644 index 06e21a8cb..000000000 --- a/src/k8s/pkg/k8sd/types/cluster_config_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package types_test - -import ( - "testing" - - "github.com/canonical/k8s/pkg/k8sd/types" - . "github.com/onsi/gomega" -) - -func TestClusterConfigEmpty(t *testing.T) { - g := NewWithT(t) - - g.Expect(types.ClusterConfig{}.Empty()).To(BeTrue()) -} diff --git a/src/k8s/pkg/utils/mapstructure.go b/src/k8s/pkg/utils/mapstructure.go index c6ecf1fbc..60e650b45 100644 --- a/src/k8s/pkg/utils/mapstructure.go +++ b/src/k8s/pkg/utils/mapstructure.go @@ -42,3 +42,45 @@ func StringToFieldsSliceHookFunc(r rune) mapstructure.DecodeHookFunc { return strings.FieldsFunc(raw, func(this rune) bool { return this == r || unicode.IsSpace(this) }), nil } } + +// YAMLToStringMapHookFunc returns a mapstructure.DecodeHookFunc that converts string to map[string]string by parsing YAML. +func YAMLToStringMapHookFunc(f reflect.Kind, t reflect.Kind, data interface{}) (interface{}, error) { + if f != reflect.String || t != reflect.Map { + return data, nil + } + + if data.(string) == "" { + return map[string]string{}, nil + } + + var result map[string]string + if err := yaml.Unmarshal([]byte(data.(string)), &result); err != nil { + return data, nil + } + + return result, nil +} + +// StringToStringMapHookFunc is like StringToFieldsSliceHookFunc(). It splits map entries on ',' and then key-value pairs on '='. +func StringToStringMapHookFunc(f reflect.Kind, t reflect.Kind, data interface{}) (interface{}, error) { + if f != reflect.String || t != reflect.Map { + return data, nil + } + + raw := data.(string) + if raw == "" { + return map[string]string{}, nil + } + + fields := strings.FieldsFunc(raw, func(this rune) bool { return this == ',' || unicode.IsSpace(this) }) + result := make(map[string]string, len(fields)) + for _, kv := range strings.FieldsFunc(raw, func(this rune) bool { return this == ',' || unicode.IsSpace(this) }) { + parts := strings.SplitN(kv, "=", 2) + if len(parts) < 2 { + return data, nil + } + result[parts[0]] = parts[1] + } + + return result, nil +}