From 28a95f9f0df3cc869a3f005fce937832bac0394d Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:24:17 -0500 Subject: [PATCH 1/4] Simplify labels to support service, inject or disable. --- api/v1/config.go | 95 ++++++++++++++++++++++------------------------- api/v1/webhook.go | 39 +++++++++++++------ 2 files changed, 72 insertions(+), 62 deletions(-) diff --git a/api/v1/config.go b/api/v1/config.go index 2a819f0..ab57580 100644 --- a/api/v1/config.go +++ b/api/v1/config.go @@ -9,20 +9,22 @@ import ( "sigs.k8s.io/yaml" ) -const GATEWAY_ANNOTATIONS_CONFIGMAP = "qtap-operator-gateway-pod-annotations-configmap" -const INJECTION_ANNOTATIONS_CONFIGMAP = "qtap-operator-injection-pod-annotations-configmap" +const SERVICE_ANNOTATIONS_CONFIGMAP = "qtap-operator-service-pod-annotations-configmap" +const INJECT_ANNOTATIONS_CONFIGMAP = "qtap-operator-inject-pod-annotations-configmap" const NAMESPACE_EGRESS_LABEL = "qpoint-egress" -const NAMESPACE_INJECTION_LABEL = "qpoint-injection" -const POD_EGRESS_ANNOTATION = "qpoint.io/egress" -const POD_INJECTION_LABEL = "sidecar.qpoint.io/inject" -const ENABLED = "enabled" -const DISABLED = "disabled" -const TRUE = "true" -const FALSE = "false" +const POD_EGRESS_LABEL = "qpoint.io/egress" + +type EgressType string + +const ( + EgressType_UNDEFINED EgressType = "undefined" + EgressType_DISABLED EgressType = "disabled" + EgressType_SERVICE EgressType = "service" + EgressType_INJECT EgressType = "inject" +) type Config struct { - EnabledEgress bool // Egress routing is enabled - EnabledInjection bool // Sidecar injection is enabled + EgressType EgressType InjectCa bool Namespace string OperatorNamespace string @@ -33,8 +35,8 @@ type Config struct { } // Config scenarios: -// a) Egress routing is enabled and gateway is disabled via the namespace label or pod annotation. This means that the egress traffic is being routed to the qtap service running somewhere else in the cluster. -// b) Egress routing is enabled and gateway is enabled via the namespace label or pod annotation. This means that the egress traffic is being routed through the qtap sidecar proxy. +// a) Egress routing is enabled and gateway is disabled via the namespace label or pod label. This means that the egress traffic is being routed to the qtap service running somewhere else in the cluster. +// b) Egress routing is enabled and gateway is enabled via the namespace label or pod label. This means that the egress traffic is being routed through the qtap sidecar proxy. // // Egress routing is always controlled by the qtap-init container which manipulates iptables rules for routing egress traffic to one of the above qtap setups. @@ -45,50 +47,41 @@ func (c *Config) Init(pod *corev1.Pod) error { return fmt.Errorf("fetching namespace '%s' from the api: %w", c.Namespace, err) } - // if the namespace is labeled for egress, then we enable. A pod annotation override will be checked below - if namespace.Labels[NAMESPACE_EGRESS_LABEL] == ENABLED { - c.EnabledEgress = true - } else if namespace.Labels[NAMESPACE_EGRESS_LABEL] == DISABLED { - c.EnabledEgress = false - } - - // check to see if an annotation is set on the pod to enable or disable egress while also verifying - // if it was enabled for the namespace but needs to be disabled for the pod. If the annotation doesn't exist nothing else needs to be checked - if egress, exists := pod.Annotations[POD_EGRESS_ANNOTATION]; exists { - if c.EnabledEgress && egress == DISABLED { - c.EnabledEgress = false - } - - if !c.EnabledEgress && egress == ENABLED { - c.EnabledEgress = true - } + namespaceEgressType := EgressType_UNDEFINED + configMapName := "" + + switch v := namespace.Labels[NAMESPACE_EGRESS_LABEL]; EgressType(v) { + case EgressType_DISABLED: + return nil + case EgressType_SERVICE: + namespaceEgressType = EgressType_SERVICE + configMapName = SERVICE_ANNOTATIONS_CONFIGMAP + case EgressType_INJECT: + namespaceEgressType = EgressType_INJECT + configMapName = INJECT_ANNOTATIONS_CONFIGMAP } - // if we're enabled - if c.EnabledEgress { - configMapName := GATEWAY_ANNOTATIONS_CONFIGMAP + podEgressType := EgressType_UNDEFINED - // if the namespace is labeled for injection, then we enable. A pod annotation override will be checked below - if namespace.Labels[NAMESPACE_INJECTION_LABEL] == ENABLED { - c.EnabledInjection = true - configMapName = INJECTION_ANNOTATIONS_CONFIGMAP - } else if namespace.Labels[NAMESPACE_INJECTION_LABEL] == DISABLED { - c.EnabledInjection = false - } + // order matters as pods override namespaces - // check to see if an label is set on the pod to enable or disable injection while also verifying - // if it was enabled for the namespace but needs to be disabled for the pod. If the label doesn't exist nothing else needs to be checked - if inject, exists := pod.Labels[POD_INJECTION_LABEL]; exists { - if c.EnabledInjection && inject == FALSE { - c.EnabledInjection = false - } + switch v := pod.Labels[POD_EGRESS_LABEL]; EgressType(v) { + case EgressType_DISABLED: + return nil + case EgressType_SERVICE: + podEgressType = EgressType_SERVICE + configMapName = SERVICE_ANNOTATIONS_CONFIGMAP + case EgressType_INJECT: + podEgressType = EgressType_INJECT + configMapName = INJECT_ANNOTATIONS_CONFIGMAP + } - if !c.EnabledInjection && inject == TRUE { - c.EnabledInjection = true - configMapName = INJECTION_ANNOTATIONS_CONFIGMAP - } - } + // egress is undefined for the entire namespace (regardless of what the pod label says) or pod and thus return immediately + if namespaceEgressType == EgressType_UNDEFINED && podEgressType == EgressType_UNDEFINED { + return nil + } + if configMapName != "" { // let's fetch the default settings in the configmap configMap := &corev1.ConfigMap{} if err := c.Client.Get(c.Ctx, client.ObjectKey{Name: configMapName, Namespace: c.OperatorNamespace}, configMap); err != nil { diff --git a/api/v1/webhook.go b/api/v1/webhook.go index ff4edb3..9c7c9f3 100644 --- a/api/v1/webhook.go +++ b/api/v1/webhook.go @@ -35,10 +35,9 @@ func (w *Webhook) Handle(ctx context.Context, req admission.Request) admission.R // initialize a config with defaults config := &Config{ + EgressType: EgressType_UNDEFINED, Namespace: req.Namespace, OperatorNamespace: w.Namespace, - EnabledEgress: false, - EnabledInjection: false, InjectCa: false, Client: w.ApiClient, Ctx: ctx, @@ -50,8 +49,11 @@ func (w *Webhook) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } - if config.EnabledEgress { - webhookLog.Info("Qpoint egress enabled, mutating...") + switch v := config.EgressType; EgressType(v) { + case EgressType_SERVICE: + // for this case the pod is mutated for service egress + + webhookLog.Info("Qpoint egress to service enabled, mutating...") // mutate the pod to include egress through the gateway if err := MutateEgress(pod, config); err != nil { @@ -70,21 +72,36 @@ func (w *Webhook) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } } + case EgressType_INJECT: + // for this case the pod is mutated for sidecar egress - } else { - webhookLog.Info("Qpoint egress not enabled, ignoring...") - } + webhookLog.Info("Qpoint egress to sidecar enabled, mutating...") - if config.EnabledInjection { - webhookLog.Info("Qpoint injection enabled, mutating...") + // mutate the pod to include egress through the gateway + if err := MutateEgress(pod, config); err != nil { + webhookLog.Error(err, "failed to mutate pod for egress") + return admission.Errored(http.StatusInternalServerError, err) + } // mutate the pod to include the sidecar if err := MutateInjection(pod, config); err != nil { webhookLog.Error(err, "failed to mutate pod for injection") return admission.Errored(http.StatusInternalServerError, err) } - } else { - webhookLog.Info("Qpoint injection not enabled, ignoring...") + + if config.InjectCa { + if err := EnsureAssetsInNamespace(config); err != nil { + webhookLog.Error(err, "failed to add assets to namespace for ca injection") + return admission.Errored(http.StatusInternalServerError, err) + } + + if err := MutateCaInjection(pod, config); err != nil { + webhookLog.Error(err, "failed to mutate pod for ca injection") + return admission.Errored(http.StatusInternalServerError, err) + } + } + default: + webhookLog.Info("Qpoint egress not enabled, ignoring...") } marshaledPod, err := json.Marshal(pod) From c5a99d0fa9fa52bf58e9cdbd5a7b85ece7ea58c5 Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:29:51 -0500 Subject: [PATCH 2/4] Update the config maps. --- config/webhook/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/webhook/configmap.yaml b/config/webhook/configmap.yaml index 56a4482..a1db2c1 100644 --- a/config/webhook/configmap.yaml +++ b/config/webhook/configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: gateway-pod-annotations-configmap + name: service-pod-annotations-configmap namespace: system data: annotations.yaml: | @@ -13,7 +13,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: injection-pod-annotations-configmap + name: inject-pod-annotations-configmap namespace: system data: annotations.yaml: | From 0ea6e9f550b77c21a7a405261df4993ae1eecb69 Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Fri, 1 Dec 2023 23:49:25 -0500 Subject: [PATCH 3/4] Set egress type on config. --- api/v1/config.go | 7 +++++++ api/v1/webhook.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/api/v1/config.go b/api/v1/config.go index ab57580..b26e0a3 100644 --- a/api/v1/config.go +++ b/api/v1/config.go @@ -52,11 +52,14 @@ func (c *Config) Init(pod *corev1.Pod) error { switch v := namespace.Labels[NAMESPACE_EGRESS_LABEL]; EgressType(v) { case EgressType_DISABLED: + c.EgressType = EgressType_DISABLED return nil case EgressType_SERVICE: + c.EgressType = EgressType_SERVICE namespaceEgressType = EgressType_SERVICE configMapName = SERVICE_ANNOTATIONS_CONFIGMAP case EgressType_INJECT: + c.EgressType = EgressType_INJECT namespaceEgressType = EgressType_INJECT configMapName = INJECT_ANNOTATIONS_CONFIGMAP } @@ -67,17 +70,21 @@ func (c *Config) Init(pod *corev1.Pod) error { switch v := pod.Labels[POD_EGRESS_LABEL]; EgressType(v) { case EgressType_DISABLED: + c.EgressType = EgressType_DISABLED return nil case EgressType_SERVICE: + c.EgressType = EgressType_SERVICE podEgressType = EgressType_SERVICE configMapName = SERVICE_ANNOTATIONS_CONFIGMAP case EgressType_INJECT: + c.EgressType = EgressType_INJECT podEgressType = EgressType_INJECT configMapName = INJECT_ANNOTATIONS_CONFIGMAP } // egress is undefined for the entire namespace (regardless of what the pod label says) or pod and thus return immediately if namespaceEgressType == EgressType_UNDEFINED && podEgressType == EgressType_UNDEFINED { + c.EgressType = EgressType_UNDEFINED return nil } diff --git a/api/v1/webhook.go b/api/v1/webhook.go index 9c7c9f3..82f1200 100644 --- a/api/v1/webhook.go +++ b/api/v1/webhook.go @@ -100,6 +100,8 @@ func (w *Webhook) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } } + case EgressType_DISABLED: + webhookLog.Info("Qpoint egress disabled, ignoring...") default: webhookLog.Info("Qpoint egress not enabled, ignoring...") } From 281928bc0d04e6f195eb3db09aebd7758a70e29c Mon Sep 17 00:00:00 2001 From: Marc Barry <4965634+marc-barry@users.noreply.github.com> Date: Sat, 2 Dec 2023 00:31:34 -0500 Subject: [PATCH 4/4] Use disable instead of disabled. --- api/v1/config.go | 10 +++++----- api/v1/webhook.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1/config.go b/api/v1/config.go index b26e0a3..0fc743b 100644 --- a/api/v1/config.go +++ b/api/v1/config.go @@ -18,7 +18,7 @@ type EgressType string const ( EgressType_UNDEFINED EgressType = "undefined" - EgressType_DISABLED EgressType = "disabled" + EgressType_DISABLE EgressType = "disable" EgressType_SERVICE EgressType = "service" EgressType_INJECT EgressType = "inject" ) @@ -51,8 +51,8 @@ func (c *Config) Init(pod *corev1.Pod) error { configMapName := "" switch v := namespace.Labels[NAMESPACE_EGRESS_LABEL]; EgressType(v) { - case EgressType_DISABLED: - c.EgressType = EgressType_DISABLED + case EgressType_DISABLE: + c.EgressType = EgressType_DISABLE return nil case EgressType_SERVICE: c.EgressType = EgressType_SERVICE @@ -69,8 +69,8 @@ func (c *Config) Init(pod *corev1.Pod) error { // order matters as pods override namespaces switch v := pod.Labels[POD_EGRESS_LABEL]; EgressType(v) { - case EgressType_DISABLED: - c.EgressType = EgressType_DISABLED + case EgressType_DISABLE: + c.EgressType = EgressType_DISABLE return nil case EgressType_SERVICE: c.EgressType = EgressType_SERVICE diff --git a/api/v1/webhook.go b/api/v1/webhook.go index 82f1200..9ffab4d 100644 --- a/api/v1/webhook.go +++ b/api/v1/webhook.go @@ -100,7 +100,7 @@ func (w *Webhook) Handle(ctx context.Context, req admission.Request) admission.R return admission.Errored(http.StatusInternalServerError, err) } } - case EgressType_DISABLED: + case EgressType_DISABLE: webhookLog.Info("Qpoint egress disabled, ignoring...") default: webhookLog.Info("Qpoint egress not enabled, ignoring...")