Introduce sidecar injection for the gateway

README.md

@@ -12,7 +12,6 @@ helm install qtap-operator qpoint/qtap-operator --namespace qpoint
 
 Manual
 
-
 The pre-built Docker container can be found at us-docker.pkg.dev/qpoint-edge/public/kubernetes-qtap-operator and uses the tag for the release <https://github.com/qpoint-io/kubernetes-qtap-operator/releases>. See <https://github.com/qpoint-io/helm-charts/blob/main/charts/qtap-operator/templates/deployment.yaml> for an example of a Deployment.
 
 ## Configure Egress

api/v1/config.go

@@ -9,10 +9,20 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-const ANNOTATIONS_CONFIGMAP = "qtap-operator-default-pod-annotations-configmap"
+const GATEWAY_ANNOTATIONS_CONFIGMAP = "qtap-operator-gateway-pod-annotations-configmap"
+const INJECTION_ANNOTATIONS_CONFIGMAP = "qtap-operator-injection-pod-annotations-configmap"
+const NAMESPACE_EGRESS_LABEL = "qpoint-egress"
+const NAMESPACE_INJECTION_LABEL = "qpoint-injection"
+const POD_EGRESS_ANNOTATION = "qpoint.io/egress"
+const POD_INJECTION_LABEL = "sidecar.qpoint.io/inject"
+const ENABLED = "enabled"
+const DISABLED = "disabled"
+const TRUE = "true"
+const FALSE = "false"
 
 type Config struct {
-	Enabled           bool
+	EnabledEgress     bool // Egress routing is enabled
+	EnabledInjection  bool // Sidecar injection is enabled
 	InjectCa          bool
 	Namespace         string
 	OperatorNamespace string
@@ -22,39 +32,67 @@ type Config struct {
 	annotations map[string]string
 }
 
+// Config scenarios:
+// a) Egress routing is enabled and gateway is disabled via the namespace label or pod annotation. This means that the egress traffic is being routed to the qtap service running somewhere else in the cluster.
+// b) Egress routing is enabled and gateway is enabled via the namespace label or pod annotation. This means that the egress traffic is being routed through the qtap sidecar proxy.
+//
+// Egress routing is always controlled by the qtap-init container which manipulates iptables rules for routing egress traffic to one of the above qtap setups.
+
 func (c *Config) Init(pod *corev1.Pod) error {
 	// first check if the namespace has the label. If it does then assume that egress is enabled
 	namespace := &corev1.Namespace{}
 	if err := c.Client.Get(c.Ctx, client.ObjectKey{Name: c.Namespace}, namespace); err != nil {
 		return fmt.Errorf("fetching namespace '%s' from the api: %w", c.Namespace, err)
 	}
 
-	// if the namespace is labeled, then we enable. A pod annotation override will be checked below
-	if namespace.Labels["qpoint-egress"] == "enabled" {
-		c.Enabled = true
+	// if the namespace is labeled for egress, then we enable. A pod annotation override will be checked below
+	if namespace.Labels[NAMESPACE_EGRESS_LABEL] == ENABLED {
+		c.EnabledEgress = true
+	} else if namespace.Labels[NAMESPACE_EGRESS_LABEL] == DISABLED {
+		c.EnabledEgress = false
 	}
 
 	// check to see if an annotation is set on the pod to enable or disable egress while also verifying
-	// if it was enabled for the namespace but needs to be disabled for the pod
-	egress, exists := pod.Annotations["qpoint.io/egress"]
-
-	// if the annotation doesn't exist nothing else needs to be checked
-	if exists {
-		if c.Enabled && egress != "enabled" {
-			c.Enabled = false
+	// if it was enabled for the namespace but needs to be disabled for the pod. If the annotation doesn't exist nothing else needs to be checked
+	if egress, exists := pod.Annotations[POD_EGRESS_ANNOTATION]; exists {
+		if c.EnabledEgress && egress == DISABLED {
+			c.EnabledEgress = false
 		}
 
-		if !c.Enabled && egress == "enabled" {
-			c.Enabled = true
+		if !c.EnabledEgress && egress == ENABLED {
+			c.EnabledEgress = true
 		}
 	}
 
 	// if we're enabled
-	if c.Enabled {
+	if c.EnabledEgress {
+		configMapName := GATEWAY_ANNOTATIONS_CONFIGMAP
+
+		// if the namespace is labeled for injection, then we enable. A pod annotation override will be checked below
+		if namespace.Labels[NAMESPACE_INJECTION_LABEL] == ENABLED {
+			c.EnabledInjection = true
+			configMapName = INJECTION_ANNOTATIONS_CONFIGMAP
+		} else if namespace.Labels[NAMESPACE_INJECTION_LABEL] == DISABLED {
+			c.EnabledInjection = false
+		}
+
+		// check to see if an label is set on the pod to enable or disable injection while also verifying
+		// if it was enabled for the namespace but needs to be disabled for the pod. If the label doesn't exist nothing else needs to be checked
+		if inject, exists := pod.Labels[POD_INJECTION_LABEL]; exists {
+			if c.EnabledInjection && inject == FALSE {
+				c.EnabledInjection = false
+			}
+
+			if !c.EnabledInjection && inject == TRUE {
+				c.EnabledInjection = true
+				configMapName = INJECTION_ANNOTATIONS_CONFIGMAP
+			}
+		}
+
 		// let's fetch the default settings in the configmap
 		configMap := &corev1.ConfigMap{}
-		if err := c.Client.Get(c.Ctx, client.ObjectKey{Name: ANNOTATIONS_CONFIGMAP, Namespace: c.OperatorNamespace}, configMap); err != nil {
-			return fmt.Errorf("fetching configmap '%s' at namespace '%s' from the api: %w", ANNOTATIONS_CONFIGMAP, c.OperatorNamespace, err)
+		if err := c.Client.Get(c.Ctx, client.ObjectKey{Name: configMapName, Namespace: c.OperatorNamespace}, configMap); err != nil {
+			return fmt.Errorf("fetching configmap '%s' at namespace '%s' from the api: %w", configMapName, c.OperatorNamespace, err)
 		}
 
 		// unmarshal the data as yaml
@@ -80,13 +118,13 @@ func (c *Config) Init(pod *corev1.Pod) error {
 	}
 
 	// determine if we should inject the certificate authority
-	if c.Get("inject-ca") == "true" {
+	if c.GetAnnotation("inject-ca") == "true" {
 		c.InjectCa = true
 	}
 
 	return nil
 }
 
-func (c *Config) Get(key string) string {
+func (c *Config) GetAnnotation(key string) string {
 	return c.annotations[fmt.Sprintf("qpoint.io/%s", key)]
 }

api/v1/egress.go

@@ -4,19 +4,22 @@ import (
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const INIT_IMAGE = "us-docker.pkg.dev/qpoint-edge/public/kubernetes-qtap-init"
+const QTAP_IMAGE = "us-docker.pkg.dev/qpoint-edge/public/qtap"
 
 var (
-	RUN_AS_USER     int64 = 0     // The root user
-	RUN_AS_GROUP    int64 = 0     // The root group
-	RUN_AS_NON_ROOT       = false // Allow running as root
+	ROOT_USER       int64 = 0 // The root user
+	ROOT_GROUP      int64 = 0 // The root group
+	RUN_AS_NON_ROOT       = false
 )
 
 func MutateEgress(pod *corev1.Pod, config *Config) error {
 	// fetch the init image tag
-	tag := config.Get("egress-init-tag")
+	tag := config.GetAnnotation("egress-init-tag")
 
 	// create an init container
 	initContainer := corev1.Container{
@@ -29,51 +32,46 @@ func MutateEgress(pod *corev1.Pod, config *Config) error {
 			},
 			// The init container needs to run as root as it modifies the network
 			// for the pod
-			RunAsUser:    &RUN_AS_USER,
-			RunAsGroup:   &RUN_AS_GROUP,
-			RunAsNonRoot: &RUN_AS_NON_ROOT,
+			RunAsUser:    &ROOT_USER,
+			RunAsGroup:   &ROOT_GROUP,
+			RunAsNonRoot: &RUN_AS_NON_ROOT, // Allow running as root
 		},
 	}
 
 	// TO_ADDR
-	toAddr := config.Get("egress-to-addr")
-	if toAddr != "" {
+	if toAddr := config.GetAnnotation("egress-to-addr"); toAddr != "" {
 		initContainer.Env = append(initContainer.Env, corev1.EnvVar{
 			Name:  "TO_ADDR",
 			Value: toAddr,
 		})
 	}
 
 	// TO_DOMAIN
-	toDomain := config.Get("egress-to-domain")
-	if toAddr == "" && toDomain != "" {
+	if toDomain := config.GetAnnotation("egress-to-domain"); toDomain != "" {
 		initContainer.Env = append(initContainer.Env, corev1.EnvVar{
 			Name:  "TO_DOMAIN",
 			Value: toDomain,
 		})
 	}
 
 	// PORT_MAPPING
-	portMapping := config.Get("egress-port-mapping")
-	if portMapping != "" {
+	if portMapping := config.GetAnnotation("egress-port-mapping"); portMapping != "" {
 		initContainer.Env = append(initContainer.Env, corev1.EnvVar{
 			Name:  "PORT_MAPPING",
 			Value: portMapping,
 		})
 	}
 
 	// ACCEPT_UIDS
-	acceptUids := config.Get("egress-accept-uids")
-	if acceptUids != "" {
+	if acceptUids := config.GetAnnotation("egress-accept-uids"); acceptUids != "" {
 		initContainer.Env = append(initContainer.Env, corev1.EnvVar{
 			Name:  "ACCEPT_UIDS",
 			Value: acceptUids,
 		})
 	}
 
 	// ACCEPT_GIDS
-	acceptGids := config.Get("egress-accept-gids")
-	if acceptGids != "" {
+	if acceptGids := config.GetAnnotation("egress-accept-gids"); acceptGids != "" {
 		initContainer.Env = append(initContainer.Env, corev1.EnvVar{
 			Name:  "ACCEPT_GIDS",
 			Value: acceptGids,
@@ -91,3 +89,158 @@ func MutateEgress(pod *corev1.Pod, config *Config) error {
 	// gtg
 	return nil
 }
+
+func MutateInjection(pod *corev1.Pod, config *Config) error {
+	// in order to start qtap a token is needed. This token can be found at a defined secret name token
+	secret := &corev1.Secret{}
+	if err := config.Client.Get(config.Ctx, client.ObjectKey{Name: "token", Namespace: config.OperatorNamespace}, secret); err != nil {
+		return fmt.Errorf("fetching secret '%s' at namespace '%s' from the api: %w", "token", config.OperatorNamespace, err)
+	}
+
+	tokenBytes, exists := secret.Data["token"]
+	if !exists {
+		return fmt.Errorf("token not found in secret '%s'", "token")
+	}
+
+	// convert the []byte data to a string
+	token := string(tokenBytes)
+
+	// fetch the init image tag
+	tag := config.GetAnnotation("qtap-tag")
+
+	// create an init container
+	qtapContainer := corev1.Container{
+		Name:  "qtap",
+		Image: fmt.Sprintf("%s:%s", QTAP_IMAGE, tag),
+		Args:  []string{"gateway"},
+		Env: []corev1.EnvVar{
+			{
+				Name:  "TOKEN",
+				Value: token,
+			},
+		},
+		StartupProbe: &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: "/readyz",
+					Port: intstr.IntOrString{
+						IntVal: 8080,
+					},
+				},
+			},
+			InitialDelaySeconds: 3,
+			PeriodSeconds:       5,
+			TimeoutSeconds:      2,
+			SuccessThreshold:    1,
+			FailureThreshold:    20,
+		},
+		ReadinessProbe: &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: "/readyz",
+					Port: intstr.IntOrString{
+						IntVal: 8080,
+					},
+				},
+			},
+			InitialDelaySeconds: 3,
+			PeriodSeconds:       5,
+			TimeoutSeconds:      2,
+			SuccessThreshold:    1,
+			FailureThreshold:    1,
+		},
+		LivenessProbe: &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: "/healthz",
+					Port: intstr.IntOrString{
+						IntVal: 8080,
+					},
+				},
+			},
+			InitialDelaySeconds: 3,
+			PeriodSeconds:       10,
+			TimeoutSeconds:      2,
+			SuccessThreshold:    1,
+			FailureThreshold:    3,
+		},
+	}
+
+	// LOG_LEVEL
+	if logLevel := config.GetAnnotation("log-level"); logLevel != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "LOG_LEVEL",
+			Value: logLevel,
+		})
+	}
+
+	// LOG_ENCODING
+	if logEncoding := config.GetAnnotation("log-encoding"); logEncoding != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "LOG_ENCODING",
+			Value: logEncoding,
+		})
+	}
+
+	// LOG_CALLER
+	if logCaller := config.GetAnnotation("log-caller"); logCaller != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "LOG_CALLER",
+			Value: logCaller,
+		})
+	}
+
+	// HTTP_LISTEN
+	if httpListen := config.GetAnnotation("http-listen"); httpListen != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "HTTP_LISTEN",
+			Value: httpListen,
+		})
+	}
+
+	// HTTPS_LISTEN
+	if httpsListen := config.GetAnnotation("https-listen"); httpsListen != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "HTTPS_LISTEN",
+			Value: httpsListen,
+		})
+	}
+
+	// TCP_LISTEN
+	if tcpListen := config.GetAnnotation("tcp-listen"); tcpListen != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "TCP_LISTEN",
+			Value: tcpListen,
+		})
+	}
+
+	// BLOCK_UNKNOWN
+	if blockUnknown := config.GetAnnotation("block-unknown"); blockUnknown != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "BLOCK_UNKNOWN",
+			Value: blockUnknown,
+		})
+	}
+
+	// ENVOY_LOG_LEVEL
+	if envoyLogLevel := config.GetAnnotation("envoy-log-level"); envoyLogLevel != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "ENVOY_LOG_LEVEL",
+			Value: envoyLogLevel,
+		})
+	}
+
+	// DNS_LOOKUP_FAMILY
+	if dnsLookupFamily := config.GetAnnotation("dns-lookup-family"); dnsLookupFamily != "" {
+		qtapContainer.Env = append(qtapContainer.Env, corev1.EnvVar{
+			Name:  "DNS_LOOKUP_FAMILY",
+			Value: dnsLookupFamily,
+		})
+	}
+
+	// append to the list
+	pod.Spec.Containers = append(pod.Spec.Containers, qtapContainer)
+
+	// gtg
+	return nil
+}