Use TLS via OpenShift service annotation when gateway/multitenancy is… (

#962) * Use TLS via OpenShift service annotation when gateway/multitenancy is disabled Signed-off-by: Ruben Vargas <[email protected]> * Use TLS via OpenShift service annotation when gateway/multitenancy is disabled Signed-off-by: Ruben Vargas <[email protected]> * Use TLS via OpenShift service annotation when gateway/multitenancy is disabled Signed-off-by: Ruben Vargas <[email protected]> * Add comments on the feature gate Signed-off-by: Ruben Vargas <[email protected]> * Add E2E tests with openshift TLS enabled Signed-off-by: Ruben Vargas <[email protected]> * Improve changelog Signed-off-by: Ruben Vargas <[email protected]> * fix ci Signed-off-by: Ruben Vargas <[email protected]> * Improve changelog entry Signed-off-by: Ruben Vargas <[email protected]> * Use manifestutils and naming functions Signed-off-by: Ruben Vargas <[email protected]> * Use same servingCertsService flag Signed-off-by: Ruben Vargas <[email protected]> * Fix e2e tests Signed-off-by: Ruben Vargas <[email protected]> * Add more description to the changelog Signed-off-by: Ruben Vargas <[email protected]> * Add documemntatioon direct into the CRD Signed-off-by: Ruben Vargas <[email protected]> --------- Signed-off-by: Ruben Vargas <[email protected]>
grafana · Jul 9, 2024 · 75de22c · 75de22c
1 parent e122de4
commit 75de22c
Show file tree

Hide file tree

Showing 25 changed files with 490 additions and 23 deletions.
diff --git a/.chloggen/ingest_tls_openshift.yaml b/.chloggen/ingest_tls_openshift.yaml
@@ -0,0 +1,43 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempostack
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use TLS via OpenShift service annotation when gateway/multitenancy is disabled
+
+# One or more tracking issues related to the change
+issues: [963]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  On OpenShift when operator config `servingCertsService` is enabled and the following TempoStack CR is used.
+  The operator provisions OpenShift serving certificates for the distributor ingest APIs
+  ```
+    apiVersion: tempo.grafana.com/v1alpha1
+    kind:  TempoStack
+    spec:
+      template:
+        distributor:
+          tls:
+            enabled: true
+  ```
+  No `certName` and `caName` should be provided, If you specify it, those will be used instead.
+
+  In order to use this on the client side, the openshift CA certificate should be used, there are two ways of get
+  access to it. You can mount the configmap generated by the operator, which will have the name `<tempostack-name>-serving-cabundle`
+  Or you can access to it on `var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt`.
+  
+  An example of OTel configuration used:
+  
+  ```
+     exporters:
+      otlp:
+        endpoint: tempo-simplest-distributor.chainsaw-tls-singletenant.svc.cluster.local:4317
+        tls:
+          insecure: false
+          ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
+  ```
diff --git a/apis/config/v1alpha1/projectconfig_types.go b/apis/config/v1alpha1/projectconfig_types.go
@@ -75,10 +75,14 @@ type BuiltInCertManagement struct {
 
 // OpenShiftFeatureGates is the supported set of all operator features gates on OpenShift.
 type OpenShiftFeatureGates struct {
-	// ServingCertsService enables OpenShift service-ca annotations on the TempoStack gateway service only
+	// ServingCertsService enables OpenShift service-ca annotations on the TempoStack
 	// to use the in-platform CA and generate a TLS cert/key pair per service for
 	// in-cluster data-in-transit encryption.
 	// More details: https://docs.openshift.com/container-platform/latest/security/certificate_types_descriptions/service-ca-certificates.html
+	//
+	// Currently is only used in two cases:
+	//   - If gateway is enabled, it will be used by the gateway component
+	//   - If the gateway is disabled and TLS is enabled on the distributor but no caName and certName are specified
 	ServingCertsService bool `json:"servingCertsService,omitempty"`
 
 	// OpenShiftRoute enables creating OpenShift Route objects.

diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go
@@ -493,6 +493,9 @@ type TempoDistributorSpec struct {
 	//
 	// +optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="TLS"
+	//
+	// If openshift feature flag `servingCertsService` is enabled and TLS is enabled but no
+	// certName or caName is specified, OpenShift service serving certificates will  be used.
 	TLS TLSSpec `json:"tls,omitempty"`
 }
 

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.11.1
-    createdAt: "2024-07-08T10:12:34Z"
+    createdAt: "2024-07-09T01:46:53Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -850,7 +850,10 @@ spec:
           the calculated resources derived from total
         displayName: Resources
         path: template.distributor.resources
-      - description: TLS defines TLS configuration for distributor receivers
+      - description: "TLS defines TLS configuration for distributor receivers \n If
+          openshift feature flag `servingCertsService` is enabled and TLS is enabled
+          but no certName or caName is specified, OpenShift service serving certificates
+          will  be used."
         displayName: TLS
         path: template.distributor.tls
       - description: CA is the name of a ConfigMap containing a CA certificate (service-ca.crt).

diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml
@@ -710,8 +710,12 @@ spec:
                             x-kubernetes-list-type: atomic
                         type: object
                       tls:
-                        description: TLS defines TLS configuration for distributor
-                          receivers
+                        description: |-
+                          TLS defines TLS configuration for distributor receivers
+
+
+                          If openshift feature flag `servingCertsService` is enabled and TLS is enabled but no
+                          certName or caName is specified, OpenShift service serving certificates will  be used.
                         properties:
                           caName:
                             description: |-

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.11.1
-    createdAt: "2024-07-08T10:12:32Z"
+    createdAt: "2024-07-09T01:46:51Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -850,7 +850,10 @@ spec:
           the calculated resources derived from total
         displayName: Resources
         path: template.distributor.resources
-      - description: TLS defines TLS configuration for distributor receivers
+      - description: "TLS defines TLS configuration for distributor receivers \n If
+          openshift feature flag `servingCertsService` is enabled and TLS is enabled
+          but no certName or caName is specified, OpenShift service serving certificates
+          will  be used."
         displayName: TLS
         path: template.distributor.tls
       - description: CA is the name of a ConfigMap containing a CA certificate (service-ca.crt).

diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml
@@ -710,8 +710,12 @@ spec:
                             x-kubernetes-list-type: atomic
                         type: object
                       tls:
-                        description: TLS defines TLS configuration for distributor
-                          receivers
+                        description: |-
+                          TLS defines TLS configuration for distributor receivers
+
+
+                          If openshift feature flag `servingCertsService` is enabled and TLS is enabled but no
+                          certName or caName is specified, OpenShift service serving certificates will  be used.
                         properties:
                           caName:
                             description: |-

diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml
@@ -706,8 +706,12 @@ spec:
                             x-kubernetes-list-type: atomic
                         type: object
                       tls:
-                        description: TLS defines TLS configuration for distributor
-                          receivers
+                        description: |-
+                          TLS defines TLS configuration for distributor receivers
+
+
+                          If openshift feature flag `servingCertsService` is enabled and TLS is enabled but no
+                          certName or caName is specified, OpenShift service serving certificates will  be used.
                         properties:
                           caName:
                             description: |-

diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml
@@ -779,7 +779,10 @@ spec:
           the calculated resources derived from total
         displayName: Resources
         path: template.distributor.resources
-      - description: TLS defines TLS configuration for distributor receivers
+      - description: "TLS defines TLS configuration for distributor receivers \n If
+          openshift feature flag `servingCertsService` is enabled and TLS is enabled
+          but no certName or caName is specified, OpenShift service serving certificates
+          will  be used."
         displayName: TLS
         path: template.distributor.tls
       - description: CA is the name of a ConfigMap containing a CA certificate (service-ca.crt).

diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml
@@ -779,7 +779,10 @@ spec:
           the calculated resources derived from total
         displayName: Resources
         path: template.distributor.resources
-      - description: TLS defines TLS configuration for distributor receivers
+      - description: "TLS defines TLS configuration for distributor receivers \n If
+          openshift feature flag `servingCertsService` is enabled and TLS is enabled
+          but no certName or caName is specified, OpenShift service serving certificates
+          will  be used."
         displayName: TLS
         path: template.distributor.tls
       - description: CA is the name of a ConfigMap containing a CA certificate (service-ca.crt).

diff --git a/docs/operator/config.yaml b/docs/operator/config.yaml
@@ -153,10 +153,14 @@ featureGates:
     # More details: https://docs.openshift.com/container-platform/latest/networking/understanding-networking.html
     openshiftRoute: false
 
-    # ServingCertsService enables OpenShift service-ca annotations on the TempoStack gateway service only
+    # ServingCertsService enables OpenShift service-ca annotations on the TempoStack
     # to use the in-platform CA and generate a TLS cert/key pair per service for
     # in-cluster data-in-transit encryption.
     # More details: https://docs.openshift.com/container-platform/latest/security/certificate_types_descriptions/service-ca-certificates.html
+    #
+    # Currently is only used in two cases:
+    #   - If gateway is enabled, it will be used by the gateway component
+    #   - If the gateway is disabled and TLS is enabled on the distributor but no caName and certName are specified
     servingCertsService: false
 
   # PrometheusOperator defines whether the Prometheus Operator CRD exists in the cluster.

diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml
@@ -104,7 +104,7 @@ spec:                                    # TempoStackSpec defines the desired st
             cpu: "500m"
             memory: "1Gi"
         tolerations: {}                  # Tolerations defines component-specific pod tolerations.
-      tls:                               # TLS defines TLS configuration for distributor receivers
+      tls:                               # TLS defines TLS configuration for distributor receivers   If openshift feature flag `servingCertsService` is enabled and TLS is enabled but no certName or caName is specified, OpenShift service serving certificates will  be used.
         enabled: false                   # Enabled defines if TLS is enabled.
         caName: ""                       # CA is the name of a ConfigMap containing a CA certificate (service-ca.crt). It needs to be in the same namespace as the Tempo custom resource.
         certName: ""                     # Cert is the name of a Secret containing a certificate (tls.crt) and private key (tls.key). It needs to be in the same namespace as the Tempo custom resource.

diff --git a/internal/manifests/distributor/distributor.go b/internal/manifests/distributor/distributor.go
@@ -17,6 +17,8 @@ import (
 	"github.com/grafana/tempo-operator/internal/manifests/naming"
 )
 
+const openshiftServiceTLSAnnotation = "service.beta.openshift.io/serving-cert-secret-name"
+
 // BuildDistributor creates distributor objects.
 func BuildDistributor(params manifestutils.Params) ([]client.Object, error) {
 	dep := deployment(params)
@@ -38,14 +40,31 @@ func BuildDistributor(params manifestutils.Params) ([]client.Object, error) {
 		}
 	}
 
+	distributorService := service(tempo)
+	objects := []client.Object{dep, distributorService}
+
 	if tempo.Spec.Template.Distributor.TLS.Enabled {
-		err = configureReceiversTLS(dep, tempo)
-		if err != nil {
-			return nil, err
+		if params.CtrlConfig.Gates.OpenShift.ServingCertsService {
+			caSecretName := naming.ServingCABundleName(tempo.Name)
+			certSecretName := naming.ServingCertName(manifestutils.DistributorComponentName, tempo.Name)
+			err = configureReceiversTLS(dep, caSecretName, certSecretName)
+			if err != nil {
+				return nil, err
+			}
+			distributorService.Annotations = map[string]string{openshiftServiceTLSAnnotation: certSecretName}
+			objects = append(objects, getConfigmapCABundle(tempo))
+		} else {
+			caSecretName := tempo.Spec.Template.Distributor.TLS.CA
+			certSecretName := tempo.Spec.Template.Distributor.TLS.Cert
+			err = configureReceiversTLS(dep, caSecretName, certSecretName)
+			if err != nil {
+				return nil, err
+			}
 		}
+
 	}
 
-	return []client.Object{dep, service(tempo)}, nil
+	return objects, nil
 }
 
 func resources(tempo v1alpha1.TempoStack) corev1.ResourceRequirements {
@@ -55,9 +74,7 @@ func resources(tempo v1alpha1.TempoStack) corev1.ResourceRequirements {
 	return *tempo.Spec.Template.Distributor.Resources
 }
 
-func configureReceiversTLS(dep *v1.Deployment, tempo v1alpha1.TempoStack) error {
-	caSecretName := tempo.Spec.Template.Distributor.TLS.CA
-	certSecretName := tempo.Spec.Template.Distributor.TLS.Cert
+func configureReceiversTLS(dep *v1.Deployment, caSecretName, certSecretName string) error {
 	podSpec := &dep.Spec.Template.Spec
 	if caSecretName != "" {
 		/*Configure CA*/
@@ -338,3 +355,10 @@ func service(tempo v1alpha1.TempoStack) *corev1.Service {
 		},
 	}
 }
+
+func getConfigmapCABundle(tempo v1alpha1.TempoStack) *corev1.ConfigMap {
+	return manifestutils.NewConfigMapCABundle(tempo.Namespace,
+		naming.ServingCABundleName(tempo.Name),
+		manifestutils.ComponentLabels(manifestutils.DistributorComponentName, tempo.Name),
+	)
+}
diff --git a/internal/webhooks/tempostack_webhook.go b/internal/webhooks/tempostack_webhook.go
@@ -409,7 +409,7 @@ func (v *validator) validateDeprecatedFields(tempo v1alpha1.TempoStack) field.Er
 
 func (v *validator) validateReceiverTLS(tempo v1alpha1.TempoStack) field.ErrorList {
 	spec := tempo.Spec.Template.Distributor.TLS
-	if spec.Enabled {
+	if spec.Enabled && !v.ctrlConfig.Gates.OpenShift.ServingCertsService {
 		if spec.Cert == "" {
 			return field.ErrorList{
 				field.Invalid(

diff --git a/tests/e2e-openshift/tls-singletenant/00-assert.yaml b/tests/e2e-openshift/tls-singletenant/00-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  namespace: chainsaw-tls-singletenant
+status:
+  readyReplicas: 1
diff --git a/tests/e2e-openshift/tls-singletenant/00-install-storage.yaml b/tests/e2e-openshift/tls-singletenant/00-install-storage.yaml
@@ -0,0 +1,85 @@
+# The namespace is auto-deleted by chainsaw after the test run.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: chainsaw-tls-singletenant
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app.kubernetes.io/name: minio
+  name: minio
+  namespace: chainsaw-tls-singletenant
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  namespace: chainsaw-tls-singletenant
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: minio
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: minio
+    spec:
+      containers:
+        - command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /storage/tempo && \
+              minio server /storage
+          env:
+            - name: MINIO_ACCESS_KEY
+              value: tempo
+            - name: MINIO_SECRET_KEY
+              value: supersecret
+          image: minio/minio
+          name: minio
+          ports:
+            - containerPort: 9000
+          volumeMounts:
+            - mountPath: /storage
+              name: storage
+      volumes:
+        - name: storage
+          persistentVolumeClaim:
+            claimName: minio
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: chainsaw-tls-singletenant
+spec:
+  ports:
+    - port: 9000
+      protocol: TCP
+      targetPort: 9000
+  selector:
+    app.kubernetes.io/name: minio
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio
+  namespace: chainsaw-tls-singletenant
+stringData:
+  endpoint: http://minio:9000
+  bucket: tempo
+  access_key_id: tempo
+  access_key_secret: supersecret
+type: Opaque