Ensure that NROP metrics are served securely

This commit consist of the following changes:

* Reenabled kube-rbac-proxy sidecar container to securely expose the /metrics endpoint for Prometheus scraping.
* Added a secret to enforce HTTPS-only access to the /metrics endpoint, restricted to the Prometheus service account.
* modified ServiceMonitor resource to enable Prometheus pods to scrape metrics.
* Added an annotation to the deployment Service, which is monitored by the Service CA operator. This operator will generate the tls.key and tls.crt files inside the secret-kube-rbac-proxy-tls secret, which is used by the kube-rbac-proxy container.
* Added Role and RoleBinding resources to grant the necessary permissions to the Prometheus service account.

Most of this configuration was based on this guide:
https://rhobs-handbook.netlify.app/products/openshiftmonitoring/collecting_metrics.md/

Signed-off-by: Ronny Baturov <[email protected]>

config/default/kustomization.yaml

@@ -22,13 +22,13 @@ bases:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-#- manager_auth_proxy_patch.yaml
+- manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

config/default/manager_auth_proxy_patch.yaml

@@ -9,20 +9,41 @@ spec:
   template:
     spec:
       containers:
+      - name: manager
       - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
+        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8080/"
+        - "--config-file=/etc/kube-rbac-proxy/config.yaml"
+        - "--tls-cert-file=/etc/tls/private/tls.crt"
+        - "--tls-private-key-file=/etc/tls/private/tls.key"
+        - "--allow-paths=/metrics"
         - "--logtostderr=true"
         - "-v=10"
         ports:
         - containerPort: 8443
           protocol: TCP
           name: https
-      - name: manager
-        args:
-        - "--platform=kubernetes"
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /etc/kube-rbac-proxy
+          name: secret-kube-rbac-proxy-metric
+          readOnly: true
+        - mountPath: /etc/tls/private
+          name: secret-kube-rbac-proxy-tls
+          readOnly: true
+      serviceAccountName: controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: secret-kube-rbac-proxy-tls
+        secret:
+          secretName: secret-kube-rbac-proxy-tls
+      - name: secret-kube-rbac-proxy-metric
+        secret:
+          secretName: secret-kube-rbac-proxy-metric

config/manager/manager.yaml

@@ -5,6 +5,7 @@ metadata:
     workload.openshift.io/allowed: management
   labels:
     control-plane: controller-manager
+    openshift.io/cluster-monitoring: "true"
   name: system
 ---
 apiVersion: apps/v1

config/prometheus/kustomization.yaml

@@ -1,2 +1,5 @@
 resources:
-- monitor.yaml
+- rbac.yaml
+- secret-kube-rbac-proxy.yaml
+# Please uncomment monitor.yaml to enable prometheus pods to scrape the metrics periodically.
+# - monitor.yaml

config/prometheus/monitor.yaml

@@ -1,20 +1,24 @@
 
-# Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-monitor
+  name: controller-manager
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
-      port: https
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
+  - interval: 30s
+    # Matches the name of the service's port.
+    targetPort: 8443
+    path: /metrics
+    scheme: https
+    bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+    tlsConfig:
+      # The CA file used by Prometheus to verify the server's certificate.
+      # It's the cluster's CA bundle from the service CA operator.
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      # The name of the server (CN) in the server's certificate.
+      serverName: numaresources-controller-manager-metrics-service.numaresources.svc
+      insecureSkipVerify: false
   selector:
     matchLabels:
       control-plane: controller-manager

config/prometheus/rbac.yaml

@@ -0,0 +1,31 @@
+# creates Role and RoleBinding for prometheus-k8s service account to access our namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring

config/prometheus/secret-kube-rbac-proxy.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secret-kube-rbac-proxy-metric
+  namespace: system
+stringData:
+  config.yaml: |-
+    "authorization":
+      "static":
+      - "path": "/metrics"
+        "resourceRequest": false
+        "user":
+          "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
+        "verb": "get"    
+type: Opaque

config/rbac/auth_proxy_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: secret-kube-rbac-proxy-tls
   labels:
     control-plane: controller-manager
   name: controller-manager-metrics-service

config/rbac/kustomization.yaml

@@ -12,7 +12,7 @@ resources:
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
-#- auth_proxy_service.yaml
-#- auth_proxy_role.yaml
-#- auth_proxy_role_binding.yaml
-#- auth_proxy_client_clusterrole.yaml
+- auth_proxy_service.yaml
+- auth_proxy_role.yaml
+- auth_proxy_role_binding.yaml
+- auth_proxy_client_clusterrole.yaml

main.go

@@ -71,11 +71,11 @@ const (
 )
 
 const (
-	defaultWebhookPort = 9443
-	defaultMetricsAddr = ":8080"
+	defaultWebhookPort    = 9443
+	defaultMetricsAddr    = ":8080"
 	defaultMetricsEnabled = true
-	defaultProbeAddr   = ":8081"
-	defaultNamespace   = "numaresources-operator"
+	defaultProbeAddr      = ":8081"
+	defaultNamespace      = "numaresources-operator"
 )
 
 var (