feat/fix: enhance cert-manager integration for metrics endpoints (fol…

…low-up to PR #4243)

This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to #3871 and #4003

Key enhancements:
- Added support for configuring certificate integration via a Kustomize patch.
- Introduced configurable flags for greater flexibility in customization.
- Use Certwatcher to allow certificate rotation

This configuration provides an option for users to be production-ready

.github/workflows/test-e2e-samples.yml

@@ -41,9 +41,10 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment all cert-manager injections
-          sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4/
           go mod tidy
 
@@ -85,10 +86,11 @@ jobs:
           # Uncomment only ValidatingWebhookConfiguration
           # from cert-manager replaces; we are leaving defaulting uncommented
           # since this sample has no defaulting webhooks
-          sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment only --conversion webhooks CA injection
-          sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-with-plugins/
           go mod tidy
 
@@ -127,9 +129,10 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          # Uncomment all cert-manager injections
-          sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+          # Uncomment all cert-manager injections for webhooks only
+          sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-multigroup
           go mod tidy
 

docs/book/src/cronjob-tutorial/running.md

@@ -26,7 +26,7 @@ cluster with, so we don't need to worry about RBAC just yet.
 
 If you want to run the webhooks locally, you'll have to generate
 certificates for serving the webhooks, and place them in the right
-directory (`/tmp/k8s-webhook-server/serving-certs/tls.{crt,key}`, by
+directory (`/tmp/k8s-metrics-server/metrics-certs/tls.{crt,key}`, by
 default).
 
 If you're not running a local API server, you'll also need to figure out

docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"path/filepath"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -30,6 +31,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`.
 */
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = ctrl.Log.WithName("setup")
+	scheme      = runtime.NewScheme()
+	setupLog    = ctrl.Log.WithName("setup")
+	certWatcher *certwatcher.CertWatcher
 )
 
 func init() {
@@ -74,6 +77,9 @@ func main() {
 	/*
 	 */
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -87,6 +93,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+	flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -133,16 +143,27 @@ func main() {
 		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
 		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
 
-		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+		// If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically
 		// generate self-signed certificates for the metrics server. While convenient for development and testing,
 		// this setup is not recommended for production.
-
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
-
+		//
+		// TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+		// to generate and use certificates managed by cert-manager for the metrics server.
+		if len(certDir) > 0 {
+			setupLog.Info("using certificates for the metrics server",
+				"cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+			var err error
+			certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+			if err != nil {
+				setupLog.Error(err, "to initialize certificate watcher", "error", err)
+				os.Exit(1)
+			}
+
+			metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+				config.GetCertificate = certWatcher.GetCertificate
+			})
+		}
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -196,6 +217,14 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	if secureMetrics && certWatcher != nil {
+		setupLog.Info("Adding certificate watcher to manager")
+		if err := mgr.Add(certWatcher); err != nil {
+			setupLog.Error(err, "unable to add certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml

@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/name: project
+    app.kubernetes.io/managed-by: kustomize
+  name: metrics-certs  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  dnsNames:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  # replacements in the config/default/kustomization.yaml file.
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml

@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/name: project
+    app.kubernetes.io/managed-by: kustomize
+  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  # replacements in the config/default/kustomization.yaml file.
+  dnsNames:
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml

@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    app.kubernetes.io/name: project
+    app.kubernetes.io/managed-by: kustomize
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml

@@ -1,5 +1,7 @@
 resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
 
 configurations:
 - kustomizeconfig.yaml

docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml

@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /var/metrics/certs/
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/var/metrics/certs/
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false
+      items:
+        - key: ca.crt
+          path: ca.crt
+        - key: tls.crt
+          path: tls.crt
+        - key: tls.key
+          path: tls.key