feat/fix: enhance cert-manager integration for metrics endpoints (fol…

…low-up to PR kubernetes-sigs#4243)

This commit is a follow-up to PR kubernetes-sigs#4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to kubernetes-sigs#3871 and kubernetes-sigs#4003

Key enhancements:
- Added support for configuring certificate integration via a Kustomize patch.
- Introduced configurable flags for greater flexibility in customization.
- Use Certwatcher to allow certificate rotation

This configuration provides an option for users to be production-ready

.github/workflows/test-e2e-samples.yml

@@ -41,9 +41,10 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment all cert-manager injections
-          sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4/
           go mod tidy
 
@@ -85,10 +86,11 @@ jobs:
           # Uncomment only ValidatingWebhookConfiguration
           # from cert-manager replaces; we are leaving defaulting uncommented
           # since this sample has no defaulting webhooks
-          sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment only --conversion webhooks CA injection
-          sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-with-plugins/
           go mod tidy
 
@@ -127,9 +129,10 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          # Uncomment all cert-manager injections
-          sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+          # Uncomment all cert-manager injections for webhooks only
+          sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-multigroup
           go mod tidy
 

Makefile

@@ -105,6 +105,11 @@ lint: golangci-lint yamllint ## Run golangci-lint linter & yamllint
 lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 	$(GOLANGCI_LINT) run --fix
 
+.PHONY: lint-config
+lint-config: golangci-lint  ## Verify golangci-lint linter configuration
+	$(GOLANGCI_LINT) config verify
+
+
 .PHONY: yamllint
 yamllint:
 	@files=$$(find testdata -name '*.yaml' ! -path 'testdata/*/dist/*'); \

docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"path/filepath"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -30,6 +31,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -74,6 +76,7 @@ func main() {
 	/*
 	 */
 	var metricsAddr string
+	var metricsCertPath, metricsCertName, metricsCertKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -87,6 +90,9 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+	flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+	flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -112,6 +118,9 @@ func main() {
 		tlsOpts = append(tlsOpts, disableHTTP2)
 	}
 
+	// Create watchers for metrics certificates
+	var metricsCertWatcher *certwatcher.CertWatcher
+
 	webhookServer := webhook.NewServer(webhook.Options{
 		TLSOpts: tlsOpts,
 	})
@@ -124,25 +133,38 @@ func main() {
 		BindAddress:   metricsAddr,
 		SecureServing: secureMetrics,
 		TLSOpts:       tlsOpts,
-	}
-
-	if secureMetrics {
 		// FilterProvider is used to protect the metrics endpoint with authn/authz.
 		// These configurations ensure that only authorized users and service accounts
 		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
 		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
-		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-
-		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
-		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
+		FilterProvider: filters.WithAuthenticationAndAuthorization,
+	}
 
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
+	// If the certificate is not specified, controller-runtime will automatically
+	// generate self-signed certificates for the metrics server. While convenient for development and testing,
+	// this setup is not recommended for production.
+	//
+	// TODO(user): If you enable certManager, uncomment the following lines:
+	// - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+	// managed by cert-manager for the metrics server.
+	// - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+	if len(metricsCertPath) > 0 {
+		certName := filepath.Join(metricsCertPath, metricsCertName)
+		certKey := filepath.Join(metricsCertPath, metricsCertKey)
+
+		setupLog.Info("metrics server is serving securely using provided certificates",
+			"metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+
+		var err error
+		metricsCertWatcher, err = certwatcher.New(certName, certKey)
+		if err != nil {
+			setupLog.Error(err, "to initialize certificate watcher", "error", err)
+			os.Exit(1)
+		}
 
+		metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+			config.GetCertificate = metricsCertWatcher.GetCertificate
+		})
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -196,6 +218,14 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	if metricsCertWatcher != nil {
+		setupLog.Info("Adding metrics certificate watcher to manager")
+		if err := mgr.Add(metricsCertWatcher); err != nil {
+			setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml

@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/name: project
+    app.kubernetes.io/managed-by: kustomize
+  name: metrics-certs  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  dnsNames:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  # replacements in the config/default/kustomization.yaml file.
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml

@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/name: project
+    app.kubernetes.io/managed-by: kustomize
+  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  # replacements in the config/default/kustomization.yaml file.
+  dnsNames:
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml

@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    app.kubernetes.io/name: project
+    app.kubernetes.io/managed-by: kustomize
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}

docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml

@@ -1,5 +1,7 @@
 resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
 
 configurations:
 - kustomizeconfig.yaml

docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml

@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/metrics-certs
+    name: metrics-certs
+    readOnly: true
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false
+      items:
+        - key: ca.crt
+          path: ca.crt
+        - key: tls.crt
+          path: tls.crt
+        - key: tls.key
+          path: tls.key