From 732a5ee446a1473e1e785475b734e7d7e7cc0187 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:10:28 +0000 Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics endpoints (follow-up to PR #4243) This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready --- .github/workflows/test-e2e-samples.yml | 19 +- .../testdata/project/cmd/main.go | 56 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------ .../project/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../project/config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- .../testdata/project/dist/install.yaml | 92 ++++++++++ .../project/internal/controller/suite_test.go | 1 - .../testdata/project/cmd/main.go | 58 +++++-- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../project/config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- .../testdata/project/cmd/main.go | 56 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------ .../project/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../project/config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- .../testdata/project/dist/install.yaml | 25 +-- docs/book/src/reference/metrics.md | 163 +++++++++++------- .../cronjob-tutorial/generate_cronjob.go | 51 +++--- hack/docs/internal/cronjob-tutorial/sample.go | 48 +++++- .../multiversion-tutorial/kustomize.go | 4 +- .../config/certmanager/certificate_metrics.go | 68 ++++++++ ...{certificate.go => certificate_webhook.go} | 43 +---- .../templates/config/certmanager/issuer.go | 60 +++++++ .../config/certmanager/kustomization.go | 4 +- ...patch.go => cert_metrics_manager_patch.go} | 55 +++--- .../config/kdefault/kustomization.go | 58 ++++++- .../config/prometheus/kustomization.go | 2 +- .../common/kustomize/v2/scaffolds/webhook.go | 2 + .../scaffolds/internal/templates/cmd/main.go | 62 +++++-- test/e2e/v4/generate_test.go | 81 +++++++-- testdata/project-v4-multigroup/cmd/main.go | 54 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------ .../config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- testdata/project-v4-with-plugins/cmd/main.go | 56 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------ .../config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- testdata/project-v4/cmd/main.go | 54 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------ .../project-v4/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- 73 files changed, 1646 insertions(+), 768 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/{certificate.go => certificate_webhook.go} (57%) create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (56%) create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index b0db7dc21c6..6d84a992a2f 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -41,9 +41,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ go mod tidy @@ -85,10 +86,11 @@ jobs: # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces; we are leaving defaulting uncommented # since this sample has no defaulting webhooks - sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only --conversion webhooks CA injection - sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-with-plugins/ go mod tidy @@ -127,9 +129,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + # Uncomment all cert-manager injections for webhooks only + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-multigroup go mod tidy diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index 458a9f8c3e2..3a94d5a170a 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -30,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -74,6 +76,7 @@ func main() { /* */ var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -87,6 +90,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -112,6 +118,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -124,25 +133,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -196,6 +218,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index b5cf1e32e2f..0cf156e6276 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block to enable certificates for metrics + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml index e00ddc958dc..8126ea89b1a 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml index c20c83edbc5..27dbb9bb5ab 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml @@ -4118,6 +4118,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs command: - /manager image: controller:latest @@ -4154,6 +4155,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -4165,10 +4169,96 @@ spec: secret: defaultMode: 420 secretName: webhook-server-cert + - name: metrics-certs + secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-metrics-certs + namespace: project-system +spec: + dnsNames: + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-serving-cert + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-selfsigned-issuer + namespace: project-system +spec: + selfSigned: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + control-plane: controller-manager + name: project-controller-manager-metrics-monitor + namespace: project-system +spec: + endpoints: + - tlsConfig: + ca: + secret: + key: ca.crt + name: metrics-server-cert + cert: + secret: + key: tls.crt + name: metrics-server-cert + insecureSkipVerify: false + keySecret: + key: tls.key + name: metrics-server-cert + selector: + matchLabels: + control-plane: controller-manager --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-mutating-webhook-configuration webhooks: - admissionReviewVersions: @@ -4195,6 +4285,8 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-validating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go index 7404dd56613..edb60cb7d8d 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go @@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the First, it will contain the necessary imports. */ - package controller import ( diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index 7347d215463..fa36de337c5 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -54,6 +56,7 @@ func init() { func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -67,6 +70,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -92,6 +98,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -104,25 +113,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + FilterProvider: filters.WithAuthenticationAndAuthorization, + } + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -158,6 +180,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 317242ed92c..75241299903 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml index f040fd58623..fdc5481b103 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go index a199eec734b..5db483c0065 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -73,6 +75,7 @@ func main() { /* */ var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -86,6 +89,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -111,6 +117,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -123,25 +132,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -199,6 +221,14 @@ func main() { /* */ + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml index 16e19c0840f..dcdae5c7e21 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block to enable certificates for metrics + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: @@ -171,7 +213,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml index e00ddc958dc..8126ea89b1a 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml index 8a9b9c9f3ab..f3716ea8657 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml @@ -7929,6 +7929,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs command: - /manager image: controller:latest @@ -7981,24 +7982,28 @@ spec: secretName: webhook-server-cert - name: metrics-certs secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false secretName: metrics-server-cert --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: metrics-certs app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-metrics-certs namespace: project-system spec: dnsNames: - - project-webhook-service.project-system.svc - - project-webhook-service.project-system.svc.cluster.local + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local issuerRef: kind: Issuer name: project-selfsigned-issuer @@ -8008,12 +8013,8 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: serving-cert app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-serving-cert namespace: project-system spec: diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index dbb85222709..f345712630a 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -126,57 +126,37 @@ spec: sleep 60; done ``` -