From dcf49ca98bf4235af1f4fcad88df0f9f3ee267d3 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:10:28 +0000 Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics endpoints (follow-up to PR #4243) This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready --- .github/workflows/test-e2e-samples.yml | 19 ++-- docs/book/src/cronjob-tutorial/running.md | 2 +- .../testdata/project/cmd/main.go | 49 ++++++++-- .../certmanager/certificate-metrics.yaml | 20 ++++ .../certmanager/certificate-webhook.yaml | 20 ++++ .../config/certmanager/certificate.yaml | 57 ----------- .../project/config/certmanager/issuer.yaml | 13 +++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 +++++++ .../certmanager_metrics_manager_patch.yaml | 21 ----- .../project/config/default/kustomization.yaml | 58 ++++++++++-- .../config/default/manager_webhook_patch.yaml | 2 +- .../testdata/project/dist/install.yaml | 94 ++++++++++++++++++- .../project/internal/controller/suite_test.go | 1 - .../testdata/project/cmd/main.go | 49 ++++++++-- .../default/cert_metrics_manager_patch.yaml | 35 +++++++ .../certmanager_metrics_manager_patch.yaml | 21 ----- .../project/config/default/kustomization.yaml | 58 ++++++++++-- .../testdata/project/cmd/main.go | 49 ++++++++-- .../certmanager/certificate-metrics.yaml | 20 ++++ .../certmanager/certificate-webhook.yaml | 20 ++++ .../config/certmanager/certificate.yaml | 57 ----------- .../project/config/certmanager/issuer.yaml | 13 +++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 +++++++ .../certmanager_metrics_manager_patch.yaml | 21 ----- .../project/config/default/kustomization.yaml | 58 ++++++++++-- .../config/default/manager_webhook_patch.yaml | 2 +- .../testdata/project/dist/install.yaml | 29 +++--- .../cronjob-tutorial/generate_cronjob.go | 51 +++++----- hack/docs/internal/cronjob-tutorial/sample.go | 48 +++++++++- .../multiversion-tutorial/kustomize.go | 4 +- .../config/certmanager/certificate_metrics.go | 68 ++++++++++++++ ...{certificate.go => certificate_webhook.go} | 43 +-------- .../templates/config/certmanager/issuer.go | 60 ++++++++++++ .../config/certmanager/kustomization.go | 4 +- ...patch.go => cert_metrics_manager_patch.go} | 59 +++++++----- .../config/kdefault/kustomization.go | 58 ++++++++++-- .../config/kdefault/webhook_manager_patch.go | 2 +- .../common/kustomize/v2/scaffolds/webhook.go | 2 + .../scaffolds/internal/templates/cmd/main.go | 45 +++++++-- .../chart-templates/manager/manager.go | 2 +- test/e2e/v4/generate_test.go | 77 +++++++++++---- testdata/project-v4-multigroup/cmd/main.go | 45 +++++++-- .../certmanager/certificate-metrics.yaml | 20 ++++ .../certmanager/certificate-webhook.yaml | 20 ++++ .../config/certmanager/certificate.yaml | 57 ----------- .../config/certmanager/issuer.yaml | 13 +++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 +++++++ .../certmanager_metrics_manager_patch.yaml | 21 ----- .../config/default/kustomization.yaml | 58 ++++++++++-- .../config/default/manager_webhook_patch.yaml | 2 +- .../project-v4-multigroup/dist/install.yaml | 2 +- testdata/project-v4-with-plugins/cmd/main.go | 49 ++++++++-- .../certmanager/certificate-metrics.yaml | 20 ++++ .../certmanager/certificate-webhook.yaml | 20 ++++ .../config/certmanager/certificate.yaml | 57 ----------- .../config/certmanager/issuer.yaml | 13 +++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 +++++++ .../certmanager_metrics_manager_patch.yaml | 21 ----- .../config/default/kustomization.yaml | 58 ++++++++++-- .../config/default/manager_webhook_patch.yaml | 2 +- .../dist/chart/templates/manager/manager.yaml | 2 +- .../project-v4-with-plugins/dist/install.yaml | 2 +- testdata/project-v4/cmd/main.go | 45 +++++++-- .../certmanager/certificate-metrics.yaml | 20 ++++ .../certmanager/certificate-webhook.yaml | 20 ++++ .../config/certmanager/certificate.yaml | 57 ----------- .../project-v4/config/certmanager/issuer.yaml | 13 +++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 +++++++ .../certmanager_metrics_manager_patch.yaml | 21 ----- .../config/default/kustomization.yaml | 58 ++++++++++-- .../config/default/manager_webhook_patch.yaml | 2 +- testdata/project-v4/dist/install.yaml | 2 +- 77 files changed, 1540 insertions(+), 686 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/{certificate.go => certificate_webhook.go} (57%) create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (53%) create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index b0db7dc21c6..6d84a992a2f 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -41,9 +41,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ go mod tidy @@ -85,10 +86,11 @@ jobs: # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces; we are leaving defaulting uncommented # since this sample has no defaulting webhooks - sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only --conversion webhooks CA injection - sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-with-plugins/ go mod tidy @@ -127,9 +129,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + # Uncomment all cert-manager injections for webhooks only + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-multigroup go mod tidy diff --git a/docs/book/src/cronjob-tutorial/running.md b/docs/book/src/cronjob-tutorial/running.md index 22c42297028..323e70691ea 100644 --- a/docs/book/src/cronjob-tutorial/running.md +++ b/docs/book/src/cronjob-tutorial/running.md @@ -26,7 +26,7 @@ cluster with, so we don't need to worry about RBAC just yet. If you want to run the webhooks locally, you'll have to generate certificates for serving the webhooks, and place them in the right -directory (`/tmp/k8s-webhook-server/serving-certs/tls.{crt,key}`, by +directory (`/tmp/k8s-metrics-server/metrics-certs/tls.{crt,key}`, by default). If you're not running a local API server, you'll also need to figure out diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index 458a9f8c3e2..1e9587f5303 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -30,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`. */ var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -74,6 +77,9 @@ func main() { /* */ var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -87,6 +93,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -133,16 +143,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -196,6 +217,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..0aecedcb53d --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index b5cf1e32e2f..196211b6054 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_webhook_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_webhook_patch.yaml index 06ab33e4e87..760bd5c13ea 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_webhook_patch.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_webhook_patch.yaml @@ -16,7 +16,7 @@ spec: name: webhook-server protocol: TCP volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true volumes: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml index c20c83edbc5..b3d3f8e15be 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml @@ -4118,6 +4118,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --cert-dir=/var/metrics/certs/ command: - /manager image: controller:latest @@ -4151,9 +4152,12 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true + - mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -4165,10 +4169,96 @@ spec: secret: defaultMode: 420 secretName: webhook-server-cert + - name: metrics-certs + secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-metrics-certs + namespace: project-system +spec: + dnsNames: + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-serving-cert + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-selfsigned-issuer + namespace: project-system +spec: + selfSigned: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + control-plane: controller-manager + name: project-controller-manager-metrics-monitor + namespace: project-system +spec: + endpoints: + - tlsConfig: + ca: + secret: + key: ca.crt + name: metrics-server-cert + cert: + secret: + key: tls.crt + name: metrics-server-cert + insecureSkipVerify: false + keySecret: + key: tls.key + name: metrics-server-cert + selector: + matchLabels: + control-plane: controller-manager --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-mutating-webhook-configuration webhooks: - admissionReviewVersions: @@ -4195,6 +4285,8 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-validating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go index 7404dd56613..edb60cb7d8d 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go @@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the First, it will contain the necessary imports. */ - package controller import ( diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index 7347d215463..60911f0a0c2 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -41,8 +43,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -54,6 +57,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -67,6 +73,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -113,16 +123,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -158,6 +179,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..0aecedcb53d --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 317242ed92c..304f01b1eb9 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go index a199eec734b..4cb6e1cf73a 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -51,8 +53,9 @@ import ( */ var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -73,6 +76,9 @@ func main() { /* */ var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -86,6 +92,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -132,16 +142,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -199,6 +220,14 @@ func main() { /* */ + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..0aecedcb53d --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml index 16e19c0840f..815dddacff7 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: @@ -171,7 +213,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_webhook_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_webhook_patch.yaml index 06ab33e4e87..760bd5c13ea 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_webhook_patch.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_webhook_patch.yaml @@ -16,7 +16,7 @@ spec: name: webhook-server protocol: TCP volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true volumes: diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml index 8a9b9c9f3ab..0368756f0bf 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml @@ -7929,6 +7929,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --cert-dir=/var/metrics/certs/ command: - /manager image: controller:latest @@ -7962,10 +7963,10 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true - - mountPath: /tmp/k8s-metrics-server/metrics-certs + - mountPath: /var/metrics/certs/ name: metrics-certs readOnly: true securityContext: @@ -7981,24 +7982,28 @@ spec: secretName: webhook-server-cert - name: metrics-certs secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false secretName: metrics-server-cert --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: metrics-certs app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-metrics-certs namespace: project-system spec: dnsNames: - - project-webhook-service.project-system.svc - - project-webhook-service.project-system.svc.cluster.local + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local issuerRef: kind: Issuer name: project-selfsigned-issuer @@ -8008,12 +8013,8 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: serving-cert app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-serving-cert namespace: project-system spec: diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go index 685666bf101..5a7f996517b 100644 --- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go +++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go @@ -90,7 +90,18 @@ func (sp *Sample) UpdateTutorial() { // 4. update makefile sp.updateMakefile() // 5. generate extra files - sp.codeGen() + cmd := exec.Command("go", "mod", "tidy") + _, err := sp.ctx.Run(cmd) + hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) + + cmd = exec.Command("go", "get", "github.com/robfig/cron") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("Failed to get package robfig/cron", err) + + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) + // 6. compensate other intro in API sp.updateAPIStuff() // 7. update reconciliation and main.go @@ -99,7 +110,9 @@ func (sp *Sample) UpdateTutorial() { // 7.2 update main.go sp.updateMain() // 8. generate extra files - sp.codeGen() + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) // 9. update suite_test explanation sp.updateSuiteTest() // 10. uncomment kustomization @@ -113,25 +126,14 @@ func (sp *Sample) UpdateTutorial() { // CodeGen is a noop for this sample, just to make generation of all samples // more efficient. We may want to refactor `UpdateTutorial` some day to take // advantage of a separate call, but it is not necessary. -func (sp *Sample) CodeGen() {} - -func (sp *Sample) codeGen() { - cmd := exec.Command("go", "mod", "tidy") +func (sp *Sample) CodeGen() { + cmd := exec.Command("make", "all") _, err := sp.ctx.Run(cmd) - hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) - - cmd = exec.Command("go", "get", "github.com/robfig/cron") - _, err = sp.ctx.Run(cmd) - hackutils.CheckError("Failed to get package robfig/cron", err) - - cmd = exec.Command("make", "all") - _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make all for cronjob tutorial", err) cmd = exec.Command("make", "build-installer") _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make build-installer for cronjob tutorial", err) - } // insert code to fix docs @@ -355,15 +357,6 @@ CronJob controller's`+" `"+`SetupWithManager`+"`"+` method. }`, ` // +kubebuilder:docs-gen:collapse=old stuff`) hackutils.CheckError("fixing main.go", err) - - // Enabling metrics with certs - err = pluginutil.UncommentCode( - filepath.Join(sp.ctx.Dir, "cmd/main.go"), - `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"`, ` - // `) - hackutils.CheckError("enabling metrics service options into main.go", err) } func (sp *Sample) updateMakefile() { @@ -601,8 +594,10 @@ func (sp *Sample) updateKustomization() { err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment`, `#`) + hackutils.CheckError("enabling cert_metrics_manager_patch.yaml", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/prometheus/kustomization.yaml"), @@ -610,11 +605,11 @@ func (sp *Sample) updateKustomization() { # - path: monitor_tls_patch.yaml # target: # kind: ServiceMonitor`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + hackutils.CheckError("enabling monitor tls patch", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - certmanagerForWebhooks, `#`) + certManagerForMetricsAndWebhooks, `#`) hackutils.CheckError("fixing default/kustomization", err) } diff --git a/hack/docs/internal/cronjob-tutorial/sample.go b/hack/docs/internal/cronjob-tutorial/sample.go index c7ee019a09b..8fa8a6afe2c 100644 --- a/hack/docs/internal/cronjob-tutorial/sample.go +++ b/hack/docs/internal/cronjob-tutorial/sample.go @@ -33,7 +33,45 @@ const CronjobSample = ` - date; echo Hello from the Kubernetes cluster restartPolicy: OnFailure` -const certmanagerForWebhooks = `#replacements: +const certManagerForMetricsAndWebhooks = `#replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -44,6 +82,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -61,6 +100,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -88,7 +128,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -104,7 +144,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -119,7 +159,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: diff --git a/hack/docs/internal/multiversion-tutorial/kustomize.go b/hack/docs/internal/multiversion-tutorial/kustomize.go index 45abfa37ad3..c1a0d10d952 100644 --- a/hack/docs/internal/multiversion-tutorial/kustomize.go +++ b/hack/docs/internal/multiversion-tutorial/kustomize.go @@ -21,7 +21,7 @@ const caInjectionNamespace = `# # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -38,7 +38,7 @@ const caInjectionCert = `# - source: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go new file mode 100644 index 00000000000..8bd38eec138 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go @@ -0,0 +1,68 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &MetricsCertificate{} + +// MetricsCertificate scaffolds a file that defines the issuer CR and the metrics certificate CR +type MetricsCertificate struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements machinery.Template +func (f *MetricsCertificate) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "certmanager", "certificate-metrics.yaml") + } + + f.TemplateBody = metricsCertManagerTemplate + + // If file exists, skip creation. + f.IfExistsAction = machinery.SkipFile + + return nil +} + +// nolint:lll +const metricsCertManagerTemplate = `# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go similarity index 57% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go index 2fc28f5da7b..955a204aa7a 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go @@ -33,7 +33,7 @@ type Certificate struct { // SetTemplateDefaults implements machinery.Template func (f *Certificate) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "certmanager", "certificate.yaml") + f.Path = filepath.Join("config", "certmanager", "certificate-webhook.yaml") } f.TemplateBody = certManagerTemplate @@ -46,32 +46,17 @@ func (f *Certificate) SetTemplateDefaults() error { const certManagerTemplate = `# The following manifests contain a self-signed issuer CR and a certificate CR. # More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: {{ .ProjectName }} - app.kubernetes.io/part-of: {{ .ProjectName }} + app.kubernetes.io/name: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. dnsNames: - SERVICE_NAME.SERVICE_NAMESPACE.svc - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local @@ -79,26 +64,4 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: {{ .ProjectName }} - app.kubernetes.io/part-of: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go new file mode 100644 index 00000000000..67374db98b2 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go @@ -0,0 +1,60 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &Issuer{} + +// Issuer scaffolds a file that defines the self-signed Issuer CR +type Issuer struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements machinery.Template +func (f *Issuer) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "certmanager", "issuer.yaml") + } + + f.TemplateBody = issuerTemplate + + // If file exists, skip creation. + f.IfExistsAction = machinery.SkipFile + + return nil +} + +const issuerTemplate = `# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go index 5b1399bcd15..8d8ac05a9fd 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go @@ -44,7 +44,9 @@ func (f *Kustomization) SetTemplateDefaults() error { } const kustomizationTemplate = `resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go similarity index 53% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go index 31859c3fd89..96af20ec75c 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go @@ -35,7 +35,7 @@ type CertManagerMetricsPatch struct { // SetTemplateDefaults implements machinery.Template func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml") + f.Path = filepath.Join("config", "default", "cert_metrics_manager_patch.yaml") } f.TemplateBody = metricsManagerPatchTemplate @@ -50,25 +50,40 @@ func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { return nil } -const metricsManagerPatchTemplate = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert +// nolint:lll +const metricsManagerPatchTemplate = `# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index 636bae6bbb0..4cc96ab4011 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -87,9 +87,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -98,6 +100,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -108,6 +148,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -125,6 +166,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -152,7 +194,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -168,7 +210,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -183,7 +225,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -199,7 +241,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -207,7 +249,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/webhook_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/webhook_manager_patch.go index 8249cc02428..645a01d484b 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/webhook_manager_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/webhook_manager_patch.go @@ -68,7 +68,7 @@ spec: name: webhook-server protocol: TCP volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true volumes: diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go index 869b58b63e9..8078eb545b7 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go @@ -84,6 +84,8 @@ func (s *webhookScaffolder) Scaffold() error { &webhook.KustomizeConfig{}, &webhook.Service{}, &certmanager.Certificate{}, + &certmanager.Issuer{}, + &certmanager.MetricsCertificate{}, &certmanager.Kustomization{}, &certmanager.KustomizeConfig{}, &network_policy.NetworkPolicyAllowWebhooks{}, diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go index 136be031810..999a9cdddfc 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go @@ -226,6 +226,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -235,6 +236,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -246,6 +248,7 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -256,6 +259,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -269,6 +275,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -315,16 +325,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -357,6 +378,14 @@ func main() { %s + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/pkg/plugins/optional/helm/v1alpha/scaffolds/internal/templates/chart-templates/manager/manager.go b/pkg/plugins/optional/helm/v1alpha/scaffolds/internal/templates/chart-templates/manager/manager.go index de01d434230..39d54b6f065 100644 --- a/pkg/plugins/optional/helm/v1alpha/scaffolds/internal/templates/chart-templates/manager/manager.go +++ b/pkg/plugins/optional/helm/v1alpha/scaffolds/internal/templates/chart-templates/manager/manager.go @@ -119,7 +119,7 @@ spec: {{- if .HasWebhooks }} {{ "{{- if and .Values.webhook.enable .Values.certmanager.enable }}" }} - name: webhook-cert - mountPath: /tmp/k8s-webhook-server/serving-certs + mountPath: /tmp/k8s-metrics-server/metrics-certs readOnly: true {{ "{{- end }}" }} {{- end }} diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index 89e61054294..143a40f5056 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -64,6 +64,8 @@ func GenerateV4(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( @@ -71,10 +73,7 @@ func GenerateV4(kbc *utils.TestContext) { monitorTlsPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) + metricsCertPatch, "#")).To(Succeed()) uncommentKustomizeCoversion(kbc) @@ -111,6 +110,8 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) // Disable metrics @@ -177,18 +178,21 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) { metricsTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + metricsCertPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), monitorTlsPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) + filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), + metricsCertReplaces, "#")).To(Succeed()) + By("uncomment kustomization.yaml to enable network policy") ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../network-policy", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) @@ -243,8 +247,7 @@ const metricsTarget = `- path: manager_metrics_patch.yaml kind: Deployment` //nolint:lll -const certManagerTarget = `#replacements: -# - source: # Uncomment the following block if you have any webhook +const certManagerTarget = `# - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 # name: webhook-service @@ -254,6 +257,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -271,6 +275,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -298,7 +303,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -314,7 +319,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -329,7 +334,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -345,7 +350,7 @@ const certNamespace = `# - source: # Uncomment the following block if you have a # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -362,7 +367,7 @@ const certName = `# - source: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -452,6 +457,44 @@ const monitorTlsPatch = `#patches: # target: # kind: ServiceMonitor` -const tlsConfigManager = `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"` +const metricsCertPatch = `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment` + +const metricsCertReplaces = `# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true` diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go index 275bd9065c7..d33c92c9f70 100644 --- a/testdata/project-v4-multigroup/cmd/main.go +++ b/testdata/project-v4-multigroup/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -70,8 +72,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -95,6 +98,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -108,6 +114,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -154,16 +164,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -348,6 +369,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..2019d532f05 --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2d174f830a9 --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml deleted file mode 100644 index 73765b7407e..00000000000 --- a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/part-of: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/part-of: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/issuer.yaml b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..e266ca5119c --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..0aecedcb53d --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index a6b8d9bc0de..00000000000 --- a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 78743d725ce..2079d714859 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4-multigroup/config/default/manager_webhook_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_webhook_patch.yaml index 9fd690c819f..978b5ff1ae2 100644 --- a/testdata/project-v4-multigroup/config/default/manager_webhook_patch.yaml +++ b/testdata/project-v4-multigroup/config/default/manager_webhook_patch.yaml @@ -16,7 +16,7 @@ spec: name: webhook-server protocol: TCP volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true volumes: diff --git a/testdata/project-v4-multigroup/dist/install.yaml b/testdata/project-v4-multigroup/dist/install.yaml index 9e6adf59b9d..ab3a18328d5 100644 --- a/testdata/project-v4-multigroup/dist/install.yaml +++ b/testdata/project-v4-multigroup/dist/install.yaml @@ -2153,7 +2153,7 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true securityContext: diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go index f01870085dc..afe685c5391 100644 --- a/testdata/project-v4-with-plugins/cmd/main.go +++ b/testdata/project-v4-with-plugins/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -45,8 +47,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -60,6 +63,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -73,6 +79,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -119,16 +129,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -194,6 +215,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..b38d5abec0f --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..8604f1cc0c5 --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml deleted file mode 100644 index d4580c48964..00000000000 --- a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-with-plugins - app.kubernetes.io/part-of: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-with-plugins - app.kubernetes.io/part-of: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..05e6df0e47f --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..0aecedcb53d --- /dev/null +++ b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index ed2f033ddbb..00000000000 --- a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml index d9a6926909f..39a8fdd01f7 100644 --- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4-with-plugins/config/default/manager_webhook_patch.yaml b/testdata/project-v4-with-plugins/config/default/manager_webhook_patch.yaml index 34989fbbeb1..5ac92950761 100644 --- a/testdata/project-v4-with-plugins/config/default/manager_webhook_patch.yaml +++ b/testdata/project-v4-with-plugins/config/default/manager_webhook_patch.yaml @@ -16,7 +16,7 @@ spec: name: webhook-server protocol: TCP volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true volumes: diff --git a/testdata/project-v4-with-plugins/dist/chart/templates/manager/manager.yaml b/testdata/project-v4-with-plugins/dist/chart/templates/manager/manager.yaml index f37cfc16711..8f3c80293d7 100644 --- a/testdata/project-v4-with-plugins/dist/chart/templates/manager/manager.yaml +++ b/testdata/project-v4-with-plugins/dist/chart/templates/manager/manager.yaml @@ -59,7 +59,7 @@ spec: volumeMounts: {{- if and .Values.webhook.enable .Values.certmanager.enable }} - name: webhook-cert - mountPath: /tmp/k8s-webhook-server/serving-certs + mountPath: /tmp/k8s-metrics-server/metrics-certs readOnly: true {{- end }} {{- if and .Values.metrics.enable .Values.certmanager.enable }} diff --git a/testdata/project-v4-with-plugins/dist/install.yaml b/testdata/project-v4-with-plugins/dist/install.yaml index 79528f60b21..292eb9afdc0 100644 --- a/testdata/project-v4-with-plugins/dist/install.yaml +++ b/testdata/project-v4-with-plugins/dist/install.yaml @@ -846,7 +846,7 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true securityContext: diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go index 1a72e4c73c1..11b79fe2ef0 100644 --- a/testdata/project-v4/cmd/main.go +++ b/testdata/project-v4/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -48,8 +50,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -63,6 +66,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -76,6 +82,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -122,16 +132,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -230,6 +251,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4/config/certmanager/certificate-metrics.yaml b/testdata/project-v4/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..6c2488daae5 --- /dev/null +++ b/testdata/project-v4/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/certificate-webhook.yaml b/testdata/project-v4/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..b3d765563ea --- /dev/null +++ b/testdata/project-v4/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/certificate.yaml b/testdata/project-v4/config/certmanager/certificate.yaml deleted file mode 100644 index 0cffc07a915..00000000000 --- a/testdata/project-v4/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/part-of: project-v4 - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/part-of: project-v4 - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/issuer.yaml b/testdata/project-v4/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..65f38af73de --- /dev/null +++ b/testdata/project-v4/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4/config/certmanager/kustomization.yaml b/testdata/project-v4/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4/config/certmanager/kustomization.yaml +++ b/testdata/project-v4/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..0aecedcb53d --- /dev/null +++ b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /var/metrics/certs/ + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/var/metrics/certs/ +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 862a49388f2..00000000000 --- a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index 5451766b0b3..a252b019971 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4/config/default/manager_webhook_patch.yaml b/testdata/project-v4/config/default/manager_webhook_patch.yaml index df50ae835d6..e3b00daccdf 100644 --- a/testdata/project-v4/config/default/manager_webhook_patch.yaml +++ b/testdata/project-v4/config/default/manager_webhook_patch.yaml @@ -16,7 +16,7 @@ spec: name: webhook-server protocol: TCP volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true volumes: diff --git a/testdata/project-v4/dist/install.yaml b/testdata/project-v4/dist/install.yaml index 33f1336fa30..b5f89774204 100644 --- a/testdata/project-v4/dist/install.yaml +++ b/testdata/project-v4/dist/install.yaml @@ -711,7 +711,7 @@ spec: drop: - ALL volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs + - mountPath: /tmp/k8s-metrics-server/metrics-certs name: cert readOnly: true securityContext: