From 027f1482db3125c7b2fbf742baf7dfbf4bd49ca4 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:10:28 +0000 Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics endpoints (follow-up to PR #4243) This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready --- .github/workflows/test-e2e-samples.yml | 19 ++- Makefile | 5 + .../testdata/project/cmd/main.go | 56 +++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------- .../project/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../project/config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- .../testdata/project/dist/install.yaml | 92 +++++++++++ .../project/internal/controller/suite_test.go | 1 - .../testdata/project/cmd/main.go | 58 +++++-- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../project/config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- .../testdata/project/cmd/main.go | 56 +++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------- .../project/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../project/config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- .../testdata/project/dist/install.yaml | 25 +-- docs/book/src/reference/metrics.md | 154 +++++++++++------- .../cronjob-tutorial/generate_cronjob.go | 51 +++--- hack/docs/internal/cronjob-tutorial/sample.go | 48 +++++- .../multiversion-tutorial/kustomize.go | 4 +- .../config/certmanager/certificate_metrics.go | 68 ++++++++ ...{certificate.go => certificate_webhook.go} | 43 +---- .../templates/config/certmanager/issuer.go | 60 +++++++ .../config/certmanager/kustomization.go | 4 +- ...patch.go => cert_metrics_manager_patch.go} | 55 ++++--- .../config/kdefault/kustomization.go | 58 ++++++- .../config/prometheus/kustomization.go | 2 +- .../common/kustomize/v2/scaffolds/webhook.go | 2 + .../scaffolds/internal/templates/cmd/main.go | 62 +++++-- test/e2e/v4/generate_test.go | 81 +++++++-- testdata/project-v4-multigroup/cmd/main.go | 54 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------- .../config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- testdata/project-v4-with-plugins/cmd/main.go | 56 +++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------- .../config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- testdata/project-v4/cmd/main.go | 54 ++++-- .../certmanager/certificate-metrics.yaml | 20 +++ .../certmanager/certificate-webhook.yaml | 20 +++ .../config/certmanager/certificate.yaml | 57 ------- .../project-v4/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 31 ++++ .../certmanager_metrics_manager_patch.yaml | 21 --- .../config/default/kustomization.yaml | 58 ++++++- .../config/prometheus/kustomization.yaml | 2 +- 74 files changed, 1640 insertions(+), 770 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/{certificate.go => certificate_webhook.go} (57%) create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (56%) create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index b0db7dc21c6..6d84a992a2f 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -41,9 +41,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ go mod tidy @@ -85,10 +86,11 @@ jobs: # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces; we are leaving defaulting uncommented # since this sample has no defaulting webhooks - sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only --conversion webhooks CA injection - sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-with-plugins/ go mod tidy @@ -127,9 +129,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + # Uncomment all cert-manager injections for webhooks only + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-multigroup go mod tidy diff --git a/Makefile b/Makefile index 7d1db87f3b5..51782cb5f47 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,11 @@ lint: golangci-lint yamllint ## Run golangci-lint linter & yamllint lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify + + .PHONY: yamllint yamllint: @files=$$(find testdata -name '*.yaml' ! -path 'testdata/*/dist/*'); \ diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index 458a9f8c3e2..58fa2fd8fc0 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -30,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -74,6 +76,7 @@ func main() { /* */ var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -87,6 +90,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -112,6 +118,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -124,25 +133,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) + + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -196,6 +218,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index b5cf1e32e2f..0cf156e6276 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block to enable certificates for metrics + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml index e00ddc958dc..8126ea89b1a 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml index c20c83edbc5..27dbb9bb5ab 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml @@ -4118,6 +4118,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs command: - /manager image: controller:latest @@ -4154,6 +4155,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -4165,10 +4169,96 @@ spec: secret: defaultMode: 420 secretName: webhook-server-cert + - name: metrics-certs + secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-metrics-certs + namespace: project-system +spec: + dnsNames: + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-serving-cert + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-selfsigned-issuer + namespace: project-system +spec: + selfSigned: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + control-plane: controller-manager + name: project-controller-manager-metrics-monitor + namespace: project-system +spec: + endpoints: + - tlsConfig: + ca: + secret: + key: ca.crt + name: metrics-server-cert + cert: + secret: + key: tls.crt + name: metrics-server-cert + insecureSkipVerify: false + keySecret: + key: tls.key + name: metrics-server-cert + selector: + matchLabels: + control-plane: controller-manager --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-mutating-webhook-configuration webhooks: - admissionReviewVersions: @@ -4195,6 +4285,8 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-validating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go index 7404dd56613..edb60cb7d8d 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go @@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the First, it will contain the necessary imports. */ - package controller import ( diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index 7347d215463..3cd126e887a 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -54,6 +56,7 @@ func init() { func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -67,6 +70,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -92,6 +98,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -104,25 +113,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + FilterProvider: filters.WithAuthenticationAndAuthorization, + } + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) + + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -158,6 +180,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 317242ed92c..75241299903 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml index f040fd58623..fdc5481b103 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go index a199eec734b..d56885473b2 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -73,6 +75,7 @@ func main() { /* */ var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -86,6 +89,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -111,6 +117,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -123,25 +132,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) + + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -199,6 +221,14 @@ func main() { /* */ + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml index 16e19c0840f..dcdae5c7e21 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block to enable certificates for metrics + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: @@ -171,7 +213,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml index e00ddc958dc..8126ea89b1a 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml index 8a9b9c9f3ab..f3716ea8657 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml @@ -7929,6 +7929,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs command: - /manager image: controller:latest @@ -7981,24 +7982,28 @@ spec: secretName: webhook-server-cert - name: metrics-certs secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false secretName: metrics-server-cert --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: metrics-certs app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-metrics-certs namespace: project-system spec: dnsNames: - - project-webhook-service.project-system.svc - - project-webhook-service.project-system.svc.cluster.local + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local issuerRef: kind: Issuer name: project-selfsigned-issuer @@ -8008,12 +8013,8 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: serving-cert app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-serving-cert namespace: project-system spec: diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index dbb85222709..0da7d5ab5b8 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -126,57 +126,28 @@ spec: sleep 60; done ``` + +### **(Recommended)** Enabling certificates for Production (Disabled by default) + - - - -### By exposing the metrics endpoint using HTTPS and Cert-Manager - -Integrating `cert-manager` with your metrics service enables secure -HTTPS access via TLS encryption. Follow the steps below to configure -your project to expose the metrics endpoint using HTTPS with cert-manager. +Projects built with Kubebuilder releases `4.4.0` and above have the logic scaffolded +to enable the usage of certificates managed by [CertManager](https://cert-manager.io/) +for securing the metrics server. Following the steps below, you can configure your +project to use certificates managed by CertManager. 1. **Enable Cert-Manager in `config/default/kustomization.yaml`:** - Uncomment the cert-manager resource to include it in your project: @@ -185,36 +156,80 @@ your project to expose the metrics endpoint using HTTPS with cert-manager. - ../certmanager ``` -2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:** - - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification: +2. **Enable the Patch to configure the usage of the certs in the Controller Deployment in `config/default/kustomization.yaml`:** + - Uncomment the `cert_metrics_manager_patch.yaml` to mount the `serving-cert` secret in the Manager Deployment. ```yaml - - path: monitor_tls_patch.yaml + # Uncomment the patches line if you enable Metrics and CertManager + # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. + # This patch will protect the metrics with certManager self-signed certs. + - path: cert_metrics_manager_patch.yaml target: - kind: ServiceMonitor + kind: Deployment ``` - -3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:** - - Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment. +3. **Enable the CertManager replaces for the Metrics Server certificates in `config/default/kustomization.yaml`:** + - Uncomment the replacements block bellow. It is required to properly set the DNS names for the certificates configured under `config/certmanager`. ```yaml - - path: manager_webhook_patch.yaml + # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. + # Uncomment the following replacements to add the cert-manager CA injection annotations + #replacements: + # - source: # Uncomment the following block to enable certificates for metrics + # kind: Service + # version: v1 + # name: controller-manager-metrics-service + # fieldPath: metadata.name + # targets: + # - select: + # kind: Certificate + # group: cert-manager.io + # version: v1 + # name: metrics-certs + # fieldPaths: + # - spec.dnsNames.0 + # - spec.dnsNames.1 + # options: + # delimiter: '.' + # index: 0 + # create: true + # + # - source: + # kind: Service + # version: v1 + # name: controller-manager-metrics-service + # fieldPath: metadata.namespace + # targets: + # - select: + # kind: Certificate + # group: cert-manager.io + # version: v1 + # name: metrics-certs + # fieldPaths: + # - spec.dnsNames.0 + # - spec.dnsNames.1 + # options: + # delimiter: '.' + # index: 1 + # create: true + # ``` -4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:** - - Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates. - Uncomment the lines for `CertDir`, `CertName`, and `KeyName`: - - ```go - if secureMetrics { - ... - metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - metricsServerOptions.CertName = "tls.crt" - metricsServerOptions.KeyName = "tls.key" - } +4. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:** + - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification: + + ```yaml + # [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus + # to securely reference certificates created and managed by cert-manager. + # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml + # to mount the "metrics-server-cert" secret in the Manager Deployment. + patches: + - path: monitor_tls_patch.yaml + target: + kind: ServiceMonitor ``` -### By using Network Policy (You can optionally enable) + +### **(Optional)** By using Network Policy (Disabled by default) NetworkPolicy acts as a basic firewall for pods within a Kubernetes cluster, controlling traffic flow at the IP address or port level. However, it doesn't handle `authn/authz`. @@ -327,3 +342,18 @@ Those metrics will be available for prometheus or other openmetrics systems to scrape. ![Screen Shot 2021-06-14 at 10 15 59 AM](https://user-images.githubusercontent.com/37827279/121932262-8843cd80-ccf9-11eb-9c8e-98d0eda80169.png) + + \ No newline at end of file diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go index 685666bf101..5a7f996517b 100644 --- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go +++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go @@ -90,7 +90,18 @@ func (sp *Sample) UpdateTutorial() { // 4. update makefile sp.updateMakefile() // 5. generate extra files - sp.codeGen() + cmd := exec.Command("go", "mod", "tidy") + _, err := sp.ctx.Run(cmd) + hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) + + cmd = exec.Command("go", "get", "github.com/robfig/cron") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("Failed to get package robfig/cron", err) + + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) + // 6. compensate other intro in API sp.updateAPIStuff() // 7. update reconciliation and main.go @@ -99,7 +110,9 @@ func (sp *Sample) UpdateTutorial() { // 7.2 update main.go sp.updateMain() // 8. generate extra files - sp.codeGen() + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) // 9. update suite_test explanation sp.updateSuiteTest() // 10. uncomment kustomization @@ -113,25 +126,14 @@ func (sp *Sample) UpdateTutorial() { // CodeGen is a noop for this sample, just to make generation of all samples // more efficient. We may want to refactor `UpdateTutorial` some day to take // advantage of a separate call, but it is not necessary. -func (sp *Sample) CodeGen() {} - -func (sp *Sample) codeGen() { - cmd := exec.Command("go", "mod", "tidy") +func (sp *Sample) CodeGen() { + cmd := exec.Command("make", "all") _, err := sp.ctx.Run(cmd) - hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) - - cmd = exec.Command("go", "get", "github.com/robfig/cron") - _, err = sp.ctx.Run(cmd) - hackutils.CheckError("Failed to get package robfig/cron", err) - - cmd = exec.Command("make", "all") - _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make all for cronjob tutorial", err) cmd = exec.Command("make", "build-installer") _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make build-installer for cronjob tutorial", err) - } // insert code to fix docs @@ -355,15 +357,6 @@ CronJob controller's`+" `"+`SetupWithManager`+"`"+` method. }`, ` // +kubebuilder:docs-gen:collapse=old stuff`) hackutils.CheckError("fixing main.go", err) - - // Enabling metrics with certs - err = pluginutil.UncommentCode( - filepath.Join(sp.ctx.Dir, "cmd/main.go"), - `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"`, ` - // `) - hackutils.CheckError("enabling metrics service options into main.go", err) } func (sp *Sample) updateMakefile() { @@ -601,8 +594,10 @@ func (sp *Sample) updateKustomization() { err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment`, `#`) + hackutils.CheckError("enabling cert_metrics_manager_patch.yaml", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/prometheus/kustomization.yaml"), @@ -610,11 +605,11 @@ func (sp *Sample) updateKustomization() { # - path: monitor_tls_patch.yaml # target: # kind: ServiceMonitor`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + hackutils.CheckError("enabling monitor tls patch", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - certmanagerForWebhooks, `#`) + certManagerForMetricsAndWebhooks, `#`) hackutils.CheckError("fixing default/kustomization", err) } diff --git a/hack/docs/internal/cronjob-tutorial/sample.go b/hack/docs/internal/cronjob-tutorial/sample.go index c7ee019a09b..f963d137e74 100644 --- a/hack/docs/internal/cronjob-tutorial/sample.go +++ b/hack/docs/internal/cronjob-tutorial/sample.go @@ -33,7 +33,45 @@ const CronjobSample = ` - date; echo Hello from the Kubernetes cluster restartPolicy: OnFailure` -const certmanagerForWebhooks = `#replacements: +const certManagerForMetricsAndWebhooks = `#replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -44,6 +82,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -61,6 +100,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -88,7 +128,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -104,7 +144,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -119,7 +159,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: diff --git a/hack/docs/internal/multiversion-tutorial/kustomize.go b/hack/docs/internal/multiversion-tutorial/kustomize.go index 45abfa37ad3..c1a0d10d952 100644 --- a/hack/docs/internal/multiversion-tutorial/kustomize.go +++ b/hack/docs/internal/multiversion-tutorial/kustomize.go @@ -21,7 +21,7 @@ const caInjectionNamespace = `# # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -38,7 +38,7 @@ const caInjectionCert = `# - source: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go new file mode 100644 index 00000000000..9b82f883206 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &MetricsCertificate{} + +// MetricsCertificate scaffolds a file that defines the issuer CR and the metrics certificate CR +type MetricsCertificate struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements machinery.Template +func (f *MetricsCertificate) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "certmanager", "certificate-metrics.yaml") + } + + f.TemplateBody = metricsCertManagerTemplate + + // If file exists, skip creation. + f.IfExistsAction = machinery.SkipFile + + return nil +} + +// nolint:lll +const metricsCertManagerTemplate = `# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go similarity index 57% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go index 2fc28f5da7b..955a204aa7a 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go @@ -33,7 +33,7 @@ type Certificate struct { // SetTemplateDefaults implements machinery.Template func (f *Certificate) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "certmanager", "certificate.yaml") + f.Path = filepath.Join("config", "certmanager", "certificate-webhook.yaml") } f.TemplateBody = certManagerTemplate @@ -46,32 +46,17 @@ func (f *Certificate) SetTemplateDefaults() error { const certManagerTemplate = `# The following manifests contain a self-signed issuer CR and a certificate CR. # More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: {{ .ProjectName }} - app.kubernetes.io/part-of: {{ .ProjectName }} + app.kubernetes.io/name: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. dnsNames: - SERVICE_NAME.SERVICE_NAMESPACE.svc - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local @@ -79,26 +64,4 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: {{ .ProjectName }} - app.kubernetes.io/part-of: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go new file mode 100644 index 00000000000..bef8a07681b --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &Issuer{} + +// Issuer scaffolds a file that defines the self-signed Issuer CR +type Issuer struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements machinery.Template +func (f *Issuer) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "certmanager", "issuer.yaml") + } + + f.TemplateBody = issuerTemplate + + // If file exists, skip creation. + f.IfExistsAction = machinery.SkipFile + + return nil +} + +const issuerTemplate = `# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go index 5b1399bcd15..8d8ac05a9fd 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go @@ -44,7 +44,9 @@ func (f *Kustomization) SetTemplateDefaults() error { } const kustomizationTemplate = `resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go similarity index 56% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go index 31859c3fd89..505fc10832a 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go @@ -35,7 +35,7 @@ type CertManagerMetricsPatch struct { // SetTemplateDefaults implements machinery.Template func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml") + f.Path = filepath.Join("config", "default", "cert_metrics_manager_patch.yaml") } f.TemplateBody = metricsManagerPatchTemplate @@ -50,25 +50,36 @@ func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { return nil } -const metricsManagerPatchTemplate = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert +// nolint:lll +const metricsManagerPatchTemplate = `# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index 636bae6bbb0..6c29ad48359 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -87,9 +87,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -98,6 +100,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -108,6 +148,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -125,6 +166,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -152,7 +194,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -168,7 +210,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -183,7 +225,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -199,7 +241,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -207,7 +249,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go index 71dbbef423b..f5f5dffc61f 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go @@ -43,7 +43,7 @@ func (f *Kustomization) SetTemplateDefaults() error { const kustomizationTemplate = `resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go index 869b58b63e9..8078eb545b7 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go @@ -84,6 +84,8 @@ func (s *webhookScaffolder) Scaffold() error { &webhook.KustomizeConfig{}, &webhook.Service{}, &certmanager.Certificate{}, + &certmanager.Issuer{}, + &certmanager.MetricsCertificate{}, &certmanager.Kustomization{}, &certmanager.KustomizeConfig{}, &network_policy.NetworkPolicyAllowWebhooks{}, diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go index 136be031810..4100d6d14f5 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go @@ -226,6 +226,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -235,6 +236,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -256,6 +258,7 @@ func init() { func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -269,6 +272,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -294,37 +300,53 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/server + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html metricsServerOptions := metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, - } - - if secureMetrics { + TLSOpts: tlsOpts, // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) + + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -357,6 +379,14 @@ func main() { %s + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index 89e61054294..2cc62bd3601 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -64,6 +64,8 @@ func GenerateV4(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( @@ -71,11 +73,10 @@ func GenerateV4(kbc *utils.TestContext) { monitorTlsPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + metricsCertPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) - + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + metricsCertReplaces, "#")).To(Succeed()) uncommentKustomizeCoversion(kbc) } @@ -111,6 +112,8 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) // Disable metrics @@ -177,18 +180,21 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) { metricsTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + metricsCertPatch, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + metricsCertReplaces, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), monitorTlsPatch, "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) + By("uncomment kustomization.yaml to enable network policy") ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../network-policy", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) @@ -243,8 +249,7 @@ const metricsTarget = `- path: manager_metrics_patch.yaml kind: Deployment` //nolint:lll -const certManagerTarget = `#replacements: -# - source: # Uncomment the following block if you have any webhook +const certManagerTarget = `# - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 # name: webhook-service @@ -254,6 +259,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -271,6 +277,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -298,7 +305,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -314,7 +321,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -329,7 +336,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -345,7 +352,7 @@ const certNamespace = `# - source: # Uncomment the following block if you have a # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -362,7 +369,7 @@ const certName = `# - source: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -452,6 +459,44 @@ const monitorTlsPatch = `#patches: # target: # kind: ServiceMonitor` -const tlsConfigManager = `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"` +const metricsCertPatch = `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment` + +const metricsCertReplaces = `# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true` diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go index 275bd9065c7..f453acd426d 100644 --- a/testdata/project-v4-multigroup/cmd/main.go +++ b/testdata/project-v4-multigroup/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -95,6 +97,7 @@ func init() { func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -108,6 +111,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -133,6 +139,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -145,25 +154,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -348,6 +370,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..2019d532f05 --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2d174f830a9 --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml deleted file mode 100644 index 73765b7407e..00000000000 --- a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/part-of: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/part-of: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/issuer.yaml b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..e266ca5119c --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index a6b8d9bc0de..00000000000 --- a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 78743d725ce..e659d82f08d 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml b/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml index f040fd58623..fdc5481b103 100644 --- a/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go index f01870085dc..7d4943094c6 100644 --- a/testdata/project-v4-with-plugins/cmd/main.go +++ b/testdata/project-v4-with-plugins/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -60,6 +62,7 @@ func init() { func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -73,6 +76,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -98,6 +104,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -110,25 +119,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) + + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -194,6 +216,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..b38d5abec0f --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..8604f1cc0c5 --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml deleted file mode 100644 index d4580c48964..00000000000 --- a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-with-plugins - app.kubernetes.io/part-of: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-with-plugins - app.kubernetes.io/part-of: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..05e6df0e47f --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index ed2f033ddbb..00000000000 --- a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml index d9a6926909f..bec1b771a28 100644 --- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml b/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml index f040fd58623..fdc5481b103 100644 --- a/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment. diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go index 1a72e4c73c1..f3b74461813 100644 --- a/testdata/project-v4/cmd/main.go +++ b/testdata/project-v4/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -63,6 +65,7 @@ func init() { func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -76,6 +79,9 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -101,6 +107,9 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics certificates + var metricsCertWatcher *certwatcher.CertWatcher + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -113,25 +122,38 @@ func main() { BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, - } - - if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, + } - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + certName := filepath.Join(metricsCertPath, metricsCertName) + certKey := filepath.Join(metricsCertPath, metricsCertKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + setupLog.Info("metrics server is serving securely using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey) + var err error + metricsCertWatcher, err = certwatcher.New(certName, certKey) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -230,6 +252,14 @@ func main() { } // +kubebuilder:scaffold:builder + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4/config/certmanager/certificate-metrics.yaml b/testdata/project-v4/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..6c2488daae5 --- /dev/null +++ b/testdata/project-v4/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/certificate-webhook.yaml b/testdata/project-v4/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..b3d765563ea --- /dev/null +++ b/testdata/project-v4/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/certificate.yaml b/testdata/project-v4/config/certmanager/certificate.yaml deleted file mode 100644 index 0cffc07a915..00000000000 --- a/testdata/project-v4/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/part-of: project-v4 - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/part-of: project-v4 - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/issuer.yaml b/testdata/project-v4/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..65f38af73de --- /dev/null +++ b/testdata/project-v4/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4/config/certmanager/kustomization.yaml b/testdata/project-v4/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4/config/certmanager/kustomization.yaml +++ b/testdata/project-v4/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..8400eb42a7b --- /dev/null +++ b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,31 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs +- op: add + path: /spec/template/spec/volumes + value: [] +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 862a49388f2..00000000000 --- a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index 5451766b0b3..469070a7ed9 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4/config/prometheus/kustomization.yaml b/testdata/project-v4/config/prometheus/kustomization.yaml index f040fd58623..fdc5481b103 100644 --- a/testdata/project-v4/config/prometheus/kustomization.yaml +++ b/testdata/project-v4/config/prometheus/kustomization.yaml @@ -1,7 +1,7 @@ resources: - monitor.yaml -# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus # to securely reference certificates created and managed by cert-manager. # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml # to mount the "metrics-server-cert" secret in the Manager Deployment.