From 8531ea8fc216867d05b9c0532e66789deb813e34 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:10:28 +0000 Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics endpoints (follow-up to PR #4243) This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready --- .github/workflows/test-e2e-samples.yml | 19 +- Makefile | 5 + .../testdata/project/cmd/main.go | 49 ++++- .../certmanager/certificate-metrics.yaml | 20 ++ .../certmanager/certificate-webhook.yaml | 20 ++ .../config/certmanager/certificate.yaml | 57 ------ .../project/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 ++++ .../certmanager_metrics_manager_patch.yaml | 21 -- .../project/config/default/kustomization.yaml | 58 +++++- .../testdata/project/dist/install.yaml | 92 +++++++++ .../project/internal/controller/suite_test.go | 1 - .../testdata/project/cmd/main.go | 49 ++++- .../default/cert_metrics_manager_patch.yaml | 35 ++++ .../certmanager_metrics_manager_patch.yaml | 21 -- .../project/config/default/kustomization.yaml | 58 +++++- .../testdata/project/cmd/main.go | 49 ++++- .../certmanager/certificate-metrics.yaml | 20 ++ .../certmanager/certificate-webhook.yaml | 20 ++ .../config/certmanager/certificate.yaml | 57 ------ .../project/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 ++++ .../certmanager_metrics_manager_patch.yaml | 21 -- .../project/config/default/kustomization.yaml | 58 +++++- .../testdata/project/dist/install.yaml | 25 +-- docs/book/src/reference/metrics.md | 187 ++++++++++++------ .../cronjob-tutorial/generate_cronjob.go | 51 +++-- hack/docs/internal/cronjob-tutorial/sample.go | 48 ++++- .../multiversion-tutorial/kustomize.go | 4 +- .../config/certmanager/certificate_metrics.go | 68 +++++++ ...{certificate.go => certificate_webhook.go} | 43 +--- .../templates/config/certmanager/issuer.go | 60 ++++++ .../config/certmanager/kustomization.go | 4 +- ...patch.go => cert_metrics_manager_patch.go} | 59 +++--- .../config/kdefault/kustomization.go | 58 +++++- .../common/kustomize/v2/scaffolds/webhook.go | 2 + .../scaffolds/internal/templates/cmd/main.go | 45 ++++- test/e2e/v4/generate_test.go | 81 ++++++-- testdata/project-v4-multigroup/cmd/main.go | 45 ++++- .../certmanager/certificate-metrics.yaml | 20 ++ .../certmanager/certificate-webhook.yaml | 20 ++ .../config/certmanager/certificate.yaml | 57 ------ .../config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 ++++ .../certmanager_metrics_manager_patch.yaml | 21 -- .../config/default/kustomization.yaml | 58 +++++- testdata/project-v4-with-plugins/cmd/main.go | 49 ++++- .../certmanager/certificate-metrics.yaml | 20 ++ .../certmanager/certificate-webhook.yaml | 20 ++ .../config/certmanager/certificate.yaml | 57 ------ .../config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 ++++ .../certmanager_metrics_manager_patch.yaml | 21 -- .../config/default/kustomization.yaml | 58 +++++- testdata/project-v4/cmd/main.go | 45 ++++- .../certmanager/certificate-metrics.yaml | 20 ++ .../certmanager/certificate-webhook.yaml | 20 ++ .../config/certmanager/certificate.yaml | 57 ------ .../project-v4/config/certmanager/issuer.yaml | 13 ++ .../config/certmanager/kustomization.yaml | 4 +- .../default/cert_metrics_manager_patch.yaml | 35 ++++ .../certmanager_metrics_manager_patch.yaml | 21 -- .../config/default/kustomization.yaml | 58 +++++- 67 files changed, 1658 insertions(+), 734 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/{certificate.go => certificate_webhook.go} (57%) create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (52%) create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-multigroup/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4-with-plugins/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-metrics.yaml create mode 100644 testdata/project-v4/config/certmanager/certificate-webhook.yaml delete mode 100644 testdata/project-v4/config/certmanager/certificate.yaml create mode 100644 testdata/project-v4/config/certmanager/issuer.yaml create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index b0db7dc21c6..6d84a992a2f 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -41,9 +41,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ go mod tidy @@ -85,10 +86,11 @@ jobs: # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces; we are leaving defaulting uncommented # since this sample has no defaulting webhooks - sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only --conversion webhooks CA injection - sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-with-plugins/ go mod tidy @@ -127,9 +129,10 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - # Uncomment all cert-manager injections - sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH + # Uncomment all cert-manager injections for webhooks only + sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-multigroup go mod tidy diff --git a/Makefile b/Makefile index 7d1db87f3b5..51782cb5f47 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,11 @@ lint: golangci-lint yamllint ## Run golangci-lint linter & yamllint lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify + + .PHONY: yamllint yamllint: @files=$$(find testdata -name '*.yaml' ! -path 'testdata/*/dist/*'); \ diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index 458a9f8c3e2..93ca4108ec8 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -30,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`. */ var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -74,6 +77,9 @@ func main() { /* */ var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -87,6 +93,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -133,16 +143,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -196,6 +217,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2b160b644e --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index b5cf1e32e2f..196211b6054 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml index c20c83edbc5..1b24ecd0b56 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml @@ -4118,6 +4118,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --cert-dir=/tmp/k8s-metrics-server/metrics-certs command: - /manager image: controller:latest @@ -4154,6 +4155,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -4165,10 +4169,96 @@ spec: secret: defaultMode: 420 secretName: webhook-server-cert + - name: metrics-certs + secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-metrics-certs + namespace: project-system +spec: + dnsNames: + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-serving-cert + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-selfsigned-issuer + namespace: project-system +spec: + selfSigned: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + control-plane: controller-manager + name: project-controller-manager-metrics-monitor + namespace: project-system +spec: + endpoints: + - tlsConfig: + ca: + secret: + key: ca.crt + name: metrics-server-cert + cert: + secret: + key: tls.crt + name: metrics-server-cert + insecureSkipVerify: false + keySecret: + key: tls.key + name: metrics-server-cert + selector: + matchLabels: + control-plane: controller-manager --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-mutating-webhook-configuration webhooks: - admissionReviewVersions: @@ -4195,6 +4285,8 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-validating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go index 7404dd56613..edb60cb7d8d 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go @@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the First, it will contain the necessary imports. */ - package controller import ( diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index 7347d215463..53b5d4ca6ab 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -41,8 +43,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -54,6 +57,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -67,6 +73,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -113,16 +123,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -158,6 +179,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2b160b644e --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 317242ed92c..304f01b1eb9 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -162,7 +204,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go index a199eec734b..17d18c8ca26 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -51,8 +53,9 @@ import ( */ var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -73,6 +76,9 @@ func main() { /* */ var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -86,6 +92,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -132,16 +142,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -199,6 +220,14 @@ func main() { /* */ + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..406becb9ead --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2fc59c2f7d7 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml deleted file mode 100644 index ce60daeb22c..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/part-of: project - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..1c600ce5a67 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2b160b644e --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml index 16e19c0840f..815dddacff7 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations replacements: + - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.name + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + + - source: + kind: Service + version: v1 + name: controller-manager-metrics-service + fieldPath: metadata.namespace + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + name: metrics-certs + fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true + - source: # Uncomment the following block if you have any webhook kind: Service version: v1 @@ -63,6 +103,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -80,6 +121,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -107,7 +149,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -123,7 +165,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: - select: @@ -138,7 +180,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: - select: @@ -154,7 +196,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.namespace # Namespace of the certificate CR targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: @@ -171,7 +213,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 - name: serving-cert # This name should match the one in certificate.yaml + name: serving-cert fieldPath: .metadata.name targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. - select: diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml index 8a9b9c9f3ab..c2d17d750ae 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml @@ -7929,6 +7929,7 @@ spec: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --cert-dir=/tmp/k8s-metrics-server/metrics-certs command: - /manager image: controller:latest @@ -7981,24 +7982,28 @@ spec: secretName: webhook-server-cert - name: metrics-certs secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false secretName: metrics-server-cert --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: metrics-certs app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-metrics-certs namespace: project-system spec: dnsNames: - - project-webhook-service.project-system.svc - - project-webhook-service.project-system.svc.cluster.local + - project-controller-manager-metrics-service.project-system.svc + - project-controller-manager-metrics-service.project-system.svc.cluster.local issuerRef: kind: Issuer name: project-selfsigned-issuer @@ -8008,12 +8013,8 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project - app.kubernetes.io/instance: serving-cert app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: certificate - app.kubernetes.io/part-of: project + app.kubernetes.io/name: project name: project-serving-cert namespace: project-system spec: diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index dbb85222709..a39055fc90a 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -126,57 +126,28 @@ spec: sleep 60; done ``` + +### **(Recommended)** Enabling certificates for Production (Disabled by default) + - - - -### By exposing the metrics endpoint using HTTPS and Cert-Manager - -Integrating `cert-manager` with your metrics service enables secure -HTTPS access via TLS encryption. Follow the steps below to configure -your project to expose the metrics endpoint using HTTPS with cert-manager. +Projects built with Kubebuilder releases `4.4.0` and above have the logic scaffolded +to enable the usage of certificates managed by [CertManager](https://cert-manager.io/) +for securing the metrics server. Following the steps below, you can configure your +project to use certificates managed by CertManager. 1. **Enable Cert-Manager in `config/default/kustomization.yaml`:** - Uncomment the cert-manager resource to include it in your project: @@ -185,36 +156,113 @@ your project to expose the metrics endpoint using HTTPS with cert-manager. - ../certmanager ``` -2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:** - - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification: +2. **Enable the Patch to configure the usage of the certs in the Controller Deployment in `config/default/kustomization.yaml`:** + - Uncomment the `cert_metrics_manager_patch.yaml` to mount the `serving-cert` secret in the Manager Deployment. ```yaml - - path: monitor_tls_patch.yaml + # Uncomment the patches line if you enable Metrics and CertManager + # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. + # This patch will protect the metrics with certManager self-signed certs. + - path: cert_metrics_manager_patch.yaml target: - kind: ServiceMonitor + kind: Deployment ``` - -3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:** - - Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment. +3. **Enable the CertManager replaces for the Metrics Server certificates in `config/default/kustomization.yaml`:** + - Uncomment the replacements block bellow. It is required to properly set the DNS names for the certificates configured under `config/certmanager`. ```yaml - - path: manager_webhook_patch.yaml + # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. + # Uncomment the following replacements to add the cert-manager CA injection annotations + #replacements: + # - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] + # kind: Service + # version: v1 + # name: controller-manager-metrics-service + # fieldPath: metadata.name + # targets: + # - select: + # kind: Certificate + # group: cert-manager.io + # version: v1 + # name: metrics-certs + # fieldPaths: + # - spec.dnsNames.0 + # - spec.dnsNames.1 + # options: + # delimiter: '.' + # index: 0 + # create: true + # + # - source: + # kind: Service + # version: v1 + # name: controller-manager-metrics-service + # fieldPath: metadata.namespace + # targets: + # - select: + # kind: Certificate + # group: cert-manager.io + # version: v1 + # name: metrics-certs + # fieldPaths: + # - spec.dnsNames.0 + # - spec.dnsNames.1 + # options: + # delimiter: '.' + # index: 1 + # create: true + # ``` -4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:** - - Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates. - Uncomment the lines for `CertDir`, `CertName`, and `KeyName`: - - ```go - if secureMetrics { - ... - metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - metricsServerOptions.CertName = "tls.crt" - metricsServerOptions.KeyName = "tls.key" - } +4. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:** + - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification: + + ```yaml + # [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus + # to securely reference certificates created and managed by cert-manager. + # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml + # to mount the "metrics-server-cert" secret in the Manager Deployment. + patches: + - path: monitor_tls_patch.yaml + target: + kind: ServiceMonitor ``` -### By using Network Policy (You can optionally enable) +Now, that you properly enable this option check the `cmd/main.go` to see how it will be used:** + +```go + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + + // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } + } +``` + +### **(Optional)** By using Network Policy (Disabled by default) NetworkPolicy acts as a basic firewall for pods within a Kubernetes cluster, controlling traffic flow at the IP address or port level. However, it doesn't handle `authn/authz`. @@ -327,3 +375,18 @@ Those metrics will be available for prometheus or other openmetrics systems to scrape. ![Screen Shot 2021-06-14 at 10 15 59 AM](https://user-images.githubusercontent.com/37827279/121932262-8843cd80-ccf9-11eb-9c8e-98d0eda80169.png) + + \ No newline at end of file diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go index 685666bf101..5a7f996517b 100644 --- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go +++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go @@ -90,7 +90,18 @@ func (sp *Sample) UpdateTutorial() { // 4. update makefile sp.updateMakefile() // 5. generate extra files - sp.codeGen() + cmd := exec.Command("go", "mod", "tidy") + _, err := sp.ctx.Run(cmd) + hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) + + cmd = exec.Command("go", "get", "github.com/robfig/cron") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("Failed to get package robfig/cron", err) + + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) + // 6. compensate other intro in API sp.updateAPIStuff() // 7. update reconciliation and main.go @@ -99,7 +110,9 @@ func (sp *Sample) UpdateTutorial() { // 7.2 update main.go sp.updateMain() // 8. generate extra files - sp.codeGen() + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) // 9. update suite_test explanation sp.updateSuiteTest() // 10. uncomment kustomization @@ -113,25 +126,14 @@ func (sp *Sample) UpdateTutorial() { // CodeGen is a noop for this sample, just to make generation of all samples // more efficient. We may want to refactor `UpdateTutorial` some day to take // advantage of a separate call, but it is not necessary. -func (sp *Sample) CodeGen() {} - -func (sp *Sample) codeGen() { - cmd := exec.Command("go", "mod", "tidy") +func (sp *Sample) CodeGen() { + cmd := exec.Command("make", "all") _, err := sp.ctx.Run(cmd) - hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) - - cmd = exec.Command("go", "get", "github.com/robfig/cron") - _, err = sp.ctx.Run(cmd) - hackutils.CheckError("Failed to get package robfig/cron", err) - - cmd = exec.Command("make", "all") - _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make all for cronjob tutorial", err) cmd = exec.Command("make", "build-installer") _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make build-installer for cronjob tutorial", err) - } // insert code to fix docs @@ -355,15 +357,6 @@ CronJob controller's`+" `"+`SetupWithManager`+"`"+` method. }`, ` // +kubebuilder:docs-gen:collapse=old stuff`) hackutils.CheckError("fixing main.go", err) - - // Enabling metrics with certs - err = pluginutil.UncommentCode( - filepath.Join(sp.ctx.Dir, "cmd/main.go"), - `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"`, ` - // `) - hackutils.CheckError("enabling metrics service options into main.go", err) } func (sp *Sample) updateMakefile() { @@ -601,8 +594,10 @@ func (sp *Sample) updateKustomization() { err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment`, `#`) + hackutils.CheckError("enabling cert_metrics_manager_patch.yaml", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/prometheus/kustomization.yaml"), @@ -610,11 +605,11 @@ func (sp *Sample) updateKustomization() { # - path: monitor_tls_patch.yaml # target: # kind: ServiceMonitor`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + hackutils.CheckError("enabling monitor tls patch", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - certmanagerForWebhooks, `#`) + certManagerForMetricsAndWebhooks, `#`) hackutils.CheckError("fixing default/kustomization", err) } diff --git a/hack/docs/internal/cronjob-tutorial/sample.go b/hack/docs/internal/cronjob-tutorial/sample.go index c7ee019a09b..8fa8a6afe2c 100644 --- a/hack/docs/internal/cronjob-tutorial/sample.go +++ b/hack/docs/internal/cronjob-tutorial/sample.go @@ -33,7 +33,45 @@ const CronjobSample = ` - date; echo Hello from the Kubernetes cluster restartPolicy: OnFailure` -const certmanagerForWebhooks = `#replacements: +const certManagerForMetricsAndWebhooks = `#replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -44,6 +82,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -61,6 +100,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -88,7 +128,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -104,7 +144,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -119,7 +159,7 @@ const certmanagerForWebhooks = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: diff --git a/hack/docs/internal/multiversion-tutorial/kustomize.go b/hack/docs/internal/multiversion-tutorial/kustomize.go index 45abfa37ad3..c1a0d10d952 100644 --- a/hack/docs/internal/multiversion-tutorial/kustomize.go +++ b/hack/docs/internal/multiversion-tutorial/kustomize.go @@ -21,7 +21,7 @@ const caInjectionNamespace = `# # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -38,7 +38,7 @@ const caInjectionCert = `# - source: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go new file mode 100644 index 00000000000..9b82f883206 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &MetricsCertificate{} + +// MetricsCertificate scaffolds a file that defines the issuer CR and the metrics certificate CR +type MetricsCertificate struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements machinery.Template +func (f *MetricsCertificate) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "certmanager", "certificate-metrics.yaml") + } + + f.TemplateBody = metricsCertManagerTemplate + + // If file exists, skip creation. + f.IfExistsAction = machinery.SkipFile + + return nil +} + +// nolint:lll +const metricsCertManagerTemplate = `# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go similarity index 57% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go index 2fc28f5da7b..955a204aa7a 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go @@ -33,7 +33,7 @@ type Certificate struct { // SetTemplateDefaults implements machinery.Template func (f *Certificate) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "certmanager", "certificate.yaml") + f.Path = filepath.Join("config", "certmanager", "certificate-webhook.yaml") } f.TemplateBody = certManagerTemplate @@ -46,32 +46,17 @@ func (f *Certificate) SetTemplateDefaults() error { const certManagerTemplate = `# The following manifests contain a self-signed issuer CR and a certificate CR. # More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: {{ .ProjectName }} - app.kubernetes.io/part-of: {{ .ProjectName }} + app.kubernetes.io/name: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml namespace: system spec: # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. dnsNames: - SERVICE_NAME.SERVICE_NAMESPACE.svc - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local @@ -79,26 +64,4 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: {{ .ProjectName }} - app.kubernetes.io/part-of: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go new file mode 100644 index 00000000000..bef8a07681b --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &Issuer{} + +// Issuer scaffolds a file that defines the self-signed Issuer CR +type Issuer struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements machinery.Template +func (f *Issuer) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "certmanager", "issuer.yaml") + } + + f.TemplateBody = issuerTemplate + + // If file exists, skip creation. + f.IfExistsAction = machinery.SkipFile + + return nil +} + +const issuerTemplate = `# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go index 5b1399bcd15..8d8ac05a9fd 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go @@ -44,7 +44,9 @@ func (f *Kustomization) SetTemplateDefaults() error { } const kustomizationTemplate = `resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go similarity index 52% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go index 31859c3fd89..cfab8380714 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go @@ -35,7 +35,7 @@ type CertManagerMetricsPatch struct { // SetTemplateDefaults implements machinery.Template func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml") + f.Path = filepath.Join("config", "default", "cert_metrics_manager_patch.yaml") } f.TemplateBody = metricsManagerPatchTemplate @@ -50,25 +50,40 @@ func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { return nil } -const metricsManagerPatchTemplate = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert +// nolint:lll +const metricsManagerPatchTemplate = `# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index 636bae6bbb0..4cc96ab4011 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -87,9 +87,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -98,6 +100,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -108,6 +148,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -125,6 +166,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -152,7 +194,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -168,7 +210,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -183,7 +225,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -199,7 +241,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionns @@ -207,7 +249,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go index 869b58b63e9..8078eb545b7 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go @@ -84,6 +84,8 @@ func (s *webhookScaffolder) Scaffold() error { &webhook.KustomizeConfig{}, &webhook.Service{}, &certmanager.Certificate{}, + &certmanager.Issuer{}, + &certmanager.MetricsCertificate{}, &certmanager.Kustomization{}, &certmanager.KustomizeConfig{}, &network_policy.NetworkPolicyAllowWebhooks{}, diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go index 136be031810..e0d495554bb 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go @@ -226,6 +226,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -235,6 +236,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -246,6 +248,7 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -256,6 +259,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -269,6 +275,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -315,16 +325,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -357,6 +378,14 @@ func main() { %s + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index 89e61054294..87fbae2d7b7 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -64,6 +64,8 @@ func GenerateV4(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( @@ -71,11 +73,10 @@ func GenerateV4(kbc *utils.TestContext) { monitorTlsPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + metricsCertPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) - + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + metricsCertReplaces, "#")).To(Succeed()) uncommentKustomizeCoversion(kbc) } @@ -111,6 +112,8 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) // Disable metrics @@ -177,18 +180,21 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) { metricsTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + metricsCertPatch, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + metricsCertReplaces, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), monitorTlsPatch, "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) + By("uncomment kustomization.yaml to enable network policy") ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../network-policy", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `#replacements:`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) @@ -243,8 +249,7 @@ const metricsTarget = `- path: manager_metrics_patch.yaml kind: Deployment` //nolint:lll -const certManagerTarget = `#replacements: -# - source: # Uncomment the following block if you have any webhook +const certManagerTarget = `# - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 # name: webhook-service @@ -254,6 +259,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -271,6 +277,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -298,7 +305,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -314,7 +321,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -329,7 +336,7 @@ const certManagerTarget = `#replacements: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -345,7 +352,7 @@ const certNamespace = `# - source: # Uncomment the following block if you have a # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -362,7 +369,7 @@ const certName = `# - source: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -452,6 +459,44 @@ const monitorTlsPatch = `#patches: # target: # kind: ServiceMonitor` -const tlsConfigManager = `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"` +const metricsCertPatch = `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment` + +const metricsCertReplaces = `# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true` diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go index 275bd9065c7..f21806bfc64 100644 --- a/testdata/project-v4-multigroup/cmd/main.go +++ b/testdata/project-v4-multigroup/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -70,8 +72,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -95,6 +98,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -108,6 +114,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -154,16 +164,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -348,6 +369,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..2019d532f05 --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..2d174f830a9 --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml deleted file mode 100644 index 73765b7407e..00000000000 --- a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/part-of: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/part-of: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/certmanager/issuer.yaml b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..e266ca5119c --- /dev/null +++ b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2b160b644e --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index a6b8d9bc0de..00000000000 --- a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 78743d725ce..2079d714859 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go index f01870085dc..3dd0660a535 100644 --- a/testdata/project-v4-with-plugins/cmd/main.go +++ b/testdata/project-v4-with-plugins/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -45,8 +47,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -60,6 +63,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -73,6 +79,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -119,16 +129,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -194,6 +215,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..b38d5abec0f --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..8604f1cc0c5 --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml deleted file mode 100644 index d4580c48964..00000000000 --- a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-with-plugins - app.kubernetes.io/part-of: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4-with-plugins - app.kubernetes.io/part-of: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..05e6df0e47f --- /dev/null +++ b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2b160b644e --- /dev/null +++ b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index ed2f033ddbb..00000000000 --- a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml index d9a6926909f..39a8fdd01f7 100644 --- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go index 1a72e4c73c1..c843015c5c0 100644 --- a/testdata/project-v4/cmd/main.go +++ b/testdata/project-v4/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -48,8 +50,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -63,6 +66,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -76,6 +82,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -122,16 +132,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + // + // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" + // to generate and use certificates managed by cert-manager for the metrics server. + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -230,6 +251,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4/config/certmanager/certificate-metrics.yaml b/testdata/project-v4/config/certmanager/certificate-metrics.yaml new file mode 100644 index 00000000000..6c2488daae5 --- /dev/null +++ b/testdata/project-v4/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/certificate-webhook.yaml b/testdata/project-v4/config/certmanager/certificate-webhook.yaml new file mode 100644 index 00000000000..b3d765563ea --- /dev/null +++ b/testdata/project-v4/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/certificate.yaml b/testdata/project-v4/config/certmanager/certificate.yaml deleted file mode 100644 index 0cffc07a915..00000000000 --- a/testdata/project-v4/config/certmanager/certificate.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: serving-cert - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/part-of: project-v4 - app.kubernetes.io/managed-by: kustomize - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - SERVICE_NAME.SERVICE_NAMESPACE.svc - - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/name: certificate - app.kubernetes.io/instance: metrics-certs - app.kubernetes.io/component: certificate - app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/part-of: project-v4 - app.kubernetes.io/managed-by: kustomize - name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - dnsNames: - - controller-manager-metrics-service.system.svc - - controller-manager-metrics-service.system.svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/certmanager/issuer.yaml b/testdata/project-v4/config/certmanager/issuer.yaml new file mode 100644 index 00000000000..65f38af73de --- /dev/null +++ b/testdata/project-v4/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/testdata/project-v4/config/certmanager/kustomization.yaml b/testdata/project-v4/config/certmanager/kustomization.yaml index bebea5a595e..fcb7498e468 100644 --- a/testdata/project-v4/config/certmanager/kustomization.yaml +++ b/testdata/project-v4/config/certmanager/kustomization.yaml @@ -1,5 +1,7 @@ resources: -- certificate.yaml +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml configurations: - kustomizeconfig.yaml diff --git a/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2b160b644e --- /dev/null +++ b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 862a49388f2..00000000000 --- a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index 5451766b0b3..a252b019971 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml @@ -53,6 +55,44 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations #replacements: +# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS] +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -63,6 +103,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -80,6 +121,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 +# name: serving-cert # fieldPaths: # - .spec.dnsNames.0 # - .spec.dnsNames.1 @@ -107,7 +149,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -123,7 +165,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # - select: @@ -138,7 +180,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # - select: @@ -154,7 +196,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.namespace # Namespace of the certificate CR # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: @@ -171,7 +213,7 @@ patches: # kind: Certificate # group: cert-manager.io # version: v1 -# name: serving-cert # This name should match the one in certificate.yaml +# name: serving-cert # fieldPath: .metadata.name # targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. # - select: