From 8531ea8fc216867d05b9c0532e66789deb813e34 Mon Sep 17 00:00:00 2001
From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com>
Date: Sun, 1 Dec 2024 16:10:28 +0000
Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics
endpoints (follow-up to PR #4243)
This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003
Key enhancements:
- Added support for configuring certificate integration via a Kustomize patch.
- Introduced configurable flags for greater flexibility in customization.
- Use Certwatcher to allow certificate rotation
This configuration provides an option for users to be production-ready
---
.github/workflows/test-e2e-samples.yml | 19 +-
Makefile | 5 +
.../testdata/project/cmd/main.go | 49 ++++-
.../certmanager/certificate-metrics.yaml | 20 ++
.../certmanager/certificate-webhook.yaml | 20 ++
.../config/certmanager/certificate.yaml | 57 ------
.../project/config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 35 ++++
.../certmanager_metrics_manager_patch.yaml | 21 --
.../project/config/default/kustomization.yaml | 58 +++++-
.../testdata/project/dist/install.yaml | 92 +++++++++
.../project/internal/controller/suite_test.go | 1 -
.../testdata/project/cmd/main.go | 49 ++++-
.../default/cert_metrics_manager_patch.yaml | 35 ++++
.../certmanager_metrics_manager_patch.yaml | 21 --
.../project/config/default/kustomization.yaml | 58 +++++-
.../testdata/project/cmd/main.go | 49 ++++-
.../certmanager/certificate-metrics.yaml | 20 ++
.../certmanager/certificate-webhook.yaml | 20 ++
.../config/certmanager/certificate.yaml | 57 ------
.../project/config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 35 ++++
.../certmanager_metrics_manager_patch.yaml | 21 --
.../project/config/default/kustomization.yaml | 58 +++++-
.../testdata/project/dist/install.yaml | 25 +--
docs/book/src/reference/metrics.md | 187 ++++++++++++------
.../cronjob-tutorial/generate_cronjob.go | 51 +++--
hack/docs/internal/cronjob-tutorial/sample.go | 48 ++++-
.../multiversion-tutorial/kustomize.go | 4 +-
.../config/certmanager/certificate_metrics.go | 68 +++++++
...{certificate.go => certificate_webhook.go} | 43 +---
.../templates/config/certmanager/issuer.go | 60 ++++++
.../config/certmanager/kustomization.go | 4 +-
...patch.go => cert_metrics_manager_patch.go} | 59 +++---
.../config/kdefault/kustomization.go | 58 +++++-
.../common/kustomize/v2/scaffolds/webhook.go | 2 +
.../scaffolds/internal/templates/cmd/main.go | 45 ++++-
test/e2e/v4/generate_test.go | 81 ++++++--
testdata/project-v4-multigroup/cmd/main.go | 45 ++++-
.../certmanager/certificate-metrics.yaml | 20 ++
.../certmanager/certificate-webhook.yaml | 20 ++
.../config/certmanager/certificate.yaml | 57 ------
.../config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 35 ++++
.../certmanager_metrics_manager_patch.yaml | 21 --
.../config/default/kustomization.yaml | 58 +++++-
testdata/project-v4-with-plugins/cmd/main.go | 49 ++++-
.../certmanager/certificate-metrics.yaml | 20 ++
.../certmanager/certificate-webhook.yaml | 20 ++
.../config/certmanager/certificate.yaml | 57 ------
.../config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 35 ++++
.../certmanager_metrics_manager_patch.yaml | 21 --
.../config/default/kustomization.yaml | 58 +++++-
testdata/project-v4/cmd/main.go | 45 ++++-
.../certmanager/certificate-metrics.yaml | 20 ++
.../certmanager/certificate-webhook.yaml | 20 ++
.../config/certmanager/certificate.yaml | 57 ------
.../project-v4/config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 35 ++++
.../certmanager_metrics_manager_patch.yaml | 21 --
.../config/default/kustomization.yaml | 58 +++++-
67 files changed, 1658 insertions(+), 734 deletions(-)
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go
rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/{certificate.go => certificate_webhook.go} (57%)
create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go
rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (52%)
create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml
create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml
delete mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate.yaml
create mode 100644 testdata/project-v4-multigroup/config/certmanager/issuer.yaml
create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml
create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml
delete mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate.yaml
create mode 100644 testdata/project-v4-with-plugins/config/certmanager/issuer.yaml
create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 testdata/project-v4/config/certmanager/certificate-metrics.yaml
create mode 100644 testdata/project-v4/config/certmanager/certificate-webhook.yaml
delete mode 100644 testdata/project-v4/config/certmanager/certificate.yaml
create mode 100644 testdata/project-v4/config/certmanager/issuer.yaml
create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml
diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml
index b0db7dc21c6..6d84a992a2f 100644
--- a/.github/workflows/test-e2e-samples.yml
+++ b/.github/workflows/test-e2e-samples.yml
@@ -41,9 +41,10 @@ jobs:
run: |
KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH
# Uncomment all cert-manager injections
- sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
- sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4/
go mod tidy
@@ -85,10 +86,11 @@ jobs:
# Uncomment only ValidatingWebhookConfiguration
# from cert-manager replaces; we are leaving defaulting uncommented
# since this sample has no defaulting webhooks
- sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH
# Uncomment only --conversion webhooks CA injection
- sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH
- sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4-with-plugins/
go mod tidy
@@ -127,9 +129,10 @@ jobs:
run: |
KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
- # Uncomment all cert-manager injections
- sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
- sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+ # Uncomment all cert-manager injections for webhooks only
+ sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4-multigroup
go mod tidy
diff --git a/Makefile b/Makefile
index 7d1db87f3b5..51782cb5f47 100644
--- a/Makefile
+++ b/Makefile
@@ -105,6 +105,11 @@ lint: golangci-lint yamllint ## Run golangci-lint linter & yamllint
lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
$(GOLANGCI_LINT) run --fix
+.PHONY: lint-config
+lint-config: golangci-lint ## Verify golangci-lint linter configuration
+ $(GOLANGCI_LINT) config verify
+
+
.PHONY: yamllint
yamllint:
@files=$$(find testdata -name '*.yaml' ! -path 'testdata/*/dist/*'); \
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
index 458a9f8c3e2..93ca4108ec8 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
+++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
@@ -21,6 +21,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -30,6 +31,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`.
*/
var (
- scheme = runtime.NewScheme()
- setupLog = ctrl.Log.WithName("setup")
+ scheme = runtime.NewScheme()
+ setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -74,6 +77,9 @@ func main() {
/*
*/
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -87,6 +93,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -133,16 +143,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
-
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
-
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -196,6 +217,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..406becb9ead
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..2fc59c2f7d7
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml
deleted file mode 100644
index ce60daeb22c..00000000000
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..1c600ce5a67
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..a2b160b644e
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 9cee4b4f580..00000000000
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
index b5cf1e32e2f..196211b6054 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+- path: cert_metrics_manager_patch.yaml
+ target:
+ kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
replacements:
+ - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.name
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 0
+ create: true
+
+ - source:
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.namespace
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 1
+ create: true
+
- source: # Uncomment the following block if you have any webhook
kind: Service
version: v1
@@ -63,6 +103,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -80,6 +121,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -107,7 +149,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -123,7 +165,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.namespace # Namespace of the certificate CR
targets:
- select:
@@ -138,7 +180,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -154,7 +196,7 @@ replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionns
@@ -162,7 +204,7 @@ replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
index c20c83edbc5..1b24ecd0b56 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
@@ -4118,6 +4118,7 @@ spec:
- --metrics-bind-address=:8443
- --leader-elect
- --health-probe-bind-address=:8081
+ - --cert-dir=/tmp/k8s-metrics-server/metrics-certs
command:
- /manager
image: controller:latest
@@ -4154,6 +4155,9 @@ spec:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
+ - mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
securityContext:
runAsNonRoot: true
seccompProfile:
@@ -4165,10 +4169,96 @@ spec:
secret:
defaultMode: 420
secretName: webhook-server-cert
+ - name: metrics-certs
+ secret:
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
+ optional: false
+ secretName: metrics-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ name: project-metrics-certs
+ namespace: project-system
+spec:
+ dnsNames:
+ - project-controller-manager-metrics-service.project-system.svc
+ - project-controller-manager-metrics-service.project-system.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: project-selfsigned-issuer
+ secretName: metrics-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ name: project-serving-cert
+ namespace: project-system
+spec:
+ dnsNames:
+ - project-webhook-service.project-system.svc
+ - project-webhook-service.project-system.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: project-selfsigned-issuer
+ secretName: webhook-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ name: project-selfsigned-issuer
+ namespace: project-system
+spec:
+ selfSigned: {}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ control-plane: controller-manager
+ name: project-controller-manager-metrics-monitor
+ namespace: project-system
+spec:
+ endpoints:
+ - tlsConfig:
+ ca:
+ secret:
+ key: ca.crt
+ name: metrics-server-cert
+ cert:
+ secret:
+ key: tls.crt
+ name: metrics-server-cert
+ insecureSkipVerify: false
+ keySecret:
+ key: tls.key
+ name: metrics-server-cert
+ selector:
+ matchLabels:
+ control-plane: controller-manager
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
+ annotations:
+ cert-manager.io/inject-ca-from: project-system/project-serving-cert
name: project-mutating-webhook-configuration
webhooks:
- admissionReviewVersions:
@@ -4195,6 +4285,8 @@ webhooks:
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
+ annotations:
+ cert-manager.io/inject-ca-from: project-system/project-serving-cert
name: project-validating-webhook-configuration
webhooks:
- admissionReviewVersions:
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
index 7404dd56613..edb60cb7d8d 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
+++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
@@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the
First, it will contain the necessary imports.
*/
-
package controller
import (
diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go
index 7347d215463..53b5d4ca6ab 100644
--- a/docs/book/src/getting-started/testdata/project/cmd/main.go
+++ b/docs/book/src/getting-started/testdata/project/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -41,8 +43,9 @@ import (
)
var (
- scheme = runtime.NewScheme()
- setupLog = ctrl.Log.WithName("setup")
+ scheme = runtime.NewScheme()
+ setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -54,6 +57,9 @@ func init() {
func main() {
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -67,6 +73,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -113,16 +123,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
-
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
-
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -158,6 +179,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..a2b160b644e
--- /dev/null
+++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 9cee4b4f580..00000000000
--- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
index 317242ed92c..304f01b1eb9 100644
--- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
+++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionns
@@ -162,7 +204,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
index a199eec734b..17d18c8ca26 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
+++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
@@ -21,6 +21,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -31,6 +32,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -51,8 +53,9 @@ import (
*/
var (
- scheme = runtime.NewScheme()
- setupLog = ctrl.Log.WithName("setup")
+ scheme = runtime.NewScheme()
+ setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -73,6 +76,9 @@ func main() {
/*
*/
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -86,6 +92,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -132,16 +142,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
-
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
-
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -199,6 +220,14 @@ func main() {
/*
*/
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..406becb9ead
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..2fc59c2f7d7
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml
deleted file mode 100644
index ce60daeb22c..00000000000
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..1c600ce5a67
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..a2b160b644e
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 9cee4b4f580..00000000000
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml
index 16e19c0840f..815dddacff7 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+- path: cert_metrics_manager_patch.yaml
+ target:
+ kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
replacements:
+ - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.name
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 0
+ create: true
+
+ - source:
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.namespace
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 1
+ create: true
+
- source: # Uncomment the following block if you have any webhook
kind: Service
version: v1
@@ -63,6 +103,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -80,6 +121,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -107,7 +149,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -123,7 +165,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.namespace # Namespace of the certificate CR
targets:
- select:
@@ -138,7 +180,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -154,7 +196,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.namespace # Namespace of the certificate CR
targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
- select:
@@ -171,7 +213,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
- select:
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml
index 8a9b9c9f3ab..c2d17d750ae 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml
@@ -7929,6 +7929,7 @@ spec:
- --metrics-bind-address=:8443
- --leader-elect
- --health-probe-bind-address=:8081
+ - --cert-dir=/tmp/k8s-metrics-server/metrics-certs
command:
- /manager
image: controller:latest
@@ -7981,24 +7982,28 @@ spec:
secretName: webhook-server-cert
- name: metrics-certs
secret:
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
+ optional: false
secretName: metrics-server-cert
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/instance: metrics-certs
app.kubernetes.io/managed-by: kustomize
- app.kubernetes.io/name: certificate
- app.kubernetes.io/part-of: project
+ app.kubernetes.io/name: project
name: project-metrics-certs
namespace: project-system
spec:
dnsNames:
- - project-webhook-service.project-system.svc
- - project-webhook-service.project-system.svc.cluster.local
+ - project-controller-manager-metrics-service.project-system.svc
+ - project-controller-manager-metrics-service.project-system.svc.cluster.local
issuerRef:
kind: Issuer
name: project-selfsigned-issuer
@@ -8008,12 +8013,8 @@ apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/instance: serving-cert
app.kubernetes.io/managed-by: kustomize
- app.kubernetes.io/name: certificate
- app.kubernetes.io/part-of: project
+ app.kubernetes.io/name: project
name: project-serving-cert
namespace: project-system
spec:
diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md
index dbb85222709..a39055fc90a 100644
--- a/docs/book/src/reference/metrics.md
+++ b/docs/book/src/reference/metrics.md
@@ -126,57 +126,28 @@ spec:
sleep 60;
done
```
+
+### **(Recommended)** Enabling certificates for Production (Disabled by default)
+
-
-
-
-### By exposing the metrics endpoint using HTTPS and Cert-Manager
-
-Integrating `cert-manager` with your metrics service enables secure
-HTTPS access via TLS encryption. Follow the steps below to configure
-your project to expose the metrics endpoint using HTTPS with cert-manager.
+Projects built with Kubebuilder releases `4.4.0` and above have the logic scaffolded
+to enable the usage of certificates managed by [CertManager](https://cert-manager.io/)
+for securing the metrics server. Following the steps below, you can configure your
+project to use certificates managed by CertManager.
1. **Enable Cert-Manager in `config/default/kustomization.yaml`:**
- Uncomment the cert-manager resource to include it in your project:
@@ -185,36 +156,113 @@ your project to expose the metrics endpoint using HTTPS with cert-manager.
- ../certmanager
```
-2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:**
- - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification:
+2. **Enable the Patch to configure the usage of the certs in the Controller Deployment in `config/default/kustomization.yaml`:**
+ - Uncomment the `cert_metrics_manager_patch.yaml` to mount the `serving-cert` secret in the Manager Deployment.
```yaml
- - path: monitor_tls_patch.yaml
+ # Uncomment the patches line if you enable Metrics and CertManager
+ # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+ # This patch will protect the metrics with certManager self-signed certs.
+ - path: cert_metrics_manager_patch.yaml
target:
- kind: ServiceMonitor
+ kind: Deployment
```
-
-3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:**
- - Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment.
+3. **Enable the CertManager replaces for the Metrics Server certificates in `config/default/kustomization.yaml`:**
+ - Uncomment the replacements block bellow. It is required to properly set the DNS names for the certificates configured under `config/certmanager`.
```yaml
- - path: manager_webhook_patch.yaml
+ # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
+ # Uncomment the following replacements to add the cert-manager CA injection annotations
+ #replacements:
+ # - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+ # kind: Service
+ # version: v1
+ # name: controller-manager-metrics-service
+ # fieldPath: metadata.name
+ # targets:
+ # - select:
+ # kind: Certificate
+ # group: cert-manager.io
+ # version: v1
+ # name: metrics-certs
+ # fieldPaths:
+ # - spec.dnsNames.0
+ # - spec.dnsNames.1
+ # options:
+ # delimiter: '.'
+ # index: 0
+ # create: true
+ #
+ # - source:
+ # kind: Service
+ # version: v1
+ # name: controller-manager-metrics-service
+ # fieldPath: metadata.namespace
+ # targets:
+ # - select:
+ # kind: Certificate
+ # group: cert-manager.io
+ # version: v1
+ # name: metrics-certs
+ # fieldPaths:
+ # - spec.dnsNames.0
+ # - spec.dnsNames.1
+ # options:
+ # delimiter: '.'
+ # index: 1
+ # create: true
+ #
```
-4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:**
- - Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates.
- Uncomment the lines for `CertDir`, `CertName`, and `KeyName`:
-
- ```go
- if secureMetrics {
- ...
- metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- metricsServerOptions.CertName = "tls.crt"
- metricsServerOptions.KeyName = "tls.key"
- }
+4. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:**
+ - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification:
+
+ ```yaml
+ # [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+ # to securely reference certificates created and managed by cert-manager.
+ # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
+ # to mount the "metrics-server-cert" secret in the Manager Deployment.
+ patches:
+ - path: monitor_tls_patch.yaml
+ target:
+ kind: ServiceMonitor
```
-### By using Network Policy (You can optionally enable)
+Now, that you properly enable this option check the `cmd/main.go` to see how it will be used:**
+
+```go
+ if secureMetrics {
+ // FilterProvider is used to protect the metrics endpoint with authn/authz.
+ // These configurations ensure that only authorized users and service accounts
+ // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+ // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
+ metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+ // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
+ }
+```
+
+### **(Optional)** By using Network Policy (Disabled by default)
NetworkPolicy acts as a basic firewall for pods within a Kubernetes cluster, controlling traffic
flow at the IP address or port level. However, it doesn't handle `authn/authz`.
@@ -327,3 +375,18 @@ Those metrics will be available for prometheus or
other openmetrics systems to scrape.
![Screen Shot 2021-06-14 at 10 15 59 AM](https://user-images.githubusercontent.com/37827279/121932262-8843cd80-ccf9-11eb-9c8e-98d0eda80169.png)
+
+
\ No newline at end of file
diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go
index 685666bf101..5a7f996517b 100644
--- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go
+++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go
@@ -90,7 +90,18 @@ func (sp *Sample) UpdateTutorial() {
// 4. update makefile
sp.updateMakefile()
// 5. generate extra files
- sp.codeGen()
+ cmd := exec.Command("go", "mod", "tidy")
+ _, err := sp.ctx.Run(cmd)
+ hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err)
+
+ cmd = exec.Command("go", "get", "github.com/robfig/cron")
+ _, err = sp.ctx.Run(cmd)
+ hackutils.CheckError("Failed to get package robfig/cron", err)
+
+ cmd = exec.Command("make", "generate", "manifests")
+ _, err = sp.ctx.Run(cmd)
+ hackutils.CheckError("run make generate and manifests", err)
+
// 6. compensate other intro in API
sp.updateAPIStuff()
// 7. update reconciliation and main.go
@@ -99,7 +110,9 @@ func (sp *Sample) UpdateTutorial() {
// 7.2 update main.go
sp.updateMain()
// 8. generate extra files
- sp.codeGen()
+ cmd = exec.Command("make", "generate", "manifests")
+ _, err = sp.ctx.Run(cmd)
+ hackutils.CheckError("run make generate and manifests", err)
// 9. update suite_test explanation
sp.updateSuiteTest()
// 10. uncomment kustomization
@@ -113,25 +126,14 @@ func (sp *Sample) UpdateTutorial() {
// CodeGen is a noop for this sample, just to make generation of all samples
// more efficient. We may want to refactor `UpdateTutorial` some day to take
// advantage of a separate call, but it is not necessary.
-func (sp *Sample) CodeGen() {}
-
-func (sp *Sample) codeGen() {
- cmd := exec.Command("go", "mod", "tidy")
+func (sp *Sample) CodeGen() {
+ cmd := exec.Command("make", "all")
_, err := sp.ctx.Run(cmd)
- hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err)
-
- cmd = exec.Command("go", "get", "github.com/robfig/cron")
- _, err = sp.ctx.Run(cmd)
- hackutils.CheckError("Failed to get package robfig/cron", err)
-
- cmd = exec.Command("make", "all")
- _, err = sp.ctx.Run(cmd)
hackutils.CheckError("Failed to run make all for cronjob tutorial", err)
cmd = exec.Command("make", "build-installer")
_, err = sp.ctx.Run(cmd)
hackutils.CheckError("Failed to run make build-installer for cronjob tutorial", err)
-
}
// insert code to fix docs
@@ -355,15 +357,6 @@ CronJob controller's`+" `"+`SetupWithManager`+"`"+` method.
}`, `
// +kubebuilder:docs-gen:collapse=old stuff`)
hackutils.CheckError("fixing main.go", err)
-
- // Enabling metrics with certs
- err = pluginutil.UncommentCode(
- filepath.Join(sp.ctx.Dir, "cmd/main.go"),
- `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"`, `
- // `)
- hackutils.CheckError("enabling metrics service options into main.go", err)
}
func (sp *Sample) updateMakefile() {
@@ -601,8 +594,10 @@ func (sp *Sample) updateKustomization() {
err = pluginutil.UncommentCode(
filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"),
- `#- path: certmanager_metrics_manager_patch.yaml`, `#`)
- hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err)
+ `#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment`, `#`)
+ hackutils.CheckError("enabling cert_metrics_manager_patch.yaml", err)
err = pluginutil.UncommentCode(
filepath.Join(sp.ctx.Dir, "config/prometheus/kustomization.yaml"),
@@ -610,11 +605,11 @@ func (sp *Sample) updateKustomization() {
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor`, `#`)
- hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err)
+ hackutils.CheckError("enabling monitor tls patch", err)
err = pluginutil.UncommentCode(
filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"),
- certmanagerForWebhooks, `#`)
+ certManagerForMetricsAndWebhooks, `#`)
hackutils.CheckError("fixing default/kustomization", err)
}
diff --git a/hack/docs/internal/cronjob-tutorial/sample.go b/hack/docs/internal/cronjob-tutorial/sample.go
index c7ee019a09b..8fa8a6afe2c 100644
--- a/hack/docs/internal/cronjob-tutorial/sample.go
+++ b/hack/docs/internal/cronjob-tutorial/sample.go
@@ -33,7 +33,45 @@ const CronjobSample = `
- date; echo Hello from the Kubernetes cluster
restartPolicy: OnFailure`
-const certmanagerForWebhooks = `#replacements:
+const certManagerForMetricsAndWebhooks = `#replacements:
+# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -44,6 +82,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -61,6 +100,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -88,7 +128,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -104,7 +144,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -119,7 +159,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
diff --git a/hack/docs/internal/multiversion-tutorial/kustomize.go b/hack/docs/internal/multiversion-tutorial/kustomize.go
index 45abfa37ad3..c1a0d10d952 100644
--- a/hack/docs/internal/multiversion-tutorial/kustomize.go
+++ b/hack/docs/internal/multiversion-tutorial/kustomize.go
@@ -21,7 +21,7 @@ const caInjectionNamespace = `#
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -38,7 +38,7 @@ const caInjectionCert = `# - source:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go
new file mode 100644
index 00000000000..9b82f883206
--- /dev/null
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certmanager
+
+import (
+ "path/filepath"
+
+ "sigs.k8s.io/kubebuilder/v4/pkg/machinery"
+)
+
+var _ machinery.Template = &MetricsCertificate{}
+
+// MetricsCertificate scaffolds a file that defines the issuer CR and the metrics certificate CR
+type MetricsCertificate struct {
+ machinery.TemplateMixin
+ machinery.ProjectNameMixin
+}
+
+// SetTemplateDefaults implements machinery.Template
+func (f *MetricsCertificate) SetTemplateDefaults() error {
+ if f.Path == "" {
+ f.Path = filepath.Join("config", "certmanager", "certificate-metrics.yaml")
+ }
+
+ f.TemplateBody = metricsCertManagerTemplate
+
+ // If file exists, skip creation.
+ f.IfExistsAction = machinery.SkipFile
+
+ return nil
+}
+
+// nolint:lll
+const metricsCertManagerTemplate = `# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: {{ .ProjectName }}
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
+`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go
similarity index 57%
rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go
rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go
index 2fc28f5da7b..955a204aa7a 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go
@@ -33,7 +33,7 @@ type Certificate struct {
// SetTemplateDefaults implements machinery.Template
func (f *Certificate) SetTemplateDefaults() error {
if f.Path == "" {
- f.Path = filepath.Join("config", "certmanager", "certificate.yaml")
+ f.Path = filepath.Join("config", "certmanager", "certificate-webhook.yaml")
}
f.TemplateBody = certManagerTemplate
@@ -46,32 +46,17 @@ func (f *Certificate) SetTemplateDefaults() error {
const certManagerTemplate = `# The following manifests contain a self-signed issuer CR and a certificate CR.
# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: {{ .ProjectName }}
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: {{ .ProjectName }}
- app.kubernetes.io/part-of: {{ .ProjectName }}
+ app.kubernetes.io/name: {{ .ProjectName }}
app.kubernetes.io/managed-by: kustomize
name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
namespace: system
spec:
# SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
dnsNames:
- SERVICE_NAME.SERVICE_NAMESPACE.svc
- SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
@@ -79,26 +64,4 @@ spec:
kind: Issuer
name: selfsigned-issuer
secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: {{ .ProjectName }}
- app.kubernetes.io/part-of: {{ .ProjectName }}
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go
new file mode 100644
index 00000000000..bef8a07681b
--- /dev/null
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go
@@ -0,0 +1,60 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certmanager
+
+import (
+ "path/filepath"
+
+ "sigs.k8s.io/kubebuilder/v4/pkg/machinery"
+)
+
+var _ machinery.Template = &Issuer{}
+
+// Issuer scaffolds a file that defines the self-signed Issuer CR
+type Issuer struct {
+ machinery.TemplateMixin
+ machinery.ProjectNameMixin
+}
+
+// SetTemplateDefaults implements machinery.Template
+func (f *Issuer) SetTemplateDefaults() error {
+ if f.Path == "" {
+ f.Path = filepath.Join("config", "certmanager", "issuer.yaml")
+ }
+
+ f.TemplateBody = issuerTemplate
+
+ // If file exists, skip creation.
+ f.IfExistsAction = machinery.SkipFile
+
+ return nil
+}
+
+const issuerTemplate = `# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: {{ .ProjectName }}
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
+`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go
index 5b1399bcd15..8d8ac05a9fd 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go
@@ -44,7 +44,9 @@ func (f *Kustomization) SetTemplateDefaults() error {
}
const kustomizationTemplate = `resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go
similarity index 52%
rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go
rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go
index 31859c3fd89..cfab8380714 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go
@@ -35,7 +35,7 @@ type CertManagerMetricsPatch struct {
// SetTemplateDefaults implements machinery.Template
func (f *CertManagerMetricsPatch) SetTemplateDefaults() error {
if f.Path == "" {
- f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml")
+ f.Path = filepath.Join("config", "default", "cert_metrics_manager_patch.yaml")
}
f.TemplateBody = metricsManagerPatchTemplate
@@ -50,25 +50,40 @@ func (f *CertManagerMetricsPatch) SetTemplateDefaults() error {
return nil
}
-const metricsManagerPatchTemplate = `apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: {{ .ProjectName }}
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
+// nolint:lll
+const metricsManagerPatchTemplate = `# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go
index 636bae6bbb0..4cc96ab4011 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go
@@ -87,9 +87,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -98,6 +100,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -108,6 +148,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -125,6 +166,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -152,7 +194,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -168,7 +210,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -183,7 +225,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -199,7 +241,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionns
@@ -207,7 +249,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go
index 869b58b63e9..8078eb545b7 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go
@@ -84,6 +84,8 @@ func (s *webhookScaffolder) Scaffold() error {
&webhook.KustomizeConfig{},
&webhook.Service{},
&certmanager.Certificate{},
+ &certmanager.Issuer{},
+ &certmanager.MetricsCertificate{},
&certmanager.Kustomization{},
&certmanager.KustomizeConfig{},
&network_policy.NetworkPolicyAllowWebhooks{},
diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go
index 136be031810..e0d495554bb 100644
--- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go
+++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go
@@ -226,6 +226,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -235,6 +236,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -246,6 +248,7 @@ import (
var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -256,6 +259,9 @@ func init() {
func main() {
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -269,6 +275,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -315,16 +325,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
-
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
-
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -357,6 +378,14 @@ func main() {
%s
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go
index 89e61054294..87fbae2d7b7 100644
--- a/test/e2e/v4/generate_test.go
+++ b/test/e2e/v4/generate_test.go
@@ -64,6 +64,8 @@ func GenerateV4(kbc *utils.TestContext) {
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
"#- ../prometheus", "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ `#replacements:`, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
certManagerTarget, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
@@ -71,11 +73,10 @@ func GenerateV4(kbc *utils.TestContext) {
monitorTlsPatch, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
- `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed())
+ metricsCertPatch, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
- filepath.Join(kbc.Dir, "cmd", "main.go"),
- tlsConfigManager, "// ")).To(Succeed())
-
+ filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ metricsCertReplaces, "#")).To(Succeed())
uncommentKustomizeCoversion(kbc)
}
@@ -111,6 +112,8 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) {
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
"#- ../prometheus", "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ `#replacements:`, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
certManagerTarget, "#")).To(Succeed())
// Disable metrics
@@ -177,18 +180,21 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) {
metricsTarget, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
- `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed())
+ metricsCertPatch, "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(
+ filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ metricsCertReplaces, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"),
monitorTlsPatch, "#")).To(Succeed())
- ExpectWithOffset(1, pluginutil.UncommentCode(
- filepath.Join(kbc.Dir, "cmd", "main.go"),
- tlsConfigManager, "// ")).To(Succeed())
+
By("uncomment kustomization.yaml to enable network policy")
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
"#- ../network-policy", "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ `#replacements:`, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
certManagerTarget, "#")).To(Succeed())
@@ -243,8 +249,7 @@ const metricsTarget = `- path: manager_metrics_patch.yaml
kind: Deployment`
//nolint:lll
-const certManagerTarget = `#replacements:
-# - source: # Uncomment the following block if you have any webhook
+const certManagerTarget = `# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
# name: webhook-service
@@ -254,6 +259,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -271,6 +277,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -298,7 +305,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -314,7 +321,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -329,7 +336,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -345,7 +352,7 @@ const certNamespace = `# - source: # Uncomment the following block if you have a
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -362,7 +369,7 @@ const certName = `# - source:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -452,6 +459,44 @@ const monitorTlsPatch = `#patches:
# target:
# kind: ServiceMonitor`
-const tlsConfigManager = `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"`
+const metricsCertPatch = `#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment`
+
+const metricsCertReplaces = `# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true`
diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go
index 275bd9065c7..f21806bfc64 100644
--- a/testdata/project-v4-multigroup/cmd/main.go
+++ b/testdata/project-v4-multigroup/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -70,8 +72,9 @@ import (
)
var (
- scheme = runtime.NewScheme()
- setupLog = ctrl.Log.WithName("setup")
+ scheme = runtime.NewScheme()
+ setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -95,6 +98,9 @@ func init() {
func main() {
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -108,6 +114,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -154,16 +164,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -348,6 +369,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..2019d532f05
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-multigroup
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..2d174f830a9
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-multigroup
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml
deleted file mode 100644
index 73765b7407e..00000000000
--- a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-multigroup
- app.kubernetes.io/part-of: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-multigroup
- app.kubernetes.io/part-of: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-multigroup/config/certmanager/issuer.yaml b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..e266ca5119c
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-multigroup
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml
+++ b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..a2b160b644e
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index a6b8d9bc0de..00000000000
--- a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml
index 78743d725ce..2079d714859 100644
--- a/testdata/project-v4-multigroup/config/default/kustomization.yaml
+++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -171,7 +213,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go
index f01870085dc..3dd0660a535 100644
--- a/testdata/project-v4-with-plugins/cmd/main.go
+++ b/testdata/project-v4-with-plugins/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -45,8 +47,9 @@ import (
)
var (
- scheme = runtime.NewScheme()
- setupLog = ctrl.Log.WithName("setup")
+ scheme = runtime.NewScheme()
+ setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -60,6 +63,9 @@ func init() {
func main() {
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -73,6 +79,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -119,16 +129,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
-
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
-
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -194,6 +215,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..b38d5abec0f
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-with-plugins
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..8604f1cc0c5
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-with-plugins
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml
deleted file mode 100644
index d4580c48964..00000000000
--- a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-with-plugins
- app.kubernetes.io/part-of: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-with-plugins
- app.kubernetes.io/part-of: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..05e6df0e47f
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-with-plugins
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml
+++ b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..a2b160b644e
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index ed2f033ddbb..00000000000
--- a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml
index d9a6926909f..39a8fdd01f7 100644
--- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml
+++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -171,7 +213,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go
index 1a72e4c73c1..c843015c5c0 100644
--- a/testdata/project-v4/cmd/main.go
+++ b/testdata/project-v4/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -48,8 +50,9 @@ import (
)
var (
- scheme = runtime.NewScheme()
- setupLog = ctrl.Log.WithName("setup")
+ scheme = runtime.NewScheme()
+ setupLog = ctrl.Log.WithName("setup")
+ certWatcher *certwatcher.CertWatcher
)
func init() {
@@ -63,6 +66,9 @@ func init() {
func main() {
var metricsAddr string
+ var certDir string
+ var certName string
+ var certKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -76,6 +82,10 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&certDir, "cert-dir", "",
+ "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+ flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+ flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -122,16 +132,27 @@ func main() {
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+ // If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
+ //
+ // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml"
+ // to generate and use certificates managed by cert-manager for the metrics server.
+ if len(certDir) > 0 {
+ setupLog.Info("using certificates for the metrics server",
+ "cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ var err error
+ certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = certWatcher.GetCertificate
+ })
+ }
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -230,6 +251,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if secureMetrics && certWatcher != nil {
+ setupLog.Info("Adding certificate watcher to manager")
+ if err := mgr.Add(certWatcher); err != nil {
+ setupLog.Error(err, "unable to add certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/testdata/project-v4/config/certmanager/certificate-metrics.yaml b/testdata/project-v4/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..6c2488daae5
--- /dev/null
+++ b/testdata/project-v4/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4/config/certmanager/certificate-webhook.yaml b/testdata/project-v4/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..b3d765563ea
--- /dev/null
+++ b/testdata/project-v4/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4/config/certmanager/certificate.yaml b/testdata/project-v4/config/certmanager/certificate.yaml
deleted file mode 100644
index 0cffc07a915..00000000000
--- a/testdata/project-v4/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project-v4
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4
- app.kubernetes.io/part-of: project-v4
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4
- app.kubernetes.io/part-of: project-v4
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4/config/certmanager/issuer.yaml b/testdata/project-v4/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..65f38af73de
--- /dev/null
+++ b/testdata/project-v4/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/testdata/project-v4/config/certmanager/kustomization.yaml b/testdata/project-v4/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/testdata/project-v4/config/certmanager/kustomization.yaml
+++ b/testdata/project-v4/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..a2b160b644e
--- /dev/null
+++ b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+# Add the volume mount for the serving certificates
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+# Add the cert-dir argument
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+# Add the volume for the serving certificates
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 862a49388f2..00000000000
--- a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project-v4
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml
index 5451766b0b3..a252b019971 100644
--- a/testdata/project-v4/config/default/kustomization.yaml
+++ b/testdata/project-v4/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block if you enable [METRICS-WITH-CERTS]
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -171,7 +213,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select: