From 027f1482db3125c7b2fbf742baf7dfbf4bd49ca4 Mon Sep 17 00:00:00 2001
From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com>
Date: Sun, 1 Dec 2024 16:10:28 +0000
Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics
endpoints (follow-up to PR #4243)
This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003
Key enhancements:
- Added support for configuring certificate integration via a Kustomize patch.
- Introduced configurable flags for greater flexibility in customization.
- Use Certwatcher to allow certificate rotation
This configuration provides an option for users to be production-ready
---
.github/workflows/test-e2e-samples.yml | 19 ++-
Makefile | 5 +
.../testdata/project/cmd/main.go | 56 +++++--
.../certmanager/certificate-metrics.yaml | 20 +++
.../certmanager/certificate-webhook.yaml | 20 +++
.../config/certmanager/certificate.yaml | 57 -------
.../project/config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 31 ++++
.../certmanager_metrics_manager_patch.yaml | 21 ---
.../project/config/default/kustomization.yaml | 58 ++++++-
.../config/prometheus/kustomization.yaml | 2 +-
.../testdata/project/dist/install.yaml | 92 +++++++++++
.../project/internal/controller/suite_test.go | 1 -
.../testdata/project/cmd/main.go | 58 +++++--
.../default/cert_metrics_manager_patch.yaml | 31 ++++
.../certmanager_metrics_manager_patch.yaml | 21 ---
.../project/config/default/kustomization.yaml | 58 ++++++-
.../config/prometheus/kustomization.yaml | 2 +-
.../testdata/project/cmd/main.go | 56 +++++--
.../certmanager/certificate-metrics.yaml | 20 +++
.../certmanager/certificate-webhook.yaml | 20 +++
.../config/certmanager/certificate.yaml | 57 -------
.../project/config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 31 ++++
.../certmanager_metrics_manager_patch.yaml | 21 ---
.../project/config/default/kustomization.yaml | 58 ++++++-
.../config/prometheus/kustomization.yaml | 2 +-
.../testdata/project/dist/install.yaml | 25 +--
docs/book/src/reference/metrics.md | 154 +++++++++++-------
.../cronjob-tutorial/generate_cronjob.go | 51 +++---
hack/docs/internal/cronjob-tutorial/sample.go | 48 +++++-
.../multiversion-tutorial/kustomize.go | 4 +-
.../config/certmanager/certificate_metrics.go | 68 ++++++++
...{certificate.go => certificate_webhook.go} | 43 +----
.../templates/config/certmanager/issuer.go | 60 +++++++
.../config/certmanager/kustomization.go | 4 +-
...patch.go => cert_metrics_manager_patch.go} | 55 ++++---
.../config/kdefault/kustomization.go | 58 ++++++-
.../config/prometheus/kustomization.go | 2 +-
.../common/kustomize/v2/scaffolds/webhook.go | 2 +
.../scaffolds/internal/templates/cmd/main.go | 62 +++++--
test/e2e/v4/generate_test.go | 81 +++++++--
testdata/project-v4-multigroup/cmd/main.go | 54 ++++--
.../certmanager/certificate-metrics.yaml | 20 +++
.../certmanager/certificate-webhook.yaml | 20 +++
.../config/certmanager/certificate.yaml | 57 -------
.../config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 31 ++++
.../certmanager_metrics_manager_patch.yaml | 21 ---
.../config/default/kustomization.yaml | 58 ++++++-
.../config/prometheus/kustomization.yaml | 2 +-
testdata/project-v4-with-plugins/cmd/main.go | 56 +++++--
.../certmanager/certificate-metrics.yaml | 20 +++
.../certmanager/certificate-webhook.yaml | 20 +++
.../config/certmanager/certificate.yaml | 57 -------
.../config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 31 ++++
.../certmanager_metrics_manager_patch.yaml | 21 ---
.../config/default/kustomization.yaml | 58 ++++++-
.../config/prometheus/kustomization.yaml | 2 +-
testdata/project-v4/cmd/main.go | 54 ++++--
.../certmanager/certificate-metrics.yaml | 20 +++
.../certmanager/certificate-webhook.yaml | 20 +++
.../config/certmanager/certificate.yaml | 57 -------
.../project-v4/config/certmanager/issuer.yaml | 13 ++
.../config/certmanager/kustomization.yaml | 4 +-
.../default/cert_metrics_manager_patch.yaml | 31 ++++
.../certmanager_metrics_manager_patch.yaml | 21 ---
.../config/default/kustomization.yaml | 58 ++++++-
.../config/prometheus/kustomization.yaml | 2 +-
74 files changed, 1640 insertions(+), 770 deletions(-)
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml
create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml
create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go
rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/{certificate.go => certificate_webhook.go} (57%)
create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go
rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (56%)
create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml
create mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml
delete mode 100644 testdata/project-v4-multigroup/config/certmanager/certificate.yaml
create mode 100644 testdata/project-v4-multigroup/config/certmanager/issuer.yaml
create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml
create mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml
delete mode 100644 testdata/project-v4-with-plugins/config/certmanager/certificate.yaml
create mode 100644 testdata/project-v4-with-plugins/config/certmanager/issuer.yaml
create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml
create mode 100644 testdata/project-v4/config/certmanager/certificate-metrics.yaml
create mode 100644 testdata/project-v4/config/certmanager/certificate-webhook.yaml
delete mode 100644 testdata/project-v4/config/certmanager/certificate.yaml
create mode 100644 testdata/project-v4/config/certmanager/issuer.yaml
create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml
delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml
diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml
index b0db7dc21c6..6d84a992a2f 100644
--- a/.github/workflows/test-e2e-samples.yml
+++ b/.github/workflows/test-e2e-samples.yml
@@ -41,9 +41,10 @@ jobs:
run: |
KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '47,49s/^#//' $KUSTOMIZATION_FILE_PATH
# Uncomment all cert-manager injections
- sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
- sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '57,210s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4/
go mod tidy
@@ -85,10 +86,11 @@ jobs:
# Uncomment only ValidatingWebhookConfiguration
# from cert-manager replaces; we are leaving defaulting uncommented
# since this sample has no defaulting webhooks
- sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '133,162s/^#//' $KUSTOMIZATION_FILE_PATH
# Uncomment only --conversion webhooks CA injection
- sed -i '153,168s/^#//' $KUSTOMIZATION_FILE_PATH
- sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '195,210s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4-with-plugins/
go mod tidy
@@ -127,9 +129,10 @@ jobs:
run: |
KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
- # Uncomment all cert-manager injections
- sed -i '55,168s/^#//' $KUSTOMIZATION_FILE_PATH
- sed -i '170,185s/^#//' $KUSTOMIZATION_FILE_PATH
+ # Uncomment all cert-manager injections for webhooks only
+ sed -i '57,57s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '96,210s/^#//' $KUSTOMIZATION_FILE_PATH
+ sed -i '212,227s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4-multigroup
go mod tidy
diff --git a/Makefile b/Makefile
index 7d1db87f3b5..51782cb5f47 100644
--- a/Makefile
+++ b/Makefile
@@ -105,6 +105,11 @@ lint: golangci-lint yamllint ## Run golangci-lint linter & yamllint
lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
$(GOLANGCI_LINT) run --fix
+.PHONY: lint-config
+lint-config: golangci-lint ## Verify golangci-lint linter configuration
+ $(GOLANGCI_LINT) config verify
+
+
.PHONY: yamllint
yamllint:
@files=$$(find testdata -name '*.yaml' ! -path 'testdata/*/dist/*'); \
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
index 458a9f8c3e2..58fa2fd8fc0 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
+++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
@@ -21,6 +21,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -30,6 +31,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -74,6 +76,7 @@ func main() {
/*
*/
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -87,6 +90,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -112,6 +118,9 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
@@ -124,25 +133,38 @@ func main() {
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
+
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -196,6 +218,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..406becb9ead
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..2fc59c2f7d7
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml
deleted file mode 100644
index ce60daeb22c..00000000000
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..1c600ce5a67
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..8400eb42a7b
--- /dev/null
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 9cee4b4f580..00000000000
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
index b5cf1e32e2f..0cf156e6276 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+- path: cert_metrics_manager_patch.yaml
+ target:
+ kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
replacements:
+ - source: # Uncomment the following block to enable certificates for metrics
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.name
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 0
+ create: true
+
+ - source:
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.namespace
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 1
+ create: true
+
- source: # Uncomment the following block if you have any webhook
kind: Service
version: v1
@@ -63,6 +103,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -80,6 +121,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -107,7 +149,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -123,7 +165,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.namespace # Namespace of the certificate CR
targets:
- select:
@@ -138,7 +180,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -154,7 +196,7 @@ replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionns
@@ -162,7 +204,7 @@ replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml
index e00ddc958dc..8126ea89b1a 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml
@@ -1,7 +1,7 @@
resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
index c20c83edbc5..27dbb9bb5ab 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
+++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
@@ -4118,6 +4118,7 @@ spec:
- --metrics-bind-address=:8443
- --leader-elect
- --health-probe-bind-address=:8081
+ - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
command:
- /manager
image: controller:latest
@@ -4154,6 +4155,9 @@ spec:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
+ - mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
securityContext:
runAsNonRoot: true
seccompProfile:
@@ -4165,10 +4169,96 @@ spec:
secret:
defaultMode: 420
secretName: webhook-server-cert
+ - name: metrics-certs
+ secret:
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
+ optional: false
+ secretName: metrics-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ name: project-metrics-certs
+ namespace: project-system
+spec:
+ dnsNames:
+ - project-controller-manager-metrics-service.project-system.svc
+ - project-controller-manager-metrics-service.project-system.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: project-selfsigned-issuer
+ secretName: metrics-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ name: project-serving-cert
+ namespace: project-system
+spec:
+ dnsNames:
+ - project-webhook-service.project-system.svc
+ - project-webhook-service.project-system.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: project-selfsigned-issuer
+ secretName: webhook-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ name: project-selfsigned-issuer
+ namespace: project-system
+spec:
+ selfSigned: {}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: project
+ control-plane: controller-manager
+ name: project-controller-manager-metrics-monitor
+ namespace: project-system
+spec:
+ endpoints:
+ - tlsConfig:
+ ca:
+ secret:
+ key: ca.crt
+ name: metrics-server-cert
+ cert:
+ secret:
+ key: tls.crt
+ name: metrics-server-cert
+ insecureSkipVerify: false
+ keySecret:
+ key: tls.key
+ name: metrics-server-cert
+ selector:
+ matchLabels:
+ control-plane: controller-manager
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
+ annotations:
+ cert-manager.io/inject-ca-from: project-system/project-serving-cert
name: project-mutating-webhook-configuration
webhooks:
- admissionReviewVersions:
@@ -4195,6 +4285,8 @@ webhooks:
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
+ annotations:
+ cert-manager.io/inject-ca-from: project-system/project-serving-cert
name: project-validating-webhook-configuration
webhooks:
- admissionReviewVersions:
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
index 7404dd56613..edb60cb7d8d 100644
--- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
+++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
@@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the
First, it will contain the necessary imports.
*/
-
package controller
import (
diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go
index 7347d215463..3cd126e887a 100644
--- a/docs/book/src/getting-started/testdata/project/cmd/main.go
+++ b/docs/book/src/getting-started/testdata/project/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -54,6 +56,7 @@ func init() {
func main() {
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -67,6 +70,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -92,6 +98,9 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
@@ -104,25 +113,38 @@ func main() {
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
-
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
+
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -158,6 +180,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..8400eb42a7b
--- /dev/null
+++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 9cee4b4f580..00000000000
--- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
index 317242ed92c..75241299903 100644
--- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
+++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionns
@@ -162,7 +204,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml
index f040fd58623..fdc5481b103 100644
--- a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml
+++ b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml
@@ -1,7 +1,7 @@
resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
index a199eec734b..d56885473b2 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
+++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
@@ -21,6 +21,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -31,6 +32,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -73,6 +75,7 @@ func main() {
/*
*/
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -86,6 +89,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -111,6 +117,9 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
@@ -123,25 +132,38 @@ func main() {
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
+
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -199,6 +221,14 @@ func main() {
/*
*/
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..406becb9ead
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..2fc59c2f7d7
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml
deleted file mode 100644
index ce60daeb22c..00000000000
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/part-of: project
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..1c600ce5a67
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..8400eb42a7b
--- /dev/null
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 9cee4b4f580..00000000000
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml
index 16e19c0840f..dcdae5c7e21 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+- path: cert_metrics_manager_patch.yaml
+ target:
+ kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
replacements:
+ - source: # Uncomment the following block to enable certificates for metrics
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.name
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 0
+ create: true
+
+ - source:
+ kind: Service
+ version: v1
+ name: controller-manager-metrics-service
+ fieldPath: metadata.namespace
+ targets:
+ - select:
+ kind: Certificate
+ group: cert-manager.io
+ version: v1
+ name: metrics-certs
+ fieldPaths:
+ - spec.dnsNames.0
+ - spec.dnsNames.1
+ options:
+ delimiter: '.'
+ index: 1
+ create: true
+
- source: # Uncomment the following block if you have any webhook
kind: Service
version: v1
@@ -63,6 +103,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -80,6 +121,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
+ name: serving-cert
fieldPaths:
- .spec.dnsNames.0
- .spec.dnsNames.1
@@ -107,7 +149,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -123,7 +165,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.namespace # Namespace of the certificate CR
targets:
- select:
@@ -138,7 +180,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets:
- select:
@@ -154,7 +196,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.namespace # Namespace of the certificate CR
targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
- select:
@@ -171,7 +213,7 @@ replacements:
kind: Certificate
group: cert-manager.io
version: v1
- name: serving-cert # This name should match the one in certificate.yaml
+ name: serving-cert
fieldPath: .metadata.name
targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
- select:
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml
index e00ddc958dc..8126ea89b1a 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml
@@ -1,7 +1,7 @@
resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml
index 8a9b9c9f3ab..f3716ea8657 100644
--- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml
+++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml
@@ -7929,6 +7929,7 @@ spec:
- --metrics-bind-address=:8443
- --leader-elect
- --health-probe-bind-address=:8081
+ - --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
command:
- /manager
image: controller:latest
@@ -7981,24 +7982,28 @@ spec:
secretName: webhook-server-cert
- name: metrics-certs
secret:
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
+ optional: false
secretName: metrics-server-cert
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/instance: metrics-certs
app.kubernetes.io/managed-by: kustomize
- app.kubernetes.io/name: certificate
- app.kubernetes.io/part-of: project
+ app.kubernetes.io/name: project
name: project-metrics-certs
namespace: project-system
spec:
dnsNames:
- - project-webhook-service.project-system.svc
- - project-webhook-service.project-system.svc.cluster.local
+ - project-controller-manager-metrics-service.project-system.svc
+ - project-controller-manager-metrics-service.project-system.svc.cluster.local
issuerRef:
kind: Issuer
name: project-selfsigned-issuer
@@ -8008,12 +8013,8 @@ apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project
- app.kubernetes.io/instance: serving-cert
app.kubernetes.io/managed-by: kustomize
- app.kubernetes.io/name: certificate
- app.kubernetes.io/part-of: project
+ app.kubernetes.io/name: project
name: project-serving-cert
namespace: project-system
spec:
diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md
index dbb85222709..0da7d5ab5b8 100644
--- a/docs/book/src/reference/metrics.md
+++ b/docs/book/src/reference/metrics.md
@@ -126,57 +126,28 @@ spec:
sleep 60;
done
```
+
+### **(Recommended)** Enabling certificates for Production (Disabled by default)
+
-
-
-
-### By exposing the metrics endpoint using HTTPS and Cert-Manager
-
-Integrating `cert-manager` with your metrics service enables secure
-HTTPS access via TLS encryption. Follow the steps below to configure
-your project to expose the metrics endpoint using HTTPS with cert-manager.
+Projects built with Kubebuilder releases `4.4.0` and above have the logic scaffolded
+to enable the usage of certificates managed by [CertManager](https://cert-manager.io/)
+for securing the metrics server. Following the steps below, you can configure your
+project to use certificates managed by CertManager.
1. **Enable Cert-Manager in `config/default/kustomization.yaml`:**
- Uncomment the cert-manager resource to include it in your project:
@@ -185,36 +156,80 @@ your project to expose the metrics endpoint using HTTPS with cert-manager.
- ../certmanager
```
-2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:**
- - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification:
+2. **Enable the Patch to configure the usage of the certs in the Controller Deployment in `config/default/kustomization.yaml`:**
+ - Uncomment the `cert_metrics_manager_patch.yaml` to mount the `serving-cert` secret in the Manager Deployment.
```yaml
- - path: monitor_tls_patch.yaml
+ # Uncomment the patches line if you enable Metrics and CertManager
+ # [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+ # This patch will protect the metrics with certManager self-signed certs.
+ - path: cert_metrics_manager_patch.yaml
target:
- kind: ServiceMonitor
+ kind: Deployment
```
-
-3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:**
- - Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment.
+3. **Enable the CertManager replaces for the Metrics Server certificates in `config/default/kustomization.yaml`:**
+ - Uncomment the replacements block bellow. It is required to properly set the DNS names for the certificates configured under `config/certmanager`.
```yaml
- - path: manager_webhook_patch.yaml
+ # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
+ # Uncomment the following replacements to add the cert-manager CA injection annotations
+ #replacements:
+ # - source: # Uncomment the following block to enable certificates for metrics
+ # kind: Service
+ # version: v1
+ # name: controller-manager-metrics-service
+ # fieldPath: metadata.name
+ # targets:
+ # - select:
+ # kind: Certificate
+ # group: cert-manager.io
+ # version: v1
+ # name: metrics-certs
+ # fieldPaths:
+ # - spec.dnsNames.0
+ # - spec.dnsNames.1
+ # options:
+ # delimiter: '.'
+ # index: 0
+ # create: true
+ #
+ # - source:
+ # kind: Service
+ # version: v1
+ # name: controller-manager-metrics-service
+ # fieldPath: metadata.namespace
+ # targets:
+ # - select:
+ # kind: Certificate
+ # group: cert-manager.io
+ # version: v1
+ # name: metrics-certs
+ # fieldPaths:
+ # - spec.dnsNames.0
+ # - spec.dnsNames.1
+ # options:
+ # delimiter: '.'
+ # index: 1
+ # create: true
+ #
```
-4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:**
- - Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates.
- Uncomment the lines for `CertDir`, `CertName`, and `KeyName`:
-
- ```go
- if secureMetrics {
- ...
- metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- metricsServerOptions.CertName = "tls.crt"
- metricsServerOptions.KeyName = "tls.key"
- }
+4. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:**
+ - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification:
+
+ ```yaml
+ # [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
+ # to securely reference certificates created and managed by cert-manager.
+ # Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
+ # to mount the "metrics-server-cert" secret in the Manager Deployment.
+ patches:
+ - path: monitor_tls_patch.yaml
+ target:
+ kind: ServiceMonitor
```
-### By using Network Policy (You can optionally enable)
+
+### **(Optional)** By using Network Policy (Disabled by default)
NetworkPolicy acts as a basic firewall for pods within a Kubernetes cluster, controlling traffic
flow at the IP address or port level. However, it doesn't handle `authn/authz`.
@@ -327,3 +342,18 @@ Those metrics will be available for prometheus or
other openmetrics systems to scrape.
![Screen Shot 2021-06-14 at 10 15 59 AM](https://user-images.githubusercontent.com/37827279/121932262-8843cd80-ccf9-11eb-9c8e-98d0eda80169.png)
+
+
\ No newline at end of file
diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go
index 685666bf101..5a7f996517b 100644
--- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go
+++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go
@@ -90,7 +90,18 @@ func (sp *Sample) UpdateTutorial() {
// 4. update makefile
sp.updateMakefile()
// 5. generate extra files
- sp.codeGen()
+ cmd := exec.Command("go", "mod", "tidy")
+ _, err := sp.ctx.Run(cmd)
+ hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err)
+
+ cmd = exec.Command("go", "get", "github.com/robfig/cron")
+ _, err = sp.ctx.Run(cmd)
+ hackutils.CheckError("Failed to get package robfig/cron", err)
+
+ cmd = exec.Command("make", "generate", "manifests")
+ _, err = sp.ctx.Run(cmd)
+ hackutils.CheckError("run make generate and manifests", err)
+
// 6. compensate other intro in API
sp.updateAPIStuff()
// 7. update reconciliation and main.go
@@ -99,7 +110,9 @@ func (sp *Sample) UpdateTutorial() {
// 7.2 update main.go
sp.updateMain()
// 8. generate extra files
- sp.codeGen()
+ cmd = exec.Command("make", "generate", "manifests")
+ _, err = sp.ctx.Run(cmd)
+ hackutils.CheckError("run make generate and manifests", err)
// 9. update suite_test explanation
sp.updateSuiteTest()
// 10. uncomment kustomization
@@ -113,25 +126,14 @@ func (sp *Sample) UpdateTutorial() {
// CodeGen is a noop for this sample, just to make generation of all samples
// more efficient. We may want to refactor `UpdateTutorial` some day to take
// advantage of a separate call, but it is not necessary.
-func (sp *Sample) CodeGen() {}
-
-func (sp *Sample) codeGen() {
- cmd := exec.Command("go", "mod", "tidy")
+func (sp *Sample) CodeGen() {
+ cmd := exec.Command("make", "all")
_, err := sp.ctx.Run(cmd)
- hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err)
-
- cmd = exec.Command("go", "get", "github.com/robfig/cron")
- _, err = sp.ctx.Run(cmd)
- hackutils.CheckError("Failed to get package robfig/cron", err)
-
- cmd = exec.Command("make", "all")
- _, err = sp.ctx.Run(cmd)
hackutils.CheckError("Failed to run make all for cronjob tutorial", err)
cmd = exec.Command("make", "build-installer")
_, err = sp.ctx.Run(cmd)
hackutils.CheckError("Failed to run make build-installer for cronjob tutorial", err)
-
}
// insert code to fix docs
@@ -355,15 +357,6 @@ CronJob controller's`+" `"+`SetupWithManager`+"`"+` method.
}`, `
// +kubebuilder:docs-gen:collapse=old stuff`)
hackutils.CheckError("fixing main.go", err)
-
- // Enabling metrics with certs
- err = pluginutil.UncommentCode(
- filepath.Join(sp.ctx.Dir, "cmd/main.go"),
- `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"`, `
- // `)
- hackutils.CheckError("enabling metrics service options into main.go", err)
}
func (sp *Sample) updateMakefile() {
@@ -601,8 +594,10 @@ func (sp *Sample) updateKustomization() {
err = pluginutil.UncommentCode(
filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"),
- `#- path: certmanager_metrics_manager_patch.yaml`, `#`)
- hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err)
+ `#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment`, `#`)
+ hackutils.CheckError("enabling cert_metrics_manager_patch.yaml", err)
err = pluginutil.UncommentCode(
filepath.Join(sp.ctx.Dir, "config/prometheus/kustomization.yaml"),
@@ -610,11 +605,11 @@ func (sp *Sample) updateKustomization() {
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor`, `#`)
- hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err)
+ hackutils.CheckError("enabling monitor tls patch", err)
err = pluginutil.UncommentCode(
filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"),
- certmanagerForWebhooks, `#`)
+ certManagerForMetricsAndWebhooks, `#`)
hackutils.CheckError("fixing default/kustomization", err)
}
diff --git a/hack/docs/internal/cronjob-tutorial/sample.go b/hack/docs/internal/cronjob-tutorial/sample.go
index c7ee019a09b..f963d137e74 100644
--- a/hack/docs/internal/cronjob-tutorial/sample.go
+++ b/hack/docs/internal/cronjob-tutorial/sample.go
@@ -33,7 +33,45 @@ const CronjobSample = `
- date; echo Hello from the Kubernetes cluster
restartPolicy: OnFailure`
-const certmanagerForWebhooks = `#replacements:
+const certManagerForMetricsAndWebhooks = `#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -44,6 +82,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -61,6 +100,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -88,7 +128,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -104,7 +144,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -119,7 +159,7 @@ const certmanagerForWebhooks = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
diff --git a/hack/docs/internal/multiversion-tutorial/kustomize.go b/hack/docs/internal/multiversion-tutorial/kustomize.go
index 45abfa37ad3..c1a0d10d952 100644
--- a/hack/docs/internal/multiversion-tutorial/kustomize.go
+++ b/hack/docs/internal/multiversion-tutorial/kustomize.go
@@ -21,7 +21,7 @@ const caInjectionNamespace = `#
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -38,7 +38,7 @@ const caInjectionCert = `# - source:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go
new file mode 100644
index 00000000000..9b82f883206
--- /dev/null
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_metrics.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certmanager
+
+import (
+ "path/filepath"
+
+ "sigs.k8s.io/kubebuilder/v4/pkg/machinery"
+)
+
+var _ machinery.Template = &MetricsCertificate{}
+
+// MetricsCertificate scaffolds a file that defines the issuer CR and the metrics certificate CR
+type MetricsCertificate struct {
+ machinery.TemplateMixin
+ machinery.ProjectNameMixin
+}
+
+// SetTemplateDefaults implements machinery.Template
+func (f *MetricsCertificate) SetTemplateDefaults() error {
+ if f.Path == "" {
+ f.Path = filepath.Join("config", "certmanager", "certificate-metrics.yaml")
+ }
+
+ f.TemplateBody = metricsCertManagerTemplate
+
+ // If file exists, skip creation.
+ f.IfExistsAction = machinery.SkipFile
+
+ return nil
+}
+
+// nolint:lll
+const metricsCertManagerTemplate = `# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: {{ .ProjectName }}
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
+`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go
similarity index 57%
rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go
rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go
index 2fc28f5da7b..955a204aa7a 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate_webhook.go
@@ -33,7 +33,7 @@ type Certificate struct {
// SetTemplateDefaults implements machinery.Template
func (f *Certificate) SetTemplateDefaults() error {
if f.Path == "" {
- f.Path = filepath.Join("config", "certmanager", "certificate.yaml")
+ f.Path = filepath.Join("config", "certmanager", "certificate-webhook.yaml")
}
f.TemplateBody = certManagerTemplate
@@ -46,32 +46,17 @@ func (f *Certificate) SetTemplateDefaults() error {
const certManagerTemplate = `# The following manifests contain a self-signed issuer CR and a certificate CR.
# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: {{ .ProjectName }}
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: {{ .ProjectName }}
- app.kubernetes.io/part-of: {{ .ProjectName }}
+ app.kubernetes.io/name: {{ .ProjectName }}
app.kubernetes.io/managed-by: kustomize
name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
namespace: system
spec:
# SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
dnsNames:
- SERVICE_NAME.SERVICE_NAMESPACE.svc
- SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
@@ -79,26 +64,4 @@ spec:
kind: Issuer
name: selfsigned-issuer
secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: {{ .ProjectName }}
- app.kubernetes.io/part-of: {{ .ProjectName }}
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go
new file mode 100644
index 00000000000..bef8a07681b
--- /dev/null
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/issuer.go
@@ -0,0 +1,60 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certmanager
+
+import (
+ "path/filepath"
+
+ "sigs.k8s.io/kubebuilder/v4/pkg/machinery"
+)
+
+var _ machinery.Template = &Issuer{}
+
+// Issuer scaffolds a file that defines the self-signed Issuer CR
+type Issuer struct {
+ machinery.TemplateMixin
+ machinery.ProjectNameMixin
+}
+
+// SetTemplateDefaults implements machinery.Template
+func (f *Issuer) SetTemplateDefaults() error {
+ if f.Path == "" {
+ f.Path = filepath.Join("config", "certmanager", "issuer.yaml")
+ }
+
+ f.TemplateBody = issuerTemplate
+
+ // If file exists, skip creation.
+ f.IfExistsAction = machinery.SkipFile
+
+ return nil
+}
+
+const issuerTemplate = `# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: {{ .ProjectName }}
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
+`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go
index 5b1399bcd15..8d8ac05a9fd 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/kustomization.go
@@ -44,7 +44,9 @@ func (f *Kustomization) SetTemplateDefaults() error {
}
const kustomizationTemplate = `resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go
similarity index 56%
rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go
rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go
index 31859c3fd89..505fc10832a 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go
@@ -35,7 +35,7 @@ type CertManagerMetricsPatch struct {
// SetTemplateDefaults implements machinery.Template
func (f *CertManagerMetricsPatch) SetTemplateDefaults() error {
if f.Path == "" {
- f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml")
+ f.Path = filepath.Join("config", "default", "cert_metrics_manager_patch.yaml")
}
f.TemplateBody = metricsManagerPatchTemplate
@@ -50,25 +50,36 @@ func (f *CertManagerMetricsPatch) SetTemplateDefaults() error {
return nil
}
-const metricsManagerPatchTemplate = `apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: {{ .ProjectName }}
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
+// nolint:lll
+const metricsManagerPatchTemplate = `# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
`
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go
index 636bae6bbb0..6c29ad48359 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go
@@ -87,9 +87,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -98,6 +100,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -108,6 +148,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -125,6 +166,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -152,7 +194,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -168,7 +210,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -183,7 +225,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -199,7 +241,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionns
@@ -207,7 +249,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go
index 71dbbef423b..f5f5dffc61f 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go
@@ -43,7 +43,7 @@ func (f *Kustomization) SetTemplateDefaults() error {
const kustomizationTemplate = `resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go
index 869b58b63e9..8078eb545b7 100644
--- a/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go
+++ b/pkg/plugins/common/kustomize/v2/scaffolds/webhook.go
@@ -84,6 +84,8 @@ func (s *webhookScaffolder) Scaffold() error {
&webhook.KustomizeConfig{},
&webhook.Service{},
&certmanager.Certificate{},
+ &certmanager.Issuer{},
+ &certmanager.MetricsCertificate{},
&certmanager.Kustomization{},
&certmanager.KustomizeConfig{},
&network_policy.NetworkPolicyAllowWebhooks{},
diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go
index 136be031810..4100d6d14f5 100644
--- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go
+++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go
@@ -226,6 +226,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -235,6 +236,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -256,6 +258,7 @@ func init() {
func main() {
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -269,6 +272,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -294,37 +300,53 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
// More info:
- // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/server
+ // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server
// - https://book.kubebuilder.io/reference/metrics.html
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
- TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
+ TLSOpts: tlsOpts,
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
- // https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
+ // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
+
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -357,6 +379,14 @@ func main() {
%s
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go
index 89e61054294..2cc62bd3601 100644
--- a/test/e2e/v4/generate_test.go
+++ b/test/e2e/v4/generate_test.go
@@ -64,6 +64,8 @@ func GenerateV4(kbc *utils.TestContext) {
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
"#- ../prometheus", "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ `#replacements:`, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
certManagerTarget, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
@@ -71,11 +73,10 @@ func GenerateV4(kbc *utils.TestContext) {
monitorTlsPatch, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
- `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed())
+ metricsCertPatch, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
- filepath.Join(kbc.Dir, "cmd", "main.go"),
- tlsConfigManager, "// ")).To(Succeed())
-
+ filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ metricsCertReplaces, "#")).To(Succeed())
uncommentKustomizeCoversion(kbc)
}
@@ -111,6 +112,8 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) {
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
"#- ../prometheus", "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ `#replacements:`, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
certManagerTarget, "#")).To(Succeed())
// Disable metrics
@@ -177,18 +180,21 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) {
metricsTarget, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
- `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed())
+ metricsCertPatch, "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(
+ filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ metricsCertReplaces, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"),
monitorTlsPatch, "#")).To(Succeed())
- ExpectWithOffset(1, pluginutil.UncommentCode(
- filepath.Join(kbc.Dir, "cmd", "main.go"),
- tlsConfigManager, "// ")).To(Succeed())
+
By("uncomment kustomization.yaml to enable network policy")
ExpectWithOffset(1, pluginutil.UncommentCode(
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
"#- ../network-policy", "#")).To(Succeed())
+ ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
+ `#replacements:`, "#")).To(Succeed())
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
certManagerTarget, "#")).To(Succeed())
@@ -243,8 +249,7 @@ const metricsTarget = `- path: manager_metrics_patch.yaml
kind: Deployment`
//nolint:lll
-const certManagerTarget = `#replacements:
-# - source: # Uncomment the following block if you have any webhook
+const certManagerTarget = `# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
# name: webhook-service
@@ -254,6 +259,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -271,6 +277,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -298,7 +305,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -314,7 +321,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -329,7 +336,7 @@ const certManagerTarget = `#replacements:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -345,7 +352,7 @@ const certNamespace = `# - source: # Uncomment the following block if you have a
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -362,7 +369,7 @@ const certName = `# - source:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -452,6 +459,44 @@ const monitorTlsPatch = `#patches:
# target:
# kind: ServiceMonitor`
-const tlsConfigManager = `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"`
+const metricsCertPatch = `#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment`
+
+const metricsCertReplaces = `# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true`
diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go
index 275bd9065c7..f453acd426d 100644
--- a/testdata/project-v4-multigroup/cmd/main.go
+++ b/testdata/project-v4-multigroup/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -95,6 +97,7 @@ func init() {
func main() {
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -108,6 +111,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -133,6 +139,9 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
@@ -145,25 +154,38 @@ func main() {
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -348,6 +370,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..2019d532f05
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-multigroup
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..2d174f830a9
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-multigroup
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml
deleted file mode 100644
index 73765b7407e..00000000000
--- a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-multigroup
- app.kubernetes.io/part-of: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-multigroup
- app.kubernetes.io/part-of: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-multigroup/config/certmanager/issuer.yaml b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..e266ca5119c
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-multigroup
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml
+++ b/testdata/project-v4-multigroup/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..8400eb42a7b
--- /dev/null
+++ b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index a6b8d9bc0de..00000000000
--- a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project-v4-multigroup
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml
index 78743d725ce..e659d82f08d 100644
--- a/testdata/project-v4-multigroup/config/default/kustomization.yaml
+++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -171,7 +213,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml b/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml
index f040fd58623..fdc5481b103 100644
--- a/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml
+++ b/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml
@@ -1,7 +1,7 @@
resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go
index f01870085dc..7d4943094c6 100644
--- a/testdata/project-v4-with-plugins/cmd/main.go
+++ b/testdata/project-v4-with-plugins/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -60,6 +62,7 @@ func init() {
func main() {
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -73,6 +76,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -98,6 +104,9 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
@@ -110,25 +119,38 @@ func main() {
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
+
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -194,6 +216,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..b38d5abec0f
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-with-plugins
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..8604f1cc0c5
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-with-plugins
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml
deleted file mode 100644
index d4580c48964..00000000000
--- a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-with-plugins
- app.kubernetes.io/part-of: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4-with-plugins
- app.kubernetes.io/part-of: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..05e6df0e47f
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4-with-plugins
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml
+++ b/testdata/project-v4-with-plugins/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..8400eb42a7b
--- /dev/null
+++ b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index ed2f033ddbb..00000000000
--- a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project-v4-with-plugins
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml
index d9a6926909f..bec1b771a28 100644
--- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml
+++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -171,7 +213,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml b/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml
index f040fd58623..fdc5481b103 100644
--- a/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml
+++ b/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml
@@ -1,7 +1,7 @@
resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go
index 1a72e4c73c1..f3b74461813 100644
--- a/testdata/project-v4/cmd/main.go
+++ b/testdata/project-v4/cmd/main.go
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"flag"
"os"
+ "path/filepath"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
@@ -29,6 +30,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -63,6 +65,7 @@ func init() {
func main() {
var metricsAddr string
+ var metricsCertPath, metricsCertName, metricsCertKey string
var enableLeaderElection bool
var probeAddr string
var secureMetrics bool
@@ -76,6 +79,9 @@ func main() {
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+ flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+ flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+ flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
@@ -101,6 +107,9 @@ func main() {
tlsOpts = append(tlsOpts, disableHTTP2)
}
+ // Create watchers for metrics certificates
+ var metricsCertWatcher *certwatcher.CertWatcher
+
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
@@ -113,25 +122,38 @@ func main() {
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
- }
-
- if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
- metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+ FilterProvider: filters.WithAuthenticationAndAuthorization,
+ }
- // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
- // generate self-signed certificates for the metrics server. While convenient for development and testing,
- // this setup is not recommended for production.
+ // If the certificate is not specified, controller-runtime will automatically
+ // generate self-signed certificates for the metrics server. While convenient for development and testing,
+ // this setup is not recommended for production.
+ //
+ // TODO(user): If you enable certManager, uncomment the following lines:
+ // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
+ // managed by cert-manager for the metrics server.
+ // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
+ if len(metricsCertPath) > 0 {
+ certName := filepath.Join(metricsCertPath, metricsCertName)
+ certKey := filepath.Join(metricsCertPath, metricsCertKey)
- // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
- // you can uncomment the following lines to use the certificate managed by cert-manager.
- // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
- // metricsServerOptions.CertName = "tls.crt"
- // metricsServerOptions.KeyName = "tls.key"
+ setupLog.Info("metrics server is serving securely using provided certificates",
+ "metrics-cert-path", metricsCertPath, "metrics-cert-name", certName, "metrics-cert-key", certKey)
+ var err error
+ metricsCertWatcher, err = certwatcher.New(certName, certKey)
+ if err != nil {
+ setupLog.Error(err, "to initialize certificate watcher", "error", err)
+ os.Exit(1)
+ }
+
+ metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+ config.GetCertificate = metricsCertWatcher.GetCertificate
+ })
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -230,6 +252,14 @@ func main() {
}
// +kubebuilder:scaffold:builder
+ if metricsCertWatcher != nil {
+ setupLog.Info("Adding metrics certificate watcher to manager")
+ if err := mgr.Add(metricsCertWatcher); err != nil {
+ setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+ os.Exit(1)
+ }
+ }
+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
diff --git a/testdata/project-v4/config/certmanager/certificate-metrics.yaml b/testdata/project-v4/config/certmanager/certificate-metrics.yaml
new file mode 100644
index 00000000000..6c2488daae5
--- /dev/null
+++ b/testdata/project-v4/config/certmanager/certificate-metrics.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4
+ app.kubernetes.io/managed-by: kustomize
+ name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ dnsNames:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4/config/certmanager/certificate-webhook.yaml b/testdata/project-v4/config/certmanager/certificate-webhook.yaml
new file mode 100644
index 00000000000..b3d765563ea
--- /dev/null
+++ b/testdata/project-v4/config/certmanager/certificate-webhook.yaml
@@ -0,0 +1,20 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4
+ app.kubernetes.io/managed-by: kustomize
+ name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
+ namespace: system
+spec:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ # replacements in the config/default/kustomization.yaml file.
+ dnsNames:
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc
+ - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: selfsigned-issuer
+ secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4/config/certmanager/certificate.yaml b/testdata/project-v4/config/certmanager/certificate.yaml
deleted file mode 100644
index 0cffc07a915..00000000000
--- a/testdata/project-v4/config/certmanager/certificate.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
- labels:
- app.kubernetes.io/name: project-v4
- app.kubernetes.io/managed-by: kustomize
- name: selfsigned-issuer
- namespace: system
-spec:
- selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: serving-cert
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4
- app.kubernetes.io/part-of: project-v4
- app.kubernetes.io/managed-by: kustomize
- name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - SERVICE_NAME.SERVICE_NAMESPACE.svc
- - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- labels:
- app.kubernetes.io/name: certificate
- app.kubernetes.io/instance: metrics-certs
- app.kubernetes.io/component: certificate
- app.kubernetes.io/created-by: project-v4
- app.kubernetes.io/part-of: project-v4
- app.kubernetes.io/managed-by: kustomize
- name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
- namespace: system
-spec:
- # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
- dnsNames:
- - controller-manager-metrics-service.system.svc
- - controller-manager-metrics-service.system.svc.cluster.local
- issuerRef:
- kind: Issuer
- name: selfsigned-issuer
- secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
diff --git a/testdata/project-v4/config/certmanager/issuer.yaml b/testdata/project-v4/config/certmanager/issuer.yaml
new file mode 100644
index 00000000000..65f38af73de
--- /dev/null
+++ b/testdata/project-v4/config/certmanager/issuer.yaml
@@ -0,0 +1,13 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ labels:
+ app.kubernetes.io/name: project-v4
+ app.kubernetes.io/managed-by: kustomize
+ name: selfsigned-issuer
+ namespace: system
+spec:
+ selfSigned: {}
diff --git a/testdata/project-v4/config/certmanager/kustomization.yaml b/testdata/project-v4/config/certmanager/kustomization.yaml
index bebea5a595e..fcb7498e468 100644
--- a/testdata/project-v4/config/certmanager/kustomization.yaml
+++ b/testdata/project-v4/config/certmanager/kustomization.yaml
@@ -1,5 +1,7 @@
resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
configurations:
- kustomizeconfig.yaml
diff --git a/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..8400eb42a7b
--- /dev/null
+++ b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,31 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts
+ value: []
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+- op: add
+ path: /spec/template/spec/volumes
+ value: []
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml
deleted file mode 100644
index 862a49388f2..00000000000
--- a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: controller-manager
- namespace: system
- labels:
- app.kubernetes.io/name: project-v4
- app.kubernetes.io/managed-by: kustomize
-spec:
- template:
- spec:
- containers:
- - name: manager
- volumeMounts:
- - mountPath: /tmp/k8s-metrics-server/metrics-certs
- name: metrics-certs
- readOnly: true
- volumes:
- - name: metrics-certs
- secret:
- secretName: metrics-server-cert
diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml
index 5451766b0b3..469070a7ed9 100644
--- a/testdata/project-v4/config/default/kustomization.yaml
+++ b/testdata/project-v4/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
kind: Deployment
# Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-#- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
@@ -53,6 +55,44 @@ patches:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
# Uncomment the following replacements to add the cert-manager CA injection annotations
#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+#
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+#
# - source: # Uncomment the following block if you have any webhook
# kind: Service
# version: v1
@@ -63,6 +103,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -80,6 +121,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
+# name: serving-cert
# fieldPaths:
# - .spec.dnsNames.0
# - .spec.dnsNames.1
@@ -107,7 +149,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -123,7 +165,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets:
# - select:
@@ -138,7 +180,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets:
# - select:
@@ -154,7 +196,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.namespace # Namespace of the certificate CR
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
@@ -171,7 +213,7 @@ patches:
# kind: Certificate
# group: cert-manager.io
# version: v1
-# name: serving-cert # This name should match the one in certificate.yaml
+# name: serving-cert
# fieldPath: .metadata.name
# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
# - select:
diff --git a/testdata/project-v4/config/prometheus/kustomization.yaml b/testdata/project-v4/config/prometheus/kustomization.yaml
index f040fd58623..fdc5481b103 100644
--- a/testdata/project-v4/config/prometheus/kustomization.yaml
+++ b/testdata/project-v4/config/prometheus/kustomization.yaml
@@ -1,7 +1,7 @@
resources:
- monitor.yaml
-# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.