forked from kubernetes-sigs/kubebuilder
-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat/fix: enhance cert-manager integration for metrics endpoints (fol…
…low-up to PR kubernetes-sigs#4243) This commit is a follow-up to PR kubernetes-sigs#4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to kubernetes-sigs#3871 and kubernetes-sigs#4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready
- Loading branch information
1 parent
d240946
commit 2f7ee7d
Showing
44 changed files
with
717 additions
and
272 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,7 @@ import ( | |
| "crypto/tls" | ||
| "flag" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) | ||
| // to ensure that exec-entrypoint and run can make use of them. | ||
|
|
@@ -30,6 +31,7 @@ import ( | |
| utilruntime "k8s.io/apimachinery/pkg/util/runtime" | ||
| clientgoscheme "k8s.io/client-go/kubernetes/scheme" | ||
| ctrl "sigs.k8s.io/controller-runtime" | ||
| "sigs.k8s.io/controller-runtime/pkg/certwatcher" | ||
| "sigs.k8s.io/controller-runtime/pkg/healthz" | ||
| "sigs.k8s.io/controller-runtime/pkg/log/zap" | ||
| "sigs.k8s.io/controller-runtime/pkg/metrics/filters" | ||
|
|
@@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`. | |
| */ | ||
|
|
||
| var ( | ||
| scheme = runtime.NewScheme() | ||
| setupLog = ctrl.Log.WithName("setup") | ||
| scheme = runtime.NewScheme() | ||
| setupLog = ctrl.Log.WithName("setup") | ||
| certWatcher *certwatcher.CertWatcher | ||
| ) | ||
|
|
||
| func init() { | ||
|
|
@@ -74,6 +77,9 @@ func main() { | |
| /* | ||
| */ | ||
| var metricsAddr string | ||
| var certDir string | ||
| var certName string | ||
| var certKey string | ||
| var enableLeaderElection bool | ||
| var probeAddr string | ||
| var secureMetrics bool | ||
|
|
@@ -87,6 +93,10 @@ func main() { | |
| "Enabling this will ensure there is only one active controller manager.") | ||
| flag.BoolVar(&secureMetrics, "metrics-secure", true, | ||
| "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") | ||
| flag.StringVar(&certDir, "cert-dir", "", | ||
| "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") | ||
| flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") | ||
| flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") | ||
| flag.BoolVar(&enableHTTP2, "enable-http2", false, | ||
| "If set, HTTP/2 will be enabled for the metrics and webhook servers") | ||
| opts := zap.Options{ | ||
|
|
@@ -133,16 +143,27 @@ func main() { | |
| // https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization | ||
| metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization | ||
|
|
||
| // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically | ||
| // If cert-name, cert-key, and cert-name are not specified, controller-runtime will automatically | ||
| // generate self-signed certificates for the metrics server. While convenient for development and testing, | ||
| // this setup is not recommended for production. | ||
|
|
||
| // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, | ||
| // you can uncomment the following lines to use the certificate managed by cert-manager. | ||
| // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" | ||
| // metricsServerOptions.CertName = "tls.crt" | ||
| // metricsServerOptions.KeyName = "tls.key" | ||
|
|
||
| // | ||
| // TODO(user): If you are using cert-manager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml" | ||
| // to generate and use certificates managed by cert-manager for the metrics server. | ||
| if len(certDir) > 0 { | ||
| setupLog.Info("using certificates for the metrics server", | ||
| "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) | ||
|
|
||
| var err error | ||
| certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) | ||
| if err != nil { | ||
| setupLog.Error(err, "to initialize certificate watcher", "error", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { | ||
| config.GetCertificate = certWatcher.GetCertificate | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ | ||
|
|
@@ -196,6 +217,14 @@ func main() { | |
| } | ||
| // +kubebuilder:scaffold:builder | ||
|
|
||
| if secureMetrics && certWatcher != nil { | ||
| setupLog.Info("Adding certificate watcher to manager") | ||
| if err := mgr.Add(certWatcher); err != nil { | ||
| setupLog.Error(err, "unable to add certificate watcher to manager") | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { | ||
| setupLog.Error(err, "unable to set up health check") | ||
| os.Exit(1) | ||
|
|
||
35 changes: 35 additions & 0 deletions
35
...book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| # This patch adds the args and volumes to allow the manager to use the metrics-server certs | ||
| # Ensure the volumeMounts field exists by creating it if missing | ||
| - op: add | ||
| path: /spec/template/spec/containers/0/volumeMounts | ||
| value: [] | ||
| # Add the volume mount for the serving certificates | ||
| - op: add | ||
| path: /spec/template/spec/containers/0/volumeMounts/- | ||
| value: | ||
| mountPath: /tmp/k8s-metrics-server/serving-certs | ||
| name: metrics-certs | ||
| readOnly: true | ||
| # Add the cert-dir argument | ||
| - op: add | ||
| path: /spec/template/spec/containers/0/args/- | ||
| value: --cert-dir=/tmp/k8s-metrics-server/serving-certs | ||
| # Ensure the volumes field exists by creating it if missing | ||
| - op: add | ||
| path: /spec/template/spec/volumes | ||
| value: [] | ||
| # Add the volume for the serving certificates | ||
| - op: add | ||
| path: /spec/template/spec/volumes/- | ||
| value: | ||
| name: metrics-certs | ||
| secret: | ||
| secretName: metrics-server-cert | ||
| optional: false | ||
| items: | ||
| - key: ca.crt | ||
| path: ca.crt | ||
| - key: tls.crt | ||
| path: tls.crt | ||
| - key: tls.key | ||
| path: tls.key |
21 changes: 0 additions & 21 deletions
21
...c/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.