Skip to content

Commit

Permalink
Add Patch to give optional option to enable ServiceMonitor to use cer…
Browse files Browse the repository at this point in the history
…t-manager-managed serving-cert with TLS verification

Adds a patch to configure ServiceMonitor with  `insecureSkipVerify: false`  to ensure TLS verification using cert-manager certificates. Updates documentation and corrects misaligned comments.
  • Loading branch information
camilamacedo86 committed Nov 9, 2024
1 parent a9ee390 commit 57fdc48
Show file tree
Hide file tree
Showing 54 changed files with 859 additions and 84 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/test-e2e-samples.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
run: |
KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '50,177s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4/
go mod tidy
Expand Down Expand Up @@ -82,8 +82,8 @@ jobs:
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
# Uncomment only ValidatingWebhookConfiguration
# from cert-manager replaces
sed -i '50,116s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '148,177s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '153,182s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4-with-plugins/
go mod tidy
Expand Down Expand Up @@ -122,7 +122,7 @@ jobs:
run: |
KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '50,177s/^#//' $KUSTOMIZATION_FILE_PATH
sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
cd testdata/project-v4-multigroup
go mod tidy
Expand Down
7 changes: 7 additions & 0 deletions docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,13 @@ func main() {
// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.

// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,25 @@ spec:
kind: Issuer
name: selfsigned-issuer
secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
app.kubernetes.io/name: certificate
app.kubernetes.io/instance: metrics-certs
app.kubernetes.io/component: certificate
app.kubernetes.io/created-by: project
app.kubernetes.io/part-of: project
app.kubernetes.io/managed-by: kustomize
name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
namespace: system
spec:
# SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
dnsNames:
- controller-manager-metrics-service.system.svc
- controller-manager-metrics-service.system.svc.cluster.local
issuerRef:
kind: Issuer
name: selfsigned-issuer
secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
labels:
app.kubernetes.io/name: project
app.kubernetes.io/managed-by: kustomize
spec:
template:
spec:
containers:
- name: manager
volumeMounts:
- mountPath: /tmp/k8s-metrics-server/metrics-certs
name: metrics-certs
readOnly: true
volumes:
- name: metrics-certs
secret:
defaultMode: 420
secretName: metrics-server-cert
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@ resources:
# be able to communicate with the Webhook Server.
#- ../network-policy

# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
# Uncomment the patches line if you enable Metrics
patches:
# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
# More info: https://book.kubebuilder.io/reference/metrics
- path: manager_metrics_patch.yaml
target:
kind: Deployment

# Uncomment the patches line if you enable Metrics and CertManager
# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
# This patch will protect the metrics with certmanager self-signed certs.
- path: certmanager_metrics_manager_patch.yaml

# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
- path: manager_webhook_patch.yaml
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
patches:
- path: monitor_tls_patch.yaml
target:
kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'metrics-server-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: metrics-server-cert
key: ca.crt
cert:
secret:
name: metrics-server-cert
key: tls.crt
keySecret:
name: metrics-server-cert
key: tls.key
7 changes: 7 additions & 0 deletions docs/book/src/getting-started/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ func main() {
// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.

// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
labels:
app.kubernetes.io/name: project
app.kubernetes.io/managed-by: kustomize
spec:
template:
spec:
containers:
- name: manager
volumeMounts:
- mountPath: /tmp/k8s-metrics-server/metrics-certs
name: metrics-certs
readOnly: true
volumes:
- name: metrics-certs
secret:
defaultMode: 420
secretName: metrics-server-cert
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@ resources:
# be able to communicate with the Webhook Server.
#- ../network-policy

# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
# Uncomment the patches line if you enable Metrics
patches:
# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
# More info: https://book.kubebuilder.io/reference/metrics
- path: manager_metrics_patch.yaml
target:
kind: Deployment

# Uncomment the patches line if you enable Metrics and CertManager
# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
# This patch will protect the metrics with certmanager self-signed certs.
#- path: certmanager_metrics_manager_patch.yaml

# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
#- path: manager_webhook_patch.yaml
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'metrics-server-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: metrics-server-cert
key: ca.crt
cert:
secret:
name: metrics-server-cert
key: tls.crt
keySecret:
name: metrics-server-cert
key: tls.key
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,13 @@ func main() {
// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.

// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,25 @@ spec:
kind: Issuer
name: selfsigned-issuer
secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
labels:
app.kubernetes.io/name: certificate
app.kubernetes.io/instance: metrics-certs
app.kubernetes.io/component: certificate
app.kubernetes.io/created-by: project
app.kubernetes.io/part-of: project
app.kubernetes.io/managed-by: kustomize
name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml
namespace: system
spec:
# SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
dnsNames:
- controller-manager-metrics-service.system.svc
- controller-manager-metrics-service.system.svc.cluster.local
issuerRef:
kind: Issuer
name: selfsigned-issuer
secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
labels:
app.kubernetes.io/name: project
app.kubernetes.io/managed-by: kustomize
spec:
template:
spec:
containers:
- name: manager
volumeMounts:
- mountPath: /tmp/k8s-metrics-server/metrics-certs
name: metrics-certs
readOnly: true
volumes:
- name: metrics-certs
secret:
defaultMode: 420
secretName: metrics-server-cert
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@ resources:
# be able to communicate with the Webhook Server.
#- ../network-policy

# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
# Uncomment the patches line if you enable Metrics
patches:
# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
# More info: https://book.kubebuilder.io/reference/metrics
- path: manager_metrics_patch.yaml
target:
kind: Deployment

# Uncomment the patches line if you enable Metrics and CertManager
# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
# This patch will protect the metrics with certmanager self-signed certs.
- path: certmanager_metrics_manager_patch.yaml

# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
- path: manager_webhook_patch.yaml
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
# to mount the "metrics-server-cert" secret in the Manager Deployment.
patches:
- path: monitor_tls_patch.yaml
target:
kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'metrics-server-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Loading

0 comments on commit 57fdc48

Please sign in to comment.