From 4bfc2be0272d201a0e3e6c415c6badff4a90bf86 Mon Sep 17 00:00:00 2001 From: Niraj Yadav Date: Wed, 4 Dec 2024 19:19:55 +0530 Subject: [PATCH] Discontinue kube-rbac-proxy This patch drops support for kube-rbac-proxy and uses controller manager's WithAuthenticationAndAuthorization. Closes: #643 Signed-off-by: Niraj Yadav --- Makefile | 8 +-- cmd/manager/main.go | 33 +++++++++++-- config/default/kustomization.yaml | 6 +-- config/manager/controller_manager_config.yaml | 2 +- config/manager/kustomization.yaml | 10 ++-- config/manager/manager.yaml | 6 +++ config/manager/manager_auth_proxy_patch.yaml | 41 ---------------- .../manager_metrics_patch.yaml | 2 +- .../{default => manager}/metrics_service.yaml | 8 +-- config/rbac/auth_proxy_role.yaml | 18 ------- config/rbac/auth_proxy_service.yaml | 16 ------ config/rbac/kustomization.yaml | 47 +++++++++--------- config/rbac/metrics_auth_role.yaml | 17 +++++++ config/rbac/metrics_auth_role_binding.yaml | 12 +++++ ...sterrole.yaml => metrics_reader_role.yaml} | 9 ++-- ....yaml => metrics_reader_role_binding.yaml} | 5 +- deploy/controller/rbac.yaml | 49 +++++++++---------- deploy/controller/setup-controller.yaml | 49 ++++++++++--------- 18 files changed, 158 insertions(+), 180 deletions(-) delete mode 100644 config/manager/manager_auth_proxy_patch.yaml rename config/{default => manager}/manager_metrics_patch.yaml (77%) rename config/{default => manager}/metrics_service.yaml (80%) delete mode 100644 config/rbac/auth_proxy_role.yaml delete mode 100644 config/rbac/auth_proxy_service.yaml create mode 100644 config/rbac/metrics_auth_role.yaml create mode 100644 config/rbac/metrics_auth_role_binding.yaml rename config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_reader_role.yaml} (59%) rename config/rbac/{auth_proxy_role_binding.yaml => metrics_reader_role_binding.yaml} (79%) diff --git a/Makefile b/Makefile index 35ef530ec..63eae240a 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,6 @@ REPLACES ?= # but can skip several. This can be accomplished using the skipRange annotation: SKIP_RANGE ?= -# By setting RBAC_PROXY_IMG to a different container-image, new versions of -# the kube-rbac-proxy can easily be tested. Products that include CSI-Addons -# may want to provide a different location of the container-image. -# The default value is set in config/default/kustomization.yaml -RBAC_PROXY_IMG ?= quay.io/brancz/kube-rbac-proxy:v0.18.0 - # The default version of the bundle (CSV) can be found in # config/manifests/bases/csi-addons.clusterserviceversion.yaml . When tagging a # release, the bundle will be versioned with the same value as well. @@ -120,7 +114,7 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen kustomize ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="{./api/...,./cmd/...,./internal/controller/...,./internal/...,./sidecar/...}" output:crd:artifacts:config=config/crd/bases - cd config/manager && $(KUSTOMIZE) edit set image controller=${CONTROLLER_IMG} rbac-proxy=${RBAC_PROXY_IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=${CONTROLLER_IMG} $(KUSTOMIZE) build config/crd > deploy/controller/crds.yaml $(KUSTOMIZE) build config/rbac > deploy/controller/rbac.yaml $(KUSTOMIZE) build config/manager > deploy/controller/setup-controller.yaml diff --git a/cmd/manager/main.go b/cmd/manager/main.go index df218544b..de88a6ef7 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -18,6 +18,7 @@ package main import ( "context" + "crypto/tls" "flag" "os" "time" @@ -43,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" //+kubebuilder:scaffold:imports @@ -70,15 +72,18 @@ func main() { metricsAddr string probeAddr string enableLeaderElection bool + enableHTTP2 bool leaderElectionLeaseDuration time.Duration leaderElectionRenewDeadline time.Duration leaderElectionRetryPeriod time.Duration showVersion bool + secureMetrics bool enableAdmissionWebhooks bool ctx = context.Background() cfg = util.NewConfig() + tlsOpts []func(*tls.Config) ) - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metrics endpoint binds to. Use :8443 for HTTPS or :8080 for HTTP, or 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -89,7 +94,9 @@ func main() { flag.DurationVar(&cfg.ReclaimSpaceTimeout, "reclaim-space-timeout", cfg.ReclaimSpaceTimeout, "Timeout for reclaimspace operation") flag.IntVar(&cfg.MaxConcurrentReconciles, "max-concurrent-reconciles", cfg.MaxConcurrentReconciles, "Maximum number of concurrent reconciles") flag.StringVar(&cfg.Namespace, "namespace", cfg.Namespace, "Namespace where the CSIAddons pod is deployed") + flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks") + flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&showVersion, "version", false, "Print Version details") flag.StringVar(&cfg.SchedulePrecedence, "schedule-precedence", "", "The order of precedence in which schedule of reclaimspace and keyrotation is considered. Possible values are sc-only") opts := zap.Options{ @@ -128,8 +135,27 @@ func main() { os.Exit(1) } + disableHTTP2 := func(config *tls.Config) { + setupLog.Info("disabling HTTP/2") + config.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ - Metrics: metricsserver.Options{BindAddress: metricsAddr}, + Metrics: metricsServerOptions, Scheme: scheme, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, @@ -138,7 +164,8 @@ func main() { RenewDeadline: &leaderElectionRenewDeadline, RetryPeriod: &leaderElectionRetryPeriod, WebhookServer: webhook.NewServer(webhook.Options{ - Port: 9443, + Port: 9443, + TLSOpts: tlsOpts, }), }) if err != nil { diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 33ef12e51..6daa79173 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -8,9 +8,9 @@ namespace: csi-addons-system # someName: someValue resources: -- ../crd -- ../rbac -- ../manager + - ../crd + - ../rbac + - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- ../webhook diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index fc6133790..ef9ad2af7 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -4,7 +4,7 @@ kind: ControllerManagerConfig health: healthProbeBindAddress: :8081 metrics: - bindAddress: 127.0.0.1:8080 + bindAddress: 127.0.0.1:8443 webhook: port: 9443 leaderElection: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 4cbda177d..5aded0f99 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -10,7 +10,12 @@ namePrefix: csi-addons- resources: - manager.yaml +- metrics_service.yaml +patches: +- path: manager_metrics_patch.yaml + target: + kind: Deployment # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. @@ -28,8 +33,3 @@ images: - name: controller newName: quay.io/csiaddons/k8s-controller newTag: latest -- name: rbac-proxy - newName: quay.io/brancz/kube-rbac-proxy - newTag: v0.18.0 -patches: -- path: manager_auth_proxy_patch.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index fd1c45f20..1b813f7e9 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -25,6 +25,7 @@ spec: kubectl.kubernetes.io/default-container: manager labels: app.kubernetes.io/name: csi-addons + control-plane: controller-manager spec: securityContext: runAsNonRoot: true @@ -32,9 +33,14 @@ spec: - command: - /csi-addons-manager args: + - --namespace=$(POD_NAMESPACE) - --leader-elect image: controller:latest name: manager + ports: + - name: metrics + containerPort: 8443 + protocol: TCP env: - name: POD_NAMESPACE valueFrom: diff --git a/config/manager/manager_auth_proxy_patch.yaml b/config/manager/manager_auth_proxy_patch.yaml deleted file mode 100644 index 7647ba969..000000000 --- a/config/manager/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - image: rbac-proxy - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=10" - ports: - - containerPort: 8443 - protocol: TCP - name: https - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - - name: manager - args: - - "--namespace=$(POD_NAMESPACE)" - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/config/default/manager_metrics_patch.yaml b/config/manager/manager_metrics_patch.yaml similarity index 77% rename from config/default/manager_metrics_patch.yaml rename to config/manager/manager_metrics_patch.yaml index 6c546ae4c..3c659c18c 100644 --- a/config/default/manager_metrics_patch.yaml +++ b/config/manager/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ # This patch adds the args to allow exposing the metrics endpoint securely - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/config/default/metrics_service.yaml b/config/manager/metrics_service.yaml similarity index 80% rename from config/default/metrics_service.yaml rename to config/manager/metrics_service.yaml index 80a526e18..b97fa56cf 100644 --- a/config/default/metrics_service.yaml +++ b/config/manager/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 - protocol: TCP - targetPort: 8080 + - name: http + port: 8443 + protocol: TCP + targetPort: 8443 selector: control-plane: controller-manager diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 449cb4bfb..000000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: - - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create - - apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 6d93adb22..000000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/name: csi-addons - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/name: csi-addons diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 9a9639b88..c92c0096f 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -10,29 +10,30 @@ namespace: csi-addons-system namePrefix: csi-addons- resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml -# For each CRD, "Editor" and "Viewer" roles are scaffolded by -# default, aiding admins in cluster management. Those roles are -# not used by the Project itself. You can comment the following lines -# if you do not want those helpers be installed with your Project. -- csiaddons_networkfenceclass_editor_role.yaml -- csiaddons_networkfenceclass_viewer_role.yaml + # All RBAC will be applied under this service account in + # the deployment namespace. You may comment out this resource + # if your manager will use a service account that exists at + # runtime. Be sure to update RoleBinding and ClusterRoleBinding + # subjects if changing service account names. + - service_account.yaml + - role.yaml + - role_binding.yaml + - leader_election_role.yaml + - leader_election_role_binding.yaml + # The following RBAC configurations are used to protect + # the metrics endpoint with authn/authz. These configurations + # ensure that only authorized users and service accounts + # can access the metrics endpoint. + - metrics_auth_role.yaml + - metrics_auth_role_binding.yaml + - metrics_reader_role.yaml + - metrics_reader_role_binding.yaml + # For each CRD, "Editor" and "Viewer" roles are scaffolded by + # default, aiding admins in cluster management. Those roles are + # not used by the Project itself. You can comment the following lines + # if you do not want those helpers be installed with your Project. + - csiaddons_networkfenceclass_editor_role.yaml + - csiaddons_networkfenceclass_viewer_role.yaml # - csiaddons_encryptionkeyrotationcronjob_editor_role.yaml # - csiaddons_encryptionkeyrotationcronjob_viewer_role.yaml # - csiaddons_encryptionkeyrotationjob_editor_role.yaml diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 000000000..32d2e4ec6 --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 000000000..e775d67ff --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/metrics_reader_role.yaml similarity index 59% rename from config/rbac/auth_proxy_client_clusterrole.yaml rename to config/rbac/metrics_reader_role.yaml index 8e6154f36..51a75db47 100644 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/config/rbac/metrics_reader_role.yaml @@ -1,10 +1,9 @@ ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: metrics-reader rules: - - nonResourceURLs: - - "/metrics" - verbs: - - get +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/metrics_reader_role_binding.yaml similarity index 79% rename from config/rbac/auth_proxy_role_binding.yaml rename to config/rbac/metrics_reader_role_binding.yaml index 08cb903f7..66abe26e2 100644 --- a/config/rbac/auth_proxy_role_binding.yaml +++ b/config/rbac/metrics_reader_role_binding.yaml @@ -1,12 +1,11 @@ ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: proxy-rolebinding + name: metrics-reader-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-reader subjects: - kind: ServiceAccount name: controller-manager diff --git a/deploy/controller/rbac.yaml b/deploy/controller/rbac.yaml index 8dbaf23c3..4f7201a45 100644 --- a/deploy/controller/rbac.yaml +++ b/deploy/controller/rbac.yaml @@ -236,17 +236,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: csi-addons-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: csi-addons-proxy-role + name: csi-addons-metrics-auth-role rules: - apiGroups: - authentication.k8s.io @@ -262,6 +252,16 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-addons-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: csi-addons-leader-election-rolebinding @@ -291,28 +291,25 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: csi-addons-proxy-rolebinding + name: csi-addons-metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: csi-addons-proxy-role + name: csi-addons-metrics-auth-role subjects: - kind: ServiceAccount name: csi-addons-controller-manager namespace: csi-addons-system --- -apiVersion: v1 -kind: Service +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - labels: - app.kubernetes.io/name: csi-addons - name: csi-addons-controller-manager-metrics-service + name: csi-addons-metrics-reader-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-addons-metrics-reader +subjects: +- kind: ServiceAccount + name: csi-addons-controller-manager namespace: csi-addons-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/name: csi-addons diff --git a/deploy/controller/setup-controller.yaml b/deploy/controller/setup-controller.yaml index d602b6024..7f5c450a3 100644 --- a/deploy/controller/setup-controller.yaml +++ b/deploy/controller/setup-controller.yaml @@ -16,7 +16,7 @@ data: health: healthProbeBindAddress: :8081 metrics: - bindAddress: 127.0.0.1:8080 + bindAddress: 127.0.0.1:8443 webhook: port: 9443 leaderElection: @@ -27,6 +27,24 @@ metadata: name: csi-addons-manager-config namespace: csi-addons-system --- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: kubernetes-csi-addons + control-plane: controller-manager + name: csi-addons-controller-manager-metrics-service + namespace: csi-addons-system +spec: + ports: + - name: http + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -45,33 +63,12 @@ spec: kubectl.kubernetes.io/default-container: manager labels: app.kubernetes.io/name: csi-addons + control-plane: controller-manager spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=10 - image: quay.io/brancz/kube-rbac-proxy:v0.18.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - - args: + - --metrics-bind-address=:8443 - --namespace=$(POD_NAMESPACE) - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect command: - /csi-addons-manager @@ -88,6 +85,10 @@ spec: initialDelaySeconds: 15 periodSeconds: 20 name: manager + ports: + - containerPort: 8443 + name: metrics + protocol: TCP readinessProbe: httpGet: path: /readyz