Discontinue kube-rbac-proxy

This patch drops support for kube-rbac-proxy
and uses controller manager's
WithAuthenticationAndAuthorization.

Closes: csi-addons#643

Signed-off-by: Niraj Yadav <[email protected]>

Makefile

@@ -43,12 +43,6 @@ REPLACES ?=
 # but can skip several. This can be accomplished using the skipRange annotation:
 SKIP_RANGE ?=
 
-# By setting RBAC_PROXY_IMG to a different container-image, new versions of
-# the kube-rbac-proxy can easily be tested. Products that include CSI-Addons
-# may want to provide a different location of the container-image.
-# The default value is set in config/default/kustomization.yaml
-RBAC_PROXY_IMG ?= quay.io/brancz/kube-rbac-proxy:v0.18.0
-
 # The default version of the bundle (CSV) can be found in
 # config/manifests/bases/csi-addons.clusterserviceversion.yaml . When tagging a
 # release, the bundle will be versioned with the same value as well.
@@ -120,7 +114,7 @@ help: ## Display this help.
 .PHONY: manifests
 manifests: controller-gen kustomize ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="{./api/...,./cmd/...,./internal/controller/...,./internal/...,./sidecar/...}" output:crd:artifacts:config=config/crd/bases
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${CONTROLLER_IMG} rbac-proxy=${RBAC_PROXY_IMG}
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${CONTROLLER_IMG}
 	$(KUSTOMIZE) build config/crd > deploy/controller/crds.yaml
 	$(KUSTOMIZE) build config/rbac > deploy/controller/rbac.yaml
 	$(KUSTOMIZE) build config/manager > deploy/controller/setup-controller.yaml

cmd/manager/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"os"
 	"time"
@@ -43,6 +44,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	//+kubebuilder:scaffold:imports
@@ -70,15 +72,18 @@ func main() {
 		metricsAddr                 string
 		probeAddr                   string
 		enableLeaderElection        bool
+		enableHTTP2                 bool
 		leaderElectionLeaseDuration time.Duration
 		leaderElectionRenewDeadline time.Duration
 		leaderElectionRetryPeriod   time.Duration
 		showVersion                 bool
+		secureMetrics               bool
 		enableAdmissionWebhooks     bool
 		ctx                         = context.Background()
 		cfg                         = util.NewConfig()
+		tlsOpts                     []func(*tls.Config)
 	)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metrics endpoint binds to. Use :8443 for HTTPS or :8080 for HTTP, or 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -89,7 +94,9 @@ func main() {
 	flag.DurationVar(&cfg.ReclaimSpaceTimeout, "reclaim-space-timeout", cfg.ReclaimSpaceTimeout, "Timeout for reclaimspace operation")
 	flag.IntVar(&cfg.MaxConcurrentReconciles, "max-concurrent-reconciles", cfg.MaxConcurrentReconciles, "Maximum number of concurrent reconciles")
 	flag.StringVar(&cfg.Namespace, "namespace", cfg.Namespace, "Namespace where the CSIAddons pod is deployed")
+	flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks")
+	flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&showVersion, "version", false, "Print Version details")
 	flag.StringVar(&cfg.SchedulePrecedence, "schedule-precedence", "", "The order of precedence in which schedule of reclaimspace and keyrotation is considered. Possible values are sc-only")
 	opts := zap.Options{
@@ -128,8 +135,27 @@ func main() {
 		os.Exit(1)
 	}
 
+	disableHTTP2 := func(config *tls.Config) {
+		setupLog.Info("disabling HTTP/2")
+		config.NextProtos = []string{"http/1.1"}
+	}
+
+	if !enableHTTP2 {
+		tlsOpts = append(tlsOpts, disableHTTP2)
+	}
+
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+
+	if secureMetrics {
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+	}
+
 	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
-		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		Scheme:                 scheme,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
@@ -138,7 +164,8 @@ func main() {
 		RenewDeadline:          &leaderElectionRenewDeadline,
 		RetryPeriod:            &leaderElectionRetryPeriod,
 		WebhookServer: webhook.NewServer(webhook.Options{
-			Port: 9443,
+			Port:    9443,
+			TLSOpts: tlsOpts,
 		}),
 	})
 	if err != nil {

config/default/kustomization.yaml

@@ -8,9 +8,9 @@ namespace: csi-addons-system
 #    someName: someValue
 
 resources:
-- ../crd
-- ../rbac
-- ../manager
+  - ../crd
+  - ../rbac
+  - ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 #- ../webhook

config/manager/controller_manager_config.yaml

@@ -4,7 +4,7 @@ kind: ControllerManagerConfig
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: 127.0.0.1:8080
+  bindAddress: 127.0.0.1:8443
 webhook:
   port: 9443
 leaderElection:

config/manager/kustomization.yaml

@@ -10,7 +10,12 @@ namePrefix: csi-addons-
 
 resources:
 - manager.yaml
+- metrics_service.yaml
 
+patches:
+- path: manager_metrics_patch.yaml
+  target:
+    kind: Deployment
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
@@ -28,8 +33,3 @@ images:
 - name: controller
   newName: quay.io/csiaddons/k8s-controller
   newTag: latest
-- name: rbac-proxy
-  newName: quay.io/brancz/kube-rbac-proxy
-  newTag: v0.18.0
-patches:
-- path: manager_auth_proxy_patch.yaml

config/manager/manager.yaml

@@ -25,16 +25,22 @@ spec:
         kubectl.kubernetes.io/default-container: manager
       labels:
         app.kubernetes.io/name: csi-addons
+        control-plane: controller-manager
     spec:
       securityContext:
         runAsNonRoot: true
       containers:
         - command:
             - /csi-addons-manager
           args:
+            - --namespace=$(POD_NAMESPACE)
             - --leader-elect
           image: controller:latest
           name: manager
+          ports:
+            - name: metrics
+              containerPort: 8443
+              protocol: TCP
           env:
             - name: POD_NAMESPACE
               valueFrom:

config/default/manager_metrics_patch.yaml → config/manager/manager_metrics_patch.yaml

@@ -1,4 +1,4 @@
 # This patch adds the args to allow exposing the metrics endpoint securely
 - op: add
   path: /spec/template/spec/containers/0/args/0
-  value: --metrics-bind-address=:8080
+  value: --metrics-bind-address=:8443

config/default/metrics_service.yaml → config/manager/metrics_service.yaml

@@ -9,9 +9,9 @@ metadata:
   namespace: system
 spec:
   ports:
-  - name: http
-    port: 8080
-    protocol: TCP
-    targetPort: 8080
+    - name: http
+      port: 8443
+      protocol: TCP
+      targetPort: 8443
   selector:
     control-plane: controller-manager

config/rbac/kustomization.yaml

@@ -10,29 +10,30 @@ namespace: csi-addons-system
 namePrefix: csi-addons-
 
 resources:
-# All RBAC will be applied under this service account in
-# the deployment namespace. You may comment out this resource
-# if your manager will use a service account that exists at
-# runtime. Be sure to update RoleBinding and ClusterRoleBinding
-# subjects if changing service account names.
-- service_account.yaml
-- role.yaml
-- role_binding.yaml
-- leader_election_role.yaml
-- leader_election_role_binding.yaml
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
-# For each CRD, "Editor" and "Viewer" roles are scaffolded by
-# default, aiding admins in cluster management. Those roles are
-# not used by the Project itself. You can comment the following lines
-# if you do not want those helpers be installed with your Project.
-- csiaddons_networkfenceclass_editor_role.yaml
-- csiaddons_networkfenceclass_viewer_role.yaml
+  # All RBAC will be applied under this service account in
+  # the deployment namespace. You may comment out this resource
+  # if your manager will use a service account that exists at
+  # runtime. Be sure to update RoleBinding and ClusterRoleBinding
+  # subjects if changing service account names.
+  - service_account.yaml
+  - role.yaml
+  - role_binding.yaml
+  - leader_election_role.yaml
+  - leader_election_role_binding.yaml
+  # The following RBAC configurations are used to protect
+  # the metrics endpoint with authn/authz. These configurations
+  # ensure that only authorized users and service accounts
+  # can access the metrics endpoint.
+  - metrics_auth_role.yaml
+  - metrics_auth_role_binding.yaml
+  - metrics_reader_role.yaml
+  - metrics_reader_role_binding.yaml
+  # For each CRD, "Editor" and "Viewer" roles are scaffolded by
+  # default, aiding admins in cluster management. Those roles are
+  # not used by the Project itself. You can comment the following lines
+  # if you do not want those helpers be installed with your Project.
+  - csiaddons_networkfenceclass_editor_role.yaml
+  - csiaddons_networkfenceclass_viewer_role.yaml
 # - csiaddons_encryptionkeyrotationcronjob_editor_role.yaml
 # - csiaddons_encryptionkeyrotationcronjob_viewer_role.yaml
 # - csiaddons_encryptionkeyrotationjob_editor_role.yaml

config/rbac/metrics_auth_role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-auth-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

config/rbac/metrics_auth_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-auth-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-auth-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/auth_proxy_client_clusterrole.yaml → config/rbac/metrics_reader_role.yaml

@@ -1,10 +1,9 @@
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: metrics-reader
 rules:
-  - nonResourceURLs:
-      - "/metrics"
-    verbs:
-      - get
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get

config/rbac/auth_proxy_role_binding.yaml → config/rbac/metrics_reader_role_binding.yaml

@@ -1,12 +1,11 @@
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-reader-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-reader
 subjects:
   - kind: ServiceAccount
     name: controller-manager