fix: Ingress rbac and support ingressClass (#1133)

* fix: ingress controller rbac issue Signed-off-by: ericsyh <[email protected]> * update the ingress Signed-off-by: ericsyh <[email protected]> * support ingressClass Signed-off-by: ericsyh <[email protected]> * update the image logic Signed-off-by: ericsyh <[email protected]> * apply the change to sn-platform chart Signed-off-by: ericsyh <[email protected]> --------- Signed-off-by: ericsyh <[email protected]>
streamnative · Dec 21, 2023 · dc7d1f5 · dc7d1f5
1 parent 704d63f
commit dc7d1f5
Show file tree

Hide file tree

Showing 10 changed files with 209 additions and 47 deletions.
diff --git a/charts/sn-platform-slim/templates/control-center/_control_center.tpl b/charts/sn-platform-slim/templates/control-center/_control_center.tpl
@@ -111,13 +111,13 @@ Get ingress image according to the k8s version.
 When k8s version is higher or equal than v1.22, ingress image should use version v1.x.x,
 otherwise it should use the default version 0.26.2 that defines in values.yaml.
 
-If k8s version is higher or equal than v1.22, but the .Values.images.nginx_ingress_controller.tag is less than v1.x.x,
-it will use registry.k8s.io/ingress-nginx/controller:v1.1.1 as default to make ingress work.
+If k8s version is higher or equal than v1.22, it will use the nginx_ingress_controller image in values. 
+otherwise, it will use quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26. as default to make ingress work.
 */}}
 {{- define "pulsar.ingress.image" -}}
-    {{- if and (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") (semverCompare "< 1.0.0" .Values.images.nginx_ingress_controller.tag )}}
-        {{- print "registry.k8s.io/ingress-nginx/controller:v1.1.1"}}
-    {{- else -}}
+    {{- if (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") }}
         {{- printf "%s:%s" .Values.images.nginx_ingress_controller.repository .Values.images.nginx_ingress_controller.tag -}}
+    {{- else -}}
+        {{- print "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.2"}}
     {{- end -}}
-{{- end -}}
+{{- end -}}
diff --git a/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml b/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml
@@ -47,11 +47,16 @@ metadata:
 {{- else }}
     ingress.kubernetes.io/ssl-redirect: "false"
 {{- end }}
+  {{- if not $isIngressAPIStable }}
     kubernetes.io/ingress.class: nginx
+  {{- end }}
   {{- with .Values.ingress.control_center.annotations }}
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+{{- if $isIngressAPIStable }}
+  ingressClassName: nginx
+{{- end }}
 {{- if and .Values.ingress.control_center.tls.enabled (not .Values.ingress.controller.tls.termination) }}
   {{- if .Values.domain.enabled }}
   tls:
@@ -61,7 +66,6 @@ spec:
 # Use the same cert for proxy which is wildcard type
       secretName: "{{ template "pulsar.proxy.tls.secret.name" . }}"
 {{- end }}
-
   {{- end }}
 {{- end }}
   rules:

diff --git a/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml b/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml
@@ -19,6 +19,11 @@
 
 {{- if .Values.ingress.controller.enabled }}
 {{- $isKubeVersionLessThanV122 := eq (include "pulsar.kubeVersion.isLessThanV122" .) "true" }}
+{{- $isIngressAPIStable := eq (include "pulsar.ingress.isStable" .) "true" -}}
+
+{{/* COMMENT */}}
+
+
 
 apiVersion: apps/v1
 kind: Deployment
@@ -70,7 +75,11 @@ spec:
         - --tcp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-tcp-services
         - --udp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-udp-services
         - --publish-service={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.ingress.controller.component }}
+        {{- if $isIngressAPIStable }}
+        - --ingress-class=nginx
+        {{- else }}
         - --annotations-prefix=nginx.ingress.kubernetes.io
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: true
           capabilities:

diff --git a/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml b/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml
@@ -46,6 +46,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -60,6 +67,15 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - "extensions"
+      - "networking.k8s.io"
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -71,18 +87,25 @@ rules:
       - "extensions"
       - "networking.k8s.io"
     resources:
-      - ingresses
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - "networking.k8s.io"
+    resources:
+      - ingressclasses
     verbs:
       - get
       - list
       - watch
   - apiGroups:
-      - "extensions"
-      - "networking.k8s.io"
+      - discovery.k8s.io
     resources:
-      - ingresses/status
+      - endpointslices
     verbs:
-      - update
+      - list
+      - watch
+      - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -94,41 +117,86 @@ metadata:
     chart: {{ template "pulsar.chart" . }}
     release: {{ .Release.Name }}
 rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - get
   - apiGroups:
       - ""
     resources:
       - configmaps
       - pods
       - secrets
-      - namespaces
+      - endpoints
     verbs:
       - get
+      - list
+      - watch
+      - update
   - apiGroups:
       - ""
     resources:
-      - configmaps
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "extensions"
+      - "networking.k8s.io"
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "extensions"
+      - "networking.k8s.io"
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
     resourceNames:
-      # Defaults to "<election-id>-<ingress-class>"
-      # Here: "<ingress-controller-leader>-<nginx>"
-      # This has to be adapted if you change either parameter
-      # when launching the nginx-ingress-controller.
       - "ingress-controller-leader-nginx"
-      # Upgrade to v1.1.1, controller will update this configmap
       - "ingress-controller-leader"
     verbs:
       - get
       - update
   - apiGroups:
-      - ""
+      - coordination.k8s.io
     resources:
-      - configmaps
+      - leases
     verbs:
       - create
   - apiGroups:
       - ""
     resources:
-      - endpoints
+      - events
+    verbs:
+      - create
+      - patch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
     verbs:
+      - list
+      - watch
       - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml
@@ -192,8 +192,8 @@ images:
     tag: "v1.5.0"
     pullPolicy: "IfNotPresent"
   nginx_ingress_controller:
-    repository: quay.io/kubernetes-ingress-controller/nginx-ingress-controller
-    tag: "0.26.2"
+    repository: registry.k8s.io/ingress-nginx/controller
+    tag: "v1.9.4"
     pullPolicy: "IfNotPresent"
   pulsar_metadata:
     repository: streamnative/sn-platform-slim

diff --git a/charts/sn-platform/templates/control-center/_control_center.tpl b/charts/sn-platform/templates/control-center/_control_center.tpl
@@ -111,13 +111,13 @@ Get ingress image according to the k8s version.
 When k8s version is higher or equal than v1.22, ingress image should use version v1.x.x,
 otherwise it should use the default version 0.26.2 that defines in values.yaml.
 
-If k8s version is higher or equal than v1.22, but the .Values.images.nginx_ingress_controller.tag is less than v1.x.x,
-it will use registry.k8s.io/ingress-nginx/controller:v1.1.1 as default to make ingress work.
+If k8s version is higher or equal than v1.22, it will use the nginx_ingress_controller image in values. 
+otherwise, it will use quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26. as default to make ingress work.
 */}}
 {{- define "pulsar.ingress.image" -}}
-    {{- if and (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") (semverCompare "< 1.0.0" .Values.images.nginx_ingress_controller.tag )}}
-        {{- print "registry.k8s.io/ingress-nginx/controller:v1.1.1"}}
-    {{- else -}}
+    {{- if (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") }}
         {{- printf "%s:%s" .Values.images.nginx_ingress_controller.repository .Values.images.nginx_ingress_controller.tag -}}
+    {{- else -}}
+        {{- print "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.2"}}
     {{- end -}}
 {{- end -}}
diff --git a/charts/sn-platform/templates/control-center/control-center-ingress.yaml b/charts/sn-platform/templates/control-center/control-center-ingress.yaml
@@ -47,11 +47,16 @@ metadata:
 {{- else }}
     ingress.kubernetes.io/ssl-redirect: "false"
 {{- end }}
+  {{- if not $isIngressAPIStable }}
     kubernetes.io/ingress.class: nginx
+  {{- end }}
   {{- with .Values.ingress.control_center.annotations }}
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+{{- if $isIngressAPIStable }}
+  ingressClassName: nginx
+{{- end }}
 {{- if and .Values.ingress.control_center.tls.enabled (not .Values.ingress.controller.tls.termination) }}
   {{- if .Values.domain.enabled }}
   tls:
@@ -61,7 +66,6 @@ spec:
 # Use the same cert for proxy which is wildcard type
       secretName: "{{ template "pulsar.proxy.tls.secret.name" . }}"
 {{- end }}
-
   {{- end }}
 {{- end }}
   rules:

diff --git a/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml b/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml
@@ -19,6 +19,11 @@
 
 {{- if .Values.ingress.controller.enabled }}
 {{- $isKubeVersionLessThanV122 := eq (include "pulsar.kubeVersion.isLessThanV122" .) "true" }}
+{{- $isIngressAPIStable := eq (include "pulsar.ingress.isStable" .) "true" -}}
+
+{{/* COMMENT */}}
+
+
 
 apiVersion: apps/v1
 kind: Deployment
@@ -70,7 +75,11 @@ spec:
         - --tcp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-tcp-services
         - --udp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-udp-services
         - --publish-service={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.ingress.controller.component }}
+        {{- if $isIngressAPIStable }}
+        - --ingress-class=nginx
+        {{- else }}
         - --annotations-prefix=nginx.ingress.kubernetes.io
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: true
           capabilities: