From 30be4ab22764b50818dc2f56229d9b6485f8c556 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Thu, 21 Dec 2023 18:40:54 +0800 Subject: [PATCH] fix: Ingress rbac and support ingressClass (#1133) * fix: ingress controller rbac issue Signed-off-by: ericsyh * update the ingress Signed-off-by: ericsyh * support ingressClass Signed-off-by: ericsyh * update the image logic Signed-off-by: ericsyh * apply the change to sn-platform chart Signed-off-by: ericsyh --------- Signed-off-by: ericsyh (cherry picked from commit dc7d1f569ff4f741c89d8be6ce230dfdadaf679f) --- .../control-center/_control_center.tpl | 12 +-- .../control-center-ingress.yaml | 6 +- .../ingress-controller-deployment.yaml | 9 ++ .../ingress-controller-rbac.yaml | 98 ++++++++++++++++--- charts/sn-platform-slim/values.yaml | 4 +- .../control-center/_control_center.tpl | 10 +- .../control-center-ingress.yaml | 6 +- .../ingress-controller-deployment.yaml | 9 ++ .../ingress-controller-rbac.yaml | 98 ++++++++++++++++--- charts/sn-platform/values.yaml | 4 +- 10 files changed, 209 insertions(+), 47 deletions(-) diff --git a/charts/sn-platform-slim/templates/control-center/_control_center.tpl b/charts/sn-platform-slim/templates/control-center/_control_center.tpl index fe930e3d0..481a326b8 100644 --- a/charts/sn-platform-slim/templates/control-center/_control_center.tpl +++ b/charts/sn-platform-slim/templates/control-center/_control_center.tpl @@ -111,13 +111,13 @@ Get ingress image according to the k8s version. When k8s version is higher or equal than v1.22, ingress image should use version v1.x.x, otherwise it should use the default version 0.26.2 that defines in values.yaml. -If k8s version is higher or equal than v1.22, but the .Values.images.nginx_ingress_controller.tag is less than v1.x.x, -it will use registry.k8s.io/ingress-nginx/controller:v1.1.1 as default to make ingress work. +If k8s version is higher or equal than v1.22, it will use the nginx_ingress_controller image in values. +otherwise, it will use quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26. as default to make ingress work. */}} {{- define "pulsar.ingress.image" -}} - {{- if and (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") (semverCompare "< 1.0.0" .Values.images.nginx_ingress_controller.tag )}} - {{- print "registry.k8s.io/ingress-nginx/controller:v1.1.1"}} - {{- else -}} + {{- if (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") }} {{- printf "%s:%s" .Values.images.nginx_ingress_controller.repository .Values.images.nginx_ingress_controller.tag -}} + {{- else -}} + {{- print "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.2"}} {{- end -}} -{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml b/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml index 374b02e86..05b90af50 100644 --- a/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml +++ b/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml @@ -47,11 +47,16 @@ metadata: {{- else }} ingress.kubernetes.io/ssl-redirect: "false" {{- end }} + {{- if not $isIngressAPIStable }} kubernetes.io/ingress.class: nginx + {{- end }} {{- with .Values.ingress.control_center.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if $isIngressAPIStable }} + ingressClassName: nginx +{{- end }} {{- if and .Values.ingress.control_center.tls.enabled (not .Values.ingress.controller.tls.termination) }} {{- if .Values.domain.enabled }} tls: @@ -61,7 +66,6 @@ spec: # Use the same cert for proxy which is wildcard type secretName: "{{ template "pulsar.proxy.tls.secret.name" . }}" {{- end }} - {{- end }} {{- end }} rules: diff --git a/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml b/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml index 8d6368487..407aa0a78 100644 --- a/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml +++ b/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml @@ -19,6 +19,11 @@ {{- if .Values.ingress.controller.enabled }} {{- $isKubeVersionLessThanV122 := eq (include "pulsar.kubeVersion.isLessThanV122" .) "true" }} +{{- $isIngressAPIStable := eq (include "pulsar.ingress.isStable" .) "true" -}} + +{{/* COMMENT */}} + + apiVersion: apps/v1 kind: Deployment @@ -70,7 +75,11 @@ spec: - --tcp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-tcp-services - --udp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-udp-services - --publish-service={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.ingress.controller.component }} + {{- if $isIngressAPIStable }} + - --ingress-class=nginx + {{- else }} - --annotations-prefix=nginx.ingress.kubernetes.io + {{- end }} securityContext: allowPrivilegeEscalation: true capabilities: diff --git a/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml b/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml index 2cb1ffaac..e0c8facad 100644 --- a/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml +++ b/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml @@ -46,6 +46,13 @@ rules: verbs: - list - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch - apiGroups: - "" resources: @@ -60,6 +67,15 @@ rules: - get - list - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -71,18 +87,25 @@ rules: - "extensions" - "networking.k8s.io" resources: - - ingresses + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses verbs: - get - list - watch - apiGroups: - - "extensions" - - "networking.k8s.io" + - discovery.k8s.io resources: - - ingresses/status + - endpointslices verbs: - - update + - list + - watch + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -94,41 +117,86 @@ metadata: chart: {{ template "pulsar.chart" . }} release: {{ .Release.Name }} rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get - apiGroups: - "" resources: - configmaps - pods - secrets - - namespaces + - endpoints verbs: - get + - list + - watch + - update - apiGroups: - "" resources: - - configmaps + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" - # Upgrade to v1.1.1, controller will update this configmap - "ingress-controller-leader" verbs: - get - update - apiGroups: - - "" + - coordination.k8s.io resources: - - configmaps + - leases verbs: - create - apiGroups: - "" resources: - - endpoints + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices verbs: + - list + - watch - get --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index e86363838..3360f6dad 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -192,8 +192,8 @@ images: tag: "v1.5.0" pullPolicy: "IfNotPresent" nginx_ingress_controller: - repository: quay.io/kubernetes-ingress-controller/nginx-ingress-controller - tag: "0.26.2" + repository: registry.k8s.io/ingress-nginx/controller + tag: "v1.9.4" pullPolicy: "IfNotPresent" pulsar_metadata: repository: streamnative/sn-platform-slim diff --git a/charts/sn-platform/templates/control-center/_control_center.tpl b/charts/sn-platform/templates/control-center/_control_center.tpl index fe930e3d0..8fcb1b450 100644 --- a/charts/sn-platform/templates/control-center/_control_center.tpl +++ b/charts/sn-platform/templates/control-center/_control_center.tpl @@ -111,13 +111,13 @@ Get ingress image according to the k8s version. When k8s version is higher or equal than v1.22, ingress image should use version v1.x.x, otherwise it should use the default version 0.26.2 that defines in values.yaml. -If k8s version is higher or equal than v1.22, but the .Values.images.nginx_ingress_controller.tag is less than v1.x.x, -it will use registry.k8s.io/ingress-nginx/controller:v1.1.1 as default to make ingress work. +If k8s version is higher or equal than v1.22, it will use the nginx_ingress_controller image in values. +otherwise, it will use quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26. as default to make ingress work. */}} {{- define "pulsar.ingress.image" -}} - {{- if and (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") (semverCompare "< 1.0.0" .Values.images.nginx_ingress_controller.tag )}} - {{- print "registry.k8s.io/ingress-nginx/controller:v1.1.1"}} - {{- else -}} + {{- if (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") }} {{- printf "%s:%s" .Values.images.nginx_ingress_controller.repository .Values.images.nginx_ingress_controller.tag -}} + {{- else -}} + {{- print "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.2"}} {{- end -}} {{- end -}} diff --git a/charts/sn-platform/templates/control-center/control-center-ingress.yaml b/charts/sn-platform/templates/control-center/control-center-ingress.yaml index 374b02e86..05b90af50 100644 --- a/charts/sn-platform/templates/control-center/control-center-ingress.yaml +++ b/charts/sn-platform/templates/control-center/control-center-ingress.yaml @@ -47,11 +47,16 @@ metadata: {{- else }} ingress.kubernetes.io/ssl-redirect: "false" {{- end }} + {{- if not $isIngressAPIStable }} kubernetes.io/ingress.class: nginx + {{- end }} {{- with .Values.ingress.control_center.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if $isIngressAPIStable }} + ingressClassName: nginx +{{- end }} {{- if and .Values.ingress.control_center.tls.enabled (not .Values.ingress.controller.tls.termination) }} {{- if .Values.domain.enabled }} tls: @@ -61,7 +66,6 @@ spec: # Use the same cert for proxy which is wildcard type secretName: "{{ template "pulsar.proxy.tls.secret.name" . }}" {{- end }} - {{- end }} {{- end }} rules: diff --git a/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml b/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml index 8d6368487..407aa0a78 100644 --- a/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml +++ b/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml @@ -19,6 +19,11 @@ {{- if .Values.ingress.controller.enabled }} {{- $isKubeVersionLessThanV122 := eq (include "pulsar.kubeVersion.isLessThanV122" .) "true" }} +{{- $isIngressAPIStable := eq (include "pulsar.ingress.isStable" .) "true" -}} + +{{/* COMMENT */}} + + apiVersion: apps/v1 kind: Deployment @@ -70,7 +75,11 @@ spec: - --tcp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-tcp-services - --udp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-udp-services - --publish-service={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.ingress.controller.component }} + {{- if $isIngressAPIStable }} + - --ingress-class=nginx + {{- else }} - --annotations-prefix=nginx.ingress.kubernetes.io + {{- end }} securityContext: allowPrivilegeEscalation: true capabilities: diff --git a/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml b/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml index 2cb1ffaac..e0c8facad 100644 --- a/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml +++ b/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml @@ -46,6 +46,13 @@ rules: verbs: - list - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch - apiGroups: - "" resources: @@ -60,6 +67,15 @@ rules: - get - list - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -71,18 +87,25 @@ rules: - "extensions" - "networking.k8s.io" resources: - - ingresses + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses verbs: - get - list - watch - apiGroups: - - "extensions" - - "networking.k8s.io" + - discovery.k8s.io resources: - - ingresses/status + - endpointslices verbs: - - update + - list + - watch + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -94,41 +117,86 @@ metadata: chart: {{ template "pulsar.chart" . }} release: {{ .Release.Name }} rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get - apiGroups: - "" resources: - configmaps - pods - secrets - - namespaces + - endpoints verbs: - get + - list + - watch + - update - apiGroups: - "" resources: - - configmaps + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" - # Upgrade to v1.1.1, controller will update this configmap - "ingress-controller-leader" verbs: - get - update - apiGroups: - - "" + - coordination.k8s.io resources: - - configmaps + - leases verbs: - create - apiGroups: - "" resources: - - endpoints + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices verbs: + - list + - watch - get --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 1381d34b8..776f37885 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -219,8 +219,8 @@ images: tag: "v1.5.0" pullPolicy: "IfNotPresent" nginx_ingress_controller: - repository: quay.io/kubernetes-ingress-controller/nginx-ingress-controller - tag: "0.26.2" + repository: registry.k8s.io/ingress-nginx/controller + tag: "v1.9.4" pullPolicy: "IfNotPresent" vault: repository: hashicorp/vault