From dc7d1f569ff4f741c89d8be6ce230dfdadaf679f Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Thu, 21 Dec 2023 18:40:54 +0800 Subject: [PATCH 1/5] fix: Ingress rbac and support ingressClass (#1133) * fix: ingress controller rbac issue Signed-off-by: ericsyh * update the ingress Signed-off-by: ericsyh * support ingressClass Signed-off-by: ericsyh * update the image logic Signed-off-by: ericsyh * apply the change to sn-platform chart Signed-off-by: ericsyh --------- Signed-off-by: ericsyh --- .../control-center/_control_center.tpl | 12 +-- .../control-center-ingress.yaml | 6 +- .../ingress-controller-deployment.yaml | 9 ++ .../ingress-controller-rbac.yaml | 98 ++++++++++++++++--- charts/sn-platform-slim/values.yaml | 4 +- .../control-center/_control_center.tpl | 10 +- .../control-center-ingress.yaml | 6 +- .../ingress-controller-deployment.yaml | 9 ++ .../ingress-controller-rbac.yaml | 98 ++++++++++++++++--- charts/sn-platform/values.yaml | 4 +- 10 files changed, 209 insertions(+), 47 deletions(-) diff --git a/charts/sn-platform-slim/templates/control-center/_control_center.tpl b/charts/sn-platform-slim/templates/control-center/_control_center.tpl index fe930e3d0..481a326b8 100644 --- a/charts/sn-platform-slim/templates/control-center/_control_center.tpl +++ b/charts/sn-platform-slim/templates/control-center/_control_center.tpl @@ -111,13 +111,13 @@ Get ingress image according to the k8s version. When k8s version is higher or equal than v1.22, ingress image should use version v1.x.x, otherwise it should use the default version 0.26.2 that defines in values.yaml. -If k8s version is higher or equal than v1.22, but the .Values.images.nginx_ingress_controller.tag is less than v1.x.x, -it will use registry.k8s.io/ingress-nginx/controller:v1.1.1 as default to make ingress work. +If k8s version is higher or equal than v1.22, it will use the nginx_ingress_controller image in values. +otherwise, it will use quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26. as default to make ingress work. */}} {{- define "pulsar.ingress.image" -}} - {{- if and (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") (semverCompare "< 1.0.0" .Values.images.nginx_ingress_controller.tag )}} - {{- print "registry.k8s.io/ingress-nginx/controller:v1.1.1"}} - {{- else -}} + {{- if (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") }} {{- printf "%s:%s" .Values.images.nginx_ingress_controller.repository .Values.images.nginx_ingress_controller.tag -}} + {{- else -}} + {{- print "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.2"}} {{- end -}} -{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml b/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml index 374b02e86..05b90af50 100644 --- a/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml +++ b/charts/sn-platform-slim/templates/control-center/control-center-ingress.yaml @@ -47,11 +47,16 @@ metadata: {{- else }} ingress.kubernetes.io/ssl-redirect: "false" {{- end }} + {{- if not $isIngressAPIStable }} kubernetes.io/ingress.class: nginx + {{- end }} {{- with .Values.ingress.control_center.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if $isIngressAPIStable }} + ingressClassName: nginx +{{- end }} {{- if and .Values.ingress.control_center.tls.enabled (not .Values.ingress.controller.tls.termination) }} {{- if .Values.domain.enabled }} tls: @@ -61,7 +66,6 @@ spec: # Use the same cert for proxy which is wildcard type secretName: "{{ template "pulsar.proxy.tls.secret.name" . }}" {{- end }} - {{- end }} {{- end }} rules: diff --git a/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml b/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml index 8d6368487..407aa0a78 100644 --- a/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml +++ b/charts/sn-platform-slim/templates/control-center/ingress-controller-deployment.yaml @@ -19,6 +19,11 @@ {{- if .Values.ingress.controller.enabled }} {{- $isKubeVersionLessThanV122 := eq (include "pulsar.kubeVersion.isLessThanV122" .) "true" }} +{{- $isIngressAPIStable := eq (include "pulsar.ingress.isStable" .) "true" -}} + +{{/* COMMENT */}} + + apiVersion: apps/v1 kind: Deployment @@ -70,7 +75,11 @@ spec: - --tcp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-tcp-services - --udp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-udp-services - --publish-service={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.ingress.controller.component }} + {{- if $isIngressAPIStable }} + - --ingress-class=nginx + {{- else }} - --annotations-prefix=nginx.ingress.kubernetes.io + {{- end }} securityContext: allowPrivilegeEscalation: true capabilities: diff --git a/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml b/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml index 2cb1ffaac..e0c8facad 100644 --- a/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml +++ b/charts/sn-platform-slim/templates/control-center/ingress-controller-rbac.yaml @@ -46,6 +46,13 @@ rules: verbs: - list - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch - apiGroups: - "" resources: @@ -60,6 +67,15 @@ rules: - get - list - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -71,18 +87,25 @@ rules: - "extensions" - "networking.k8s.io" resources: - - ingresses + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses verbs: - get - list - watch - apiGroups: - - "extensions" - - "networking.k8s.io" + - discovery.k8s.io resources: - - ingresses/status + - endpointslices verbs: - - update + - list + - watch + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -94,41 +117,86 @@ metadata: chart: {{ template "pulsar.chart" . }} release: {{ .Release.Name }} rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get - apiGroups: - "" resources: - configmaps - pods - secrets - - namespaces + - endpoints verbs: - get + - list + - watch + - update - apiGroups: - "" resources: - - configmaps + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" - # Upgrade to v1.1.1, controller will update this configmap - "ingress-controller-leader" verbs: - get - update - apiGroups: - - "" + - coordination.k8s.io resources: - - configmaps + - leases verbs: - create - apiGroups: - "" resources: - - endpoints + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices verbs: + - list + - watch - get --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index e86363838..3360f6dad 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -192,8 +192,8 @@ images: tag: "v1.5.0" pullPolicy: "IfNotPresent" nginx_ingress_controller: - repository: quay.io/kubernetes-ingress-controller/nginx-ingress-controller - tag: "0.26.2" + repository: registry.k8s.io/ingress-nginx/controller + tag: "v1.9.4" pullPolicy: "IfNotPresent" pulsar_metadata: repository: streamnative/sn-platform-slim diff --git a/charts/sn-platform/templates/control-center/_control_center.tpl b/charts/sn-platform/templates/control-center/_control_center.tpl index fe930e3d0..8fcb1b450 100644 --- a/charts/sn-platform/templates/control-center/_control_center.tpl +++ b/charts/sn-platform/templates/control-center/_control_center.tpl @@ -111,13 +111,13 @@ Get ingress image according to the k8s version. When k8s version is higher or equal than v1.22, ingress image should use version v1.x.x, otherwise it should use the default version 0.26.2 that defines in values.yaml. -If k8s version is higher or equal than v1.22, but the .Values.images.nginx_ingress_controller.tag is less than v1.x.x, -it will use registry.k8s.io/ingress-nginx/controller:v1.1.1 as default to make ingress work. +If k8s version is higher or equal than v1.22, it will use the nginx_ingress_controller image in values. +otherwise, it will use quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26. as default to make ingress work. */}} {{- define "pulsar.ingress.image" -}} - {{- if and (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") (semverCompare "< 1.0.0" .Values.images.nginx_ingress_controller.tag )}} - {{- print "registry.k8s.io/ingress-nginx/controller:v1.1.1"}} - {{- else -}} + {{- if (eq (include "pulsar.kubeVersion.isLessThanV122" .) "false") }} {{- printf "%s:%s" .Values.images.nginx_ingress_controller.repository .Values.images.nginx_ingress_controller.tag -}} + {{- else -}} + {{- print "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.2"}} {{- end -}} {{- end -}} diff --git a/charts/sn-platform/templates/control-center/control-center-ingress.yaml b/charts/sn-platform/templates/control-center/control-center-ingress.yaml index 374b02e86..05b90af50 100644 --- a/charts/sn-platform/templates/control-center/control-center-ingress.yaml +++ b/charts/sn-platform/templates/control-center/control-center-ingress.yaml @@ -47,11 +47,16 @@ metadata: {{- else }} ingress.kubernetes.io/ssl-redirect: "false" {{- end }} + {{- if not $isIngressAPIStable }} kubernetes.io/ingress.class: nginx + {{- end }} {{- with .Values.ingress.control_center.annotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if $isIngressAPIStable }} + ingressClassName: nginx +{{- end }} {{- if and .Values.ingress.control_center.tls.enabled (not .Values.ingress.controller.tls.termination) }} {{- if .Values.domain.enabled }} tls: @@ -61,7 +66,6 @@ spec: # Use the same cert for proxy which is wildcard type secretName: "{{ template "pulsar.proxy.tls.secret.name" . }}" {{- end }} - {{- end }} {{- end }} rules: diff --git a/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml b/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml index 8d6368487..407aa0a78 100644 --- a/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml +++ b/charts/sn-platform/templates/control-center/ingress-controller-deployment.yaml @@ -19,6 +19,11 @@ {{- if .Values.ingress.controller.enabled }} {{- $isKubeVersionLessThanV122 := eq (include "pulsar.kubeVersion.isLessThanV122" .) "true" }} +{{- $isIngressAPIStable := eq (include "pulsar.ingress.isStable" .) "true" -}} + +{{/* COMMENT */}} + + apiVersion: apps/v1 kind: Deployment @@ -70,7 +75,11 @@ spec: - --tcp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-tcp-services - --udp-services-configmap={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-udp-services - --publish-service={{ template "pulsar.namespace" . }}/{{ template "pulsar.fullname" . }}-{{ .Values.ingress.controller.component }} + {{- if $isIngressAPIStable }} + - --ingress-class=nginx + {{- else }} - --annotations-prefix=nginx.ingress.kubernetes.io + {{- end }} securityContext: allowPrivilegeEscalation: true capabilities: diff --git a/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml b/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml index 2cb1ffaac..e0c8facad 100644 --- a/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml +++ b/charts/sn-platform/templates/control-center/ingress-controller-rbac.yaml @@ -46,6 +46,13 @@ rules: verbs: - list - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch - apiGroups: - "" resources: @@ -60,6 +67,15 @@ rules: - get - list - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -71,18 +87,25 @@ rules: - "extensions" - "networking.k8s.io" resources: - - ingresses + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses verbs: - get - list - watch - apiGroups: - - "extensions" - - "networking.k8s.io" + - discovery.k8s.io resources: - - ingresses/status + - endpointslices verbs: - - update + - list + - watch + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -94,41 +117,86 @@ metadata: chart: {{ template "pulsar.chart" . }} release: {{ .Release.Name }} rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get - apiGroups: - "" resources: - configmaps - pods - secrets - - namespaces + - endpoints verbs: - get + - list + - watch + - update - apiGroups: - "" resources: - - configmaps + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" - # Upgrade to v1.1.1, controller will update this configmap - "ingress-controller-leader" verbs: - get - update - apiGroups: - - "" + - coordination.k8s.io resources: - - configmaps + - leases verbs: - create - apiGroups: - "" resources: - - endpoints + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices verbs: + - list + - watch - get --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 1381d34b8..776f37885 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -219,8 +219,8 @@ images: tag: "v1.5.0" pullPolicy: "IfNotPresent" nginx_ingress_controller: - repository: quay.io/kubernetes-ingress-controller/nginx-ingress-controller - tag: "0.26.2" + repository: registry.k8s.io/ingress-nginx/controller + tag: "v1.9.4" pullPolicy: "IfNotPresent" vault: repository: hashicorp/vault From a3c2540b16832e8cfe4c2874c0ad8699a3b911aa Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Thu, 21 Dec 2023 18:41:10 +0800 Subject: [PATCH 2/5] fix: scc template in slim chart (#1137) Signed-off-by: ericsyh --- .../templates/openshift/scc-rolebinding.yaml | 3 +-- charts/sn-platform-slim/templates/openshift/scc.yaml | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/charts/sn-platform-slim/templates/openshift/scc-rolebinding.yaml b/charts/sn-platform-slim/templates/openshift/scc-rolebinding.yaml index 57f3510c8..0042619d2 100644 --- a/charts/sn-platform-slim/templates/openshift/scc-rolebinding.yaml +++ b/charts/sn-platform-slim/templates/openshift/scc-rolebinding.yaml @@ -18,8 +18,7 @@ # {{- if and .Values.openshift.enabled .Values.openshift.scc.enabled -}} -{{- $sas := list (include "pulsar.vault.serviceAccount" .) -}} -{{- $sas = append $sas (include "pulsar.zookeeper.serviceAccount" .) -}} +{{- $sas := list (include "pulsar.zookeeper.serviceAccount" .) -}} {{- $sas = append $sas (include "pulsar.bookkeeper.serviceAccount" .) -}} {{- $sas = append $sas (include "pulsar.broker.serviceAccount" .) -}} {{- $sas = append $sas (include "pulsar.proxy.serviceAccount" .) -}} diff --git a/charts/sn-platform-slim/templates/openshift/scc.yaml b/charts/sn-platform-slim/templates/openshift/scc.yaml index 41dd4eb4a..ff77cb11f 100644 --- a/charts/sn-platform-slim/templates/openshift/scc.yaml +++ b/charts/sn-platform-slim/templates/openshift/scc.yaml @@ -29,11 +29,6 @@ allowHostPID: false allowHostPorts: false allowPrivilegeEscalation: true allowPrivilegedContainer: true -allowedCapabilities: - {{- if .Values.components.vault }} - - IPC_LOCK - - SETFCAP - {{- end }} defaultAddCapabilities: null fsGroup: type: RunAsAny From 80560ea8f343f36869476c2268fbe6d0f8bbd6f5 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Thu, 21 Dec 2023 18:41:24 +0800 Subject: [PATCH 3/5] chore: upgrade pulsar operator to v0.17.10 (#1135) * chore: upgrade pulsar operator to 0.17.9 Signed-off-by: ericsyh * upgrade Signed-off-by: ericsyh * update operator chart template Signed-off-by: ericsyh * change appVersion to 0.17.10 Signed-off-by: ericsyh * update Signed-off-by: ericsyh --------- Signed-off-by: ericsyh --- charts/pulsar-operator/Chart.yaml | 4 ++-- charts/pulsar-operator/templates/pulsar-operator/_helpers.tpl | 4 ++-- .../pulsar-operator/templates/zookeeper-operator/_helpers.tpl | 2 +- charts/pulsar-operator/values.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/pulsar-operator/Chart.yaml b/charts/pulsar-operator/Chart.yaml index 1c85f4769..19ca0e9db 100644 --- a/charts/pulsar-operator/Chart.yaml +++ b/charts/pulsar-operator/Chart.yaml @@ -18,8 +18,8 @@ # apiVersion: v1 -version: 0.17.0 -appVersion: "0.17.8" +version: 0.17.9 +appVersion: "0.17.10" kubeVersion: ">= 1.16.0-0 < 1.29.0-0" description: Apache Pulsar Operators Helm chart for Kubernetes name: pulsar-operator diff --git a/charts/pulsar-operator/templates/pulsar-operator/_helpers.tpl b/charts/pulsar-operator/templates/pulsar-operator/_helpers.tpl index 32d151173..eeb8fed22 100644 --- a/charts/pulsar-operator/templates/pulsar-operator/_helpers.tpl +++ b/charts/pulsar-operator/templates/pulsar-operator/_helpers.tpl @@ -3,7 +3,7 @@ install broker crd yaml file to tpl. */}} {{- define "broker.crd" -}} {{- $files := .Files }} -{{ $files.Get "crds/pulsar.streamnative.io_pulsarbrokers" }} +{{ $files.Get "crds/pulsar.streamnative.io_pulsarbrokers.yaml" }} {{- end -}} {{/* @@ -11,7 +11,7 @@ install proxy crd yaml file to tpl. */}} {{- define "proxy.crd" -}} {{- $files := .Files }} -{{ $files.Get "crds/pulsar.streamnative.io_pulsarproxies" }} +{{ $files.Get "crds/pulsar.streamnative.io_pulsarproxies.yaml" }} {{- end -}} {{/*Define the image for pulsar*/}} diff --git a/charts/pulsar-operator/templates/zookeeper-operator/_helpers.tpl b/charts/pulsar-operator/templates/zookeeper-operator/_helpers.tpl index 578cc18ee..0c0a639a3 100644 --- a/charts/pulsar-operator/templates/zookeeper-operator/_helpers.tpl +++ b/charts/pulsar-operator/templates/zookeeper-operator/_helpers.tpl @@ -3,7 +3,7 @@ install crd yaml file to tpl */}} {{- define "zookeeper.crd" -}} {{- $files := .Files }} -{{ $files.Get "crds/zookeeper.streamnative.io_zookeeperclusters" }} +{{ $files.Get "crds/zookeeper.streamnative.io_zookeeperclusters.yaml" }} {{- end -}} {{/*Define the image for zookeeper*/}} diff --git a/charts/pulsar-operator/values.yaml b/charts/pulsar-operator/values.yaml index 724253610..289545c79 100644 --- a/charts/pulsar-operator/values.yaml +++ b/charts/pulsar-operator/values.yaml @@ -45,7 +45,7 @@ components: ## Control what images to use for each component images: registry: "docker.streamnative.io" - tag: "v0.17.8" + tag: "v0.17.10" zookeeper: registry: "" From 6cd50a011a1bbaf2fec97e7ce8a5e1d741b10da4 Mon Sep 17 00:00:00 2001 From: Yisheng Cai Date: Thu, 21 Dec 2023 19:51:25 +0800 Subject: [PATCH 4/5] [sn-platform(-slim)] Support readOnlyRootFilesystem (#1136) * Expose variable KUBECTL_BIN and OUTPUT * Define KUBECTL_BIN and OUTPUT to writable path * Toolset support readOnlyRootFilesystem * Detector support readOnlyRootFilesystem * Fix zk cluster indent * Add rootless example --- .../conf/toolset/pulsar/clean_tls.sh | 6 +- .../toolset/pulsar/cleanup_helm_release.sh | 6 +- .../conf/toolset/pulsar/common_auth.sh | 1 - .../toolset/pulsar/decommission_bookies.sh | 8 +- .../conf/toolset/pulsar/generate_token.sh | 8 +- .../pulsar/generate_token_secret_key.sh | 19 ++- .../conf/toolset/pulsar/get_token.sh | 4 +- .../toolset/pulsar/gke_bootstrap_script.sh | 2 +- .../toolset/pulsar/prepare_helm_release.sh | 5 +- ...setup-clouddns-resolver-service-account.sh | 2 +- .../toolset/pulsar/upload-lets-encrypt-ca.sh | 2 +- .../conf/toolset/pulsar/upload_tls.sh | 6 +- .../detector/pulsar-detector-deployment.yaml | 10 +- .../toolset/jwt-secret-init-job.yaml | 5 +- .../toolset/toolset-statefulset.yaml | 33 ++++- charts/sn-platform-slim/values.yaml | 1 + .../conf/toolset/pulsar/clean_tls.sh | 6 +- .../toolset/pulsar/cleanup_helm_release.sh | 6 +- .../conf/toolset/pulsar/common_auth.sh | 1 - .../toolset/pulsar/decommission_bookies.sh | 8 +- .../conf/toolset/pulsar/generate_token.sh | 8 +- .../pulsar/generate_token_secret_key.sh | 16 +-- .../conf/toolset/pulsar/get_token.sh | 4 +- .../toolset/pulsar/gke_bootstrap_script.sh | 2 +- .../toolset/pulsar/prepare_helm_release.sh | 5 +- ...setup-clouddns-resolver-service-account.sh | 2 +- .../toolset/pulsar/upload-lets-encrypt-ca.sh | 2 +- .../conf/toolset/pulsar/upload_tls.sh | 6 +- .../detector/pulsar-detector-deployment.yaml | 10 +- .../toolset/jwt-secret-init-job.yaml | 5 +- .../toolset/toolset-statefulset.yaml | 33 ++++- .../zookeeper/zookeeper-cluster.yaml | 2 +- charts/sn-platform/values.yaml | 1 + examples/sn-platform/values-rootless.yaml | 128 ++++++++++++++++++ 34 files changed, 284 insertions(+), 79 deletions(-) create mode 100644 examples/sn-platform/values-rootless.yaml diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh b/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh index 884cc383e..743df35ca 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh @@ -81,14 +81,14 @@ done function delete_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl delete secret ${tls_ca_secret} -n ${namespace} + ${KUBECTL_BIN} delete secret ${tls_ca_secret} -n ${namespace} } function delete_server_cert() { local component=$1 local server_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${server_cert_secret} \ + ${KUBECTL_BIN} delete secret ${server_cert_secret} \ -n ${namespace} } @@ -96,7 +96,7 @@ function delete_client_cert() { local component=$1 local client_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${client_cert_secret} \ + ${KUBECTL_BIN} delete secret ${client_cert_secret} \ -n ${namespace} } diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh b/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh index 76f7573ca..9b2809b49 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh @@ -73,15 +73,15 @@ release=${release:-pulsar-dev} function delete_namespace() { if [[ "${delete_namespace}" == "true" ]]; then - /pulsar/kubectl delete namespace ${namespace} + ${KUBECTL_BIN} delete namespace ${namespace} fi } # delete the cc admin secrets -/pulsar/kubectl delete -n ${namespace} secret ${release}-admin-secret +${KUBECTL_BIN} delete -n ${namespace} secret ${release}-admin-secret # delete tokens -/pulsar/kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs /pulsar/kubectl delete secrets -n ${namespace} +${KUBECTL_BIN} get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs ${KUBECTL_BIN} delete secrets -n ${namespace} # delete namespace delete_namespace diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh b/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh index ede8c86eb..c0844647c 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh @@ -23,7 +23,6 @@ if [ -z "$CHART_HOME" ]; then exit 1 fi -OUTPUT=${CHART_HOME}/output OUTPUT_BIN=${OUTPUT}/bin PULSARCTL_VERSION=v2.10.2.2 PULSARCTL_BIN=/pulsar/bin/pulsarctl diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh b/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh index 55e2823f7..7c240c264 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh @@ -89,8 +89,8 @@ autorecovery_pod=${autorecovery_pod:-autorecovery} for ((i=replicas; i>=1; i--)) do j=$((i-1)) - echo /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - echo /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 - /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + echo ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + echo ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 done diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh index 7dd1b656e..d38fd0c2c 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh @@ -96,11 +96,11 @@ function pulsar::jwt::generate_symmetric_token() { trap "test -f $tmpfile && rm $tmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} ${PULSARCTL_BIN} token create -a HS256 --secret-key-file ${tmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" } function pulsar::jwt::generate_asymmetric_token() { @@ -111,11 +111,11 @@ function pulsar::jwt::generate_asymmetric_token() { trap "test -f $privatekeytmpfile && rm $privatekeytmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} ${PULSARCTL_BIN} token create -a RS256 --private-key-file ${privatekeytmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh index f5c9f28a3..411261c3e 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh @@ -18,8 +18,7 @@ # under the License. # -set -e - +set -x; CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd ${CHART_HOME} @@ -82,9 +81,9 @@ function pulsar::jwt::generate_symmetric_key() { tmpfile=$(mktemp) trap "test -f $tmpfile && rm $tmpfile" RETURN ${PULSARCTL_BIN} token create-secret-key --output-file ${tmpfile} - mv $tmpfile SECRETKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=SECRETKEY - rm SECRETKEY + mv $tmpfile ${OUTPUT}/SECRETKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=${OUTPUT}/SECRETKEY + rm ${OUTPUT}/SECRETKEY } function pulsar::jwt::generate_asymmetric_key() { @@ -95,11 +94,11 @@ function pulsar::jwt::generate_asymmetric_key() { publickeytmpfile=$(mktemp) trap "test -f $publickeytmpfile && rm $publickeytmpfile" RETURN ${PULSARCTL_BIN} token create-key-pair -a RS256 --output-private-key ${privatekeytmpfile} --output-public-key ${publickeytmpfile} - mv $privatekeytmpfile PRIVATEKEY - mv $publickeytmpfile PUBLICKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=PRIVATEKEY --from-file=PUBLICKEY - rm PRIVATEKEY - rm PUBLICKEY + mv $privatekeytmpfile $OUTPUT/PRIVATEKEY + mv $publickeytmpfile $OUTPUT/PUBLICKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/PRIVATEKEY --from-file=$OUTPUT/PUBLICKEY + rm $OUTPUT/PRIVATEKEY + rm $OUTPUT/PUBLICKEY } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh b/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh index 38c9272f7..25aeb0525 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh @@ -84,8 +84,8 @@ release=${release:-pulsar-dev} function pulsar::jwt::get_token() { local token_name="${release}-token-${role}" - local token=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) - local token_type=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) + local token=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) + local token_type=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) echo "token type: ${token_type}" echo "-------------------------" diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh b/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh index e6f20599a..86eea6538 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh @@ -58,7 +58,7 @@ function bootstrap(){ echo "Wait for metrics API service" # Helm 2.15 and 3.0 bug https://github.com/helm/helm/issues/6361#issuecomment-550503455 - /pulsar/kubectl --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io + ${KUBECTL_BIN} --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io helm repo update } diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh b/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh index 68e55a300..b61484a3c 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh @@ -18,6 +18,7 @@ # under the License. # +set -x; CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd ${CHART_HOME} @@ -94,7 +95,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function generate_gcs_offloader_service_account_keyfile() { local secret_name="${release}-gcs-offloader-service-account" - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} \ + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} \ --from-file="gcs.json=${gcs_offloader_service_account_keyfile}" } @@ -102,7 +103,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function do_create_namespace() { if [[ "${create_namespace}" == "true" ]]; then - /pulsar/kubectl create namespace ${namespace} + ${KUBECTL_BIN} create namespace ${namespace} fi } diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh b/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh index 1c4afed80..3c8a30b22 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh @@ -46,7 +46,7 @@ gcloud iam service-accounts keys create ${RESOLVER_NAME}-key.json \ --iam-account ${RESOLVER_NAME}@$PROJECT_ID.iam.gserviceaccount.com echo "Save the service account key as a kubernete secret '${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct' in namespace '${NAMESPACE}'." -/pulsar/kubectl create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ +${KUBECTL_BIN} create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ --from-file=${RESOLVER_NAME}-key.json -n ${NAMESPACE} echo "Remove the generated key." diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh b/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh index d84b9cefb..9a0be2909 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh @@ -29,5 +29,5 @@ PEM="${CA_NAME}.pem" NAMESPACE=$1 -/pulsar/kubectl create secret generic ${CA_NAME} \ +${KUBECTL_BIN} create secret generic ${CA_NAME} \ --from-file=${PEM} -n ${NAMESPACE} diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh b/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh index 94b3f764d..668001d6d 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh @@ -91,7 +91,7 @@ ca_cert_file=${tlsdir}/certs/ca.cert.pem function upload_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" + ${KUBECTL_BIN} create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" } function upload_server_cert() { @@ -100,7 +100,7 @@ function upload_server_cert() { local tls_cert_file="${tlsdir}/servers/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/servers/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${server_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${server_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ @@ -113,7 +113,7 @@ function upload_client_cert() { local tls_cert_file="${tlsdir}/clients/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/clients/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${client_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${client_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ diff --git a/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml b/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml index f3beb074f..a53c686d8 100644 --- a/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml +++ b/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml @@ -85,6 +85,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs # This init container will wait for at least one broker to be ready before # deploying the pulsar-detector - name: wait-broker-ready @@ -103,6 +106,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs {{- end }} containers: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_detector.component }}" @@ -145,8 +151,10 @@ spec: volumeMounts: {{- toYaml .Values.pulsar_detector.extraVolumeMounts | nindent 10 }} {{- end }} - {{- if .Values.pulsar_detector.extraVolumes }} volumes: + - name: tmp + emptyDir: {} + {{- if .Values.pulsar_detector.extraVolumes }} {{- toYaml .Values.pulsar_detector.extraVolumes | nindent 8 }} {{- end }} {{- end }} diff --git a/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml b/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml index 4a66b68ec..1b681ba5f 100644 --- a/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml +++ b/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml @@ -79,13 +79,14 @@ spec: args: - | set -ex; - cp /tmp/binaries/kubectl /pulsar/kubectl; - chmod +x /pulsar/kubectl; mkdir -p scripts/pulsar; cp scripts/jwt-secret-config/* scripts/pulsar; chmod +x scripts/pulsar/*; usingSecretKey={{ .Values.auth.authentication.jwt.usingSecretKey }}; ls -lh scripts/pulsar/; + export KUBECTL_BIN=/tmp/binaries/kubectl; + export OUTPUT=scripts/pulsar/output; + mkdir ${OUTPUT}; if [ "${usingSecretKey}" = "true" ]; then ./scripts/pulsar/prepare_helm_release.sh -n {{ template "pulsar.namespace" . }} -k {{ .Release.Name }} --symmetric; else diff --git a/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml b/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml index 24d4b5b98..542e5d8b6 100644 --- a/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml +++ b/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml @@ -72,8 +72,27 @@ spec: {{ toYaml .Values.toolset.tolerations | indent 8 }} {{- end }} terminationGracePeriodSeconds: {{ .Values.toolset.gracePeriod }} - {{- if .Values.toolset.installBusybox }} initContainers: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: "init-copy-config" + image: "{{ .Values.images.toolset.repository }}:{{ .Values.images.toolset.tag }}" + imagePullPolicy: {{ .Values.images.toolset.pullPolicy }} + command: + - sh + - -c + - | + set -ex; + cp -r /pulsar/conf/* /conf_tmp/; + echo OK > /conf_tmp/status; + {{- if .Values.toolset.resources }} + resources: +{{ toYaml .Values.toolset.resources | indent 10 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /conf_tmp/ + {{- end }} + {{- if .Values.toolset.installBusybox }} - name: busybox image: "{{ .Values.images.toolset.busybox.repository }}:{{ .Values.images.toolset.busybox.tag }}" imagePullPolicy: {{ .Values.images.toolset.busybox.pullPolicy }} @@ -98,7 +117,7 @@ spec: volumeMounts: - name: binaries mountPath: /home/tmp/binaries - {{- end }} + {{- end }} containers: - name: "pulsar" {{- include "pulsar.toolset.image" . | nindent 8 }} @@ -122,6 +141,12 @@ spec: {{ toYaml . | indent 8 }} {{- end }} volumeMounts: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + mountPath: /pulsar/conf + - name: tmp + mountPath: /pulsar/logs + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries mountPath: /bin/busybox @@ -138,6 +163,10 @@ spec: {{ toYaml . | indent 8 }} {{- end }} volumes: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + emptyDir: {} + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries emptyDir: {} diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index 3360f6dad..c20604952 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -1550,6 +1550,7 @@ toolset: component: toolset useProxy: false installBusybox: true + readOnlyRootFilesystem: false replicaCount: 1 # nodeSelector: # cloud.google.com/gke-nodepool: default-pool diff --git a/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh b/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh index 884cc383e..743df35ca 100755 --- a/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh +++ b/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh @@ -81,14 +81,14 @@ done function delete_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl delete secret ${tls_ca_secret} -n ${namespace} + ${KUBECTL_BIN} delete secret ${tls_ca_secret} -n ${namespace} } function delete_server_cert() { local component=$1 local server_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${server_cert_secret} \ + ${KUBECTL_BIN} delete secret ${server_cert_secret} \ -n ${namespace} } @@ -96,7 +96,7 @@ function delete_client_cert() { local component=$1 local client_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${client_cert_secret} \ + ${KUBECTL_BIN} delete secret ${client_cert_secret} \ -n ${namespace} } diff --git a/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh b/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh index 76f7573ca..9b2809b49 100755 --- a/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh +++ b/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh @@ -73,15 +73,15 @@ release=${release:-pulsar-dev} function delete_namespace() { if [[ "${delete_namespace}" == "true" ]]; then - /pulsar/kubectl delete namespace ${namespace} + ${KUBECTL_BIN} delete namespace ${namespace} fi } # delete the cc admin secrets -/pulsar/kubectl delete -n ${namespace} secret ${release}-admin-secret +${KUBECTL_BIN} delete -n ${namespace} secret ${release}-admin-secret # delete tokens -/pulsar/kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs /pulsar/kubectl delete secrets -n ${namespace} +${KUBECTL_BIN} get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs ${KUBECTL_BIN} delete secrets -n ${namespace} # delete namespace delete_namespace diff --git a/charts/sn-platform/conf/toolset/pulsar/common_auth.sh b/charts/sn-platform/conf/toolset/pulsar/common_auth.sh index ede8c86eb..c0844647c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/common_auth.sh +++ b/charts/sn-platform/conf/toolset/pulsar/common_auth.sh @@ -23,7 +23,6 @@ if [ -z "$CHART_HOME" ]; then exit 1 fi -OUTPUT=${CHART_HOME}/output OUTPUT_BIN=${OUTPUT}/bin PULSARCTL_VERSION=v2.10.2.2 PULSARCTL_BIN=/pulsar/bin/pulsarctl diff --git a/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh b/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh index 55e2823f7..7c240c264 100755 --- a/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh +++ b/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh @@ -89,8 +89,8 @@ autorecovery_pod=${autorecovery_pod:-autorecovery} for ((i=replicas; i>=1; i--)) do j=$((i-1)) - echo /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - echo /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 - /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + echo ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + echo ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 done diff --git a/charts/sn-platform/conf/toolset/pulsar/generate_token.sh b/charts/sn-platform/conf/toolset/pulsar/generate_token.sh index 7dd1b656e..d38fd0c2c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/generate_token.sh +++ b/charts/sn-platform/conf/toolset/pulsar/generate_token.sh @@ -96,11 +96,11 @@ function pulsar::jwt::generate_symmetric_token() { trap "test -f $tmpfile && rm $tmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} ${PULSARCTL_BIN} token create -a HS256 --secret-key-file ${tmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" } function pulsar::jwt::generate_asymmetric_token() { @@ -111,11 +111,11 @@ function pulsar::jwt::generate_asymmetric_token() { trap "test -f $privatekeytmpfile && rm $privatekeytmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} ${PULSARCTL_BIN} token create -a RS256 --private-key-file ${privatekeytmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh b/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh index f5c9f28a3..a3acbee8c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh +++ b/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh @@ -82,9 +82,9 @@ function pulsar::jwt::generate_symmetric_key() { tmpfile=$(mktemp) trap "test -f $tmpfile && rm $tmpfile" RETURN ${PULSARCTL_BIN} token create-secret-key --output-file ${tmpfile} - mv $tmpfile SECRETKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=SECRETKEY - rm SECRETKEY + mv $tmpfile $OUTPUT/SECRETKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/SECRETKEY + rm $OUTPUT/SECRETKEY } function pulsar::jwt::generate_asymmetric_key() { @@ -95,11 +95,11 @@ function pulsar::jwt::generate_asymmetric_key() { publickeytmpfile=$(mktemp) trap "test -f $publickeytmpfile && rm $publickeytmpfile" RETURN ${PULSARCTL_BIN} token create-key-pair -a RS256 --output-private-key ${privatekeytmpfile} --output-public-key ${publickeytmpfile} - mv $privatekeytmpfile PRIVATEKEY - mv $publickeytmpfile PUBLICKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=PRIVATEKEY --from-file=PUBLICKEY - rm PRIVATEKEY - rm PUBLICKEY + mv $privatekeytmpfile $OUTPUT/PRIVATEKEY + mv $publickeytmpfile $OUTPUT/PUBLICKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/PRIVATEKEY --from-file=$OUTPUT/PUBLICKEY + rm $OUTPUT/PRIVATEKEY + rm $OUTPUT/PUBLICKEY } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform/conf/toolset/pulsar/get_token.sh b/charts/sn-platform/conf/toolset/pulsar/get_token.sh index 38c9272f7..25aeb0525 100755 --- a/charts/sn-platform/conf/toolset/pulsar/get_token.sh +++ b/charts/sn-platform/conf/toolset/pulsar/get_token.sh @@ -84,8 +84,8 @@ release=${release:-pulsar-dev} function pulsar::jwt::get_token() { local token_name="${release}-token-${role}" - local token=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) - local token_type=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) + local token=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) + local token_type=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) echo "token type: ${token_type}" echo "-------------------------" diff --git a/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh b/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh index e6f20599a..86eea6538 100755 --- a/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh +++ b/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh @@ -58,7 +58,7 @@ function bootstrap(){ echo "Wait for metrics API service" # Helm 2.15 and 3.0 bug https://github.com/helm/helm/issues/6361#issuecomment-550503455 - /pulsar/kubectl --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io + ${KUBECTL_BIN} --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io helm repo update } diff --git a/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh b/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh index 68e55a300..b61484a3c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh +++ b/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh @@ -18,6 +18,7 @@ # under the License. # +set -x; CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd ${CHART_HOME} @@ -94,7 +95,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function generate_gcs_offloader_service_account_keyfile() { local secret_name="${release}-gcs-offloader-service-account" - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} \ + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} \ --from-file="gcs.json=${gcs_offloader_service_account_keyfile}" } @@ -102,7 +103,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function do_create_namespace() { if [[ "${create_namespace}" == "true" ]]; then - /pulsar/kubectl create namespace ${namespace} + ${KUBECTL_BIN} create namespace ${namespace} fi } diff --git a/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh b/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh index 1c4afed80..3c8a30b22 100755 --- a/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh +++ b/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh @@ -46,7 +46,7 @@ gcloud iam service-accounts keys create ${RESOLVER_NAME}-key.json \ --iam-account ${RESOLVER_NAME}@$PROJECT_ID.iam.gserviceaccount.com echo "Save the service account key as a kubernete secret '${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct' in namespace '${NAMESPACE}'." -/pulsar/kubectl create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ +${KUBECTL_BIN} create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ --from-file=${RESOLVER_NAME}-key.json -n ${NAMESPACE} echo "Remove the generated key." diff --git a/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh b/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh index d84b9cefb..9a0be2909 100755 --- a/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh +++ b/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh @@ -29,5 +29,5 @@ PEM="${CA_NAME}.pem" NAMESPACE=$1 -/pulsar/kubectl create secret generic ${CA_NAME} \ +${KUBECTL_BIN} create secret generic ${CA_NAME} \ --from-file=${PEM} -n ${NAMESPACE} diff --git a/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh b/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh index 94b3f764d..668001d6d 100755 --- a/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh +++ b/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh @@ -91,7 +91,7 @@ ca_cert_file=${tlsdir}/certs/ca.cert.pem function upload_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" + ${KUBECTL_BIN} create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" } function upload_server_cert() { @@ -100,7 +100,7 @@ function upload_server_cert() { local tls_cert_file="${tlsdir}/servers/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/servers/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${server_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${server_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ @@ -113,7 +113,7 @@ function upload_client_cert() { local tls_cert_file="${tlsdir}/clients/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/clients/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${client_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${client_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ diff --git a/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml b/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml index 1d9381a0c..2fd961e1f 100644 --- a/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml +++ b/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml @@ -85,6 +85,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs # This init container will wait for at least one broker to be ready before # deploying the pulsar-detector - name: wait-broker-ready @@ -103,6 +106,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs {{- end }} containers: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_detector.component }}" @@ -152,8 +158,10 @@ spec: volumeMounts: {{- toYaml .Values.pulsar_detector.extraVolumeMounts | nindent 10 }} {{- end }} - {{- if .Values.pulsar_detector.extraVolumes }} volumes: + - name: tmp + emptyDir: {} + {{- if .Values.pulsar_detector.extraVolumes }} {{- toYaml .Values.pulsar_detector.extraVolumes | nindent 8 }} {{- end }} {{- end }} diff --git a/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml b/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml index a99307561..312067d97 100644 --- a/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml +++ b/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml @@ -79,13 +79,14 @@ spec: args: - | set -ex; - cp /tmp/binaries/kubectl /pulsar/kubectl; - chmod +x /pulsar/kubectl; mkdir -p scripts/pulsar; cp scripts/jwt-secret-config/* scripts/pulsar; chmod +x scripts/pulsar/*; usingSecretKey={{ .Values.auth.authentication.jwt.usingSecretKey }}; ls -lh scripts/pulsar/; + export KUBECTL_BIN=/tmp/binaries/kubectl; + export OUTPUT=scripts/pulsar/output; + mkdir ${OUTPUT}; if [ "${usingSecretKey}" = "true" ]; then ./scripts/pulsar/prepare_helm_release.sh -n {{ template "pulsar.namespace" . }} -k {{ .Release.Name }} --symmetric; else diff --git a/charts/sn-platform/templates/toolset/toolset-statefulset.yaml b/charts/sn-platform/templates/toolset/toolset-statefulset.yaml index affd90b4c..b4f9e8b24 100644 --- a/charts/sn-platform/templates/toolset/toolset-statefulset.yaml +++ b/charts/sn-platform/templates/toolset/toolset-statefulset.yaml @@ -72,8 +72,27 @@ spec: {{ toYaml .Values.toolset.tolerations | indent 8 }} {{- end }} terminationGracePeriodSeconds: {{ .Values.toolset.gracePeriod }} - {{- if .Values.toolset.installBusybox }} initContainers: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: "init-copy-config" + image: "{{ .Values.images.toolset.repository }}:{{ .Values.images.toolset.tag }}" + imagePullPolicy: {{ .Values.images.toolset.pullPolicy }} + command: + - sh + - -c + - | + set -ex; + cp -r /pulsar/conf/* /conf_tmp/; + echo OK > /conf_tmp/status; + {{- if .Values.toolset.resources }} + resources: +{{ toYaml .Values.toolset.resources | indent 10 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /conf_tmp/ + {{- end }} + {{- if .Values.toolset.installBusybox }} - name: busybox image: "{{ .Values.images.toolset.busybox.repository }}:{{ .Values.images.toolset.busybox.tag }}" imagePullPolicy: {{ .Values.images.toolset.busybox.pullPolicy }} @@ -98,7 +117,7 @@ spec: volumeMounts: - name: binaries mountPath: /home/tmp/binaries - {{- end }} + {{- end }} containers: - name: "pulsar" {{- include "pulsar.toolset.image" . | nindent 8 }} @@ -122,6 +141,12 @@ spec: {{ toYaml . | indent 8 }} {{- end }} volumeMounts: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + mountPath: /pulsar/conf + - name: tmp + mountPath: /pulsar/logs + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries mountPath: /bin/busybox @@ -168,6 +193,10 @@ spec: {{- end }} {{- end }} volumes: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + emptyDir: {} + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries emptyDir: {} diff --git a/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml b/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml index 5fbac3140..dfb7c7f8a 100644 --- a/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml +++ b/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml @@ -166,7 +166,7 @@ spec: {{- range .Values.zookeeper.volumes.data.accessModes }} - {{ . | quote }} {{- end }} - resources: + resources: requests: storage: {{ .Values.zookeeper.volumes.data.size }} {{- include "pulsar.zookeeper.data.storage.class" . | nindent 6 }} diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 776f37885..e4a3fabe0 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -1627,6 +1627,7 @@ toolset: component: toolset useProxy: false installBusybox: true + readOnlyRootFilesystem: false replicaCount: 1 # nodeSelector: # cloud.google.com/gke-nodepool: default-pool diff --git a/examples/sn-platform/values-rootless.yaml b/examples/sn-platform/values-rootless.yaml new file mode 100644 index 000000000..fcb56f432 --- /dev/null +++ b/examples/sn-platform/values-rootless.yaml @@ -0,0 +1,128 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# vault is not supported in rootless env +components: + pulsar_detector: true + vault: false + +# monitoring components haven't been tested in rootless env +monitoring: + prometheus: false + grafana: false + node_exporter: false + alert_manager: false + loki: false + datadog: false + +auth: + authentication: + enabled: true + provider: "jwt" + jwt: + enabled: true + usingSecretKey: false + autoInitSecret: true + vault: + enabled: false + +zookeeper: + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 +bookkeeper: + bookKeeperCluster: + annotations: + cloud.streamnative.io/omit-init-sysctl: "true" + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + autorecovery: + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +broker: + pulsarBroker: + annotations: + cloud.streamnative.io/omit-init-sysctl: "true" + replicaCount: 2 + kop: + enabled: false + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +proxy: + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +streamnative_console: + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + configData: + INIT_DEFAULT_ENVIRONMENT: true + BACKEND_DEFAULT_SUPER_USER_ROLE: "admin,pulsar-manager-admin" + login: + sso: + pulsarJwt: + enabled: true + config: + AUTHENTICATION_CUSTOM_CLAIM: sub + # The secret should contain both `TOKEN` + SERVICE_ACCOUNT_SUPER_TOKEN_SECRET: "sn-platform-token-admin" + # AUTH_METHOD_CUSTOM_CLAIM: sub + # USERNAME_CLAIM: sub + JWT_BROKER_TOKEN_MODE: PRIVATE + # The secret should contain both `PUBLICKEY`, `PRIVATEKEY` + JWT_BROKER_PUBLIC_PRIVATE_KEY: "sn-platform-token-asymmetric-key" + # The secret should contain both `SECRETKEY` + JWT_BROKER_SECRET_KEY: "" +toolset: + readOnlyRootFilesystem: true + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +pulsar_detector: + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 From a49f514e0d85d4e6ec056b0823155708d984e672 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Fri, 22 Dec 2023 22:38:29 +0800 Subject: [PATCH 5/5] fix: update default zk size to avoid oom (#1141) * fix: update the default zk pod size to avoid oom Signed-off-by: ericsyh * bump pulsar image version Signed-off-by: ericsyh --------- Signed-off-by: ericsyh --- charts/sn-platform-slim/values.yaml | 26 ++++++++++++------------- charts/sn-platform/values.yaml | 30 ++++++++++++++--------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index c20604952..d1174f7b8 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -120,42 +120,42 @@ monitoring: images: zookeeper: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent customTools: backup: repository: "streamnative/pulsar-metadata-tool" - tag: "3.1.0.5" + tag: "3.1.1.1" restore: repository: "streamnative/pulsar-metadata-tool" - tag: "3.1.0.5" + tag: "3.1.1.1" bookie: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent autorecovery: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent broker: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent proxy: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent pulsar_detector: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent functions: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent function_worker: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent # NOTE: allow overriding the toolset image toolset: @@ -197,7 +197,7 @@ images: pullPolicy: "IfNotPresent" pulsar_metadata: repository: streamnative/sn-platform-slim - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent configmapReload: repository: jimmidyson/configmap-reload @@ -618,8 +618,8 @@ zookeeper: # See https://kubernetes.io/docs/concepts/workloads/pods/init-containers/#resources resources: requests: - memory: "256Mi" - cpu: "0.1" + memory: "512Mi" + cpu: "0.2" limits: {} # memory: "1Gi" # cpu: "0.4" diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index e4a3fabe0..131d8f5e9 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -130,22 +130,22 @@ monitoring: images: zookeeper: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent customTools: backup: repository: "streamnative/pulsar-metadata-tool" - tag: "3.1.0.5" + tag: "3.1.1.1" restore: repository: "streamnative/pulsar-metadata-tool" - tag: "3.1.0.5" + tag: "3.1.1.1" bookie: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent presto: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent exporter: repository: bitnami/jmx-exporter @@ -154,31 +154,31 @@ images: # NOTE: allow overriding the presto worker image # presto_worker: # repository: streamnative/sn-platform - # tag: 3.1.0.5 + # tag: 3.1.1.1 # pullPolicy: IfNotPresent autorecovery: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent broker: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent proxy: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent pulsar_detector: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent functions: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent function_worker: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent # NOTE: allow overriding the toolset image toolset: @@ -243,7 +243,7 @@ images: pullPolicy: IfNotPresent pulsar_metadata: repository: streamnative/sn-platform - tag: "3.1.0.5" + tag: "3.1.1.1" pullPolicy: IfNotPresent configmapReload: repository: jimmidyson/configmap-reload @@ -693,8 +693,8 @@ zookeeper: # See https://kubernetes.io/docs/concepts/workloads/pods/init-containers/#resources resources: requests: - memory: "256Mi" - cpu: "0.1" + memory: "512Mi" + cpu: "0.2" limits: {} # memory: "1Gi" # cpu: "0.4"