From 5661ca0c52ac5f87f71afcaac489be758e194207 Mon Sep 17 00:00:00 2001 From: Yisheng Cai Date: Thu, 21 Dec 2023 19:51:25 +0800 Subject: [PATCH] [sn-platform(-slim)] Support readOnlyRootFilesystem (#1136) * Expose variable KUBECTL_BIN and OUTPUT * Define KUBECTL_BIN and OUTPUT to writable path * Toolset support readOnlyRootFilesystem * Detector support readOnlyRootFilesystem * Fix zk cluster indent * Add rootless example (cherry picked from commit 6cd50a011a1bbaf2fec97e7ce8a5e1d741b10da4) --- .../conf/toolset/pulsar/clean_tls.sh | 6 +- .../toolset/pulsar/cleanup_helm_release.sh | 6 +- .../conf/toolset/pulsar/common_auth.sh | 1 - .../toolset/pulsar/decommission_bookies.sh | 8 +- .../conf/toolset/pulsar/generate_token.sh | 8 +- .../pulsar/generate_token_secret_key.sh | 19 ++- .../conf/toolset/pulsar/get_token.sh | 4 +- .../toolset/pulsar/gke_bootstrap_script.sh | 2 +- .../toolset/pulsar/prepare_helm_release.sh | 5 +- ...setup-clouddns-resolver-service-account.sh | 2 +- .../toolset/pulsar/upload-lets-encrypt-ca.sh | 2 +- .../conf/toolset/pulsar/upload_tls.sh | 6 +- .../detector/pulsar-detector-deployment.yaml | 10 +- .../toolset/jwt-secret-init-job.yaml | 5 +- .../toolset/toolset-statefulset.yaml | 33 ++++- charts/sn-platform-slim/values.yaml | 1 + .../conf/toolset/pulsar/clean_tls.sh | 6 +- .../toolset/pulsar/cleanup_helm_release.sh | 6 +- .../conf/toolset/pulsar/common_auth.sh | 1 - .../toolset/pulsar/decommission_bookies.sh | 8 +- .../conf/toolset/pulsar/generate_token.sh | 8 +- .../pulsar/generate_token_secret_key.sh | 16 +-- .../conf/toolset/pulsar/get_token.sh | 4 +- .../toolset/pulsar/gke_bootstrap_script.sh | 2 +- .../toolset/pulsar/prepare_helm_release.sh | 5 +- ...setup-clouddns-resolver-service-account.sh | 2 +- .../toolset/pulsar/upload-lets-encrypt-ca.sh | 2 +- .../conf/toolset/pulsar/upload_tls.sh | 6 +- .../detector/pulsar-detector-deployment.yaml | 10 +- .../toolset/jwt-secret-init-job.yaml | 5 +- .../toolset/toolset-statefulset.yaml | 33 ++++- .../zookeeper/zookeeper-cluster.yaml | 2 +- charts/sn-platform/values.yaml | 1 + examples/sn-platform/values-rootless.yaml | 128 ++++++++++++++++++ 34 files changed, 284 insertions(+), 79 deletions(-) create mode 100644 examples/sn-platform/values-rootless.yaml diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh b/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh index 884cc383e..743df35ca 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh @@ -81,14 +81,14 @@ done function delete_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl delete secret ${tls_ca_secret} -n ${namespace} + ${KUBECTL_BIN} delete secret ${tls_ca_secret} -n ${namespace} } function delete_server_cert() { local component=$1 local server_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${server_cert_secret} \ + ${KUBECTL_BIN} delete secret ${server_cert_secret} \ -n ${namespace} } @@ -96,7 +96,7 @@ function delete_client_cert() { local component=$1 local client_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${client_cert_secret} \ + ${KUBECTL_BIN} delete secret ${client_cert_secret} \ -n ${namespace} } diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh b/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh index 76f7573ca..9b2809b49 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh @@ -73,15 +73,15 @@ release=${release:-pulsar-dev} function delete_namespace() { if [[ "${delete_namespace}" == "true" ]]; then - /pulsar/kubectl delete namespace ${namespace} + ${KUBECTL_BIN} delete namespace ${namespace} fi } # delete the cc admin secrets -/pulsar/kubectl delete -n ${namespace} secret ${release}-admin-secret +${KUBECTL_BIN} delete -n ${namespace} secret ${release}-admin-secret # delete tokens -/pulsar/kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs /pulsar/kubectl delete secrets -n ${namespace} +${KUBECTL_BIN} get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs ${KUBECTL_BIN} delete secrets -n ${namespace} # delete namespace delete_namespace diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh b/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh index ede8c86eb..c0844647c 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh @@ -23,7 +23,6 @@ if [ -z "$CHART_HOME" ]; then exit 1 fi -OUTPUT=${CHART_HOME}/output OUTPUT_BIN=${OUTPUT}/bin PULSARCTL_VERSION=v2.10.2.2 PULSARCTL_BIN=/pulsar/bin/pulsarctl diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh b/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh index 55e2823f7..7c240c264 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh @@ -89,8 +89,8 @@ autorecovery_pod=${autorecovery_pod:-autorecovery} for ((i=replicas; i>=1; i--)) do j=$((i-1)) - echo /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - echo /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 - /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + echo ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + echo ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 done diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh index 7dd1b656e..d38fd0c2c 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh @@ -96,11 +96,11 @@ function pulsar::jwt::generate_symmetric_token() { trap "test -f $tmpfile && rm $tmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} ${PULSARCTL_BIN} token create -a HS256 --secret-key-file ${tmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" } function pulsar::jwt::generate_asymmetric_token() { @@ -111,11 +111,11 @@ function pulsar::jwt::generate_asymmetric_token() { trap "test -f $privatekeytmpfile && rm $privatekeytmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} ${PULSARCTL_BIN} token create -a RS256 --private-key-file ${privatekeytmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh index f5c9f28a3..411261c3e 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh @@ -18,8 +18,7 @@ # under the License. # -set -e - +set -x; CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd ${CHART_HOME} @@ -82,9 +81,9 @@ function pulsar::jwt::generate_symmetric_key() { tmpfile=$(mktemp) trap "test -f $tmpfile && rm $tmpfile" RETURN ${PULSARCTL_BIN} token create-secret-key --output-file ${tmpfile} - mv $tmpfile SECRETKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=SECRETKEY - rm SECRETKEY + mv $tmpfile ${OUTPUT}/SECRETKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=${OUTPUT}/SECRETKEY + rm ${OUTPUT}/SECRETKEY } function pulsar::jwt::generate_asymmetric_key() { @@ -95,11 +94,11 @@ function pulsar::jwt::generate_asymmetric_key() { publickeytmpfile=$(mktemp) trap "test -f $publickeytmpfile && rm $publickeytmpfile" RETURN ${PULSARCTL_BIN} token create-key-pair -a RS256 --output-private-key ${privatekeytmpfile} --output-public-key ${publickeytmpfile} - mv $privatekeytmpfile PRIVATEKEY - mv $publickeytmpfile PUBLICKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=PRIVATEKEY --from-file=PUBLICKEY - rm PRIVATEKEY - rm PUBLICKEY + mv $privatekeytmpfile $OUTPUT/PRIVATEKEY + mv $publickeytmpfile $OUTPUT/PUBLICKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/PRIVATEKEY --from-file=$OUTPUT/PUBLICKEY + rm $OUTPUT/PRIVATEKEY + rm $OUTPUT/PUBLICKEY } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh b/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh index 38c9272f7..25aeb0525 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh @@ -84,8 +84,8 @@ release=${release:-pulsar-dev} function pulsar::jwt::get_token() { local token_name="${release}-token-${role}" - local token=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) - local token_type=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) + local token=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) + local token_type=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) echo "token type: ${token_type}" echo "-------------------------" diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh b/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh index e6f20599a..86eea6538 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh @@ -58,7 +58,7 @@ function bootstrap(){ echo "Wait for metrics API service" # Helm 2.15 and 3.0 bug https://github.com/helm/helm/issues/6361#issuecomment-550503455 - /pulsar/kubectl --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io + ${KUBECTL_BIN} --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io helm repo update } diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh b/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh index 68e55a300..b61484a3c 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh @@ -18,6 +18,7 @@ # under the License. # +set -x; CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd ${CHART_HOME} @@ -94,7 +95,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function generate_gcs_offloader_service_account_keyfile() { local secret_name="${release}-gcs-offloader-service-account" - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} \ + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} \ --from-file="gcs.json=${gcs_offloader_service_account_keyfile}" } @@ -102,7 +103,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function do_create_namespace() { if [[ "${create_namespace}" == "true" ]]; then - /pulsar/kubectl create namespace ${namespace} + ${KUBECTL_BIN} create namespace ${namespace} fi } diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh b/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh index 1c4afed80..3c8a30b22 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh @@ -46,7 +46,7 @@ gcloud iam service-accounts keys create ${RESOLVER_NAME}-key.json \ --iam-account ${RESOLVER_NAME}@$PROJECT_ID.iam.gserviceaccount.com echo "Save the service account key as a kubernete secret '${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct' in namespace '${NAMESPACE}'." -/pulsar/kubectl create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ +${KUBECTL_BIN} create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ --from-file=${RESOLVER_NAME}-key.json -n ${NAMESPACE} echo "Remove the generated key." diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh b/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh index d84b9cefb..9a0be2909 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh @@ -29,5 +29,5 @@ PEM="${CA_NAME}.pem" NAMESPACE=$1 -/pulsar/kubectl create secret generic ${CA_NAME} \ +${KUBECTL_BIN} create secret generic ${CA_NAME} \ --from-file=${PEM} -n ${NAMESPACE} diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh b/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh index 94b3f764d..668001d6d 100755 --- a/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh +++ b/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh @@ -91,7 +91,7 @@ ca_cert_file=${tlsdir}/certs/ca.cert.pem function upload_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" + ${KUBECTL_BIN} create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" } function upload_server_cert() { @@ -100,7 +100,7 @@ function upload_server_cert() { local tls_cert_file="${tlsdir}/servers/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/servers/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${server_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${server_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ @@ -113,7 +113,7 @@ function upload_client_cert() { local tls_cert_file="${tlsdir}/clients/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/clients/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${client_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${client_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ diff --git a/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml b/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml index f3beb074f..a53c686d8 100644 --- a/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml +++ b/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml @@ -85,6 +85,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs # This init container will wait for at least one broker to be ready before # deploying the pulsar-detector - name: wait-broker-ready @@ -103,6 +106,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs {{- end }} containers: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_detector.component }}" @@ -145,8 +151,10 @@ spec: volumeMounts: {{- toYaml .Values.pulsar_detector.extraVolumeMounts | nindent 10 }} {{- end }} - {{- if .Values.pulsar_detector.extraVolumes }} volumes: + - name: tmp + emptyDir: {} + {{- if .Values.pulsar_detector.extraVolumes }} {{- toYaml .Values.pulsar_detector.extraVolumes | nindent 8 }} {{- end }} {{- end }} diff --git a/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml b/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml index 4a66b68ec..1b681ba5f 100644 --- a/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml +++ b/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml @@ -79,13 +79,14 @@ spec: args: - | set -ex; - cp /tmp/binaries/kubectl /pulsar/kubectl; - chmod +x /pulsar/kubectl; mkdir -p scripts/pulsar; cp scripts/jwt-secret-config/* scripts/pulsar; chmod +x scripts/pulsar/*; usingSecretKey={{ .Values.auth.authentication.jwt.usingSecretKey }}; ls -lh scripts/pulsar/; + export KUBECTL_BIN=/tmp/binaries/kubectl; + export OUTPUT=scripts/pulsar/output; + mkdir ${OUTPUT}; if [ "${usingSecretKey}" = "true" ]; then ./scripts/pulsar/prepare_helm_release.sh -n {{ template "pulsar.namespace" . }} -k {{ .Release.Name }} --symmetric; else diff --git a/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml b/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml index 24d4b5b98..542e5d8b6 100644 --- a/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml +++ b/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml @@ -72,8 +72,27 @@ spec: {{ toYaml .Values.toolset.tolerations | indent 8 }} {{- end }} terminationGracePeriodSeconds: {{ .Values.toolset.gracePeriod }} - {{- if .Values.toolset.installBusybox }} initContainers: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: "init-copy-config" + image: "{{ .Values.images.toolset.repository }}:{{ .Values.images.toolset.tag }}" + imagePullPolicy: {{ .Values.images.toolset.pullPolicy }} + command: + - sh + - -c + - | + set -ex; + cp -r /pulsar/conf/* /conf_tmp/; + echo OK > /conf_tmp/status; + {{- if .Values.toolset.resources }} + resources: +{{ toYaml .Values.toolset.resources | indent 10 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /conf_tmp/ + {{- end }} + {{- if .Values.toolset.installBusybox }} - name: busybox image: "{{ .Values.images.toolset.busybox.repository }}:{{ .Values.images.toolset.busybox.tag }}" imagePullPolicy: {{ .Values.images.toolset.busybox.pullPolicy }} @@ -98,7 +117,7 @@ spec: volumeMounts: - name: binaries mountPath: /home/tmp/binaries - {{- end }} + {{- end }} containers: - name: "pulsar" {{- include "pulsar.toolset.image" . | nindent 8 }} @@ -122,6 +141,12 @@ spec: {{ toYaml . | indent 8 }} {{- end }} volumeMounts: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + mountPath: /pulsar/conf + - name: tmp + mountPath: /pulsar/logs + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries mountPath: /bin/busybox @@ -138,6 +163,10 @@ spec: {{ toYaml . | indent 8 }} {{- end }} volumes: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + emptyDir: {} + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries emptyDir: {} diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index 3360f6dad..c20604952 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -1550,6 +1550,7 @@ toolset: component: toolset useProxy: false installBusybox: true + readOnlyRootFilesystem: false replicaCount: 1 # nodeSelector: # cloud.google.com/gke-nodepool: default-pool diff --git a/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh b/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh index 884cc383e..743df35ca 100755 --- a/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh +++ b/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh @@ -81,14 +81,14 @@ done function delete_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl delete secret ${tls_ca_secret} -n ${namespace} + ${KUBECTL_BIN} delete secret ${tls_ca_secret} -n ${namespace} } function delete_server_cert() { local component=$1 local server_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${server_cert_secret} \ + ${KUBECTL_BIN} delete secret ${server_cert_secret} \ -n ${namespace} } @@ -96,7 +96,7 @@ function delete_client_cert() { local component=$1 local client_cert_secret="${release}-tls-${component}" - /pulsar/kubectl delete secret ${client_cert_secret} \ + ${KUBECTL_BIN} delete secret ${client_cert_secret} \ -n ${namespace} } diff --git a/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh b/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh index 76f7573ca..9b2809b49 100755 --- a/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh +++ b/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh @@ -73,15 +73,15 @@ release=${release:-pulsar-dev} function delete_namespace() { if [[ "${delete_namespace}" == "true" ]]; then - /pulsar/kubectl delete namespace ${namespace} + ${KUBECTL_BIN} delete namespace ${namespace} fi } # delete the cc admin secrets -/pulsar/kubectl delete -n ${namespace} secret ${release}-admin-secret +${KUBECTL_BIN} delete -n ${namespace} secret ${release}-admin-secret # delete tokens -/pulsar/kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs /pulsar/kubectl delete secrets -n ${namespace} +${KUBECTL_BIN} get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs ${KUBECTL_BIN} delete secrets -n ${namespace} # delete namespace delete_namespace diff --git a/charts/sn-platform/conf/toolset/pulsar/common_auth.sh b/charts/sn-platform/conf/toolset/pulsar/common_auth.sh index ede8c86eb..c0844647c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/common_auth.sh +++ b/charts/sn-platform/conf/toolset/pulsar/common_auth.sh @@ -23,7 +23,6 @@ if [ -z "$CHART_HOME" ]; then exit 1 fi -OUTPUT=${CHART_HOME}/output OUTPUT_BIN=${OUTPUT}/bin PULSARCTL_VERSION=v2.10.2.2 PULSARCTL_BIN=/pulsar/bin/pulsarctl diff --git a/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh b/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh index 55e2823f7..7c240c264 100755 --- a/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh +++ b/charts/sn-platform/conf/toolset/pulsar/decommission_bookies.sh @@ -89,8 +89,8 @@ autorecovery_pod=${autorecovery_pod:-autorecovery} for ((i=replicas; i>=1; i--)) do j=$((i-1)) - echo /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset} - echo /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 - /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + echo ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset} + echo ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 + ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181 done diff --git a/charts/sn-platform/conf/toolset/pulsar/generate_token.sh b/charts/sn-platform/conf/toolset/pulsar/generate_token.sh index 7dd1b656e..d38fd0c2c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/generate_token.sh +++ b/charts/sn-platform/conf/toolset/pulsar/generate_token.sh @@ -96,11 +96,11 @@ function pulsar::jwt::generate_symmetric_token() { trap "test -f $tmpfile && rm $tmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile} ${PULSARCTL_BIN} token create -a HS256 --secret-key-file ${tmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric" } function pulsar::jwt::generate_asymmetric_token() { @@ -111,11 +111,11 @@ function pulsar::jwt::generate_asymmetric_token() { trap "test -f $privatekeytmpfile && rm $privatekeytmpfile" RETURN tokentmpfile=$(mktemp) trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN - /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} + ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile} ${PULSARCTL_BIN} token create -a RS256 --private-key-file ${privatekeytmpfile} --subject ${role} 2&> ${tokentmpfile} newtokentmpfile=$(mktemp) tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile} - /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" + ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric" } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh b/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh index f5c9f28a3..a3acbee8c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh +++ b/charts/sn-platform/conf/toolset/pulsar/generate_token_secret_key.sh @@ -82,9 +82,9 @@ function pulsar::jwt::generate_symmetric_key() { tmpfile=$(mktemp) trap "test -f $tmpfile && rm $tmpfile" RETURN ${PULSARCTL_BIN} token create-secret-key --output-file ${tmpfile} - mv $tmpfile SECRETKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=SECRETKEY - rm SECRETKEY + mv $tmpfile $OUTPUT/SECRETKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/SECRETKEY + rm $OUTPUT/SECRETKEY } function pulsar::jwt::generate_asymmetric_key() { @@ -95,11 +95,11 @@ function pulsar::jwt::generate_asymmetric_key() { publickeytmpfile=$(mktemp) trap "test -f $publickeytmpfile && rm $publickeytmpfile" RETURN ${PULSARCTL_BIN} token create-key-pair -a RS256 --output-private-key ${privatekeytmpfile} --output-public-key ${publickeytmpfile} - mv $privatekeytmpfile PRIVATEKEY - mv $publickeytmpfile PUBLICKEY - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=PRIVATEKEY --from-file=PUBLICKEY - rm PRIVATEKEY - rm PUBLICKEY + mv $privatekeytmpfile $OUTPUT/PRIVATEKEY + mv $publickeytmpfile $OUTPUT/PUBLICKEY + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/PRIVATEKEY --from-file=$OUTPUT/PUBLICKEY + rm $OUTPUT/PRIVATEKEY + rm $OUTPUT/PUBLICKEY } if [[ "${symmetric}" == "true" ]]; then diff --git a/charts/sn-platform/conf/toolset/pulsar/get_token.sh b/charts/sn-platform/conf/toolset/pulsar/get_token.sh index 38c9272f7..25aeb0525 100755 --- a/charts/sn-platform/conf/toolset/pulsar/get_token.sh +++ b/charts/sn-platform/conf/toolset/pulsar/get_token.sh @@ -84,8 +84,8 @@ release=${release:-pulsar-dev} function pulsar::jwt::get_token() { local token_name="${release}-token-${role}" - local token=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) - local token_type=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) + local token=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode) + local token_type=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode) echo "token type: ${token_type}" echo "-------------------------" diff --git a/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh b/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh index e6f20599a..86eea6538 100755 --- a/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh +++ b/charts/sn-platform/conf/toolset/pulsar/gke_bootstrap_script.sh @@ -58,7 +58,7 @@ function bootstrap(){ echo "Wait for metrics API service" # Helm 2.15 and 3.0 bug https://github.com/helm/helm/issues/6361#issuecomment-550503455 - /pulsar/kubectl --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io + ${KUBECTL_BIN} --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io helm repo update } diff --git a/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh b/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh index 68e55a300..b61484a3c 100755 --- a/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh +++ b/charts/sn-platform/conf/toolset/pulsar/prepare_helm_release.sh @@ -18,6 +18,7 @@ # under the License. # +set -x; CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd ${CHART_HOME} @@ -94,7 +95,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function generate_gcs_offloader_service_account_keyfile() { local secret_name="${release}-gcs-offloader-service-account" - /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} \ + ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} \ --from-file="gcs.json=${gcs_offloader_service_account_keyfile}" } @@ -102,7 +103,7 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma function do_create_namespace() { if [[ "${create_namespace}" == "true" ]]; then - /pulsar/kubectl create namespace ${namespace} + ${KUBECTL_BIN} create namespace ${namespace} fi } diff --git a/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh b/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh index 1c4afed80..3c8a30b22 100755 --- a/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh +++ b/charts/sn-platform/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh @@ -46,7 +46,7 @@ gcloud iam service-accounts keys create ${RESOLVER_NAME}-key.json \ --iam-account ${RESOLVER_NAME}@$PROJECT_ID.iam.gserviceaccount.com echo "Save the service account key as a kubernete secret '${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct' in namespace '${NAMESPACE}'." -/pulsar/kubectl create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ +${KUBECTL_BIN} create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \ --from-file=${RESOLVER_NAME}-key.json -n ${NAMESPACE} echo "Remove the generated key." diff --git a/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh b/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh index d84b9cefb..9a0be2909 100755 --- a/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh +++ b/charts/sn-platform/conf/toolset/pulsar/upload-lets-encrypt-ca.sh @@ -29,5 +29,5 @@ PEM="${CA_NAME}.pem" NAMESPACE=$1 -/pulsar/kubectl create secret generic ${CA_NAME} \ +${KUBECTL_BIN} create secret generic ${CA_NAME} \ --from-file=${PEM} -n ${NAMESPACE} diff --git a/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh b/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh index 94b3f764d..668001d6d 100755 --- a/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh +++ b/charts/sn-platform/conf/toolset/pulsar/upload_tls.sh @@ -91,7 +91,7 @@ ca_cert_file=${tlsdir}/certs/ca.cert.pem function upload_ca() { local tls_ca_secret="${release}-ca-tls" - /pulsar/kubectl create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" + ${KUBECTL_BIN} create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}" } function upload_server_cert() { @@ -100,7 +100,7 @@ function upload_server_cert() { local tls_cert_file="${tlsdir}/servers/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/servers/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${server_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${server_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ @@ -113,7 +113,7 @@ function upload_client_cert() { local tls_cert_file="${tlsdir}/clients/${component}/${component}.cert.pem" local tls_key_file="${tlsdir}/clients/${component}/${component}.key-pk8.pem" - /pulsar/kubectl create secret generic ${client_cert_secret} \ + ${KUBECTL_BIN} create secret generic ${client_cert_secret} \ -n ${namespace} \ --from-file="tls.crt=${tls_cert_file}" \ --from-file="tls.key=${tls_key_file}" \ diff --git a/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml b/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml index 1d9381a0c..2fd961e1f 100644 --- a/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml +++ b/charts/sn-platform/templates/detector/pulsar-detector-deployment.yaml @@ -85,6 +85,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs # This init container will wait for at least one broker to be ready before # deploying the pulsar-detector - name: wait-broker-ready @@ -103,6 +106,9 @@ spec: {{- if .Values.pulsar_detector.resources }} resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }} {{- end }} + volumeMounts: + - name: tmp + mountPath: /pulsar/logs {{- end }} containers: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_detector.component }}" @@ -152,8 +158,10 @@ spec: volumeMounts: {{- toYaml .Values.pulsar_detector.extraVolumeMounts | nindent 10 }} {{- end }} - {{- if .Values.pulsar_detector.extraVolumes }} volumes: + - name: tmp + emptyDir: {} + {{- if .Values.pulsar_detector.extraVolumes }} {{- toYaml .Values.pulsar_detector.extraVolumes | nindent 8 }} {{- end }} {{- end }} diff --git a/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml b/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml index a99307561..312067d97 100644 --- a/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml +++ b/charts/sn-platform/templates/toolset/jwt-secret-init-job.yaml @@ -79,13 +79,14 @@ spec: args: - | set -ex; - cp /tmp/binaries/kubectl /pulsar/kubectl; - chmod +x /pulsar/kubectl; mkdir -p scripts/pulsar; cp scripts/jwt-secret-config/* scripts/pulsar; chmod +x scripts/pulsar/*; usingSecretKey={{ .Values.auth.authentication.jwt.usingSecretKey }}; ls -lh scripts/pulsar/; + export KUBECTL_BIN=/tmp/binaries/kubectl; + export OUTPUT=scripts/pulsar/output; + mkdir ${OUTPUT}; if [ "${usingSecretKey}" = "true" ]; then ./scripts/pulsar/prepare_helm_release.sh -n {{ template "pulsar.namespace" . }} -k {{ .Release.Name }} --symmetric; else diff --git a/charts/sn-platform/templates/toolset/toolset-statefulset.yaml b/charts/sn-platform/templates/toolset/toolset-statefulset.yaml index affd90b4c..b4f9e8b24 100644 --- a/charts/sn-platform/templates/toolset/toolset-statefulset.yaml +++ b/charts/sn-platform/templates/toolset/toolset-statefulset.yaml @@ -72,8 +72,27 @@ spec: {{ toYaml .Values.toolset.tolerations | indent 8 }} {{- end }} terminationGracePeriodSeconds: {{ .Values.toolset.gracePeriod }} - {{- if .Values.toolset.installBusybox }} initContainers: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: "init-copy-config" + image: "{{ .Values.images.toolset.repository }}:{{ .Values.images.toolset.tag }}" + imagePullPolicy: {{ .Values.images.toolset.pullPolicy }} + command: + - sh + - -c + - | + set -ex; + cp -r /pulsar/conf/* /conf_tmp/; + echo OK > /conf_tmp/status; + {{- if .Values.toolset.resources }} + resources: +{{ toYaml .Values.toolset.resources | indent 10 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /conf_tmp/ + {{- end }} + {{- if .Values.toolset.installBusybox }} - name: busybox image: "{{ .Values.images.toolset.busybox.repository }}:{{ .Values.images.toolset.busybox.tag }}" imagePullPolicy: {{ .Values.images.toolset.busybox.pullPolicy }} @@ -98,7 +117,7 @@ spec: volumeMounts: - name: binaries mountPath: /home/tmp/binaries - {{- end }} + {{- end }} containers: - name: "pulsar" {{- include "pulsar.toolset.image" . | nindent 8 }} @@ -122,6 +141,12 @@ spec: {{ toYaml . | indent 8 }} {{- end }} volumeMounts: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + mountPath: /pulsar/conf + - name: tmp + mountPath: /pulsar/logs + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries mountPath: /bin/busybox @@ -168,6 +193,10 @@ spec: {{- end }} {{- end }} volumes: + {{- if .Values.toolset.readOnlyRootFilesystem }} + - name: tmp + emptyDir: {} + {{- end }} {{- if .Values.toolset.installBusybox }} - name: binaries emptyDir: {} diff --git a/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml b/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml index 5fbac3140..dfb7c7f8a 100644 --- a/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml +++ b/charts/sn-platform/templates/zookeeper/zookeeper-cluster.yaml @@ -166,7 +166,7 @@ spec: {{- range .Values.zookeeper.volumes.data.accessModes }} - {{ . | quote }} {{- end }} - resources: + resources: requests: storage: {{ .Values.zookeeper.volumes.data.size }} {{- include "pulsar.zookeeper.data.storage.class" . | nindent 6 }} diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 776f37885..e4a3fabe0 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -1627,6 +1627,7 @@ toolset: component: toolset useProxy: false installBusybox: true + readOnlyRootFilesystem: false replicaCount: 1 # nodeSelector: # cloud.google.com/gke-nodepool: default-pool diff --git a/examples/sn-platform/values-rootless.yaml b/examples/sn-platform/values-rootless.yaml new file mode 100644 index 000000000..fcb56f432 --- /dev/null +++ b/examples/sn-platform/values-rootless.yaml @@ -0,0 +1,128 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# vault is not supported in rootless env +components: + pulsar_detector: true + vault: false + +# monitoring components haven't been tested in rootless env +monitoring: + prometheus: false + grafana: false + node_exporter: false + alert_manager: false + loki: false + datadog: false + +auth: + authentication: + enabled: true + provider: "jwt" + jwt: + enabled: true + usingSecretKey: false + autoInitSecret: true + vault: + enabled: false + +zookeeper: + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 +bookkeeper: + bookKeeperCluster: + annotations: + cloud.streamnative.io/omit-init-sysctl: "true" + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + autorecovery: + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +broker: + pulsarBroker: + annotations: + cloud.streamnative.io/omit-init-sysctl: "true" + replicaCount: 2 + kop: + enabled: false + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +proxy: + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +streamnative_console: + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + configData: + INIT_DEFAULT_ENVIRONMENT: true + BACKEND_DEFAULT_SUPER_USER_ROLE: "admin,pulsar-manager-admin" + login: + sso: + pulsarJwt: + enabled: true + config: + AUTHENTICATION_CUSTOM_CLAIM: sub + # The secret should contain both `TOKEN` + SERVICE_ACCOUNT_SUPER_TOKEN_SECRET: "sn-platform-token-admin" + # AUTH_METHOD_CUSTOM_CLAIM: sub + # USERNAME_CLAIM: sub + JWT_BROKER_TOKEN_MODE: PRIVATE + # The secret should contain both `PUBLICKEY`, `PRIVATEKEY` + JWT_BROKER_PUBLIC_PRIVATE_KEY: "sn-platform-token-asymmetric-key" + # The secret should contain both `SECRETKEY` + JWT_BROKER_SECRET_KEY: "" +toolset: + readOnlyRootFilesystem: true + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000 + +pulsar_detector: + securityContext: + runAsNonRoot: true + runAsGroup: 10001 + fsGroup: 10001 + runAsUser: 10000