fix: add cleanup in the bk cluster (#1140)

* fix: add cleanup in the bk cluster Signed-off-by: ericsyh <[email protected]> * apply to sn-platform chart Signed-off-by: ericsyh <[email protected]> * update the restore configmap Signed-off-by: ericsyh <[email protected]> * Make init job for backup/restoreg * Update charts/sn-platform/values.yaml * Update charts/sn-platform-slim/values.yaml --------- Signed-off-by: ericsyh <[email protected]> Co-authored-by: zymap <[email protected]> (cherry picked from commit 4d8def2)
streamnative · Mar 11, 2024 · 42bb9e4 · 42bb9e4
1 parent 66ead5c
commit 42bb9e4
Show file tree

Hide file tree

Showing 10 changed files with 95 additions and 53 deletions.
diff --git a/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml b/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml
@@ -309,8 +309,43 @@ spec:
     pdb:
       metadata:
         name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  {{- with .Values.bookkeeper.customization }}
   customization:
+  {{- with .Values.bookkeeper.customization }}
   {{- toYaml . | nindent 2 -}}
   {{- end }}
+  {{- if .Values.zookeeper.customTools.restore.enable}}
+    - match:
+        groupVersionKinds:
+          - kind: Job
+        name: .*-bk-init
+      manifest: |
+        spec:
+          template:
+            spec:
+              initContainers:
+              - name: pulsar-metadata-cleanup
+                image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
+                    {{ if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+                    export "$(cat conf/pulsar_env.sh | xargs)";
+                    export OPTS="${PULSAR_EXTRA_OPTS} ${EXTRA_OPTS}";
+                    env;
+                    {{- end }}
+                    bin/pulsar-metadata-tool cleanup
+                env:
+                - name: METADATA_TOOL_CONF
+                  value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
+                - name: EXTRA_OPTS
+                  value: {{ .Values.zookeeper.customTools.restore.configData.OPTS }}
+                volumeMounts:
+                  - name: cleanup-config
+                    mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
+              volumes:
+                - name: cleanup-config
+                  configMap:
+                    name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
+  {{- end }}
 {{- end }}
diff --git a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml
@@ -336,30 +336,6 @@ spec:
       PULSAR_PREFIX_managedLedgerOffloadServiceEndpoint: "{{ .Values.broker.offload.azureblob.managedLedgerOffloadServiceEndpoint }}"
       {{- end}}
       {{- end }}
-  initJobPod:
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-    initContainers:
-      - name: cleanup
-        image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
-        command:
-          - bash
-          - '-c'
-        args:
-          - "bin/pulsar-metadata-tool cleanup"
-        env:
-          - name: METADATA_TOOL_CONF
-            value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
-        volumeMounts:
-          - name: cleanup-config
-            mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
-    volumes:
-      - name: cleanup-config
-        configMap:
-          name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
-    {{- end }}
-    {{- if .Values.broker.resources }}
-    resources: {{- toYaml .Values.broker.resources | nindent 10 }}
-    {{- end }}
   {{- if or .Values.tls.broker.enabled .Values.broker.extraVolumeMounts .Values.broker.extraVolumeClaimTemplates }}
   apiObjects:
     statefulSet:

diff --git a/charts/sn-platform-slim/templates/zookeeper/zookeeper-backup-statefulset.yaml b/charts/sn-platform-slim/templates/zookeeper/zookeeper-backup-statefulset.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       securityContext:
       {{- with .Values.zookeeper.securityContext }}
-      {{ toYaml . | indent 8 }}
+      {{ toYaml . | nindent 8 }}
       {{- end }}
       {{- if .Values.zookeeper.customTools.serviceAccount.use }}
       serviceAccountName: {{ template "pulsar.zookeeper.backup.serviceAccount" . }}
@@ -85,7 +85,7 @@ spec:
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.backup.component }}-service"
         image: "{{ .Values.images.zookeeper.customTools.backup.repository }}:{{ .Values.images.zookeeper.customTools.backup.tag }}"
-        imagePullPolicy: {{ .Values.zookeeper.customTools.backup.pullPolicy }}
+        imagePullPolicy: {{ .Values.images.zookeeper.customTools.backup.pullPolicy }}
         command: ["sh", "-c"]
         args:
           - >
@@ -163,6 +163,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.readiness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.readiness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.readiness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.liveness.enabled }}
         livenessProbe:
@@ -172,6 +173,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.liveness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.liveness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.liveness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.liveness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.startup.enabled }}
         startupProbe:
@@ -181,6 +183,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.startup.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.startup.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.startup.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.startup.timeoutSeconds }}
         {{- end }}
         volumeMounts:
         {{- include "pulsar.zookeeper.data.volumeMounts" . | nindent 8 }}

diff --git a/charts/sn-platform-slim/templates/zookeeper/zookeeper-restore-configmap.yaml b/charts/sn-platform-slim/templates/zookeeper/zookeeper-restore-configmap.yaml
@@ -36,5 +36,7 @@ data:
     managedLedgerPath={{ .Values.zookeeper.customTools.restore.managedLedgerPath }}
     webServerPort=8088
     restoreVersion={{ .Values.zookeeper.customTools.restore.restoreVersion }}
+    cleanupBookieMeta={{ .Values.zookeeper.customTools.restore.cleanupBookieMeta }}
+    cleanupClusterMeta={{ .Values.zookeeper.customTools.restore.cleanupClusterMeta }}
 {{ toYaml .Values.zookeeper.customTools.restore.configData | indent 2 }}
 {{- end }}
diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml
@@ -126,9 +126,11 @@ images:
       backup:
         repository: "streamnative/pulsar-metadata-tool"
         tag: "3.1.1.1"
+        pullPolicy: IfNotPresent
       restore:
         repository: "streamnative/pulsar-metadata-tool"
         tag: "3.1.1.1"
+        pullPolicy: IfNotPresent
   bookie:
     repository: streamnative/sn-platform-slim
     tag: "3.1.1.1"
@@ -583,16 +585,19 @@ zookeeper:
       failureThreshold: 10
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
     readiness:
       enabled: true
       failureThreshold: 10
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
     startup:
       enabled: false
       failureThreshold: 30
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
   affinity:
     anti_affinity: true
     zone_anti_affinity: true

diff --git a/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml b/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml
@@ -309,8 +309,43 @@ spec:
     pdb:
       metadata:
         name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  {{- with .Values.bookkeeper.customization }}
   customization:
+  {{- with .Values.bookkeeper.customization }}
   {{- toYaml . | nindent 2 -}}
   {{- end }}
+  {{- if .Values.zookeeper.customTools.restore.enable}}
+    - match:
+        groupVersionKinds:
+          - kind: Job
+        name: .*-bk-init
+      manifest: |
+        spec:
+          template:
+            spec:
+              initContainers:
+              - name: pulsar-metadata-cleanup
+                image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
+                    {{ if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+                    export "$(cat conf/pulsar_env.sh | xargs)";
+                    export OPTS="${PULSAR_EXTRA_OPTS} ${EXTRA_OPTS}";
+                    env;
+                    {{- end }}
+                    bin/pulsar-metadata-tool cleanup
+                env:
+                - name: METADATA_TOOL_CONF
+                  value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
+                - name: EXTRA_OPTS
+                  value: {{ .Values.zookeeper.customTools.restore.configData.OPTS }}
+                volumeMounts:
+                  - name: cleanup-config
+                    mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
+              volumes:
+                - name: cleanup-config
+                  configMap:
+                    name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
+  {{- end }}
 {{- end }}
diff --git a/charts/sn-platform/templates/broker/broker-cluster.yaml b/charts/sn-platform/templates/broker/broker-cluster.yaml
@@ -341,30 +341,6 @@ spec:
       PULSAR_PREFIX_managedLedgerOffloadServiceEndpoint: "{{ .Values.broker.offload.azureblob.managedLedgerOffloadServiceEndpoint }}"
       {{- end}}
       {{- end }}
-  initJobPod:
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-    initContainers:
-      - name: cleanup
-        image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
-        command:
-          - bash
-          - '-c'
-        args:
-          - "bin/pulsar-metadata-tool cleanup"
-        env:
-          - name: METADATA_TOOL_CONF
-            value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
-        volumeMounts:
-          - name: cleanup-config
-            mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
-    volumes:
-      - name: cleanup-config
-        configMap:
-          name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
-    {{- end }}
-    {{- if .Values.broker.resources }}
-    resources: {{- toYaml .Values.broker.resources | nindent 10 }}
-    {{- end }}
   {{- if or .Values.tls.broker.enabled .Values.broker.extraVolumeMounts .Values.broker.extraVolumeClaimTemplates }}
   apiObjects:
     statefulSet:

diff --git a/charts/sn-platform/templates/zookeeper/zookeeper-backup-statefulset.yaml b/charts/sn-platform/templates/zookeeper/zookeeper-backup-statefulset.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       securityContext:
       {{- with .Values.zookeeper.securityContext }}
-      {{ toYaml . | indent 8 }}
+      {{ toYaml . | nindent 8 }}
       {{- end }}
       {{- if .Values.zookeeper.customTools.serviceAccount.use }}
       serviceAccountName: {{ template "pulsar.zookeeper.backup.serviceAccount" . }}
@@ -163,6 +163,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.readiness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.readiness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.readiness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.liveness.enabled }}
         livenessProbe:
@@ -172,6 +173,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.liveness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.liveness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.liveness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.liveness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.startup.enabled }}
         startupProbe:
@@ -181,6 +183,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.startup.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.startup.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.startup.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.startup.timeoutSeconds }}
         {{- end }}
         volumeMounts:
         {{- include "pulsar.zookeeper.data.volumeMounts" . | nindent 8 }}

diff --git a/charts/sn-platform/templates/zookeeper/zookeeper-restore-configmap.yaml b/charts/sn-platform/templates/zookeeper/zookeeper-restore-configmap.yaml
@@ -36,5 +36,7 @@ data:
     managedLedgerPath={{ .Values.zookeeper.customTools.restore.managedLedgerPath }}
     webServerPort=8088
     restoreVersion={{ .Values.zookeeper.customTools.restore.restoreVersion }}
+    cleanupBookieMeta={{ .Values.zookeeper.customTools.restore.cleanupBookieMeta }}
+    cleanupClusterMeta={{ .Values.zookeeper.customTools.restore.cleanupClusterMeta }}
 {{ toYaml .Values.zookeeper.customTools.restore.configData | indent 2 }}
 {{- end }}
diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml
@@ -136,9 +136,11 @@ images:
       backup:
         repository: "streamnative/pulsar-metadata-tool"
         tag: "3.1.1.1"
+        pullPolicy: IfNotPresent
       restore:
         repository: "streamnative/pulsar-metadata-tool"
         tag: "3.1.1.1"
+        pullPolicy: IfNotPresent
   bookie:
     repository: streamnative/sn-platform
     tag: "3.1.1.1"
@@ -658,16 +660,19 @@ zookeeper:
       failureThreshold: 10
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
     readiness:
       enabled: true
       failureThreshold: 10
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
     startup:
       enabled: false
       failureThreshold: 30
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
   affinity:
     anti_affinity: true
     zone_anti_affinity: true