Make init job for backup/restoreg

streamnative · Dec 25, 2023 · a2b5d9e · a2b5d9e
1 parent 7ce6b39
commit a2b5d9e
Show file tree

Hide file tree

Showing 8 changed files with 105 additions and 105 deletions.
diff --git a/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml b/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml
@@ -53,24 +53,6 @@ spec:
     {{- toYaml . | nindent 6 -}}
     {{- end }}
   pod:
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-    initContainers:
-    - name: pulsar-metadata-cleanup
-      image: "{{ .Values.zookeeper.customTools.restore.repository }}:{{ .Values.zookeeper.customTools.restore.tag }}"
-      imagePullPolicy: "{{ .Values.zookeeper.customTools.restore.pullPolicy }}"
-      command: ["sh", "-c"]
-      args:
-        - >
-          bin/pulsar-metadata-tool cleanup
-      env:
-        - name: METADATA_TOOL_CONF
-          value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
-        - name: EXTRA_OPTS
-          value: {{ .Values.zookeeper.customTools.restore.configData.OPTS }}
-      volumeMounts:
-        - name: cleanup-config
-          mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
-    {{- end }}
     {{- with .Values.bookkeeper.topologySpreadConstraints }}
     topologySpreadConstraints:
     {{- toYaml . | nindent 4 }}
@@ -137,17 +119,10 @@ spec:
     {{- include "pulsar.bookkeeper.certs.volumes" . | nindent 4 }}
     {{- include "pulsar.bookkeeper.log.volumes" . | nindent 4 }}
     {{- end }}
-    {{- if and (not .Values.bookkeeper.operator.adopt_existing) (or .Values.zookeeper.customTools.restore.enable .Values.tls.bookie.enabled) }}
+    {{- if and (not .Values.bookkeeper.operator.adopt_existing) .Values.tls.bookie.enabled }}
     volumes:
-    {{- if .Values.tls.bookie.enabled }}
     {{- include "pulsar.bookkeeper.certs.volumes" . | nindent 4 }}
     {{- end }}
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-      - name: cleanup-config
-        configMap:
-          name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
-    {{- end }}
-    {{- end }}
   {{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence}}
   storage:
     journal:
@@ -334,8 +309,43 @@ spec:
     pdb:
       metadata:
         name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  {{- with .Values.bookkeeper.customization }}
   customization:
+  {{- with .Values.bookkeeper.customization }}
   {{- toYaml . | nindent 2 -}}
   {{- end }}
+  {{- if .Values.zookeeper.customTools.restore.enable}}
+    - match:
+        groupVersionKinds:
+          - kind: Job
+        name: .*-bk-init
+      manifest: |
+        spec:
+          template:
+            spec:
+              initContainers:
+              - name: pulsar-metadata-cleanup
+                image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
+                    {{ if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+                    export "$(cat conf/pulsar_env.sh | xargs)";
+                    export OPTS="${PULSAR_EXTRA_OPTS} ${EXTRA_OPTS}";
+                    env;
+                    {{- end }}
+                    bin/pulsar-metadata-tool cleanup
+                env:
+                - name: METADATA_TOOL_CONF
+                  value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
+                - name: EXTRA_OPTS
+                  value: {{ .Values.zookeeper.customTools.restore.configData.OPTS }}
+                volumeMounts:
+                  - name: cleanup-config
+                    mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
+              volumes:
+                - name: cleanup-config
+                  configMap:
+                    name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
+  {{- end }}
 {{- end }}
diff --git a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml
@@ -336,30 +336,6 @@ spec:
       PULSAR_PREFIX_managedLedgerOffloadServiceEndpoint: "{{ .Values.broker.offload.azureblob.managedLedgerOffloadServiceEndpoint }}"
       {{- end}}
       {{- end }}
-  initJobPod:
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-    initContainers:
-      - name: cleanup
-        image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
-        command:
-          - bash
-          - '-c'
-        args:
-          - "bin/pulsar-metadata-tool cleanup"
-        env:
-          - name: METADATA_TOOL_CONF
-            value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
-        volumeMounts:
-          - name: cleanup-config
-            mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
-    volumes:
-      - name: cleanup-config
-        configMap:
-          name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
-    {{- end }}
-    {{- if .Values.broker.resources }}
-    resources: {{- toYaml .Values.broker.resources | nindent 10 }}
-    {{- end }}
   {{- if or .Values.tls.broker.enabled .Values.broker.extraVolumeMounts .Values.broker.extraVolumeClaimTemplates }}
   apiObjects:
     statefulSet:

diff --git a/charts/sn-platform-slim/templates/zookeeper/zookeeper-backup-statefulset.yaml b/charts/sn-platform-slim/templates/zookeeper/zookeeper-backup-statefulset.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       securityContext:
       {{- with .Values.zookeeper.securityContext }}
-      {{ toYaml . | indent 8 }}
+      {{ toYaml . | nindent 8 }}
       {{- end }}
       {{- if .Values.zookeeper.customTools.serviceAccount.use }}
       serviceAccountName: {{ template "pulsar.zookeeper.backup.serviceAccount" . }}
@@ -85,7 +85,7 @@ spec:
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.backup.component }}-service"
         image: "{{ .Values.images.zookeeper.customTools.backup.repository }}:{{ .Values.images.zookeeper.customTools.backup.tag }}"
-        imagePullPolicy: {{ .Values.zookeeper.customTools.backup.pullPolicy }}
+        imagePullPolicy: {{ .Values.images.zookeeper.customTools.backup.pullPolicy }}
         command: ["sh", "-c"]
         args:
           - >
@@ -163,6 +163,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.readiness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.readiness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.readiness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.liveness.enabled }}
         livenessProbe:
@@ -172,6 +173,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.liveness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.liveness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.liveness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.liveness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.startup.enabled }}
         startupProbe:
@@ -181,6 +183,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.startup.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.startup.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.startup.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.startup.timeoutSeconds }}
         {{- end }}
         volumeMounts:
         {{- include "pulsar.zookeeper.data.volumeMounts" . | nindent 8 }}

diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml
@@ -126,9 +126,11 @@ images:
       backup:
         repository: "streamnative/pulsar-metadata-tool"
         tag: "3.1.0.5"
+        pullPolicy: IfNotPresent
       restore:
         repository: "streamnative/pulsar-metadata-tool"
         tag: "3.1.0.5"
+        pullPolicy: IfNotPresent
   bookie:
     repository: streamnative/sn-platform-slim
     tag: "3.1.0.5"
@@ -583,16 +585,19 @@ zookeeper:
       failureThreshold: 10
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
     readiness:
       enabled: true
       failureThreshold: 10
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
     startup:
       enabled: false
       failureThreshold: 30
       initialDelaySeconds: 10
       periodSeconds: 30
+      timeoutSeconds: 10
   affinity:
     anti_affinity: true
     zone_anti_affinity: true
@@ -713,6 +718,12 @@ zookeeper:
   service:
     annotations: {}
   customTools:
+  # metadata tool needs access the data directory but the user in the image doesn't have the right
+  # permission to access. Set the running user to root to fix this issue.
+  # Change the securityContext to:
+  # securityContext:
+  #   runAsUser: 0
+  #   runAsNonRoot: false
     serviceAccount:
       use: true
       create: true

diff --git a/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml b/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml
@@ -53,24 +53,6 @@ spec:
     {{- toYaml . | nindent 6 -}}
     {{- end }}
   pod:
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-    initContainers:
-    - name: pulsar-metadata-cleanup
-      image: "{{ .Values.zookeeper.customTools.restore.repository }}:{{ .Values.zookeeper.customTools.restore.tag }}"
-      imagePullPolicy: "{{ .Values.zookeeper.customTools.restore.pullPolicy }}"
-      command: ["sh", "-c"]
-      args:
-        - >
-          bin/pulsar-metadata-tool cleanup
-      env:
-        - name: METADATA_TOOL_CONF
-          value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
-        - name: EXTRA_OPTS
-          value: {{ .Values.zookeeper.customTools.restore.configData.OPTS }}
-      volumeMounts:
-        - name: cleanup-config
-          mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
-    {{- end }}
     {{- with .Values.bookkeeper.topologySpreadConstraints }}
     topologySpreadConstraints:
     {{- toYaml . | nindent 4 }}
@@ -137,17 +119,10 @@ spec:
     {{- include "pulsar.bookkeeper.certs.volumes" . | nindent 4 }}
     {{- include "pulsar.bookkeeper.log.volumes" . | nindent 4 }}
     {{- end }}
-    {{- if and (not .Values.bookkeeper.operator.adopt_existing) (or .Values.zookeeper.customTools.restore.enable .Values.tls.bookie.enabled) }}
+    {{- if and (not .Values.bookkeeper.operator.adopt_existing) .Values.tls.bookie.enabled }}
     volumes:
-    {{- if .Values.tls.bookie.enabled }}
     {{- include "pulsar.bookkeeper.certs.volumes" . | nindent 4 }}
     {{- end }}
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-      - name: cleanup-config
-        configMap:
-          name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
-    {{- end }}
-    {{- end }}
   {{- if and .Values.volumes.persistence .Values.bookkeeper.volumes.persistence}}
   storage:
     journal:
@@ -334,8 +309,43 @@ spec:
     pdb:
       metadata:
         name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
-  {{- with .Values.bookkeeper.customization }}
   customization:
+  {{- with .Values.bookkeeper.customization }}
   {{- toYaml . | nindent 2 -}}
   {{- end }}
+  {{- if .Values.zookeeper.customTools.restore.enable}}
+    - match:
+        groupVersionKinds:
+          - kind: Job
+        name: .*-bk-init
+      manifest: |
+        spec:
+          template:
+            spec:
+              initContainers:
+              - name: pulsar-metadata-cleanup
+                image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    {{- include "pulsar.toolset.zookeeper.tls.settings" . | nindent 12 }}
+                    {{ if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+                    export "$(cat conf/pulsar_env.sh | xargs)";
+                    export OPTS="${PULSAR_EXTRA_OPTS} ${EXTRA_OPTS}";
+                    env;
+                    {{- end }}
+                    bin/pulsar-metadata-tool cleanup
+                env:
+                - name: METADATA_TOOL_CONF
+                  value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
+                - name: EXTRA_OPTS
+                  value: {{ .Values.zookeeper.customTools.restore.configData.OPTS }}
+                volumeMounts:
+                  - name: cleanup-config
+                    mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
+              volumes:
+                - name: cleanup-config
+                  configMap:
+                    name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
+  {{- end }}
 {{- end }}
diff --git a/charts/sn-platform/templates/broker/broker-cluster.yaml b/charts/sn-platform/templates/broker/broker-cluster.yaml
@@ -341,30 +341,6 @@ spec:
       PULSAR_PREFIX_managedLedgerOffloadServiceEndpoint: "{{ .Values.broker.offload.azureblob.managedLedgerOffloadServiceEndpoint }}"
       {{- end}}
       {{- end }}
-  initJobPod:
-    {{- if .Values.zookeeper.customTools.restore.enable }}
-    initContainers:
-      - name: cleanup
-        image: "{{ .Values.images.zookeeper.customTools.restore.repository }}:{{ .Values.images.zookeeper.customTools.restore.tag }}"
-        command:
-          - bash
-          - '-c'
-        args:
-          - "bin/pulsar-metadata-tool cleanup"
-        env:
-          - name: METADATA_TOOL_CONF
-            value: "/pulsar-metadata-tool/conf/pulsar-metadata-tool/pulsar-metadata-tool.properties"
-        volumeMounts:
-          - name: cleanup-config
-            mountPath: /pulsar-metadata-tool/conf/pulsar-metadata-tool
-    volumes:
-      - name: cleanup-config
-        configMap:
-          name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.customTools.restore.component }}"
-    {{- end }}
-    {{- if .Values.broker.resources }}
-    resources: {{- toYaml .Values.broker.resources | nindent 10 }}
-    {{- end }}
   {{- if or .Values.tls.broker.enabled .Values.broker.extraVolumeMounts .Values.broker.extraVolumeClaimTemplates }}
   apiObjects:
     statefulSet:

diff --git a/charts/sn-platform/templates/zookeeper/zookeeper-backup-statefulset.yaml b/charts/sn-platform/templates/zookeeper/zookeeper-backup-statefulset.yaml
@@ -58,7 +58,7 @@ spec:
       {{- end }}
       securityContext:
       {{- with .Values.zookeeper.securityContext }}
-      {{ toYaml . | indent 8 }}
+      {{ toYaml . | nindent 8 }}
       {{- end }}
       {{- if .Values.zookeeper.customTools.serviceAccount.use }}
       serviceAccountName: {{ template "pulsar.zookeeper.backup.serviceAccount" . }}
@@ -163,6 +163,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.readiness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.readiness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.readiness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.liveness.enabled }}
         livenessProbe:
@@ -172,6 +173,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.liveness.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.liveness.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.liveness.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.liveness.timeoutSeconds }}
         {{- end }}
         {{- if .Values.zookeeper.probe.startup.enabled }}
         startupProbe:
@@ -181,6 +183,7 @@ spec:
           initialDelaySeconds: {{ .Values.zookeeper.probe.startup.initialDelaySeconds }}
           periodSeconds: {{ .Values.zookeeper.probe.startup.periodSeconds }}
           failureThreshold: {{ .Values.zookeeper.probe.startup.failureThreshold }}
+          timeoutSeconds: {{ .Values.zookeeper.probe.startup.timeoutSeconds }}
         {{- end }}
         volumeMounts:
         {{- include "pulsar.zookeeper.data.volumeMounts" . | nindent 8 }}