From efb364eecc9a66b39557b64ec61e2d497d3d78f8 Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@penumbralabs.xyz>
Date: Thu, 21 Mar 2024 16:23:35 -0700
Subject: [PATCH] ci: disable readinessProbes in maintenanceMode

While testing upgrades as part of [0], updating the docs in [1],
it became clear that during manual maintenance for munging state during
chain upgrades procedures, we want all the nodes to be reset together.
Therefore if maintenanceMode=true, we'll also disable the
readinessProbes, otherwise the rollout will be staged, forcing us to
upgrade each node in term, rather than being able to parallelize them.
Now, we can `pd export` and backup state, then update the statefulset to bump
the container version (to the post-upgrade version), and restart it in
maintenance mode to continue with the `pd migrate` step and resulting
copying around of emitted files.

[0] https://github.com/penumbra-zone/penumbra/pull/4055
[1] https://github.com/penumbra-zone/penumbra/wiki/Performing-upgrades
---
 .../charts/penumbra-network/templates/statefulset.yaml      | 6 ++++++
 deployments/charts/penumbra-node/templates/statefulset.yaml | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/deployments/charts/penumbra-network/templates/statefulset.yaml b/deployments/charts/penumbra-network/templates/statefulset.yaml
index fe04485102..9325bd4e48 100644
--- a/deployments/charts/penumbra-network/templates/statefulset.yaml
+++ b/deployments/charts/penumbra-network/templates/statefulset.yaml
@@ -97,11 +97,14 @@ spec:
             - name: pd-abci
               containerPort: 26658
               protocol: TCP
+          {{- if .Values.maintenanceMode }}
+          {{- else }}
           readinessProbe:
             tcpSocket:
               port: pd-grpc
             timeoutSeconds: 5
             initialDelaySeconds: 20
+          {{- end }}
           resources:
             {{- toYaml $.Values.resources | nindent 12 }}
           securityContext:
@@ -132,11 +135,14 @@ spec:
             - name: tm-metrics
               containerPort: 26660
               protocol: TCP
+          {{- if .Values.maintenanceMode }}
+          {{- else }}
           readinessProbe:
             tcpSocket:
               port: tm-rpc
             timeoutSeconds: 10
             initialDelaySeconds: 10
+          {{- end }}
           resources:
             {{- toYaml $.Values.resources | nindent 12 }}
           volumeMounts:
diff --git a/deployments/charts/penumbra-node/templates/statefulset.yaml b/deployments/charts/penumbra-node/templates/statefulset.yaml
index 3305005322..af74cd5872 100644
--- a/deployments/charts/penumbra-node/templates/statefulset.yaml
+++ b/deployments/charts/penumbra-node/templates/statefulset.yaml
@@ -208,11 +208,14 @@ spec:
               mountPath: /penumbra-config
           securityContext:
             runAsUser: {{ .Values.maintenanceMode | ternary 0 .Values.securityContext.runAsUser }}
+          {{- if .Values.maintenanceMode }}
+          {{- else }}
           readinessProbe:
             tcpSocket:
               port: pd-grpc
             timeoutSeconds: 10
             initialDelaySeconds: 10
+          {{- end }}
 
         - name: cometbft
           securityContext:
@@ -235,11 +238,14 @@ spec:
             - name: tm-rpc
               containerPort: 26657
               protocol: TCP
+          {{- if .Values.maintenanceMode }}
+          {{- else }}
           readinessProbe:
             tcpSocket:
               port: tm-rpc
             timeoutSeconds: 10
             initialDelaySeconds: 10
+          {{- end }}
           resources:
             {{- toYaml .Values.cometbft.resources | nindent 12 }}
           volumeMounts: