From 8b6aac69f51eed3f0de952afdda23e5a045f3089 Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@penumbralabs.xyz>
Date: Fri, 22 Mar 2024 14:37:32 -0700
Subject: [PATCH] ci: wait for all pods ready post-deploy

Slightly smarter CI logic, which will block until all pods are marked
Ready post-deployment. Due to an oversight, the "part-of" label wasn't
applied to the fullnode pods, so the deploy script exited after the
validators were running, but before the fullnodes were finished setting
up. That was fine, until #3336, which tacked on a subsequent deploy step
that assumes the RPC is ready to rock.

Also updates the statefulsets to deploy the child pods in parallel,
rather than serially, which shaves a few minutes off setup/teardown.
Only really affects preview env, which has frequent deploy churn.
---
 deployments/charts/penumbra-network/templates/statefulset.yaml | 1 +
 deployments/charts/penumbra-node/templates/statefulset.yaml    | 2 ++
 deployments/ci.sh                                              | 3 ++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/deployments/charts/penumbra-network/templates/statefulset.yaml b/deployments/charts/penumbra-network/templates/statefulset.yaml
index 9325bd4e48..4a230fd732 100644
--- a/deployments/charts/penumbra-network/templates/statefulset.yaml
+++ b/deployments/charts/penumbra-network/templates/statefulset.yaml
@@ -14,6 +14,7 @@ metadata:
     app.kubernetes.io/part-of: {{ include "penumbra-network.part_of" $ }}
     {{- include "penumbra-network.labels" $ | nindent 4 }}
 spec:
+  podManagementPolicy: Parallel
   replicas: {{ $count }}
   volumeClaimTemplates:
     - metadata:
diff --git a/deployments/charts/penumbra-node/templates/statefulset.yaml b/deployments/charts/penumbra-node/templates/statefulset.yaml
index 01432ae7e2..5472743bc7 100644
--- a/deployments/charts/penumbra-node/templates/statefulset.yaml
+++ b/deployments/charts/penumbra-node/templates/statefulset.yaml
@@ -11,6 +11,7 @@ metadata:
     "app.kubernetes.io/part-of": {{ .Values.part_of }}
     {{- end }}
 spec:
+  podManagementPolicy: Parallel
   replicas: {{ .Values.nodes | len | int }}
   volumeClaimTemplates:
     - metadata:
@@ -48,6 +49,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
+        {{- include "penumbra-node.labels" . | nindent 8 }}
         {{- include "penumbra-node.selectorLabels" . | nindent 8 }}
     spec:
       # Force the pods to different k8s nodes, so that egress ip is unique per CometBFT node.
diff --git a/deployments/ci.sh b/deployments/ci.sh
index 9ff6f50557..9d46cde767 100755
--- a/deployments/ci.sh
+++ b/deployments/ci.sh
@@ -54,9 +54,10 @@ function helm_install() {
 }
 
 function wait_for_pods_to_be_running() {
-    echo "Waiting for pods to be running..."
+    echo "Waiting for pods to be running ($(date))..."
     kubectl wait --for=condition=ready pods --timeout=5m \
         -l app.kubernetes.io/part-of="$HELM_RELEASE"
+    echo "Done waiting for pods to be running ($(date))"
 }
 
 # Deploy a fresh testnet, destroying all prior chain state with new genesis.