From f12d995a96511643d532a73c72445d8f347f9eff Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@penumbralabs.xyz>
Date: Fri, 7 Jun 2024 08:48:22 -0700
Subject: [PATCH] ci: fix persistence for indexing dbs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Due to an oversight in the volume-mount path, out postgres databases for
indexing ABCI events via CometBFT's psql indexer setup were not
persisting across node restarts. This was due to the fact that the
upstream `postgres` container image declares its own VOLUME at
`/var/lib/postgresql/data`, visible here:

    ❯ podman inspect docker.io/postgres:16-bookworm | jq '.[0].Config.Volumes'
    {
      "/var/lib/postgresql/data": {}
    }

which clobbered our manual mount at `/var/lib/postgresql`, a level up.
We must override the volume mountpoint in order for the PVC to take
precedence over the anonymous volume. Done by adding the fullpath
to that exact mount, and then overriding PGDATA to use a subdir therein.

Also pins a stable version of the container image tag, so we don't get
surprised by postgres jumps.

Refs #4526. Will make sure this applies cleanly automatically on preview
post-merge, then will update the deployments for currently-running
networks. Does not handle historical reingestion; that's tracked
separately in #4566.
---
 .../penumbra-node/templates/statefulset.yaml  | 23 +++++++++++++++----
 deployments/charts/penumbra-node/values.yaml  |  2 +-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/deployments/charts/penumbra-node/templates/statefulset.yaml b/deployments/charts/penumbra-node/templates/statefulset.yaml
index 40f61e8c3a..a64696ed8d 100644
--- a/deployments/charts/penumbra-node/templates/statefulset.yaml
+++ b/deployments/charts/penumbra-node/templates/statefulset.yaml
@@ -30,6 +30,12 @@ spec:
     {{- if eq .Values.cometbft.config.indexer "psql" }}
     - metadata:
         name: db
+        labels:
+          {{- include "penumbra-node.labels" . | nindent 10 }}
+          "app.kubernetes.io/component": fullnode
+          {{- if .Values.part_of }}
+          "app.kubernetes.io/part-of": {{ .Values.part_of }}
+          {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         resources:
@@ -168,14 +174,14 @@ spec:
             - sh
             - -cex
             - |
-              cert_dir="/var/lib/postgresql/certs"
+              cert_dir="/var/lib/postgresql/data/certs"
               mkdir -p "$cert_dir"
               chmod "0750" "$cert_dir"
               cp -v /opt/postgres-certificates/server.crt /opt/postgres-certificates/server.key "$cert_dir"
               chown -R 999:999 "$cert_dir"
           volumeMounts:
             - name: db
-              mountPath: /var/lib/postgresql
+              mountPath: /var/lib/postgresql/data
             - name: db-certificates
               mountPath: /opt/postgres-certificates
       {{- end }}
@@ -277,9 +283,9 @@ spec:
           - -c
           - ssl=on
           - -c
-          - ssl_cert_file=/var/lib/postgresql/certs/server.crt
+          - ssl_cert_file=/var/lib/postgresql/data/certs/server.crt
           - -c
-          - ssl_key_file=/var/lib/postgresql/certs/server.key
+          - ssl_key_file=/var/lib/postgresql/data/certs/server.key
           {{- end }}
           {{- end }}
           ports:
@@ -311,6 +317,10 @@ spec:
             - name: POSTGRES_PASSWORD
               value: penumbra
           {{- end }}
+            # Set a custom PGDATA directory inside the mountpoint, so that other db config
+            # like certificates can persist in the volume, without polluting the data dir.
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
             - name: ORDINAL_NUMBER
               valueFrom:
                 fieldRef:
@@ -330,7 +340,10 @@ spec:
               mountPath: /docker-entrypoint-initdb.d
               readOnly: true
             - name: db
-              mountPath: /var/lib/postgresql
+              # N.B. We mount `/var/lib/postgresql/data` specifically, rather than `/var/lib/postgresql`,
+              # in order to override the internal volume mount used by the Postgres container image.
+              # With the `/data` suffix, db will not persist.
+              mountPath: /var/lib/postgresql/data
       {{ end }}
 
       {{- with .Values.nodeSelector }}
diff --git a/deployments/charts/penumbra-node/values.yaml b/deployments/charts/penumbra-node/values.yaml
index 04878f5496..04c3de7060 100644
--- a/deployments/charts/penumbra-node/values.yaml
+++ b/deployments/charts/penumbra-node/values.yaml
@@ -59,7 +59,7 @@ postgres:
   image:
     repository: docker.io/library/postgres
     pullPolicy: IfNotPresent
-    tag: "latest"
+    tag: "16-bookworm"
   # In order to support TLS for an external db connection, consider reusing a `kubernetes.io/tls`
   # Secret here, which will be mounted read-only and used for SSL. Requires an externally-provisioned LB.
   certificateSecretName: ""