Merge pull request #86 from hotosm/build/pg-upgrade-bash-script

Overhaul postgres upgrade process v14 --> v16

.github/workflows/build_and_deploy.yml

@@ -17,7 +17,6 @@ jobs:
   build:
     name: Build Docker image
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     environment:
       name: ${{ github.ref_name }}
     steps:
@@ -61,7 +60,6 @@ jobs:
 
   deploy_to_vm:
     name: Deploy to VM
-    timeout-minutes: 15
     needs:
       - build
     runs-on: ubuntu-latest
@@ -92,8 +90,16 @@ jobs:
 
       - name: Deploy to VM
         run: |
-          bash contrib/pg-upgrade/upgrade-db.sh
+          # Create db data volume if not exists
+          docker volume create drone-tm-db-data || true
 
+          # Run db upgrade if necessary
+          docker compose --file contrib/pg-upgrade/docker-compose.yml \
+            --env-file .env up --abort-on-container-failure
+
+          # Cleanup db upgrade containers
+          docker compose --file contrib/pg-upgrade/docker-compose.yml down
+ 
           docker compose --file docker-compose.vm.yml --env-file .env up \
             --detach --remove-orphans --force-recreate --pull=always
         env:

contrib/pg-upgrade/Dockerfile

@@ -1,9 +1,37 @@
-FROM tianon/postgres-upgrade:14-to-16
-
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends \
-        "postgresql-14-postgis-3" \
-        "postgresql-16-postgis-3" \
-    && rm -rf /var/lib/apt/lists/*
+FROM docker.io/postgis/postgis:14-3.4-alpine as pg14
+FROM docker.io/postgis/postgis:15-3.4-alpine as pg15
+FROM docker.io/postgis/postgis:16-3.4-alpine as pg16
+FROM docker.io/pgautoupgrade/pgautoupgrade:16-alpine as pgautoupgrade
+
+# Original entrypoint copied overwritten in /usr/local/bin, so copy elsewhere
+RUN cp /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh
+
+# Remove libs for Postgres < v14
+RUN rm -rf \
+    /usr/local-pg9.5 \
+    /usr/local-pg9.6 \
+    /usr/local-pg10 \
+    /usr/local-pg11 \
+    /usr/local-pg12 \
+    /usr/local-pg13
+
+# Copy in PostGIS libs / bins
+COPY --from=pg16 /_pgis*.* /
+COPY --from=pg16 /usr/lib /usr/lib
+
+# Copy extensions for postgresql 16
+COPY --from=pg16 /usr/local /usr/local
+# Copy all extensions for postgresql 15
+COPY --from=pg15 /usr/local/lib/postgresql /usr/local-pg15/lib/postgresql
+# Copy all extensions for postgresql 14
+COPY --from=pg14 /usr/local/lib/postgresql /usr/local-pg14/lib/postgresql
+
+# Squash image to reduce size
+FROM scratch
+COPY --from=pgautoupgrade / /
+ENV \
+    PGTARGET=16 \
+    PGDATA=/var/lib/postgresql/data
+WORKDIR /var/lib/postgresql
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["postgres"]

contrib/pg-upgrade/README.md

@@ -1,16 +1,11 @@
 # Postgres Version Upgrades
 
-- Based on images from https://github.com/tianon/docker-postgres-upgrade
+- Based on images from https://github.com/pgautoupgrade/docker-pgautoupgrade
 - Adds the PostGIS dependency and builds an image for our repo.
 - The image is used for upgrading between containerised Postgres versions.
 
 ```bash
 # From the repo root
-bash contrib/pg-upgrade/upgrade-db.sh
+docker compose --file contrib/pg-upgrade/docker-compose.yml \
+    up --abort-on-container-failure
 ```
-
-This will start the upgrade, wait for completion, then mount
-the data and start the new Postgres 16 container.
-
-> Note it is important to shut down the postgres container first, or
-> a postmaster error will be encountered.

contrib/pg-upgrade/docker-compose.yml

@@ -1,19 +1,20 @@
 # The services run in sequential order, via depends_on
 
 volumes:
-  pg-14-data:
-    name: drone-tm-pg-14-data
-  pg-16-data:
+  db-data:
     external: true
-    name: drone-tm-pg-16-data
+    name: drone-tm-db-data
+
+networks:
+  pg-upgrade:
 
 services:
   # Check if the upgrade has already be complete --> v16
   db-check-upgrade:
     image: postgis/postgis:16-3.4-alpine
     volumes:
-      - pg-16-data:/var/lib/postgresql/data
-    restart: "no"
+      - db-data:/var/lib/postgresql/data
+    restart: no
     entrypoint: /bin/sh -c
     command:
       - |
@@ -36,87 +37,60 @@ services:
   # Only required as we are migrating from filesystem to volume
   db-to-volume:
     image: postgis/postgis:16-3.4-alpine
+    depends_on:
+      db-check-upgrade:
+        condition: service_completed_successfully
     volumes:
-      - ${PROJECT_DIR:-.}/DockerData/dtm_db_data:/var/lib/postgresql/old/data
-      - pg-14-data:/var/lib/postgresql/new/data
-    restart: "no"
+      - ${PROJECT_DIR:-../..}/DockerData/dtm_db_data:/db-data-old
+      - db-data:/db-data-new
+    restart: no
     entrypoint: /bin/sh -c
     command:
       - |
-        rm -rf /var/lib/postgresql/new/data
-        cp -r /var/lib/postgresql/old/data/* /var/lib/postgresql/new/data/
+        rm -rf /db-data-new/*
+        cp -r /db-data-old/* /db-data-new/
         echo 'Copied postgres data to docker volume'
 
-  # Do the actual db upgrade
-  db-upgrade-version:
-    image: ghcr.io/hotosm/drone-tm/pg-upgrade:14-to-16
-    build: contrib/pg-upgrade
+  # Do the db upgrade
+  db-upgrade:
+    image: ghcr.io/hotosm/drone-tm/pg-upgrade:16-3.4-alpine
+    build: .
     depends_on:
       db-to-volume:
         condition: service_completed_successfully
     volumes:
-      - pg-14-data:/var/lib/postgresql/14/data
-      # Volume defined in main docker-compose.yml
-      - pg-16-data:/var/lib/postgresql/16/data
-    env_file: .env
+      - db-data:/var/lib/postgresql/data
+    env_file: ../../.env
     environment:
-      PGUSER: ${POSTGRES_USER}
-      POSTGRES_INITDB_ARGS: -U ${POSTGRES_USER}
-    restart: "no"
+      PGAUTO_ONESHOT: yes
+    restart: no
 
-  # Replace the generated pg_hba.conf access file with the original
-  db-config-hba:
+  # Run maintenance tasks
+  db-maintenance:
     image: postgis/postgis:16-3.4-alpine
     depends_on:
-      db-upgrade-version:
+      db-upgrade:
         condition: service_completed_successfully
     volumes:
-      - pg-14-data:/var/lib/postgresql/14/data
-      - pg-16-data:/var/lib/postgresql/16/data
-    restart: "no"
+      - db-data:/var/lib/postgresql/data
+    env_file: ../../.env
+    restart: no
     entrypoint: /bin/sh -c
     command:
       - |
-        cp -f \
-          /var/lib/postgresql/14/data/pg_hba.conf \
-          /var/lib/postgresql/16/data/
-        echo 'Copied pg_hba.conf to new postgres dir'
+        gosu postgres pg_ctl start -D /var/lib/postgresql/data
 
-  # Start the db so we can run maintenance tasks
-  db-startup:
-    image: postgis/postgis:16-3.4-alpine
-    depends_on:
-      db-config-hba:
-        condition: service_completed_successfully
-    volumes:
-      - pg-16-data:/var/lib/postgresql/data
-    env_file: .env
-    networks:
-      - dtm-network
-    restart: unless-stopped
-    healthcheck:
-      test: pg_isready -U ${POSTGRES_USER:-dtm} -d ${POSTGRES_DB:-dtm_db}
-      start_period: 5s
-      interval: 10s
-      timeout: 5s
-      retries: 3
+        # Upgrade PostGIS extension
+        PGPASSWORD=${POSTGRES_PASSWORD:-dtm} \
+        psql --host=localhost --username=${POSTGRES_USER:-dtm} \
+        ${POSTGRES_DB:-dtm_db} -c '
+          ALTER EXTENSION "postgis" UPDATE;
+        '
 
-  # Run maintenance, db vacuum
-  db-upgrade:
-    image: postgis/postgis:16-3.4-alpine
-    depends_on:
-      db-startup:
-        condition: service_healthy
-    env_file: .env
-    networks:
-      - dtm-network
-    restart: "no"
-    entrypoint: /bin/sh -c
-    command:
-      - |
-        PGPASSWORD=${POSTGRES_PASSWORD} \
+        # Rebuild statistics use vacuum
+        PGPASSWORD=${POSTGRES_PASSWORD:-dtm} \
         vacuumdb \
-        --host=db-startup \
-        --username=${POSTGRES_USER} \
+        --host=localhost \
+        --username=${POSTGRES_USER:-dtm} \
         --all \
         --analyze-in-stages

docker-compose.vm.yml

@@ -1,4 +1,10 @@
-version: "3"
+networks:
+  dtm-network:
+    name: dtm-network
+
+volumes:
+  db-data:
+    name: drone-tm-db-data
 
 services:
   backend:
@@ -24,7 +30,7 @@ services:
   db:
     image: postgis/postgis:16-3.4-alpine
     volumes:
-      - pg-16-data:/var/lib/postgresql/data
+      - db-data:/var/lib/postgresql/data
     env_file: .env
     networks:
       - dtm-network
@@ -76,11 +82,3 @@ services:
       - dtm-network
     entrypoint: ["pdm", "run", "alembic", "upgrade", "head"]
     restart: "no"
-
-networks:
-  dtm-network:
-    name: dtm-network
-
-volumes:
-  pg-16-data:
-    name: drone-tm-pg-16-data

docker-compose.yml

@@ -1,5 +1,14 @@
+networks:
+  dtm-network:
+    name: dtm-network
+
+volumes:
+  db-data:
+    name: drone-tm-db-data
+
 services:
   backend:
+    image: ghcr.io/hotosm/drone-tm/backend:ci
     build:
       context: .
       dockerfile: src/backend/Dockerfile
@@ -31,12 +40,15 @@ services:
       - ${PROJECT_DIR:-.}/src/frontend:/app
       - /var/run/docker.sock:/var/run/docker.sock
 
-  # If error, please upgrade the db `bash contrib/pg-upgrade/upgrade-db.sh`
+  # If error, please upgrade the db with `contrib/pg-upgrade/docker-compose.yml`
   db:
     image: postgis/postgis:16-3.4-alpine
     volumes:
-      - pg-16-data:/var/lib/postgresql/data
+      - db-data:/var/lib/postgresql/data
     env_file: .env
+    environment:
+      LANG: en-GB.utf8
+      # POSTGRES_INITDB_ARGS: "--locale-provider=icu --icu-locale=en-GB"
     networks:
       - dtm-network
     restart: unless-stopped
@@ -90,11 +102,3 @@ services:
       - dtm-network
     entrypoint: ["pdm", "run", "alembic", "upgrade", "head"]
     restart: "no"
-
-networks:
-  dtm-network:
-    name: dtm-network
-
-volumes:
-  pg-16-data:
-    name: drone-tm-pg-16-data