CI/Dockerfile: port 1.8 changes to 1.9

.github/workflows/complementary-config-test.yaml

@@ -56,11 +56,12 @@ jobs:
 
       - name: Config parser check
         run: |
+          export LOCAL_UID=$(id -u $USER)
+          export LOCAL_GID=$(id -g $USER)
           cd ./datacube-ows
           export $(grep -v '^#' ./complementary_config_test/.env_complementary_config_dea_dev | xargs)
-          docker-compose -f docker-compose.yaml -f docker-compose.cleandb.yaml up -d
-          docker-compose -f docker-compose.yaml -f docker-compose.cleandb.yaml exec -T ows /bin/sh -c "cd /code; sh ./docker/ows/wait-for-db"
-          docker-compose -f docker-compose.yaml -f docker-compose.cleandb.yaml exec -T ows /bin/sh -c "datacube system init; datacube system check"
-          docker-compose -f docker-compose.yaml -f docker-compose.cleandb.yaml exec -T ows /bin/sh -c "curl https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/dev/services/wms/inventory.json -o /tmp/inventory.json"
-          docker-compose -f docker-compose.yaml -f docker-compose.db.yaml exec -T ows /bin/sh -c "cd /code; ./compare-cfg.sh"
-          docker-compose -f docker-compose.yaml -f docker-compose.cleandb.yaml down
+          docker compose -f docker-compose.yaml -f docker-compose.cleandb.yaml up -d --wait
+          docker compose -f docker-compose.yaml -f docker-compose.cleandb.yaml exec -T ows /bin/sh -c "datacube system init; datacube system check"
+          docker compose -f docker-compose.yaml -f docker-compose.cleandb.yaml exec -T ows /bin/sh -c "curl https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/dev/services/wms/inventory.json -o /tmp/inventory.json"
+          docker compose -f docker-compose.yaml -f docker-compose.db.yaml exec -T ows /bin/sh -c "cd /code; ./compare-cfg.sh"
+          docker compose -f docker-compose.yaml -f docker-compose.cleandb.yaml down

.github/workflows/pyspy-profiling.yaml

@@ -40,8 +40,10 @@ jobs:
     # Run performance profiling
     - name: setup performance profiling with py-spy (stage 1 - run profiling containers)
       run: |
+        export LOCAL_UID=$(id -u $USER)
+        export LOCAL_GID=$(id -g $USER)
         export $(grep -v '^#' .env_simple | xargs)
-        docker-compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml up -d
+        docker compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml up -d
 
     - name: Sleep for 10 seconds (stage 1 - wait for services to be ready)
       uses: whatnick/wait-action@master
@@ -51,6 +53,8 @@ jobs:
     - name: set output container pid (stage 1 - get ows container pid)
       id: set-output-container-id
       run: |
+        export LOCAL_UID=$(id -u $USER)
+        export LOCAL_GID=$(id -g $USER)
         export $(grep -v '^#' .env_simple | xargs)
         echo "::set-output name=PID::$(docker inspect --format '{{.State.Pid}}' $(docker inspect -f '{{.Name}}' \
         $(docker-compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml ps -q ows) \
@@ -60,17 +64,19 @@ jobs:
       timeout-minutes: 1
       continue-on-error: true
       run: |
+        export LOCAL_UID=$(id -u $USER)
+        export LOCAL_GID=$(id -g $USER)
         export $(grep -v '^#' .env_simple | xargs)
-        docker-compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml \
+        docker compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml \
         exec -T ows /bin/sh -c "cd /code;./test_urls.sh &"
-        docker-compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml \
+        docker compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml \
         run pyspy record -f speedscope -o ./artifacts/profile.json --duration 30 \
         --pid ${{steps.set-output-container-id.outputs.PID}} --subprocesses
 
     - name: Stop py-spy profiling after timeout (stage 1 - stop profiling)
       run: |
         export $(grep -v '^#' .env_simple | xargs)
-        docker-compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml down
+        docker compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.pyspy.yaml down
 
     - name: Upload profile to artifact (stage 1 - Upload profiling svg to artifacts)
       uses: actions/upload-artifact@v4

.github/workflows/test-prod.yaml

@@ -54,13 +54,10 @@ jobs:
       # Build prod image and tag as latest, connect to pre-indexed database
       - name: Build and run prod OWS images (stage 2)
         run: |
+          export LOCAL_UID=$(id -u $USER)
+          export LOCAL_GID=$(id -g $USER)
           export $(grep -v '^#' .env_simple | xargs)
-          docker-compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.prod.yaml up -d
-
-      - name: Sleep for 10 seconds
-        uses: whatnick/wait-action@master
-        with:
-          time: '10s'
+          docker compose -f docker-compose.yaml -f docker-compose.db.yaml -f docker-compose.prod.yaml up -d --wait
 
       # Run some tests on the images
       # These tests require a working database

.github/workflows/test.yml

@@ -55,16 +55,17 @@ jobs:
       - name: Test and lint dev OWS image
         run: |
           mkdir artifacts
-          docker run -v ${PWD}/artifacts:/mnt/artifacts ${ORG}/${IMAGE}:_builder /bin/sh -c "cd /code;./check-code.sh"
+          docker run -e LOCAL_UID=$(id -u $USER) -e LOCAL_GID=$(id -g $USER) -v ${PWD}/artifacts:/mnt/artifacts ${ORG}/${IMAGE}:_builder /bin/sh -c "cd /code;./check-code.sh"
           mv ./artifacts/coverage.xml ./artifacts/coverage-unit.xml
 
       - name: Dockerized Integration Pytest
         run: |
-          chmod a+rw artifacts
+          export LOCAL_UID=$(id -u $USER)
+          export LOCAL_GID=$(id -g $USER)
           export $(grep -v '^#' .env_simple | xargs)
-          docker-compose -f docker-compose.yaml -f docker-compose.db.yaml up -d
-          docker-compose -f docker-compose.yaml -f docker-compose.db.yaml exec -T ows /bin/sh -c "cd /code;./check-code-all.sh"
-          docker-compose -f docker-compose.yaml -f docker-compose.db.yaml down
+          docker compose -f docker-compose.yaml -f docker-compose.db.yaml up -d --wait --build
+          docker compose -f docker-compose.yaml -f docker-compose.db.yaml exec -T ows /bin/sh -c "cd /code;./check-code-all.sh"
+          docker compose -f docker-compose.yaml -f docker-compose.db.yaml down
 
       - name: Upload All coverage to Codecov
         uses: codecov/codecov-action@v4

Dockerfile

@@ -38,6 +38,16 @@ RUN pip freeze
 # Should match builder base.
 FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.5
 
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+            gosu \
+            tini \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/dpkg/* /var/tmp/* /var/log/dpkg.log
+
+# Add login-script for UID/GID-remapping.
+COPY --chown=root:root --link docker/files/remap-user.sh /usr/local/bin/remap-user.sh
+
 # all the python pip installed libraries
 COPY --from=builder  /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages
 COPY --from=builder  /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
@@ -68,7 +78,7 @@ WORKDIR /code
 COPY . /code
 
 # Configure user
-RUN useradd -m -s /bin/bash -N -g 100 -u 1001 ows
+RUN useradd -m -s /bin/bash ows
 WORKDIR "/home/ows"
 
 ENV GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR" \
@@ -78,5 +88,5 @@ ENV GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR" \
 
 RUN chown 1000:100 /dev/shm
 
-USER ows
+ENTRYPOINT ["/usr/local/bin/remap-user.sh"]
 CMD ["gunicorn", "-b", "0.0.0.0:8000", "--workers=3", "--threads=2", "-k", "gevent", "--timeout", "121", "--pid", "/home/ows/gunicorn.pid", "--log-level", "info", "--worker-tmp-dir", "/dev/shm", "--config", "python:datacube_ows.gunicorn_config", "datacube_ows.wsgi"]

check-code-all.sh

@@ -2,9 +2,6 @@
 # Convenience script for running Travis-like checks.
 set -ex
 
-# ensure db is ready
-sh ./docker/ows/wait-for-db
-
 # Initialise ODC schemas
 
 datacube system init

docker-compose.cleandb.yaml

@@ -1,19 +1,26 @@
-version: '3.4'
-
 services:
   postgres:
     # clean postgis db
     image: kartoza/postgis:16
+    hostname: postgres
     environment:
       - POSTGRES_DB=${DB_DATABASE}
       - POSTGRES_PASSWORD=${DB_PASSWORD}
       - POSTGRES_USER=${DB_USERNAME}
     ports:
       - "${DB_PORT}:5432"
     restart: always
+    healthcheck:
+      test: ["CMD", "pg_isready", "-h", "postgres", "-q", "-d", "$POSTGRES_DB", "-U", "$POSTGRES_USER"]
+      timeout: 45s
+      interval: 10s
+      retries: 10
   # Overwrite ows so it can talk to docker db
   ows:
     ports:
       - 8000:8000
     environment:
       DB_PORT: 5432
+    depends_on:
+      postgres:
+        condition: service_healthy

docker-compose.db.yaml

@@ -1,17 +1,24 @@
-version: '3.4'
-
 services:
   postgres:
     # db
     build: docker/database/
+    hostname: postgres
     environment:
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_USER=${POSTGRES_USER}
     ports:
       - "${POSTGRES_PORT}:${POSTGRES_PORT}"
     restart: always
+    healthcheck:
+      test: ["CMD", "pg_isready", "-h", "postgres", "-q", "-d", "$POSTGRES_DB", "-U", "$POSTGRES_USER"]
+      timeout: 45s
+      interval: 10s
+      retries: 10
   # Overwrite ows so it can talk to docker db
   ows:
     ports:
       - 8000:8000
+    depends_on:
+      postgres:
+        condition: service_healthy

docker-compose.index.yaml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 services:
   index:
     image: opendatacube/datacube-index:latest
@@ -11,5 +9,6 @@ services:
       DB_DATABASE: ${DB_DATABASE}
     restart: always
     depends_on:
-      - postgres
+      postgres:
+        condition: service_healthy
     command: tail -f /dev/null

docker-compose.prod.yaml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 # override default compose to change the launch command
 
 services:

docker-compose.pyspy.yaml

@@ -1,9 +1,10 @@
-version: '3.4'
-
 services:
   ows:
     cap_add:
       - SYS_PTRACE
+    depends_on:
+      postgres:
+        condition: service_healthy
 
   pyspy:
     build: docker/pyspy/

docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 # Default compose will create an ows image, with dev settings and connect to a local db
 services:
   ows:
@@ -13,6 +11,8 @@ services:
     # Uncomment for use with non-dockerised postgres (for docker-compose 1.x)
     # network_mode: host
     environment:
+      LOCAL_UID: ${LOCAL_UID:-1000}
+      LOCAL_GID: ${LOCAL_GID:-1000}
       # Defaults are defined in .env file
       AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
       AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
@@ -45,8 +45,7 @@ services:
       SENTRY_DSN: "${SENTRY_DSN}"
     volumes:
       - ${OWS_CFG_DIR}:${OWS_CFG_MOUNT_DIR}
-      - ./docker/ows/wait-for-db:/usr/local/bin/wait-for-db
       - ./:/code/
       - ./artifacts:/mnt/artifacts
     restart: always
-    command: ["wait-for-db", "flask", "run", "--host=0.0.0.0", "--port=8000"]
+    command: ["flask", "run", "--host=0.0.0.0", "--port=8000"]

docker/files/remap-user.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+# Script that gives the container user uid $LOCAL_UID and gid $LOCAL_GID.
+# If $LOCAL_UID or $LOCAL_GID are not set, they default to 1000 (default
+# for the first user created in Ubuntu).
+
+USER_ID=${LOCAL_UID:-1000}
+GROUP_ID=${LOCAL_GID:-1000}
+
+[[ "$USER_ID" == "1000" ]] || usermod -u $USER_ID -o -m -d /home/ows ows
+[[ "$GROUP_ID" == "1000" ]] || groupmod -g $GROUP_ID ows
+[[ $(id -u) != "0" ]] || GOSU="/usr/sbin/gosu ows"
+exec /usr/bin/tini -- $GOSU "$@"