Merge pull request #314 from ropable/master

Skip DFES archived/decommissioned devices, filter hidden devices from view responses, update GitHub workflows, switch to uv as build manager

.github/workflows/image-build-scan.yml

@@ -1,4 +1,4 @@
-name: "Build Docker image and run Trivy vulnerability scan"
+name: 'Build Docker image and run Trivy vulnerability scan'
 
 on:
   push:
@@ -61,6 +61,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -81,14 +82,14 @@ jobs:
         env:
           TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db
         with:
-          scan-type: "image"
-          scanners: "vuln"
+          scan-type: 'image'
+          scanners: 'vuln'
           image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          vuln-type: "os,library"
-          severity: "HIGH,CRITICAL"
-          format: "sarif"
-          output: "trivy-results.sarif"
+          vuln-type: 'os,library'
+          severity: 'HIGH,CRITICAL'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: "trivy-results.sarif"
+          sarif_file: 'trivy-results.sarif'

.github/workflows/run-tests.yml

@@ -1,11 +1,11 @@
-name: "Run unit tests"
+name: 'Run unit tests'
 
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
   workflow_dispatch:
 
 jobs:
@@ -46,34 +46,21 @@ jobs:
         with:
           python-version: '3.12'
       #----------------------------------------------
-      # Install & configure Poetry
+      # Install & configure uv
       #----------------------------------------------
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v5
         with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
+          enable-cache: true
       #----------------------------------------------
-      # Load cached venv if cache exists
+      # Install project dependencies
       #----------------------------------------------
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-      #----------------------------------------------
-      # Install project dependencies if cache does not exist
-      #----------------------------------------------
-      - name: Install project dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
+      - name: Install dependencies
+        run: uv sync
       #----------------------------------------------
       # Run unit tests
       #----------------------------------------------
       - name: Run tests
         run: |
-          source .venv/bin/activate
-          python manage.py collectstatic
-          python manage.py test --noinput --failfast --verbosity 0
+          uv run python manage.py collectstatic
+          uv run python manage.py test --noinput --failfast --verbosity 0

Dockerfile

@@ -1,52 +1,60 @@
 # syntax=docker/dockerfile:1
 # Prepare the base environment.
-FROM python:3.12.8-alpine AS builder_base
+FROM python:3.12-slim-bookworm AS builder_base
+
+# This approximately follows this guide: https://hynek.me/articles/docker-uv/
+# Which creates a standalone environment with the dependencies.
+# - Silence uv complaining about not being able to use hard links,
+# - tell uv to byte-compile packages for faster application startups,
+# - prevent uv from accidentally downloading isolated Python builds,
+# - pick a Python,
+# - and finally declare `/app` as the target for `uv sync`.
+ENV UV_LINK_MODE=copy \
+  UV_COMPILE_BYTECODE=1 \
+  UV_PYTHON_DOWNLOADS=never \
+  UV_PROJECT_ENVIRONMENT=/app/.venv
+
+COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
+
+# Since there's no point in shipping lock files, we move them
+# into a directory that is NOT copied into the runtime image.
+# The trailing slash makes COPY create `/_lock/` automagically.
+COPY pyproject.toml uv.lock /_lock/
+
+# Synchronize dependencies.
+# This layer is cached until uv.lock or pyproject.toml change.
+RUN --mount=type=cache,target=/root/.cache \
+  cd /_lock && \
+  uv sync \
+  --frozen \
+  --no-group dev
+
+##################################################################################
+
+FROM python:3.12-slim-bookworm
 LABEL [email protected]
 LABEL org.opencontainers.image.source=https://github.com/dbca-wa/resource_tracking
 
-# Install system requirements to build Python packages.
-RUN apk add --no-cache \
-  gcc \
-  libressl-dev \
-  musl-dev \
-  libffi-dev
-# Create a non-root user to run the application.
-ARG UID=10001
-ARG GID=10001
-RUN addgroup -g ${GID} appuser \
-  && adduser -H -D -u ${UID} -G appuser appuser
-
-# Install Python libs using Poetry.
-FROM builder_base AS python_libs_resourcetracking
-# Add system dependencies required to use GDAL
-# Ref: https://stackoverflow.com/a/59040511/14508
-RUN apk add --no-cache \
-  gdal \
-  geos \
-  proj \
-  binutils \
-  && ln -s /usr/lib/libproj.so.25 /usr/lib/libproj.so \
-  && ln -s /usr/lib/libgdal.so.36 /usr/lib/libgdal.so \
-  && ln -s /usr/lib/libgeos_c.so.1 /usr/lib/libgeos_c.so
-WORKDIR /app
-COPY poetry.lock pyproject.toml ./
-ARG POETRY_VERSION=1.8.5
-RUN pip install --no-cache-dir --root-user-action=ignore poetry==${POETRY_VERSION} \
-  && poetry config virtualenvs.create false \
-  && poetry install --no-interaction --no-ansi --only main
-# Remove system libraries, no longer required.
-RUN apk del \
-  gcc \
-  libressl-dev \
-  musl-dev \
-  libffi-dev
+# Install OS packages
+RUN apt-get update -y \
+  && apt-get upgrade -y \
+  && apt-get install -y gdal-bin proj-bin libmagic-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user.
+RUN groupadd -r -g 1000 app \
+  && useradd -r -u 1000 -d /app -g app -N app
+
+COPY --from=builder_base --chown=app:app /app /app
+# Make sure we use the virtualenv by default
+ENV PATH="/app/.venv/bin:$PATH"
 
 # Install the project.
-FROM python_libs_resourcetracking AS project_resourcetracking
-COPY gunicorn.py manage.py ./
+WORKDIR /app
+COPY gunicorn.py manage.py pyproject.toml ./
 COPY resource_tracking ./resource_tracking
 COPY tracking ./tracking
 RUN python manage.py collectstatic --noinput
-USER ${UID}
+USER app
 EXPOSE 8080
 CMD ["gunicorn", "resource_tracking.asgi:application", "--config", "gunicorn.py"]

README.md

@@ -6,22 +6,28 @@ variety of sources and aggregates it into a single database.
 ## Installation
 
 The recommended way to set up this project for development is using
-[Poetry](https://python-poetry.org/docs/) to install and manage a virtual Python
-environment. With Poetry installed, change into the project directory and run:
+[uv](https://docs.astral.sh/uv/)
+to install and manage a Python virtual environment.
+With uv installed, install the required Python version (see `pyproject.toml`). Example:
 
-    poetry install
+    uv python install 3.12
+
+Change into the project directory and run:
+
+    uv python pin 3.12
+    uv sync
 
 Activate the virtualenv like so:
 
-    poetry shell
+    source .venv/bin/activate
 
-To run Python commands in the activated virtualenv, thereafter run them as normal:
+To run Python commands in the activated virtualenv, thereafter run them like so:
 
     python manage.py
 
-Manage new or updating project dependencies with Poetry also, like so:
+Manage new or updated project dependencies with uv also, like so:
 
-    poetry add newpackage==1.0
+    uv add newpackage==1.0
 
 ## Environment variables
 
@@ -76,6 +82,6 @@ This project includes the following pre-commit hooks:
 Pre-commit hooks may have additional system dependencies to run. Optionally
 install pre-commit hooks locally like so:
 
-    poetry run pre-commit install
+    pre-commit install
 
 Reference: <https://pre-commit.com/>

kustomize/base/deployment.yaml

@@ -20,22 +20,22 @@ spec:
           image: ghcr.io/dbca-wa/resource_tracking
           env:
             - name: ALLOWED_HOSTS
-              value: ".dbca.wa.gov.au"
+              value: '.dbca.wa.gov.au'
             - name: CSRF_TRUSTED_ORIGINS
-              value: "https://*.dbca.wa.gov.au"
+              value: 'https://*.dbca.wa.gov.au'
             - name: CSRF_COOKIE_SECURE
-              value: "True"
+              value: 'True'
             - name: SESSION_COOKIE_SECURE
-              value: "True"
+              value: 'True'
             - name: TZ
-              value: "Australia/Perth"
+              value: 'Australia/Perth'
           resources:
             requests:
-              memory: "100Mi"
-              cpu: "5m"
+              memory: '100Mi'
+              cpu: '5m'
             limits:
-              memory: "2Gi"
-              cpu: "1000m"
+              memory: '2Gi'
+              cpu: '1000m'
           startupProbe:
             httpGet:
               path: /livez
@@ -66,6 +66,7 @@ spec:
             timeoutSeconds: 5
           securityContext:
             runAsNonRoot: true
+            runAsUser: 1000
             privileged: false
             allowPrivilegeEscalation: false
             capabilities:
@@ -78,6 +79,6 @@ spec:
       volumes:
         - name: tmpfs-ram
           emptyDir:
-            medium: "Memory"
+            medium: 'Memory'
       restartPolicy: Always
       terminationGracePeriodSeconds: 180

kustomize/overlays/prod/kustomization.yaml

@@ -29,4 +29,4 @@ patches:
   - path: service_patch.yaml
 images:
   - name: ghcr.io/dbca-wa/resource_tracking
-    newTag: 1.4.24
+    newTag: 1.4.25

kustomize/template/cronjob.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: resourcetracking-cronjob
 spec:
-  schedule: ""
+  schedule: ''
   concurrencyPolicy: Forbid
   jobTemplate:
     spec:
@@ -18,13 +18,14 @@ spec:
             - name: resourcetracking-cronjob
               image: ghcr.io/dbca-wa/resource_tracking
               imagePullPolicy: Always
-              command: ["python"]
-              args: ["--version"]
+              command: ['python']
+              args: ['--version']
               env:
                 - name: TZ
-                  value: "Australia/Perth"
+                  value: 'Australia/Perth'
               securityContext:
                 runAsNonRoot: true
+                runAsUser: 1000
                 privileged: false
                 allowPrivilegeEscalation: false
                 capabilities: