Drop Python 3.6 support

* Update pre-commit dependencies
* Run pre-commit
* Use Python 3.9 in Dockerfile
* Update toxenv
* Update GitHub workflow
* Upgrade requirements
* Fix tests
* Upgrade osdeps
* Add missing rules to jhove migration

.github/workflows/acceptance-test.yml

@@ -43,6 +43,8 @@ jobs:
       - name: "Build images"
         run: |
           make -C hack/ build
+        env:
+          PYTHON_VERSION: "3.9"
       - name: "Start services"
         run: |
           docker compose up -d

.github/workflows/test.yml

@@ -8,42 +8,24 @@ on:
       - "stable/**"
 jobs:
   test:
-    name: "Test ${{ matrix.rule }}"
+    name: "Test ${{ matrix.rule }} on ${{ matrix.python-version }}"
     runs-on: "ubuntu-22.04"
     strategy:
       fail-fast: false
       matrix:
+        rule: ["mcp-server", "mcp-client", "dashboard", "archivematica-common"]
+        python-version: ["3.8", "3.9", "3.10"]
         include:
-          - rule: mcp-server
-            coverage: true
-          - rule: mcp-client
-            coverage: true
-          - rule: dashboard
-            coverage: true
-          - rule: archivematica-common
-            coverage: true
-          - rule: storage-service
-            coverage: false
-          - rule: migrations
-            coverage: false
+          - rule: "storage-service"
+            python-version: "3.9"
+          - rule: "migrations"
+            python-version: "3.9"
     steps:
       - name: "Check out repository"
         uses: "actions/checkout@v3"
       - name: "Check out the archivematica-storage-service submodule"
         run: |
           git submodule update --init hack/submodules/archivematica-storage-service/
-      - name: "Set up pip cache"
-        uses: "actions/cache@v3"
-        with:
-          path: "~/.cache/pip"
-          key: "${{ runner.os }}-pip-${{ github.sha }}"
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: "Set up tox cache"
-        uses: "actions/cache@v3"
-        with:
-          path: ./.tox
-          key: "${{ runner.os }}-tox-${{ matrix.rule }}-${{ hashFiles('**/tox.ini', '**/requirements*.txt', '**/requirements/*.txt') }}"
       - name: "Set up buildx"
         uses: "docker/setup-buildx-action@v2"
         id: buildx
@@ -53,16 +35,19 @@ jobs:
         uses: "actions/cache@v3"
         with:
           path: /tmp/.docker-cache-old
-          key: ${{ runner.os }}-docker-${{ matrix.rule }}-${{ github.sha }}
+          key: ${{ runner.os }}-docker-${{ matrix.python-version }}-${{ matrix.rule }}-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-docker-${{ matrix.rule }}-
+            ${{ runner.os }}-docker-${{ matrix.python-version }}-${{ matrix.rule }}-
+            ${{ runner.os }}-docker-${{ matrix.python-version }}-
             ${{ runner.os }}-docker-
       - name: "Build archivematica-tests image"
         uses: "docker/build-push-action@v4"
         with:
           context: .
           file: ./hack/Dockerfile
           target: archivematica-tests
+          build-args: |
+            PYTHON_VERSION=${{ matrix.python-version }}
           tags: archivematica-tests:latest
           push: false
           load: true
@@ -72,25 +57,18 @@ jobs:
         run: |
           make -C hack/ create-volumes
       - name: "Run make rule"
-        if: "! matrix.coverage"
-        run: |
-          make -C hack/ test-${{ matrix.rule }}
-        env:
-          TOXARGS: -vv
-          PYTEST_ADDOPTS: -vv
-      - name: "Run make rule with coverage"
-        if: "matrix.coverage"
         run: |
           make -C hack/ test-${{ matrix.rule }}
         env:
           TOXARGS: -vv
           PYTEST_ADDOPTS: -vv --cov /src/src/ --cov-config=/src/.coveragerc --cov-report xml:/src/coverage.xml
+          PYTHON_VERSION: ${{ matrix.python-version }}
       - name: "Upload coverage report"
-        if: matrix.coverage && github.repository == 'artefactual/archivematica'
+        if: matrix.rule != 'storage-service' && matrix.rule != 'migrations' && github.repository == 'artefactual/archivematica'
         uses: "codecov/codecov-action@v3"
         with:
           files: ./coverage.xml
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           verbose: true
           name: ${{ matrix.rule }}
       - name: "Set newest docker cache"
@@ -129,10 +107,13 @@ jobs:
     steps:
       - name: "Check out repository"
         uses: "actions/checkout@v3"
-      - name: "Set up Python 3.8"
+      - name: "Set up Python 3.9"
         uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: |
+            requirements-dev.txt
       - name: "Install tox"
         run: |
           python -m pip install --upgrade pip

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.5.0
+  rev: v4.4.0
   hooks:
   - id: pretty-format-json
     args: [--no-ensure-ascii, --autofix]
@@ -10,28 +10,28 @@ repos:
         src/(MCPClient/MCPServer|dashboard)/osdeps/.*\.json
       )
 - repo: https://github.com/asottile/pyupgrade
-  rev: v2.31.0
+  rev: v3.10.1
   hooks:
   - id: pyupgrade
-    args: [--py3-plus, --py36-plus]
+    args: [--py38-plus]
 - repo: https://github.com/asottile/reorder_python_imports
-  rev: v2.6.0
+  rev: v3.10.0
   hooks:
   - id: reorder-python-imports
-    args: [--py3-plus, --py36-plus]
+    args: [--py38-plus]
 - repo: https://github.com/adamchainz/django-upgrade
-  rev: "1.14.0"
+  rev: "1.14.1"
   hooks:
   - id: django-upgrade
     args: [--target-version, "3.2"]
-- repo: https://github.com/ambv/black
-  rev: 22.8.0
+- repo: https://github.com/psf/black
+  rev: "23.7.0"
   hooks:
   - id: black
     args: [--safe, --quiet]
     language_version: python3
 - repo: https://github.com/pycqa/flake8
-  rev: 5.0.4
+  rev: "6.1.0"
   hooks:
   - id: flake8
     language_version: python3

hack/Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:22.04 AS base
 
 ARG USER_ID=1000
 ARG GROUP_ID=1000
-ARG PYTHON_VERSION=3.6
+ARG PYTHON_VERSION=3.9
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV PYTHONUNBUFFERED 1
@@ -54,7 +54,7 @@ RUN set -ex \
 	&& apt-get update \
 	&& apt-get install -y --no-install-recommends \
 		libyaml-dev clamav \
-	&& /src/hack/osdeps.py Ubuntu-22 1 | grep -v -E "nginx|postfix|python3.6-dev" | xargs apt-get install -y --no-install-recommends \
+	&& /src/hack/osdeps.py Ubuntu-22 1 | grep -v -E "nginx|postfix|python3.9-dev" | xargs apt-get install -y --no-install-recommends \
 	&& rm -rf /var/lib/apt/lists/*
 
 # Download ClamAV virus signatures
@@ -104,7 +104,7 @@ USER archivematica
 
 FROM base AS archivematica-mcp-client
 
-ARG PYTHON_VERSION=3.6
+ARG PYTHON_VERSION=3.9
 
 # Some scripts in archivematica-fpr-admin executed by MCPClient rely on certain
 # files being available in this image (e.g. see https://git.io/vA1wF).
@@ -123,7 +123,7 @@ ENTRYPOINT pyenv exec python${PYTHON_VERSION} /src/src/MCPClient/lib/archivemati
 
 FROM base AS archivematica-mcp-server
 
-ARG PYTHON_VERSION=3.6
+ARG PYTHON_VERSION=3.9
 
 ENV DJANGO_SETTINGS_MODULE settings.common
 ENV PYTHONPATH /src/src/MCPServer/lib/:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/
@@ -134,7 +134,7 @@ ENTRYPOINT pyenv exec python${PYTHON_VERSION} /src/src/MCPServer/lib/archivemati
 
 FROM base AS archivematica-dashboard
 
-ARG PYTHON_VERSION=3.6
+ARG PYTHON_VERSION=3.9
 
 USER root
 

hack/Makefile

@@ -11,6 +11,7 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../src)
 
 NULL :=
 SPACE := $(NULL) $(NULL)
+COMMA := ,
 
 define compose_all
 	docker compose -f docker-compose.yml -f docker-compose.acceptance-tests.yml -f docker-compose.pmm.yml $(1)
@@ -34,7 +35,7 @@ define run_toxenvs
 			--rm \
 			--entrypoint tox \
 				archivematica-tests \
-					$(subst $(SPACE), -e ,$(SPACE)$(1)) \
+					-e $(subst $(SPACE),$(COMMA),$(strip $(1))) \
 					${TOXARGS})
 endef
 
@@ -235,6 +236,7 @@ test-build:  ## Build archivematica-tests image.
 		-t archivematica-tests \
 		-f $(CURDIR)/Dockerfile \
 		--build-arg TARGET=archivematica-tests \
+		--build-arg PYTHON_VERSION=${PYTHON_VERSION} \
 			../
 
 __TOXENVS_MCPSERVER := mcp-server

hack/docker-compose.tests.yml

@@ -8,6 +8,7 @@ services:
       dockerfile: "hack/Dockerfile"
       args:
         TARGET: "archivematica-tests"
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
     volumes:
       - "../:/src"
     links:

hack/docker-compose.yml

@@ -109,7 +109,7 @@ services:
         TARGET: "archivematica-mcp-server"
         USER_ID: ${USER_ID:-1000}
         GROUP_ID: ${GROUP_ID:-1000}
-        PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
     environment:
       DJANGO_SECRET_KEY: "12345"
       DJANGO_SETTINGS_MODULE: "settings.common"
@@ -136,7 +136,7 @@ services:
         TARGET: "archivematica-mcp-client"
         USER_ID: ${USER_ID:-1000}
         GROUP_ID: ${GROUP_ID:-1000}
-        PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
     environment:
       DJANGO_SECRET_KEY: "12345"
       DJANGO_SETTINGS_MODULE: "settings.common"
@@ -178,7 +178,7 @@ services:
         TARGET: "archivematica-dashboard"
         USER_ID: ${USER_ID:-1000}
         GROUP_ID: ${GROUP_ID:-1000}
-        PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
     environment:
       FORWARDED_ALLOW_IPS: "*"
       AM_GUNICORN_ACCESSLOG: "/dev/null"
@@ -213,7 +213,7 @@ services:
         TARGET: "archivematica-storage-service"
         USER_ID: ${USER_ID:-1000}
         GROUP_ID: ${GROUP_ID:-1000}
-        PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
     environment:
       FORWARDED_ALLOW_IPS: "*"
       SS_GUNICORN_ACCESSLOG: "/dev/null"