Install Python with pyenv

This changes the Dockerfile to be a multi-stage build that:
* Uses Ubuntu 22.04 as the base image
* Installs Python 3.9 using pyenv
* Downloads Chrome and Firefox using Selenium Manager.

.github/workflows/test.yml

@@ -16,22 +16,14 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
-      - name: "Get pip cache dir"
-        id: "pip-cache"
-        run: |
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-      - name: "Cache pip packages"
-        uses: "actions/cache@v3"
-        with:
-          path: "${{ steps.pip-cache.outputs.dir }}"
-          key: "${{ runner.os }}-pip-${{ hashFiles('**/base.txt', '**/test.txt') }}"
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: |
+            requirements-dev.txt
       - name: "Install requirements"
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
       - name: "Run test"
         run: |
           ./simplebrowsertest.py
@@ -43,15 +35,17 @@ jobs:
         uses: "actions/checkout@v3"
       - name: "Build containers"
         run: |
-          docker build -t archivematica-acceptance-tests .
+          docker build -t archivematica-acceptance-tests --build-arg TARGET --build-arg PYTHON_VERSION .
         env:
           COMPOSE_DOCKER_CLI_BUILD: 1
           DOCKER_BUILDKIT: 1
+          TARGET: "archivematica-acceptance-tests"
+          PYTHON_VERSION: "3.9"
       - name: "Run test"
         run: |
           docker run \
             --rm \
-            --security-opt="seccomp=etc/docker/seccomp/chrome.json" \
+            --security-opt="seccomp=unconfined" \
             archivematica-acceptance-tests \
             /home/artefactual/acceptance-tests/simplebrowsertest.py
   lint:
@@ -63,7 +57,10 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v4"
         with:
-          python-version: "3.8"
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: |
+            requirements-dev.txt
       - name: "Install tox"
         run: |
           python -m pip install --upgrade pip

.pre-commit-config.yaml

@@ -1,12 +1,22 @@
 repos:
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.10.1
+  hooks:
+  - id: pyupgrade
+    args: [--py38-plus]
+- repo: https://github.com/asottile/reorder_python_imports
+  rev: v3.10.0
+  hooks:
+  - id: reorder-python-imports
+    args: [--py38-plus]
 - repo: https://github.com/psf/black
-  rev: 22.8.0
+  rev: "23.7.0"
   hooks:
   - id: black
     args: [--safe, --quiet]
     language_version: python3
 - repo: https://github.com/pycqa/flake8
-  rev: 5.0.4
+  rev: "6.1.0"
   hooks:
   - id: flake8
     language_version: python3

Dockerfile

@@ -1,118 +1,127 @@
-FROM ubuntu:18.04
+ARG TARGET=archivematica-acceptance-tests
 
-RUN echo "deb http://archive.ubuntu.com/ubuntu bionic main universe\n" > /etc/apt/sources.list \
-	&& echo "deb http://archive.ubuntu.com/ubuntu bionic-updates main universe\n" >> /etc/apt/sources.list \
-	&& echo "deb http://security.ubuntu.com/ubuntu bionic-security main universe\n" >> /etc/apt/sources.list
+ARG UBUNTU_VERSION=22.04
+
+FROM ubuntu:${UBUNTU_VERSION} AS base
+
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG PYTHON_VERSION=3.9
+ARG PYENV_DIR=/pyenv
+ARG SELENIUM_DIR=/selenium
 
 ENV DEBIAN_FRONTEND=noninteractive
-ENV DEBCONF_NONINTERACTIVE_SEEN=true
 
-RUN apt-get -qqy update \
+RUN set -ex \
+	&& apt-get -qqy update \
 	&& apt-get -qqy --no-install-recommends install \
-		gnupg \
-		bzip2 \
 		ca-certificates \
-		tzdata \
-		sudo \
-		unzip \
 		curl \
-		wget \
 		git \
-		build-essential \
-		locales \
-		openssh-client \
-		p7zip-full \
-		python \
-		python3-pip \
-		python3-setuptools \
-		python3-dev \
-		libxml2-dev \
-		libxslt-dev \
-		zlib1g-dev \
 		jq \
-		libnss3 \
-		libgbm1 \
-		libdrm2 \
+		locales \
 	&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
-
-# Set the locale
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-ENV TZ "UTC"
-RUN echo "${TZ}" > /etc/timezone \
-	&& dpkg-reconfigure --frontend noninteractive tzdata
-
-RUN groupadd --gid 333 archivematica \
-	&& useradd --shell /bin/bash --groups sudo,archivematica --create-home artefactual \
-	&& echo 'ALL ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers \
-	&& echo 'artefactual:secret' | chpasswd
-
-#
-# Chrome
-#
-
-ARG CHROME_VERSION="google-chrome-stable"
-RUN CHROME_STABLE_VERSION=$(wget -qO- https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels.Stable.version') \
-	&& CHROME_VERSION=$(if [ ${CHROME_VERSION:-google-chrome-stable} = "google-chrome-stable" ]; then echo $CHROME_STABLE_VERSION; else echo $CHROME_VERSION; fi) \
-	&& CHROME_DOWNLOAD_URL=$(wget -qO- https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r --arg version "$CHROME_VERSION" '.versions[] | select(.version == $version) | .downloads.chrome[] | select(.platform == "linux64") | .url') \
-	&& echo "Using chrome version: "$CHROME_VERSION \
-	&& wget --no-verbose -O /tmp/chrome_linux64.zip $CHROME_DOWNLOAD_URL \
-	&& unzip /tmp/chrome_linux64.zip -d /opt/chrome-$CHROME_VERSION \
-	&& rm /tmp/chrome_linux64.zip \
-	&& chmod 755 /opt/chrome-$CHROME_VERSION/chrome-linux64/chrome \
-	&& sudo ln -fs /opt/chrome-$CHROME_VERSION/chrome-linux64/chrome /usr/bin/google-chrome
-
-ARG CHROME_DRIVER_VERSION="latest"
-RUN CHROMEDRIVER_STABLE_VERSION=$(wget -qO- https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels.Stable.version') \
-	&& CHROMEDRIVER_VERSION=$(if [ ${CHROME_DRIVER_VERSION:-latest} = "latest" ]; then echo $CHROMEDRIVER_STABLE_VERSION; else echo $CHROME_DRIVER_VERSION; fi) \
-	&& CHROMEDRIVER_DOWNLOAD_URL=$(wget -qO- https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r --arg version "$CHROMEDRIVER_VERSION" '.versions[] | select(.version == $version) | .downloads.chromedriver[] | select(.platform == "linux64") | .url') \
-	&& echo "Using chromedriver version: "$CHROMEDRIVER_VERSION \
-	&& wget --no-verbose -O /tmp/chromedriver_linux64.zip $CHROMEDRIVER_DOWNLOAD_URL \
-	&& unzip /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION \
-	&& rm /tmp/chromedriver_linux64.zip \
-	&& chmod 755 /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver-linux64/chromedriver \
-	&& sudo ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver-linux64/chromedriver /usr/bin/chromedriver
-
-#
-# Firefox
-#
-
-ARG FIREFOX_VERSION=94.0.2
-RUN FIREFOX_DOWNLOAD_URL=$(if [ $FIREFOX_VERSION = "latest" ] || [ $FIREFOX_VERSION = "nightly-latest" ] || [ $FIREFOX_VERSION = "devedition-latest" ]; then echo "https://download.mozilla.org/?product=firefox-$FIREFOX_VERSION-ssl&os=linux64&lang=en-US"; else echo "https://download-installer.cdn.mozilla.net/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2"; fi) \
-	&& apt-get update -qqy \
-	&& apt-get -qqy --no-install-recommends install firefox \
-	&& rm -rf /var/lib/apt/lists/* /var/cache/apt/* \
-	&& wget --no-verbose -O /tmp/firefox.tar.bz2 $FIREFOX_DOWNLOAD_URL \
-	&& apt-get -y purge firefox \
-	&& rm -rf /opt/firefox \
-	&& tar -C /opt -xjf /tmp/firefox.tar.bz2 \
-	&& rm /tmp/firefox.tar.bz2 \
-	&& mv /opt/firefox /opt/firefox-$FIREFOX_VERSION \
-	&& ln -fs /opt/firefox-$FIREFOX_VERSION/firefox /usr/bin/firefox
-
-ARG GECKODRIVER_VERSION=0.30.0
-RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo $(wget -qO- "https://api.github.com/repos/mozilla/geckodriver/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([0-9.]+)".*/\1/'); else echo $GECKODRIVER_VERSION; fi) \
-	&& echo "Using GeckoDriver version: "$GK_VERSION \
-	&& wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \
-	&& rm -rf /opt/geckodriver \
-	&& tar -C /opt -zxf /tmp/geckodriver.tar.gz \
-	&& rm /tmp/geckodriver.tar.gz \
-	&& mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \
-	&& chmod 755 /opt/geckodriver-$GK_VERSION \
-	&& ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver
-
-COPY requirements /home/artefactual/acceptance-tests/requirements/
-RUN pip3 install wheel \
-	&& pip3 install -r /home/artefactual/acceptance-tests/requirements/base.txt \
-	&& pip3 install -r /home/artefactual/acceptance-tests/requirements/test.txt
-COPY . /home/artefactual/acceptance-tests
+ENV PYENV_ROOT=${PYENV_DIR}/data
+ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:${SELENIUM_DIR}/bin:$PATH
+
+# -----------------------------------------------------------------------------
+
+FROM base AS browsers-builder
+
+ARG SELENIUM_DIR=/selenium
+
+RUN set -ex \
+	&& SELENIUM_CACHE=${SELENIUM_DIR}/cache \
+	&& SELENIUM_BIN=${SELENIUM_DIR}/bin \
+	&& mkdir -p $SELENIUM_CACHE $SELENIUM_BIN \
+	&& curl -o $SELENIUM_BIN/selenium-manager -L https://github.com/SeleniumHQ/selenium/raw/trunk/common/manager/linux/selenium-manager \
+	&& chmod +x $SELENIUM_BIN/selenium-manager \
+	&& CHROME_OUTPUT=$($SELENIUM_BIN/selenium-manager --cache-path $SELENIUM_CACHE --browser chrome --output JSON) \
+	&& FIREFOX_OUTPUT=$($SELENIUM_BIN/selenium-manager --cache-path $SELENIUM_CACHE --browser firefox --output JSON) \
+	&& ln -s $(echo $CHROME_OUTPUT | jq -r '.result.browser_path') $SELENIUM_BIN/google-chrome \
+	&& ln -s $(echo $CHROME_OUTPUT | jq -r '.result.driver_path') $SELENIUM_BIN/chromedriver \
+	&& ln -s $(echo $FIREFOX_OUTPUT | jq -r '.result.browser_path') $SELENIUM_BIN/firefox \
+	&& ln -s $(echo $FIREFOX_OUTPUT | jq -r '.result.driver_path') $SELENIUM_BIN/geckodriver
+
+# -----------------------------------------------------------------------------
+
+FROM base AS pyenv-builder
+
+ARG PYTHON_VERSION=3.9
+
+RUN set -ex \
+	&& apt-get -qqy update \
+	&& apt-get -qqy --no-install-recommends install \
+		build-essential \
+		libbz2-dev \
+		libffi-dev \
+		liblzma-dev \
+		libncursesw5-dev \
+		libreadline-dev \
+		libsqlite3-dev \
+		libssl-dev \
+		libxml2-dev \
+		libxmlsec1-dev \
+		tk-dev \
+		xz-utils \
+		zlib1g-dev \
+	&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+RUN set -ex \
+	&& curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \
+	&& pyenv install ${PYTHON_VERSION} \
+	&& pyenv global ${PYTHON_VERSION}
+
+COPY requirements-dev.txt requirements-dev.txt
+
+RUN set -ex \
+	&& pyenv exec python${PYTHON_VERSION} -m pip install --upgrade pip setuptools \
+	&& pyenv exec python${PYTHON_VERSION} -m pip install --requirement requirements-dev.txt \
+	&& pyenv rehash
+
+# -----------------------------------------------------------------------------
+
+FROM base AS archivematica-acceptance-tests
+
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+RUN set -ex \
+	&& apt-get -qqy update \
+	&& apt-get -qqy --no-install-recommends install \
+		bzip2 \
+		gnupg \
+		libasound2 \
+		libdbus-glib-1-2 \
+		libdrm2 \
+		libgbm1 \
+		libglib2.0-0 \
+		libgtk-3-0 \
+		libnss3 \
+		libx11-xcb1 \
+		libxcb1 \
+		libxslt-dev \
+		libxtst6 \
+		openssh-client \
+		p7zip-full \
+		tzdata \
+		unzip \
+	&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+COPY --chown=${USER_ID}:${GROUP_ID} --from=browsers-builder --link /selenium /selenium
+COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link /pyenv /pyenv
+COPY --chown=${USER_ID}:${GROUP_ID} --link . /home/artefactual/acceptance-tests
+
+RUN set -ex \
+	&& groupadd --gid ${GROUP_ID} artefactual \
+	&& useradd --uid ${USER_ID} --gid ${GROUP_ID} --create-home artefactual
+
 WORKDIR /home/artefactual/acceptance-tests
-RUN chown -R artefactual:artefactual /home/artefactual
 
 USER artefactual
-ENV HOME /home/artefactual
-ENV USER artefactual

Makefile

@@ -0,0 +1,17 @@
+.PHONY: pip-compile
+pip-compile:  # Compile pip requirements
+	pip-compile --allow-unsafe --output-file requirements.txt requirements.in
+	pip-compile --allow-unsafe --output-file requirements-dev.txt requirements-dev.in
+
+.PHONY: pip-upgrade
+pip-upgrade:  # Upgrade pip requirements
+	pip-compile --allow-unsafe --upgrade --output-file requirements.txt requirements.in
+	pip-compile --allow-unsafe --upgrade --output-file requirements-dev.txt requirements-dev.in
+
+.PHONY: pip-sync
+pip-sync:  # Sync virtualenv
+	pip-sync requirements.txt
+
+.PHONY: pip-sync-dev
+pip-sync-dev:  # Sync dev virtualenv
+	pip-sync requirements-dev.txt

amuser/am_api_ability.py

@@ -3,7 +3,6 @@
 This module contains the ``ArchivematicaAPIAbility`` class, which represents a
 user's ability to use Archivematica's APIs to interact with Archivematica.
 """
-
 import logging
 import os
 import time
@@ -31,8 +30,8 @@ def download_aip(self, transfer_name, sip_uuid, ss_api_key):
                   ?username=<SS-USERNAME>&api_key=<SS-API-KEY>
         """
         payload = {"username": self.ss_username, "api_key": ss_api_key}
-        url = "{}api/v2/file/{}/download/".format(self.ss_url, sip_uuid)
-        aip_name = "{}-{}.7z".format(transfer_name, sip_uuid)
+        url = f"{self.ss_url}api/v2/file/{sip_uuid}/download/"
+        aip_name = f"{transfer_name}-{sip_uuid}.7z"
         aip_path = os.path.join(self.tmp_path, aip_name)
         max_attempts = self.max_download_aip_attempts
         attempt = 0
@@ -62,18 +61,16 @@ def download_aip(self, transfer_name, sip_uuid, ss_api_key):
                     r.status_code,
                     r.text,
                 )
-                raise ArchivematicaAPIAbilityError(
-                    "Unable to download AIP {}".format(sip_uuid)
-                )
+                raise ArchivematicaAPIAbilityError(f"Unable to download AIP {sip_uuid}")
 
     def download_aip_pointer_file(self, sip_uuid, ss_api_key):
         """Use the AM SS API to download the completed AIP's pointer file.
         Calls http://localhost:8000/api/v2/file/<SIP-UUID>/pointer_file/\
                   ?username=<SS-USERNAME>&api_key=<SS-API-KEY>
         """
         payload = {"username": self.ss_username, "api_key": ss_api_key}
-        url = "{}api/v2/file/{}/pointer_file/".format(self.ss_url, sip_uuid)
-        pointer_file_name = "pointer.{}.xml".format(sip_uuid)
+        url = f"{self.ss_url}api/v2/file/{sip_uuid}/pointer_file/"
+        pointer_file_name = f"pointer.{sip_uuid}.xml"
         pointer_file_path = os.path.join(self.tmp_path, pointer_file_name)
         max_attempts = self.max_download_aip_attempts
         attempt = 0
@@ -105,15 +102,15 @@ def download_aip_pointer_file(self, sip_uuid, ss_api_key):
                     r.text,
                 )
                 raise ArchivematicaAPIAbilityError(
-                    "Unable to download AIP {} pointer file".format(sip_uuid)
+                    f"Unable to download AIP {sip_uuid} pointer file"
                 )
 
     def poll_until_aip_stored(
         self, sip_uuid, ss_api_key, poll_interval=1, max_polls=None
     ):
         max_polls = max_polls or self.max_check_aip_stored_attempts
         payload = {"username": self.ss_username, "api_key": ss_api_key}
-        url = "{}api/v2/file/{}/".format(self.ss_url, sip_uuid)
+        url = f"{self.ss_url}api/v2/file/{sip_uuid}/"
         counter = 0
         while True:
             counter += 1