Upgrade python to 3.12 and upgrade related packages; Bug fix failing mime type tests

.dockerignore

@@ -0,0 +1,2 @@
+deploy/Dockerfile
+deploy/Dockerfile-rhel

.github/workflows/check-python-dependcies.yml

@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.12'
         architecture: 'x64'
     - name: Run checks for package updates
       run: bash tests/run-update-checks.sh

.github/workflows/docker.yml

@@ -10,7 +10,7 @@ on:
       # TODO: Review scope of GitHub action grows
       - '.github/workflows/docker.yml'
       - 'deploy/Dockerfile'
-      - 'entrypoints/**'
+      # - 'entrypoints/**'
       - 'requirements/**'
   # schedule:
   #   - cron: 30 9 * * WED

.github/workflows/run-django-test.yml

@@ -27,7 +27,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.12'
         architecture: 'x64'
     - name: Run Python 3/Django tests
       run: bash tests/run-django-tests.sh

.github/workflows/run-fabric-test.yml

@@ -16,7 +16,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.12'
         architecture: 'x64'
     - name: Syntax check Python 3 Fabric deployment script
       run: bash tests/run-fabric-tests.sh

CHANGELOG

@@ -1,6 +1,12 @@
 # Changelog
 
-9.8.2   TMMA-508: Resolve issue in tests after RHEL migration
+9.9.0   TMMA-511: Upgrade Python from 3.8.x to 3.12.x
+        TMMA-508: Resolve issue in tests after RHEL migration
+        Apply MySQL Connector/Python connector security update
+        TMMA-347: Set up RHEL8 development docker environment
+        Update packages used to connect to redis
+        Update Python dependencies
+        Update version of MySQL used in GitHub actions
 
 9.8.1   TMMA-509: Improve security of Python packaging tool
         Update Python packaging tool

README.md

@@ -9,8 +9,8 @@
 
 ### Prerequisites
 
-* Vagrant https://www.vagrantup.com/
-* VirtualBox https://www.virtualbox.org/ or another provider, see https://www.vagrantup.com/docs/providers/
+* Vagrant [https://www.vagrantup.com/]
+* VirtualBox [https://www.virtualbox.org/] or another provider, see [https://www.vagrantup.com/docs/providers/]
 NB: The vagrant installation also requires an additional plugin to mount the development source code cloned on your local machine.
 
 Tested with these versions:
@@ -131,23 +131,25 @@ NB: If you want to manually run migrations you need to use the --database flag
     python manage.py migrate --database=admin --settings=temmpo.settings.dev
 
 #### Updating the requirements file using pip-tools (via Vagrant VM)
+
 NB: This can take a while as we also generate hashes for additional security.
 
     fab pip_sync_requirements_file:env=dev,use_local_mode=True -f /usr/local/projects/temmpo/lib/dev/src/temmpo/deploy/fabfile.py
 
-#### Upgrading the requirements file using pip-tools (via Vagrant VM)
-NB: This can take a while as we also generate hashes for additional security.
-
 Optionally pass in a package or update them all within any requirements.in file constraints
 
     fab pip_tools_update_requirements:env=dev,use_local_mode=True,package="" -f /usr/local/projects/temmpo/lib/dev/src/temmpo/deploy/fabfile.py
 
-
-Alternatively
+Alternatively a Python (Debian) base docker image to update requirements
 
     docker build -f deploy/Dockerfile -t temmpo-web .
     docker run --rm -it -v $PWD:/srv -w /srv temmpo-web bash /srv/entrypoints/update-requirements.sh
 
+Alternatively build a RHEL base docker image to update requirements
+
+    docker compose build web
+    docker compose run --rm --no-deps web bash ./entrypoints/update-requirements.sh
+
 #### Create Docker images for different environments
 
     docker build -f deploy/Dockerfile -t temmpo-web-dev . --build-arg REQUIREMENTS_FILE=dev.txt

browser/fields.py

@@ -26,6 +26,10 @@ def _create_upload_file_from_path(self, file_path):
     def to_python(self, value):
         value = super(ExtractorFileField, self).to_python(value)
         mime_type = magic.from_buffer(value.read(1024), mime=True)
+
+        logger.debug("DEBUG: temp file path %s" % value.temporary_file_path())
+        logger.debug("DEBUG: mime %s" % mime_type)
+
         if mime_type in ('application/gzip', 'application/x-gzip', 'application/bzip', 'application/bzip2', 'application/x-bzip', 'application/x-bzip2'):
             try:
                 extracted_file_path = xtract(value.temporary_file_path())

browser/models.py

@@ -22,9 +22,8 @@
 
 def get_user_upload_location(instance, filename):
     """Based on slugify code - from django.utils.text import slugify."""
-    # TODO Test changes explictly
-    filename = re.sub('[^\.\w\s-]', '', filename).strip().lower()
-    filename = re.sub('[-\s]+', '-', filename)
+    filename = re.sub(r'[^\.\w\s-]', '', filename).strip().lower()
+    filename = re.sub(r'[-\s]+', '-', filename)
 
     return timezone.now().strftime('/'.join(['abstracts', str(instance.user.id), '%Y-%m-%d', '%H-%M-%S-' + filename]))
 

browser/probe.py

@@ -57,7 +57,7 @@ def check(self, components, label='', function=None, max_duration=5, allow_fail=
             success = function()
             end = time.time()
             if end - start > max_duration:
-                logger.warn(
+                logger.warning(
                     "Check timestamp exceeded for '%s' (%s took longer than %s)" % (label, end - start, max_duration))
         except Exception as e:
             end = time.time()
@@ -99,7 +99,7 @@ def probe_email(self):
             return True
 
     def python_version(self):
-        if sys.version_info.major == 3 and sys.version_info.minor == 8:
+        if sys.version_info.major == 3 and sys.version_info.minor == 12:
             return True
         else:
             return False

browser/validators.py

@@ -1,18 +1,19 @@
 # -*- coding: utf-8 -*-
 import logging
-import magic
 import re
 
+import magic
+
 from django.core.exceptions import ValidationError
 
 logger = logging.getLogger(__name__)
 
-OVID_MEDLINE_IDENTIFIER_PATTERN = re.compile(b"^<\d+>")
-OVID_MEDLINE_IDENTIFIER_LABEL_PATTERN = re.compile(b"^Unique Identifier")
-OVID_MEDLINE_IDENTIFIER_MESH_PATTERN = re.compile(b"^MeSH Subject Headings")
+OVID_MEDLINE_IDENTIFIER_PATTERN = re.compile(rb"^<\d+>")
+OVID_MEDLINE_IDENTIFIER_LABEL_PATTERN = re.compile(rb"^Unique Identifier")
+OVID_MEDLINE_IDENTIFIER_MESH_PATTERN = re.compile(rb"^MeSH Subject Headings")
 
-PUBMED_IDENTIFIER_PATTERN = re.compile(b"^PMID- \d+")
-PUBMED_IDENTIFIER_MH_PATTERN = re.compile(b"^MH  -")
+PUBMED_IDENTIFIER_PATTERN = re.compile(rb"^PMID- \d+")
+PUBMED_IDENTIFIER_MH_PATTERN = re.compile(rb"^MH  -")
 
 
 class MimetypeValidator(object):
@@ -25,6 +26,8 @@ def __init__(self, mimetypes):
     def __call__(self, value):
         try:
             mime = magic.from_buffer(value.read(1024), mime=True)
+            logger.debug("DEBUG: temp file path %s" % value.temporary_file_path())
+            logger.debug("DEBUG: mime %s" % mime)
             if mime not in self.mimetypes:
                 raise ValidationError('%s is not an acceptable file type. Please use a %s formatted file instead.' % (value, ' or '.join(self.mimetypes)))
         except AttributeError as e:

deploy/Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.8.20-slim
+FROM python:3.12.7-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
-ENV PIP_VERSION='24.2'
-ENV SETUPTOOLS_VERSION='74.1.2'
+ENV PIP_VERSION='24.3.1'
+ENV SETUPTOOLS_VERSION='75.2.0'
 ENV PIP_TOOLS_VERSION='7.4.1'
 # NB: Version 15.1.0 is installed on RHEL
 ENV VE_VERSION='20.26.4'

deploy/Dockerfile-rhel

@@ -0,0 +1,95 @@
+FROM redhat/ubi8:8.10
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+RUN yum -C repolist
+
+ARG RHEL_ORG_ID
+ARG RHEL_ACTIVATION_KEY
+RUN subscription-manager register --org ${RHEL_ORG_ID} --activationkey ${RHEL_ACTIVATION_KEY}
+
+RUN subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-debug-rpms
+RUN subscription-manager repos --enable rhel-8-for-x86_64-supplementary-source-rpms
+RUN subscription-manager repos --enable rhel-8-for-x86_64-baseos-e4s-source-rpms
+
+RUN yum -C repolist
+RUN subscription-manager repos --list
+
+# Install EPEL
+RUN yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
+# Update system packages and install dependencies
+RUN yum update -y && \
+    yum -y install \
+            bzip2 \
+            clamav \
+            clamav-data \
+            clamav-devel \
+            clamav-filesystem \
+            clamav-freshclam \
+            clamav-lib \
+            clamd \
+            curl \
+    #         firefox-esr \
+            gcc \
+            libffi \
+            libxml2 \
+            pkg-config \
+            mariadb-connector-c-devel \
+            perl-DBD-MySQL \
+            python312 \
+            python3.12-pip \
+            python3.12-devel \
+            python3.12-lxml \
+            python3.12-setuptools \
+            python3.12-wheel \
+            zip
+
+ENV PIP_VERSION='24.3.1'
+ENV SETUPTOOLS_VERSION='75.2.0'
+ENV PIP_TOOLS_VERSION='7.4.1'
+ENV VE_VERSION='20.26.3'
+ENV FABRIC_VERSION='1.15.0'
+
+RUN pip3 -V
+RUN pip3.12 -V
+
+RUN pip3.12 install -U pip==${PIP_VERSION}
+RUN pip3.12 install -U setuptools==${SETUPTOOLS_VERSION}
+RUN pip3.12 install pip-tools==${PIP_TOOLS_VERSION}
+RUN pip3.12 install virtualenv==${VE_VERSION}
+RUN pip3.12 install fabric==${FABRIC_VERSION}
+
+RUN pip3.12 freeze
+RUN virtualenv --version
+
+ENV ENVIRON="test"
+ENV REQUIREMENTS="test"
+
+ENV PROJECT_PATH=/usr/local/projects/temmpo
+ENV CODE_PATH=${PROJECT_PATH}/lib/${ENVIRON}/src/temmpo
+
+WORKDIR ${CODE_PATH}
+
+COPY deploy/setup-rhel-env.sh deploy/
+RUN bash deploy/setup-rhel-env.sh
+
+COPY deploy/fabfile.py deploy/
+COPY requirements/test.txt requirements/test.txt
+COPY temmpo/settings/github_actions_ci_private_settings.py temmpo/settings/
+
+RUN ln -s temmpo/settings/github_actions_ci_private_settings.py ${PROJECT_PATH}/.settings/private_settings.py
+
+RUN fab -l -f deploy/fabfile.py
+RUN fab make_virtualenv:env=${ENVIRON},configure_apache=False,clone_repo=False,branch=None,migrate_db=False,use_local_mode=True,requirements=${REQUIREMENTS},restart_rqworker=False,virtualenv=virtualenv,project_dir=${PROJECT_PATH}/ -f deploy/fabfile.py
+
+COPY . ${CODE_PATH}
+
+VOLUME [ ${CODE_PATH} ]
+
+RUN ls -l /usr/local/projects/temmpo/.settings/private_settings.py
+
+CMD [ "pip3.12", "check" ]  # TODO replace with command to run tests
+
+# CMD ../../bin/python3 manage.py test --settings=temmpo.settings.test_mysql --exclude-tag=selenium-test --exclude-tag=skip-on-ubuntu