TMMA-511: Upgrade Python 3.9 & add dev RHEL docker container WIP

.dockerignore

@@ -0,0 +1,2 @@
+deploy/Dockerfile
+deploy/Dockerfile-rhel

.github/workflows/check-python-dependcies.yml

@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.9'
         architecture: 'x64'
     - name: Run checks for package updates
       run: bash tests/run-update-checks.sh

.github/workflows/docker.yml

@@ -6,12 +6,6 @@ on:
   workflow_dispatch:
   pull_request:
   push:
-    paths:
-      # TODO: Review scope of GitHub action grows
-      - '.github/workflows/docker.yml'
-      - 'deploy/Dockerfile'
-      - 'entrypoints/**'
-      - 'requirements/**'
   # schedule:
   #   - cron: 30 9 * * WED
 

.github/workflows/run-django-test.yml

@@ -27,7 +27,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.9'
         architecture: 'x64'
     - name: Run Python 3/Django tests
       run: bash tests/run-django-tests.sh

.github/workflows/run-fabric-test.yml

@@ -16,7 +16,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.9'
         architecture: 'x64'
     - name: Syntax check Python 3 Fabric deployment script
       run: bash tests/run-fabric-tests.sh

CHANGELOG

@@ -1,5 +1,10 @@
 # Changelog
 
+9.9.0   TMMA-511: Upgrade Python from 3.8.x to 3.9+
+        TMMA-347: Set up RHEL8 development docker environment
+        Update version of MySQL used in GitHub actions
+        Update Python dependencies
+
 9.8.2   TMMA-508: Resolve issue in tests after RHEL migration
 
 9.8.1   TMMA-509: Improve security of Python packaging tool

README.md

@@ -148,6 +148,12 @@ Alternatively
     docker build -f deploy/Dockerfile -t temmpo-web .
     docker run --rm -it -v $PWD:/srv -w /srv temmpo-web bash /srv/entrypoints/update-requirements.sh
 
+Alternatively
+
+    docker compose build web
+    docker compose run --no-deps web bash /srv/entrypoints/update-requirements.sh
+
+
 #### Create Docker images for different environments
 
     docker build -f deploy/Dockerfile -t temmpo-web-dev . --build-arg REQUIREMENTS_FILE=dev.txt

browser/probe.py

@@ -99,7 +99,7 @@ def probe_email(self):
             return True
 
     def python_version(self):
-        if sys.version_info.major == 3 and sys.version_info.minor == 8:
+        if sys.version_info.major == 3 and sys.version_info.minor == 9:
             return True
         else:
             return False

deploy/Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.8.20-slim
+FROM python:3.9.20-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
-ENV PIP_VERSION='24.2'
-ENV SETUPTOOLS_VERSION='74.1.2'
+ENV PIP_VERSION='24.3.1'
+ENV SETUPTOOLS_VERSION='75.2.0'
 ENV PIP_TOOLS_VERSION='7.4.1'
 # NB: Version 15.1.0 is installed on RHEL
 ENV VE_VERSION='20.26.4'

deploy/Dockerfile-rhel

@@ -0,0 +1,91 @@
+FROM redhat/ubi8:8.10
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+RUN yum -C repolist
+
+ARG RHEL_ORG_ID
+ARG RHEL_ACTIVATION_KEY
+RUN subscription-manager register --org ${RHEL_ORG_ID} --activationkey ${RHEL_ACTIVATION_KEY}
+
+RUN subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-debug-rpms
+RUN subscription-manager repos --enable rhel-8-for-x86_64-supplementary-source-rpms
+RUN subscription-manager repos --enable rhel-8-for-x86_64-baseos-e4s-source-rpms
+
+RUN yum -C repolist
+RUN subscription-manager repos --list
+
+# Install EPEL
+RUN yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
+# Update system packages and install dependencies
+RUN yum update -y && \
+    yum -y install \
+            bzip2 \
+            clamav \
+            clamav-data \
+            clamav-devel \
+            clamav-filesystem \
+            clamav-freshclam \
+            clamav-lib \
+            clamd \
+            curl \
+    #         firefox-esr \
+            gcc \
+            libffi \
+            libxml2 \
+            pkg-config \
+            mariadb-connector-c-devel \
+            perl-DBD-MySQL \
+            python39 \
+            python39-devel \
+            python39-lxml \
+            python39-setuptools \
+            python39-wheel \
+            zip
+
+ENV PIP_VERSION='24.3.1'
+ENV SETUPTOOLS_VERSION='75.2.0'
+ENV PIP_TOOLS_VERSION='7.4.1'
+ENV VE_VERSION='20.26.3'
+ENV FABRIC_VERSION='1.15.0'
+
+RUN pip3 install -U pip==${PIP_VERSION}
+RUN pip3 install -U setuptools==${SETUPTOOLS_VERSION}
+RUN pip3 install pip-tools==${PIP_TOOLS_VERSION}
+RUN pip3 install virtualenv==${VE_VERSION}
+RUN pip3 install fabric==${FABRIC_VERSION}
+
+RUN pip3 freeze
+RUN virtualenv --version
+
+ENV ENVIRON="test"
+ENV REQUIREMENTS="test"
+
+ENV PROJECT_PATH=/usr/local/projects/temmpo
+ENV CODE_PATH=${PROJECT_PATH}/lib/${ENVIRON}/src/temmpo
+
+WORKDIR ${CODE_PATH}
+
+COPY deploy/setup-rhel-env.sh deploy/
+RUN bash deploy/setup-rhel-env.sh
+
+COPY deploy/fabfile.py deploy/
+COPY requirements/test.txt requirements/test.txt
+COPY temmpo/settings/github_actions_ci_private_settings.py temmpo/settings/
+
+RUN ln -s temmpo/settings/github_actions_ci_private_settings.py ${PROJECT_PATH}/.settings/private_settings.py
+
+RUN fab -l -f deploy/fabfile.py
+RUN fab make_virtualenv:env=${ENVIRON},configure_apache=False,clone_repo=False,branch=None,migrate_db=False,use_local_mode=True,requirements=${REQUIREMENTS},restart_rqworker=False,virtualenv=virtualenv,project_dir=${PROJECT_PATH}/ -f deploy/fabfile.py
+
+COPY . ${CODE_PATH}
+
+VOLUME [ ${CODE_PATH} ]
+
+RUN ls -l /usr/local/projects/temmpo/.settings/private_settings.py
+
+CMD [ "pip3", "check" ]  # TODO replace with command to run tests
+
+# CMD ../../bin/python3 manage.py test --settings=temmpo.settings.test_mysql --exclude-tag=selenium-test --exclude-tag=skip-on-ubuntu

deploy/Vagrantfile

@@ -5,78 +5,112 @@
 
 Vagrant.configure("2") do |config|
 
-  config.vm.box = "centos/7"
-  config.vm.box_version = "2004.01"
+  config.vm.box = "generic/rhel8"
 
   # Used to allow pass phrase free SSH access to allow
   # remote testing of Fabric scripts against Vagrant VMs
   config.ssh.insert_key = false
 
   config.vm.define "db" do |db|
 
+    db.vm.synced_folder ".", "/vagrant", # type: "sshfs",
+       owner: "vagrant", group: "vagrant"
+
     db.vm.provision "shell", inline: <<-SHELL
 
       sudo timedatectl set-timezone Europe/London
       sudo yum update
 
-      echo "Install dev utils"
-      sudo yum install -y unzip
-      sudo yum install -y nano
-
-      echo "Install essential tools & repos"
+      echo "Install essential tools & EPEL 8 repo"
       sudo yum install -y wget
-      wget -N https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-      sudo rpm -ivh --replacepkgs epel-release-latest-7.noarch.rpm
+      wget -N https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+      sudo rpm -ivh --replacepkgs epel-release-latest-8.noarch.rpm
 
-      # Install database server and client
-      sudo yum -y install mariadb-server mariadb
+      # Install database server
+      sudo yum install libaio libsepol lsof
+      wget https://repo.mysql.com/mysql84-community-release-el8-1.noarch.rpm
+      sudo yum localinstall -y mysql84-community-release-el8-1.noarch.rpm
 
-      echo "Run services"
-      sudo systemctl start mariadb
-      sudo systemctl enable mariadb
+      # sudo yum repolist enabled | grep mysql.*-community
+      # sudo yum repolist enabled | grep mysql
+      # sudo yum --disablerepo=\* --enablerepo='mysql*-community*' list available
+
+      sudo yum-config-manager --disable mysql-8.4-lts-community
+      sudo yum-config-manager --enable  mysql80-community
+
+      sudo yum update
+
+      sudo yum install -y mysql-community-libs
+      sudo yum install -y --nobest mysql-community-server
 
       echo "Allow external connections from mysql"
       cat > allow-external-connections.cnf << CONF
 [mysqld]
 bind-address=0.0.0.0
 CONF
       sudo mv allow-external-connections.cnf /etc/my.cnf.d/
-      sudo systemctl restart mariadb
-      mysql -u root < /vagrant/db-setup.sql
-    SHELL
+
+      echo "Run services"
+      sudo systemctl status mysqld
+      sudo systemctl restart mysqld
+      sudo systemctl enable mysqld
+      sudo systemctl status mysqld
+
+      # echo "Insecure installation"
+      # mysqld --initialize-insecure=ON --user=mysql -y
+      # echo "Run setup scripts"
+      # mysql -u root --skip-password < /vagrant/db-setup.sql
+      echo "Secure installation"
+      mysql_secure_installation -u mysql --password=tmp-dev-environment-12345 --use-default=true
+      echo "Run setup scripts"
+      mysql -u root --password=tmp-dev-environment-12345 < /vagrant/db-setup.sql
+
+      SHELL
 
       db.vm.network "forwarded_port", guest: 3306, host: 3306
-      db.vm.network "private_network", ip: "10.0.1.20"
+      db.vm.network "private_network", ip: "192.168.56.0"
   end
 
   # Django box
   config.vm.define "django", primary: true do |django|
 
     # Run the Django setup
-    django.vm.provision "shell", path: "deploy-centos.sh"
+    django.vm.provision "shell", path: "deploy-rhel.sh"
 
-    django.vm.synced_folder "../", "/usr/local/projects/temmpo/lib/dev/src/temmpo", type: "sshfs", 
+    django.vm.synced_folder "../", "/usr/local/projects/temmpo/lib/dev/src/temmpo", # type: "sshfs",  
        owner: "vagrant", group: "vagrant"
 
     django.vm.network "forwarded_port", guest: 59099, host: 59099
-    django.vm.network "private_network", ip: "10.0.1.21"
+    django.vm.network "private_network", ip: "192.168.56.11"
   end
 
   # Apache/Django box
   config.vm.define "apache", autostart: false do |apache|
 
     # Run the Django setup
-    apache.vm.provision "shell", path: "deploy-centos.sh"
+    apache.vm.provision "shell", path: "deploy-rhel.sh"
 
-    apache.vm.synced_folder "../", "/vagrant", type: "sshfs", 
+    apache.vm.synced_folder "../", "/vagrant", # type: "sshfs",  
        owner: "vagrant", group: "vagrant"
 
     apache.vm.network "forwarded_port", guest: 80, host: 8800
-    apache.vm.network "private_network", ip: "10.0.1.19"
+    apache.vm.network "private_network", ip: "192.168.56.19"
   end
 
   config.vm.provider "virtualbox" do |v|
     v.memory = 4096
     v.cpus = 2
   end
+
+  # config.vbguest.iso_path = "VBoxGuestAdditions_7.0.22.iso"
+  # config.vbguest.auto_update = true
+  # config.vbguest.no_remote = false
+  # no_install
+
+  # config.vm.provider "docker" do |d|
+  #   d.image = "redhat/ubi8"
+  #   # d.vagrant_vagrantfile
+  #   # v.memory = 4096
+  #   # v.cpus = 2
+  # end 
 end

deploy/deploy-centos.sh

@@ -42,32 +42,32 @@ setsebool -P httpd_can_network_connect 1
 setsebool -P httpd_can_network_connect_db 1
 setsebool -P httpd_can_sendmail 1
 
-echo "###   Install Python 3.8 and components"
+echo "###   Install Python 3.9 and components"
 
 yum -y install gcc gcc-c++ openssl-devel bzip2-devel libffi-devel zlib-devel
 cd /opt
-wget https://www.python.org/ftp/python/3.8.13/Python-3.8.13.tgz
-tar -xzf Python-3.8.13.tgz
-cd Python-3.8.13/
+wget https://www.python.org/ftp/python/3.9.20/Python-3.9.20.tgz
+tar -xzf Python-3.9.20.tgz
+cd Python-3.9.20/
 ./configure --enable-optimizations --enable-shared
 make altinstall
 # Create symlinks
-ln -sfn /usr/local/bin/python3.8 /usr/bin/python3.8
-ln -sfn /usr/local/bin/pip3.8 /usr/bin/pip3.8
+ln -sfn /usr/local/bin/python3.9 /usr/bin/python3.9
+ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3.9
 
 echo "export LD_LIBRARY_PATH=/usr/local/lib/" > ld_library.sh
 mv ld_library.sh /etc/profile.d/ld_library.sh
 export set LD_LIBRARY_PATH=/usr/local/lib/
 
 # Install symtem wide python requirements
-pip3.8 install -U pip==19.3.1 # As per app servers
-pip3.8 install Fabric==1.15.0 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
+pip3.9 install -U pip==19.3.1 # As per app servers
+pip3.9 install Fabric==1.15.0 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
 
-pip3.8 install mod_wsgi==4.9.4 # As per app servers
-ls /usr/local/lib64/python3.8/site-packages/mod_wsgi/server/
-pip3.8 install virtualenv==20.24.5 # As per app servers
+pip3.9 install mod_wsgi==4.9.4 # As per app servers
+ls /usr/local/lib64/python3.9/site-packages/mod_wsgi/server/
+pip3.9 install virtualenv==20.24.5 # As per app servers
 
-ln -s /usr/local/bin/virtualenv /usr/bin/virtualenv-3.8
+ln -s /usr/local/bin/virtualenv /usr/bin/virtualenv-3.9
 
 yum -y install python3-wheel
 yum -y install python3-lxml
@@ -161,7 +161,7 @@ chromedriver -v
 echo "###   Confirm install list"
 yum list installed 
 pip freeze
-pip3.8 freeze
+pip3.9 freeze
 
 echo "###   Create directories normally managed by Puppet"
 mkdir -p /usr/local/projects/temmpo/etc/apache/conf.d
@@ -233,7 +233,7 @@ fi
 ## TODO: Fix path to mod_wsgi module
 echo "###   Add basic catch all Apache config normally managed by Puppet"
 cat > /etc/httpd/conf.d/temmpo.conf <<APACHE_CONF
-LoadModule wsgi_module "/usr/local/lib64/python3.8/site-packages/mod_wsgi/server/mod_wsgi-py38.cpython-36m-x86_64-linux-gnu.so"
+LoadModule wsgi_module "/usr/local/lib64/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-36m-x86_64-linux-gnu.so"
 WSGIPythonHome "/usr/local/projects/temmpo/lib/dev"
 
 <VirtualHost *:*>