feat: Install and run Python 3.8 on CentOS 7

Merge pull request #28 from jaimesouza/ASP-3704_drop_python3.6

CONTRIBUTING.md

@@ -59,7 +59,7 @@ Can also be used for pull requests.
 
 * `Type: Idea Bank` - Issues that pertain to proposing potential improvement to slurmd-operator.
 
-* `Type: Enchancement` - Issues marked as an agreed upon enhancement to slurmd-operator. Can also be used for pull requests.
+* `Type: Enhancement` - Issues marked as an agreed upon enhancement to slurmd-operator. Can also be used for pull requests.
 
 * `Statues: Help wanted` - Issues where we need help from the greater slurmd-operator community to solve.
 

dispatch

@@ -1,44 +1,44 @@
 #!/bin/bash
-# This hook installs the centos dependencies needed to run the charm,
+# This hook installs the dependencies needed to run the charm,
 # creates the dispatch executable, regenerates the symlinks for start and
 # upgrade-charm, and kicks off the operator framework.
 
 set -e
 
-# Source the os-release information into the env.
+# Source the os-release information into the env
 . /etc/os-release
 
 if ! [[ -f '.installed' ]]
 then
-	# Determine if we are running in centos or ubuntu, if centos
-	# provision the needed prereqs.
-	if [[ $ID == 'ubuntu' ]]
+	if [[ $ID == 'centos' ]]
 	then
-		echo "Running Ubuntu."
-		# necessary to compile and install NHC
-		apt-get install --assume-yes make automake
-	elif [[ $ID == 'centos' ]]
+		# Install dependencies and build custom python
+		yum -y install epel-release
+		yum -y install wget gcc make tar bzip2-devel zlib-devel xz-devel openssl-devel libffi-devel sqlite-devel ncurses-devel
+
+		export PYTHON_VERSION=3.8.16
+		wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz -P /tmp
+		tar xvf /tmp/Python-${PYTHON_VERSION}.tar.xz -C /tmp
+		cd /tmp/Python-${PYTHON_VERSION}
+		./configure --enable-optimizations
+		make -C /tmp/Python-${PYTHON_VERSION} -j $(nproc) altinstall		
+		cd $OLDPWD
+		rm -rf /tmp/Python*
+
+	elif [[ $ID == 'ubuntu' ]]
 	then
-		# Determine the centos version and install prereqs accordingly
-		major=$(cat /etc/centos-release | tr -dc '0-9.'|cut -d \. -f1)
-			echo "Running CentOS$major, installing prereqs."
-		if [[ $major == "7" ]]
-		then
-			yum -y install epel-release
-			yum -y install yum-priorities python3 make automake yum-utils
-		elif [[ $major == "8" ]]
-		then
-			dnf -y install epel-release
-			dnf -y install yum-priorities python3 make automake yum-utils
-		else
-			echo "Running unsuppored version of centos: $major"
-			exit -1
-		fi
-	else
-		echo "Running unsuppored os: $ID"
-		exit -1
+		# Necessary to compile and install NHC
+		apt-get install --assume-yes make
 	fi
 	touch .installed
 fi
 
-JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv ./src/charm.py
+# set the correct python bin path
+if [[ $ID == "centos" ]]
+then
+	PYTHON_BIN="/usr/bin/env python3.8"
+else
+	PYTHON_BIN="/usr/bin/env python3"
+fi
+
+JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv $PYTHON_BIN ./src/charm.py

requirements.txt

@@ -1,2 +1,3 @@
 ops==2.*
-git+https://github.com/omnivector-solutions/[email protected]
+distro
+git+https://github.com/omnivector-solutions/[email protected]

src/charm.py

@@ -7,6 +7,7 @@
 import logging
 from pathlib import Path
 
+import distro
 from charms.fluentbit.v0.fluentbit import FluentbitClient
 from charms.operator_libs_linux.v0.juju_systemd_notices import (
     ServiceStartedEvent,
@@ -19,9 +20,14 @@
 from ops.main import main
 from ops.model import ActiveStatus, BlockedStatus, WaitingStatus
 from slurm_ops_manager import SlurmManager
-from utils import slurmd
+from utils import monkeypatch, slurmd
 
 logger = logging.getLogger(__name__)
+if distro.id() == "centos":
+    logger.debug("Monkeypatching slurmd operator to support CentOS base")
+    SystemdNotices = monkeypatch.juju_systemd_notices(SystemdNotices)
+    slurmd = monkeypatch.slurmd_override_default(slurmd)
+    slurmd = monkeypatch.slurmd_override_service(slurmd)
 
 
 class SlurmdCharm(CharmBase):

src/utils/monkeypatch.py

@@ -0,0 +1,154 @@
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Monkeypatch slurmd operator classes and methods to work on CentOS 7."""
+
+import inspect
+import logging
+import textwrap
+from pathlib import Path
+
+from charms.operator_libs_linux.v0.juju_systemd_notices import (
+    SystemdNotices,
+    _daemon_reload,
+    _enable_service,
+    _start_service,
+)
+from charms.operator_libs_linux.v1 import systemd
+
+from . import slurmd
+
+_logger = logging.getLogger(__name__)
+
+
+def juju_systemd_notices(notices: SystemdNotices) -> SystemdNotices:
+    """Patch SystemdNotices object from juju_systemd_notices.
+
+    This function will patch the subscribe method of SystemdNotices
+    to use `/usr/bin/env python3.8` as the PYTHONEXE for running the
+    juju_systemd_notices daemon when on CentOS 7.
+
+    Args:
+        notices: SystemdNotices class reference to patch.
+    """
+    _logger.debug("Monkeypatching SystemdNotices subscribe method")
+
+    def patched_subscribe(self) -> None:  # pragma: nocover
+        _logger.debug("Generating systemd notice hooks for %s", self._services)
+        start_hooks = [Path(f"hooks/service-{service}-started") for service in self._services]
+        stop_hooks = [Path(f"hooks/service-{service}-stopped") for service in self._services]
+        for hook in start_hooks + stop_hooks:
+            if hook.exists():
+                _logger.debug("Hook %s already exists. Skipping...", hook.name)
+            else:
+                hook.symlink_to(self._charm.framework.charm_dir / "dispatch")
+
+        _logger.debug("Starting %s daemon", self._service_file.name)
+        if self._service_file.exists():
+            _logger.debug("Overwriting existing service file %s", self._service_file.name)
+        self._service_file.write_text(
+            textwrap.dedent(
+                f"""
+                [Unit]
+                Description=Juju systemd notices daemon
+                After=multi-user.target
+
+                [Service]
+                Type=simple
+                Restart=always
+                WorkingDirectory={self._charm.framework.charm_dir}
+                Environment="PYTHONPATH={self._charm.framework.charm_dir / "venv"}"
+                ExecStart=/usr/bin/env python3.8 {inspect.getfile(notices)} {self._charm.unit.name}
+
+                [Install]
+                WantedBy=multi-user.target
+                """
+            ).strip()
+        )
+        _logger.debug("Service file %s written. Reloading systemd", self._service_file.name)
+        _daemon_reload()
+        # Notices daemon is enabled so that the service will start even after machine reboot.
+        # This functionality is needed in the event that a charm is rebooted to apply updates.
+        _enable_service(self._service_file.name)
+        _start_service(self._service_file.name)
+        _logger.debug("Started %s daemon", self._service_file.name)
+
+    notices.subscribe = patched_subscribe
+    return notices
+
+
+def slurmd_override_default(slurmd_module: slurmd) -> slurmd:
+    """Patch override_default function for slurmd utility.
+
+    This function will patch the override_default function of the slurmd utility
+    to save
+
+    Args:
+        slurmd_module: slurmd utility module reference to patch.
+    """
+    _logger.debug("Monkeypatching slurmd.override_default function")
+
+    def patched_override_default(host: str, port: int) -> None:  # pragma: nocover
+        _logger.debug(f"Overriding /etc/default/slurmd with hostname {host} and port {port}")
+        Path("/etc/sysconfig/slurmd").write_text(
+            textwrap.dedent(
+                f"""
+                SLURMD_OPTIONS="--conf-server {host}:{port}"
+                PYTHONPATH={Path.cwd() / "lib"}
+                """
+            ).strip()
+        )
+
+    slurmd_module.override_default = patched_override_default
+    return slurmd_module
+
+
+def slurmd_override_service(slurmd_module: slurmd) -> slurmd:
+    """Patch override_service function from slurmd utility.
+
+    This function will patch the override_service function of the slurmd utility
+    to use `/usr/bin/env python3.8` as the PYTHONEXE for running the custom slurmd
+    ExecStart script in the slurmd service file.
+
+    Args:
+        slurmd_module: slurmd utility module reference to patch.
+    """
+    _logger.debug("Monkeypatching slurmd.override_service function")
+
+    def patched_override_service() -> None:  # pragma: nocover
+        _logger.debug("Overriding default slurmd service file")
+        if not (override_dir := Path("/etc/systemd/system/slurmd.service.d")).is_dir():
+            override_dir.mkdir()
+
+        overrides = override_dir / "99-slurmd-charm.conf"
+        overrides.write_text(
+            textwrap.dedent(
+                f"""
+                    [Unit]
+                    ConditionPathExists=
+
+                    [Service]
+                    Type=forking
+                    ExecStart=
+                    ExecStart=/usr/bin/env python3.8 {inspect.getfile(slurmd_module)}
+                    LimitMEMLOCK=infinity
+                    LimitNOFILE=1048576
+                    TimeoutSec=900
+                    """
+            ).strip()
+        )
+        systemd.daemon_reload()
+
+    slurmd_module.override_service = patched_override_service
+    return slurmd_module

tests/unit/test_charm.py

@@ -66,6 +66,20 @@ def test_install_success(self, defer, *_) -> None:
         self.assertTrue(self.harness.charm._stored.slurm_installed)
         defer.assert_not_called()
 
+    @patch("slurm_ops_manager.SlurmManager.install")
+    @patch("pathlib.Path.read_text", return_value="v1.0.0")
+    @patch("ops.model.Unit.set_workload_version")
+    @patch("ops.model.Resources.fetch")
+    @patch("utils.slurmd.override_default")
+    @patch("utils.slurmd.override_service")
+    @patch("charms.operator_libs_linux.v0.juju_systemd_notices.SystemdNotices.subscribe")
+    @patch("ops.framework.EventBase.defer")
+    def test_install_success_centos(self, defer, *_) -> None:
+        """Test install success behavior on CentOS."""
+        self.harness.charm.on.install.emit()
+        self.assertTrue(self.harness.charm._stored.slurm_installed)
+        defer.assert_not_called()
+
     def test_service_slurmd_start(self) -> None:
         """Test service_slurmd_started event handler."""
         self.harness.charm.on.service_slurmd_started.emit()