Introduce remote spawner functionality

avocado/plugins/spawners/lxc.py

@@ -184,7 +184,7 @@ def is_task_alive(runtime_task):
             return False
 
         status, _, _ = LXCSpawner.run_container_cmd(
-            container, ["pgrep", "-r", "R,S", "-f", "task-run"]
+            container, ["pgrep", "-r", "R,S", "-f", runtime_task.task.identifier]
         )
         return status == 0
 

optional_plugins/spawner_remote/VERSION

@@ -0,0 +1 @@
+102.0

optional_plugins/spawner_remote/avocado_spawner_remote/__init__.py

@@ -0,0 +1,224 @@
+import asyncio
+import contextlib
+import json
+import logging
+import os
+import shlex
+
+from aexpect import exceptions, remote
+
+from avocado.core.plugin_interfaces import Init, Spawner
+from avocado.core.settings import settings
+from avocado.core.spawners.common import SpawnerMixin, SpawnMethod
+
+LOG = logging.getLogger("avocado.job." + __name__)
+
+
+class RemoteSpawnerException(Exception):
+    """Errors more closely related to the spawner functionality"""
+
+
+class RemoteSpawnerInit(Init):
+
+    description = "Remote (host) based spawner initialization"
+
+    def initialize(self):
+        section = "spawner.remote"
+
+        help_msg = "List of already available remote host slots to spawn in"
+        settings.register_option(
+            section=section, key="slots", help_msg=help_msg, key_type=list, default=[]
+        )
+
+        help_msg = "Remote host setup hook command to customize optional new hosts"
+        settings.register_option(
+            section=section, key="setup_hook", help_msg=help_msg, default=""
+        )
+
+        help_msg = "Test timeout enforced for sessions (just for this spawner)"
+        settings.register_option(
+            section=section, key="test_timeout", help_msg=help_msg, default=14400
+        )
+
+
+def with_slot_reservation(fn):
+    """
+    Decorator for slot cache context manager.
+
+    :param fn: function to run with slot reservation
+    :type fn: function
+    :returns: same function with the slot now reserved
+    :rtype: function
+
+    The main reason for the decorator is to not have to indent the entire
+    task running function in order to safely release the slot upon any error.
+    """
+
+    async def wrapper(self, runtime_task):
+        with RemoteSpawner.reserve_slot(self, runtime_task) as slot:
+            runtime_task.spawner_handle = slot
+            return await fn(self, runtime_task)
+
+    return wrapper
+
+
+class RemoteSpawner(Spawner, SpawnerMixin):
+
+    description = "Remote (host) based spawner"
+    METHODS = [SpawnMethod.STANDALONE_EXECUTABLE]
+    slots_cache = {}
+
+    @staticmethod
+    async def run_remote_cmd_async(session, command, timeout):
+        loop = asyncio.get_event_loop()
+        try:
+            status, output = await loop.run_in_executor(
+                None, session.cmd_status_output, command, timeout
+            )
+        except exceptions.ShellTimeoutError:
+            status, output = 2, f"Remote command timeout of {timeout} reached"
+        except exceptions.ShellProcessTerminatedError:
+            status, output = 2, "Remote command terminated prematurely"
+        return status, output
+
+    @contextlib.contextmanager
+    def reserve_slot(self, runtime_task):
+        """
+        Reserve a free or custom remote host slot for the runtime task.
+
+        :param runtime_task: runtime task to reserve the slot for
+        :type runtime_task: :py:class:`avocado.core.task.runtime.RuntimeTask`
+        :yields: a free slot to use if such was found
+        :raises: :py:class:`RuntimeError` if no free slot could be found
+
+        This will either use a runtime cache to find a free remote host slot to
+        run the task in or use a custom hostname/slot ID to allow for custom
+        schedulers to make their own decisions on which hosts to run and when.
+        """
+        if len(RemoteSpawner.slots_cache) == 0:
+            # TODO: consider whether to provide persistence across runs via external storage
+            for session_slot in self.config.get("spawner.remote.slots"):
+                if not session_slot:
+                    continue
+                with open(session_slot, "r", encoding="utf-8") as f:
+                    session_data = json.load(f)
+                session = remote.remote_login(**session_data)
+                RemoteSpawner.slots_cache[session] = False
+
+        if runtime_task.spawner_handle is not None:
+            slot = runtime_task.spawner_handle
+        else:
+            slots = RemoteSpawner.slots_cache
+            for key, value in slots.items():
+                if not value:
+                    slot = key
+                    slots[key] = True
+                    break
+            else:
+                raise RuntimeError(
+                    "No free slot available for the task, are "
+                    "you running with more processes than slots?"
+                )
+
+        try:
+            yield slot
+        finally:
+            RemoteSpawner.slots_cache[slot] = False
+
+    @staticmethod
+    def is_task_alive(runtime_task):
+        if runtime_task.spawner_handle is None:
+            return False
+        # NOTE: since this is called at the end of each test, it is reasonable
+        # to reuse the same session with a new command
+        session = runtime_task.spawner_handle
+        status, _ = session.cmd_status_output(
+            f"pgrep -r R,S -f {runtime_task.task.identifier}"
+        )
+        return status == 0
+
+    @with_slot_reservation
+    async def spawn_task(self, runtime_task):
+        self.create_task_output_dir(runtime_task)
+        task = runtime_task.task
+        full_module_name = (
+            runtime_task.task.runnable.pick_runner_module_from_entry_point_kind(
+                runtime_task.task.runnable.kind
+            )
+        )
+        if full_module_name is None:
+            msg = f"Could not determine Python module name for runnable with kind {runtime_task.task.runnable.kind}"
+            raise RemoteSpawnerException(msg)
+        # using the "python" symlink will result in the container default python version
+        entry_point_args = ["python3", "-m", full_module_name, "task-run"]
+        entry_point_args.extend(task.get_command_args())
+
+        session = runtime_task.spawner_handle
+        LOG.info(f"Hostname: {session.host} Port: {session.port}")
+
+        setup_hook = self.config.get("spawner.remote.setup_hook")
+        # Customize and deploy test data to the container
+        if setup_hook:
+            status, output = await RemoteSpawner.run_remote_cmd_async(
+                session, setup_hook
+            )
+            LOG.debug(f"Customization command exited with code {status}")
+            if status != 0:
+                LOG.error(
+                    f"Error exit code {status} on {session.host}:{session.port} "
+                    f"from setup hook with output:\n{output}"
+                )
+                return False
+
+        cmd = shlex.join(entry_point_args) + " > /dev/null"
+        timeout = self.config.get("spawner.remote.test_timeout")
+        status, output = await RemoteSpawner.run_remote_cmd_async(session, cmd, timeout)
+        LOG.debug(f"Command exited with code {status}")
+        if status != 0:
+            LOG.error(
+                f"Error exit code {status} on {session.host}:{session.port} "
+                f"with output:\n{output}"
+            )
+            return False
+
+        return True
+
+    def create_task_output_dir(self, runtime_task):
+        output_dir_path = self.task_output_dir(runtime_task)
+        output_lxc_path = "/tmp/.avocado_task_output_dir"
+
+        os.makedirs(output_dir_path, exist_ok=True)
+        runtime_task.task.setup_output_dir(output_lxc_path)
+
+    async def wait_task(self, runtime_task):
+        while True:
+            if not RemoteSpawner.is_task_alive(runtime_task):
+                return
+            await asyncio.sleep(0.1)
+
+    async def terminate_task(self, runtime_task):
+        session = runtime_task.spawner_handle
+        session.sendcontrol("c")
+        try:
+            session.read_up_to_prompt()
+            return True
+        except exceptions.ExpectTimeoutError:
+            LOG.error("Failed to terminate task on {session.host}")
+            return False
+
+    @staticmethod
+    async def check_task_requirements(runtime_task):
+        """Check the runtime task requirements needed to be able to run"""
+        return True
+
+    @staticmethod
+    async def is_requirement_in_cache(runtime_task):
+        return False
+
+    @staticmethod
+    async def save_requirement_in_cache(runtime_task):
+        pass
+
+    @staticmethod
+    async def update_requirement_cache(runtime_task, result):
+        pass

optional_plugins/spawner_remote/setup.py

@@ -0,0 +1,48 @@
+#!/bin/env python3
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2023
+# Author: Cleber Rosa <[email protected]>
+
+import os
+
+from setuptools import setup
+
+# Handle systems with setuptools < 40
+try:
+    from setuptools import find_namespace_packages
+except ImportError:
+    packages = ["avocado_spawner_remote"]
+else:
+    packages = find_namespace_packages(include=["avocado_spawner_remote"])
+
+BASE_PATH = os.path.dirname(__file__)
+with open(os.path.join(BASE_PATH, "VERSION"), "r", encoding="utf-8") as version_file:
+    VERSION = version_file.read().strip()
+
+
+setup(
+    name="avocado-framework-plugin-spawner-remote",
+    version=VERSION,
+    description="Remote (host) based spawner",
+    author="Avocado Developers",
+    author_email="[email protected]",
+    url="http://avocado-framework.github.io/",
+    packages=packages,
+    include_package_data=True,
+    install_requires=[f"avocado-framework=={VERSION}", "aexpect>=1.6.2"],
+    test_suite="tests",
+    entry_points={
+        "avocado.plugins.init": ["remote = avocado_spawner_remote:RemoteSpawnerInit"],
+        "avocado.plugins.spawner": ["remote = avocado_spawner_remote:RemoteSpawner"],
+    },
+)

python-avocado.spec

@@ -46,6 +46,9 @@ BuildRequires: python3-docutils
 BuildRequires: python3-lxml
 BuildRequires: python3-psutil
 BuildRequires: python3-setuptools
+%if ! 0%{?rhel}
+BuildRequires: python3-aexpect
+%endif
 
 %if ! 0%{?rhel}
 %if ! 0%{?fedora} > 35
@@ -131,6 +134,11 @@ popd
 pushd optional_plugins/result_upload
 %py3_build
 popd
+%if ! 0%{?rhel}
+pushd optional_plugins/spawner_remote
+%py3_build
+popd
+%endif
 rst2man man/avocado.rst man/avocado.1
 
 %install
@@ -166,6 +174,11 @@ popd
 pushd optional_plugins/result_upload
 %py3_install
 popd
+%if ! 0%{?rhel}
+pushd optional_plugins/spawner_remote
+%py3_install
+popd
+%endif
 mkdir -p %{buildroot}%{_mandir}/man1
 install -m 0644 man/avocado.1 %{buildroot}%{_mandir}/man1/avocado.1
 mkdir -p %{buildroot}%{_pkgdocdir}
@@ -230,6 +243,7 @@ PATH=%{buildroot}%{_bindir}:%{buildroot}%{_libexecdir}/avocado:$PATH \
 %exclude %{python3_sitelib}/avocado_framework_plugin_golang*
 %exclude %{python3_sitelib}/avocado_framework_plugin_ansible*
 %exclude %{python3_sitelib}/avocado_framework_plugin_result_upload*
+%exclude %{python3_sitelib}/avocado_framework_plugin_spawner_remote*
 %exclude %{python3_sitelib}/tests*
 
 %package -n python3-avocado-common
@@ -374,6 +388,20 @@ a dedicated sever.
 %{python3_sitelib}/avocado_result_upload*
 %{python3_sitelib}/avocado_framework_plugin_result_upload*
 
+%if ! 0%{?rhel}
+%package -n python3-avocado-plugins-spawner-remote
+Summary: Avocado Plugin to spawn tests on a remote host
+License: GPLv2+
+Requires: python3-avocado == %{version}-%{release}
+
+%description -n python3-avocado-plugins-spawner-remote
+This optional plugin is intended to spawn tests on a remote host.
+
+%files -n python3-avocado-plugins-spawner-remote
+%{python3_sitelib}/avocado_spawner_remote*
+%{python3_sitelib}/avocado_framework_plugin_spawner_remote*
+%endif
+
 %package -n python3-avocado-examples
 Summary: Avocado Test Framework Example Tests
 License: GPLv2+

selftests/check.py

@@ -26,7 +26,7 @@
     "job-api-6": 4,
     "job-api-7": 1,
     "nrunner-interface": 70,
-    "nrunner-requirement": 12,
+    "nrunner-requirement": 16,
     "unit": 667,
     "jobs": 11,
     "functional-parallel": 297,
@@ -679,6 +679,7 @@ def create_suites(args):  # pylint: disable=W0621
             {"spawner": "process"},
             {"spawner": "podman"},
             {"spawner": "lxc"},
+            {"spawner": "remote"},
         ],
     }