103.next LTS backports

Makefile

@@ -37,7 +37,7 @@ uninstall:
 	$(PYTHON) setup.py develop --uninstall $(PYTHON_DEVELOP_ARGS)
 
 requirements-dev: pip
-	- $(PYTHON) -m pip install -r requirements-dev.txt $(PYTHON_DEVELOP_ARGS)
+	$(PYTHON) -m pip install -r requirements-dev.txt $(PYTHON_DEVELOP_ARGS)
 
 smokecheck: clean uninstall develop
 	$(PYTHON) -m avocado run examples/tests/passtest.py

avocado/core/dependencies/dependency.py

@@ -0,0 +1,70 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2024
+# Authors: Jan Richter <[email protected]>
+
+from avocado.core.nrunner.runnable import Runnable
+
+
+class Dependency:
+    """
+    Data holder for dependency.
+    """
+
+    def __init__(self, kind=None, uri=None, args=(), kwargs=None):
+        self._kind = kind
+        self._uri = uri
+        self._args = args
+        self._kwargs = kwargs or {}
+
+    @property
+    def kind(self):
+        return self._kind
+
+    @property
+    def uri(self):
+        return self._uri
+
+    @property
+    def args(self):
+        return self._args
+
+    @property
+    def kwargs(self):
+        return self._kwargs
+
+    def __hash__(self):
+        return hash(
+            (
+                self.kind,
+                self.uri,
+                tuple(sorted(self.args)),
+                tuple(sorted(self.kwargs.items())),
+            )
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, Dependency):
+            return hash(self) == hash(other)
+        return False
+
+    def to_runnable(self, config):
+        return Runnable(self.kind, self.uri, *self.args, config=config, **self.kwargs)
+
+    @classmethod
+    def from_dictionary(cls, dictionary):
+        return cls(
+            dictionary.pop("type", None),
+            dictionary.pop("uri", None),
+            dictionary.pop("args", ()),
+            dictionary,
+        )

avocado/core/nrunner/runnable.py

@@ -80,7 +80,7 @@ def __init__(self, kind, uri, *args, config=None, **kwargs):
         #: attr:`avocado.core.nrunner.runner.BaseRunner.CONFIGURATION_USED`
         self._config = {}
         if config is None:
-            config = self.filter_runnable_config(kind, {})
+            config = self.add_configuration_used(kind, {})
         self.config = config or {}
         self.args = args
         self.tags = kwargs.pop("tags", None)
@@ -176,10 +176,10 @@ def config(self, config):
         configuration_used = Runnable.get_configuration_used_by_kind(self.kind)
         if not set(configuration_used).issubset(set(config.keys())):
             LOG.warning(
-                "The runnable config should have only values "
-                "essential for its runner. In the next version of "
-                "avocado, this will raise a Value Error. Please "
-                "use avocado.core.nrunner.runnable.Runnable.filter_runnable_config "
+                "The runnable config should have values essential for its runner. "
+                "In this case, it's missing some of the used configuration.  In a "
+                "future avocado version this will raise a ValueError. Please "
+                "use avocado.core.nrunner.runnable.Runnable.add_configuration_used "
                 "or avocado.core.nrunner.runnable.Runnable.from_avocado_config"
             )
         self._config = config
@@ -221,7 +221,7 @@ def from_avocado_config(cls, kind, uri, *args, config=None, **kwargs):
         """Creates runnable with only essential config for runner of specific kind."""
         if not config:
             config = {}
-        config = cls.filter_runnable_config(kind, config)
+        config = cls.add_configuration_used(kind, config)
         return cls(kind, uri, *args, config=config, **kwargs)
 
     @classmethod
@@ -245,30 +245,30 @@ def get_configuration_used_by_kind(cls, kind):
         return configuration_used
 
     @classmethod
-    def filter_runnable_config(cls, kind, config):
+    def add_configuration_used(cls, kind, config):
         """
-        Returns only essential values for specific runner.
+        Adds essential configuration values for specific runner.
 
-        It will use configuration from argument completed by values from
-        config file and avocado default configuration.
+        It will add missing configuration in the given config,
+        complementing it with values from config file and avocado default
+        configuration.
 
         :param kind: Kind of runner which should use the configuration.
         :type kind: str
-        :param config: Configuration values for runner. If some values will be
-                       missing the default ones and from config file will be
-                       used.
+        :param config: Configuration values for runner. If any used configuration
+                       values are missing, the default ones and from config file
+                       will be used.
         :type config: dict
-        :returns: Config dict, which has only values essential for runner
-                  based on STANDALONE_EXECUTABLE_CONFIG_USED
+        :returns: Config dict, which has existing entries plus values
+                  essential for runner based on
+                  STANDALONE_EXECUTABLE_CONFIG_USED
         :rtype: dict
         """
         whole_config = settings.as_dict()
-        filtered_config = {}
         for config_item in cls.get_configuration_used_by_kind(kind):
-            filtered_config[config_item] = config.get(
-                config_item, whole_config.get(config_item)
-            )
-        return filtered_config
+            if config_item not in config:
+                config[config_item] = whole_config.get(config_item)
+        return config
 
     def get_command_args(self):
         """

avocado/core/nrunner/task.py

@@ -40,29 +40,46 @@ class TaskStatusService:
 
     def __init__(self, uri):
         self.uri = uri
-        self.connection = None
+        self._connection = None
 
-    def post(self, status):
+    @property
+    def connection(self):
+        if not self._connection:
+            self._create_connection()
+        return self._connection
+
+    def _create_connection(self):
+        """
+        Creates connection with `self.uri` based on `socket.create_connection`
+        """
         if ":" in self.uri:
             host, port = self.uri.split(":")
             port = int(port)
-            if self.connection is None:
-                for _ in range(600):
-                    try:
-                        self.connection = socket.create_connection((host, port))
-                        break
-                    except ConnectionRefusedError as error:
-                        LOG.warning(error)
-                        time.sleep(1)
-                else:
-                    self.connection = socket.create_connection((host, port))
+            for _ in range(600):
+                try:
+                    self._connection = socket.create_connection((host, port))
+                    break
+                except ConnectionRefusedError as error:
+                    LOG.warning(error)
+                    time.sleep(1)
+            else:
+                self._connection = socket.create_connection((host, port))
         else:
-            if self.connection is None:
-                self.connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-                self.connection.connect(self.uri)
+            self._connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            self._connection.connect(self.uri)
 
+    def post(self, status):
         data = json_dumps(status)
-        self.connection.send(data.encode("ascii") + "\n".encode("ascii"))
+        try:
+            self.connection.send(data.encode("ascii") + "\n".encode("ascii"))
+        except BrokenPipeError:
+            try:
+                self._create_connection()
+                self.connection.send(data.encode("ascii") + "\n".encode("ascii"))
+            except ConnectionRefusedError:
+                LOG.warning(f"Connection with {self.uri} has been lost.")
+                return False
+        return True
 
     def close(self):
         if self.connection is not None:
@@ -203,12 +220,23 @@ def run(self):
         self.setup_output_dir()
         runner_klass = self.runnable.pick_runner_class()
         runner = runner_klass()
+        running_status_services = self.status_services
+        damaged_status_services = []
         for status in runner.run(self.runnable):
             if status["status"] == "started":
                 status.update({"output_dir": self.runnable.output_dir})
             status.update({"id": self.identifier})
             if self.job_id is not None:
                 status.update({"job_id": self.job_id})
-            for status_service in self.status_services:
-                status_service.post(status)
+            for status_service in running_status_services:
+                if not status_service.post(status):
+                    damaged_status_services.append(status_service)
+            if damaged_status_services:
+                running_status_services = list(
+                    filter(
+                        lambda s: s not in damaged_status_services,
+                        running_status_services,
+                    )
+                )
+                damaged_status_services.clear()
             yield status

avocado/core/safeloader/docstring.py

@@ -1,6 +1,8 @@
 import json
 import re
 
+from avocado.core.dependencies.dependency import Dependency
+
 #: Gets the docstring directive value from a string. Used to tweak
 #: test behavior in various ways
 DOCSTRING_DIRECTIVE_RE_RAW = (
@@ -78,7 +80,9 @@ def get_docstring_directives_dependencies(docstring):
         if item.startswith("dependency="):
             _, dependency_str = item.split("dependency=", 1)
             try:
-                dependencies.append(json.loads(dependency_str))
+                dependencies.append(
+                    Dependency.from_dictionary(json.loads(dependency_str))
+                )
             except json.decoder.JSONDecodeError:
                 # ignore dependencies in case of malformed dictionary
                 continue

avocado/plugins/dependency.py

@@ -12,7 +12,6 @@
 # Copyright: Red Hat Inc. 2021
 # Authors: Willian Rampazzo <[email protected]>
 
-from avocado.core.nrunner.runnable import Runnable
 from avocado.core.plugin_interfaces import PreTest
 
 
@@ -33,15 +32,7 @@ def pre_test_runnables(test_runnable, suite_config=None):  # pylint: disable=W02
         if not test_runnable.dependencies:
             return []
         dependency_runnables = []
-        for dependency in test_runnable.dependencies:
-            # make a copy to change the dictionary and do not affect the
-            # original `dependencies` dictionary from the test
-            dependency_copy = dependency.copy()
-            kind = dependency_copy.pop("type")
-            uri = dependency_copy.pop("uri", None)
-            args = dependency_copy.pop("args", ())
-            dependency_runnable = Runnable(
-                kind, uri, *args, config=test_runnable.config, **dependency_copy
-            )
-            dependency_runnables.append(dependency_runnable)
+        unique_dependencies = list(dict.fromkeys(test_runnable.dependencies))
+        for dependency in unique_dependencies:
+            dependency_runnables.append(dependency.to_runnable(test_runnable.config))
         return dependency_runnables

avocado/plugins/runner_nrunner.py

@@ -222,7 +222,7 @@ def _update_avocado_configuration_used_on_runnables(runnables, config):
         :type config: dict
         """
         for runnable in runnables:
-            runnable.config = Runnable.filter_runnable_config(runnable.kind, config)
+            runnable.config = Runnable.add_configuration_used(runnable.kind, config)
 
     def _determine_status_server(self, test_suite, config_key):
         if test_suite.config.get("run.status_server_auto"):

avocado/utils/archive.py

@@ -116,7 +116,13 @@ def is_zstd_file(path):
         return zstd_file.read(len(ZSTD_MAGIC)) == ZSTD_MAGIC
 
 
-def _probe_zstd_cmd():
+def probe_zstd_cmd():
+    """
+    Attempts to find a suitable zstd tool that behaves as expected
+
+    :rtype: str or None
+    :returns: path to a suitable zstd executable or None if not found
+    """
     zstd_cmd = shutil.which("zstd")
     if zstd_cmd is not None:
         proc = subprocess.run(
@@ -136,7 +142,7 @@ def zstd_uncompress(path, output_path=None, force=False):
     """
     Extracts a zstd compressed file.
     """
-    zstd_cmd = _probe_zstd_cmd()
+    zstd_cmd = probe_zstd_cmd()
     if not zstd_cmd:
         raise ArchiveException("Unable to find a suitable zstd compression tool")
     output_path = _decide_on_path(path, ".zst", output_path)

optional_plugins/html/tests/html_result.py

@@ -82,8 +82,8 @@ def test_output_incompatible_setup(self):
 
     def test_output_compatible_setup_2(self):
         prefix = "avocado_" + __name__
-        tmpfile = tempfile.mktemp(prefix=prefix, dir=self.tmpdir.name)
-        tmpfile2 = tempfile.mktemp(prefix=prefix, dir=self.tmpdir.name)
+        tmpfile = os.path.join(self.tmpdir.name, f"{prefix}_result.xml")
+        tmpfile2 = os.path.join(self.tmpdir.name, f"{prefix}_result.json")
         tmpdir = tempfile.mkdtemp(prefix=prefix, dir=self.tmpdir.name)
         tmpfile3 = os.path.join(tmpdir, "result.html")
         cmd_line = (

requirements-dev.txt

@@ -10,7 +10,7 @@ black==22.3.0
 coverage==5.5
 
 # To run make check
-psutil==5.8.0
+psutil==5.9.5
 
 # pycdlib is an optional requirement in production
 # but is necessary for selftests

selftests/check.py

@@ -26,11 +26,11 @@
     "job-api-6": 4,
     "job-api-7": 1,
     "nrunner-interface": 70,
-    "nrunner-requirement": 16,
-    "unit": 667,
+    "nrunner-requirement": 20,
+    "unit": 668,
     "jobs": 11,
-    "functional-parallel": 297,
-    "functional-serial": 4,
+    "functional-parallel": 298,
+    "functional-serial": 5,
     "optional-plugins": 0,
     "optional-plugins-golang": 2,
     "optional-plugins-html": 3,

selftests/functional/basic.py

@@ -819,7 +819,9 @@ def check_matplotlib_logs(file_path):
             self.assertTrue(os.path.exists(file_path))
             with open(file_path, encoding="utf-8") as file:
                 stream = file.read()
-                self.assertIn("matplotlib __init__         L0337 DEBUG|", stream)
+                self.assertTrue(
+                    re.match(r"matplotlib __init__         L[0-9]* DEBUG|", stream)
+                )
 
         log_dir = os.path.join(self.tmpdir.name, "latest")
         test_log_dir = os.path.join(