Introduce "exec-runnables-recipe" resolver

avocado/plugins/list.py

@@ -214,6 +214,21 @@ def configure(self, parser):
             allow_multiple=True,
         )
 
+        settings.add_argparser_to_option(
+            namespace="resolver.run_executables",
+            parser=parser,
+            long_arg="--resolver-run-executables",
+            allow_multiple=True,
+        )
+
+        settings.add_argparser_to_option(
+            namespace="resolver.exec_runnables_recipe.arguments",
+            metavar="ARGS",
+            parser=parser,
+            long_arg="--resolver-exec-arguments",
+            allow_multiple=True,
+        )
+
         help_msg = "Writes runnable recipe files to a directory."
         settings.register_option(
             section="list.recipes",

avocado/plugins/resolvers.py

@@ -19,10 +19,12 @@
 import json
 import os
 import re
+import shlex
+import subprocess
 
 from avocado.core.extension_manager import PluginPriority
 from avocado.core.nrunner.runnable import Runnable
-from avocado.core.plugin_interfaces import Resolver
+from avocado.core.plugin_interfaces import Init, Resolver
 from avocado.core.references import reference_split
 from avocado.core.resolver import (
     ReferenceResolution,
@@ -31,16 +33,12 @@
     get_file_assets,
 )
 from avocado.core.safeloader import find_avocado_tests, find_python_unittests
+from avocado.core.settings import settings
 
 
-class ExecTestResolver(Resolver):
-
-    name = "exec-test"
-    description = "Test resolver for executable files to be handled as tests"
-    priority = PluginPriority.VERY_LOW
-
-    def resolve(self, reference):
-
+class BaseExec:
+    @staticmethod
+    def check_exec(reference):
         criteria_check = check_file(
             reference,
             reference,
@@ -52,6 +50,18 @@ def resolve(self, reference):
         if criteria_check is not True:
             return criteria_check
 
+
+class ExecTestResolver(BaseExec, Resolver):
+
+    name = "exec-test"
+    description = "Test resolver for executable files to be handled as tests"
+    priority = PluginPriority.VERY_LOW
+
+    def resolve(self, reference):
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
+
         runnable = Runnable("exec-test", reference, assets=get_file_assets(reference))
         return ReferenceResolution(
             reference, ReferenceResolutionResult.SUCCESS, [runnable]
@@ -121,24 +131,16 @@ def resolve(self, reference):
         )
 
 
-class TapResolver(Resolver):
+class TapResolver(BaseExec, Resolver):
 
     name = "tap"
     description = "Test resolver for executable files to be handled as TAP tests"
     priority = PluginPriority.LAST_RESORT
 
     def resolve(self, reference):
-
-        criteria_check = check_file(
-            reference,
-            reference,
-            suffix=None,
-            type_name="executable file",
-            access_check=os.R_OK | os.X_OK,
-            access_name="executable",
-        )
-        if criteria_check is not True:
-            return criteria_check
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
 
         runnable = Runnable("tap", reference, assets=get_file_assets(reference))
         return ReferenceResolution(
@@ -196,3 +198,102 @@ def resolve(self, reference):
             return criteria_check
 
         return self._validate_and_load_runnables(reference)
+
+
+class ExecRunnablesRecipeInit(Init):
+    name = "exec-runnables-recipe"
+    description = 'Configuration for resolver plugin "exec-runnables-recipe" plugin'
+
+    def initialize(self):
+        help_msg = (
+            'Whether resolvers (such as "exec-runnables-recipe") should '
+            "execute files given as test references that have executable "
+            "permissions. This is disabled by default due to security "
+            "implications of running executables that may not be trusted."
+        )
+        settings.register_option(
+            section="resolver",
+            key="run_executables",
+            key_type=bool,
+            default=False,
+            help_msg=help_msg,
+        )
+
+        help_msg = (
+            "Command line options (space separated) that will be added "
+            "to the executable when executing it as a producer of "
+            "runnables-recipe JSON content."
+        )
+        settings.register_option(
+            section="resolver.exec_runnables_recipe",
+            key="arguments",
+            key_type=str,
+            default="",
+            help_msg=help_msg,
+        )
+
+
+class ExecRunnablesRecipeResolver(BaseExec, Resolver):
+    name = "exec-runnables-recipe"
+    description = "Test resolver for executables that output JSON runnable recipes"
+    priority = PluginPriority.LOW
+
+    def resolve(self, reference):
+        if not self.config.get("resolver.run_executables"):
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=(
+                    "Running executables is not enabled. Refer to "
+                    '"resolver.run_executables" configuration option'
+                ),
+            )
+
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
+
+        args = self.config.get("resolver.exec_runnables_recipe.arguments")
+        if args:
+            cmd = [reference] + shlex.split(args)
+        else:
+            cmd = reference
+        try:
+            process = subprocess.Popen(
+                cmd,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        except (FileNotFoundError, PermissionError) as exc:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=(f'Failure while running running executable "{reference}": {exc}'),
+            )
+
+        content, _ = process.communicate()
+        try:
+            runnables = json.loads(content)
+        except json.JSONDecodeError:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=f'Content generated by running executable "{reference}" is not JSON',
+            )
+
+        if not (
+            isinstance(runnables, list)
+            and all([isinstance(r, dict) for r in runnables])
+        ):
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=f"Content generated by running executable {reference} does not look like a runnables recipe JSON content",
+            )
+
+        return ReferenceResolution(
+            reference,
+            ReferenceResolutionResult.SUCCESS,
+            [Runnable.from_dict(r) for r in runnables],
+        )

avocado/plugins/run.py

@@ -267,6 +267,21 @@ def configure(self, parser):
             long_arg="--log-test-data-directories",
         )
 
+        settings.add_argparser_to_option(
+            namespace="resolver.run_executables",
+            parser=parser,
+            long_arg="--resolver-run-executables",
+            allow_multiple=True,
+        )
+
+        settings.add_argparser_to_option(
+            namespace="resolver.exec_runnables_recipe.arguments",
+            metavar="ARGS",
+            parser=parser,
+            long_arg="--resolver-exec-arguments",
+            allow_multiple=True,
+        )
+
         parser_common_args.add_tag_filter_args(parser)
 
     def run(self, config):

docs/source/guides/writer/chapters/recipes.rst

@@ -76,3 +76,46 @@ That will be parsed by the ``runnables-recipe`` resolver, like in
 
   exec-test /bin/true
   exec-test /bin/false
+
+Using dynamically generated recipes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``exec-runnables-recipe`` resolver allows a user to point to a
+file that will be executed, and that is expected to generate (on its
+``STDOUT``) content compatible with the Runnable recipe format
+mentioned previously.
+
+.. note:: For security reasons, Avocado won't execute files
+          indiscriminately when looking for tests (at the resolution
+          phase). One must set the ``--resolver-run-executables``
+          command line option (or the underlying
+          ``resolver.run_executables`` configuration option) to allow
+          running executables at the resolver stage.
+
+A script such as:
+
+.. literalinclude:: ../../../../../examples/nrunner/resolvers/exec_runnables_recipe.sh
+
+Will output JSON that is compatible with the runnable recipe format.
+That can be used directly via either ``avocado list`` or ``avocado
+run``.  Example::
+
+  $ avocado list --resolver-run-executables examples/nrunner/resolvers/exec_runnables_recipe.sh
+
+  exec-test true-test
+  exec-test false-test
+
+If the executable to be run needs arguments, you can pass it via the
+``--resolver-exec-arguments`` or the underlying
+``resolver.exec_runnable_recipe.arguments`` option.  The following
+script receives an optional parameter that can change the type of the
+tests it generates:
+
+.. literalinclude:: ../../../../../examples/nrunner/resolvers/exec_runnables_recipe_kind.sh
+
+In order to have those tests resolved as ``tap`` tests, one can run::
+
+  $ avocado list --resolver-run-executables --resolver-exec-arguments tap examples/nrunner/resolvers/exec_runnables_recipe_kind.sh
+
+  tap true-test
+  tap false-test

examples/nrunner/resolvers/exec_runnables_recipe.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo '[{"kind": "exec-test","uri": "/bin/true","identifier": "true-test"},{"kind": "exec-test","uri": "/bin/false","identifier": "false-test"}]'

examples/nrunner/resolvers/exec_runnables_recipe_kind.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+kind=${1:-exec-test}
+echo "[{\"kind\": \"$kind\",\"uri\": \"/bin/true\",\"identifier\": \"true-test\"},{\"kind\": \"$kind\",\"uri\": \"/bin/false\",\"identifier\": \"false-test\"}]"

selftests/check.py

@@ -29,7 +29,7 @@
     "nrunner-requirement": 28,
     "unit": 678,
     "jobs": 11,
-    "functional-parallel": 309,
+    "functional-parallel": 312,
     "functional-serial": 7,
     "optional-plugins": 0,
     "optional-plugins-golang": 2,