Skip to content

Commit

Permalink
Introduce "exec-runnables-recipe" resolver
Browse files Browse the repository at this point in the history
This resolver is somewhat of a hybrid between the "exec-test" and the
"runnables-recipe" resolvers.

It runs an executable, and attempts to read from its STDOUT content
that will be treated as runnables-recipe JSON content.  If that
succeeds, the content will be returned as test resolutions.  This is
useful for executable tests or test generators that will output the
tests dinamically.

Signed-off-by: Cleber Rosa <[email protected]>
  • Loading branch information
clebergnu committed Sep 18, 2024
1 parent 2920bc9 commit 6a91f4a
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 3 deletions.
7 changes: 7 additions & 0 deletions avocado/plugins/list.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,13 @@ def configure(self, parser):
allow_multiple=True,
)

settings.add_argparser_to_option(
namespace="resolver.run_executables",
parser=parser,
long_arg="--resolver-run-executables",
allow_multiple=True,
)

help_msg = "Writes runnable recipe files to a directory."
settings.register_option(
section="list.recipes",
Expand Down
85 changes: 84 additions & 1 deletion avocado/plugins/resolvers.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@
import json
import os
import re
import subprocess

from avocado.core.extension_manager import PluginPriority
from avocado.core.nrunner.runnable import Runnable
from avocado.core.plugin_interfaces import Resolver
from avocado.core.plugin_interfaces import Init, Resolver
from avocado.core.references import reference_split
from avocado.core.resolver import (
ReferenceResolution,
Expand All @@ -31,6 +32,7 @@
get_file_assets,
)
from avocado.core.safeloader import find_avocado_tests, find_python_unittests
from avocado.core.settings import settings


class BaseExec:
Expand Down Expand Up @@ -195,3 +197,84 @@ def resolve(self, reference):
return criteria_check

return self._validate_and_load_runnables(reference)


class ExecRunnablesRecipeInit(Init):
name = "exec-runnables-recipe"
description = 'Configuration for resolver plugin "exec-runnables-recipe" plugin'

def initialize(self):
help_msg = (
'Whether resolvers (such as "exec-runnables-recipe") should '
"execute files given as test references that have executable "
"permissions. This is disabled by default due to security "
"implications of running executables that may not be trusted."
)
settings.register_option(
section="resolver",
key="run_executables",
key_type=bool,
default=False,
help_msg=help_msg,
)


class ExecRunnablesRecipeResolver(BaseExec, Resolver):
name = "exec-runnables-recipe"
description = "Test resolver for executables that output JSON runnable recipes"
priority = PluginPriority.LOW

def resolve(self, reference):
if not self.config.get("resolver.run_executables"):
return ReferenceResolution(
reference,
ReferenceResolutionResult.NOTFOUND,
info=(
"Running executables is not enabled. Refer to "
'"resolver.run_executables" configuration option'
),
)

exec_criteria = self.check_exec(reference)
if exec_criteria is not None:
return exec_criteria

try:
process = subprocess.Popen(
reference,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
except (FileNotFoundError, PermissionError) as exc:
return ReferenceResolution(
reference,
ReferenceResolutionResult.NOTFOUND,
info=(f'Failure while running running executable "{reference}": {exc}'),
)

content, _ = process.communicate()
try:
runnables = json.loads(content)
except json.JSONDecodeError:
return ReferenceResolution(
reference,
ReferenceResolutionResult.NOTFOUND,
info=f'Content generated by running executable "{reference}" is not JSON',
)

if not (
isinstance(runnables, list)
and all([isinstance(r, dict) for r in runnables])
):
return ReferenceResolution(
reference,
ReferenceResolutionResult.NOTFOUND,
info=f"Content generated by running executable {reference} does not look like a runnables recipe JSON content",
)

return ReferenceResolution(
reference,
ReferenceResolutionResult.SUCCESS,
[Runnable.from_dict(r) for r in runnables],
)
7 changes: 7 additions & 0 deletions avocado/plugins/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,13 @@ def configure(self, parser):
long_arg="--log-test-data-directories",
)

settings.add_argparser_to_option(
namespace="resolver.run_executables",
parser=parser,
long_arg="--resolver-run-executables",
allow_multiple=True,
)

parser_common_args.add_tag_filter_args(parser)

def run(self, config):
Expand Down
28 changes: 28 additions & 0 deletions docs/source/guides/writer/chapters/recipes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,31 @@ That will be parsed by the ``runnables-recipe`` resolver, like in

exec-test /bin/true
exec-test /bin/false

Using dinamically generated recipes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``exec-runnables-recipe`` resolver allows a user to point to a
file that will be executed, and that is expected to generate (on its
``STDOUT``) content compatible with the Runnable recipe format
mentioned previously.

.. note:: For security reasons, Avocado won't execute files
indiscriminately when looking for tests (at the resolution
phase). One must set the ``--resolver-run-executables``
command line option (or the underlying
``resolver.run_executables`` configuration option) to allow
running executables at the resolver stage.

A script such as:

.. literalinclude:: ../../../../../examples/nrunner/resolvers/exec_runnables_recipe.sh

Will output JSON that is compatible with the runnable recipe format.
That can be used directly via either ``avocado list`` or ``avocado
run``. Example::

$ avocado list --resolver-run-executables examples/nrunner/resolvers/exec_runnables_recipe.sh

exec-test true-test
exec-test false-test
2 changes: 1 addition & 1 deletion selftests/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"nrunner-requirement": 28,
"unit": 678,
"jobs": 11,
"functional-parallel": 309,
"functional-parallel": 311,
"functional-serial": 7,
"optional-plugins": 0,
"optional-plugins-golang": 2,
Expand Down
46 changes: 45 additions & 1 deletion selftests/functional/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,13 @@
# is also the same
from selftests.functional.list import AVOCADO_TEST_OK as AVOCADO_INSTRUMENTED_TEST
from selftests.functional.list import EXEC_TEST
from selftests.utils import AVOCADO, BASEDIR, TestCaseTmpDir, python_module_available
from selftests.utils import (
AVOCADO,
BASEDIR,
TestCaseTmpDir,
python_module_available,
skipUnlessPathExists,
)


class ResolverFunctional(unittest.TestCase):
Expand Down Expand Up @@ -157,6 +163,44 @@ def test_runnable_recipe_origin(self):
result.stdout,
)

@skipUnlessPathExists("/bin/sh")
def test_exec_runnable_recipe_disabled(self):
resolver_path = os.path.join(
BASEDIR,
"examples",
"nrunner",
"resolvers",
"exec_runnables_recipe.sh",
)
cmd_line = f"{AVOCADO} -V list {resolver_path}"
result = process.run(cmd_line)
self.assertIn(
b"examples/nrunner/resolvers/exec_runnables_recipe.sh exec-test",
result.stdout,
)
self.assertIn(b"exec-test: 1\n", result.stdout)

@skipUnlessPathExists("/bin/sh")
def test_exec_runnable_recipe_enabled(self):
resolver_path = os.path.join(
BASEDIR,
"examples",
"nrunner",
"resolvers",
"exec_runnables_recipe.sh",
)
cmd_line = f"{AVOCADO} -V list --resolver-run-executables {resolver_path}"
result = process.run(cmd_line)
self.assertIn(
b"exec-test true-test /bin/true exec-runnables-recipe",
result.stdout,
)
self.assertIn(
b"exec-test false-test /bin/false exec-runnables-recipe",
result.stdout,
)
self.assertIn(b"exec-test: 2\n", result.stdout)


class ResolverFunctionalTmp(TestCaseTmpDir):
def test_runnables_recipe(self):
Expand Down
2 changes: 2 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ def run(self):
"nrunner = avocado.plugins.runner_nrunner:RunnerInit",
"testlogsui = avocado.plugins.testlogs:TestLogsUIInit",
"human = avocado.plugins.human:HumanInit",
"exec-runnables-recipe = avocado.plugins.resolvers:ExecRunnablesRecipeInit",
],
"avocado.plugins.cli": [
"xunit = avocado.plugins.xunit:XUnitCLI",
Expand Down Expand Up @@ -461,6 +462,7 @@ def run(self):
"tap = avocado.plugins.resolvers:TapResolver",
"runnable-recipe = avocado.plugins.resolvers:RunnableRecipeResolver",
"runnables-recipe = avocado.plugins.resolvers:RunnablesRecipeResolver",
"exec-runnables-recipe = avocado.plugins.resolvers:ExecRunnablesRecipeResolver",
],
"avocado.plugins.suite.runner": [
"nrunner = avocado.plugins.runner_nrunner:Runner",
Expand Down

0 comments on commit 6a91f4a

Please sign in to comment.