diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37fdca9ca2..def72c3aee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -376,12 +376,12 @@ jobs: cd examples/plugins/tests/magic python3 setup.py develop --user cd ../../../../ - python3 -m avocado -V list -- pass fail | grep "magic: 2" + python3 -m avocado -V list -- magic:pass magic:fail | grep "magic: 2" podman pull quay.io/avocado-framework/avocado-ci-magic - python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- pass - tail -n1 ~/avocado/job-results/latest/results.tap | grep "ok 1 pass" - python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- fail || true - tail -n1 ~/avocado/job-results/latest/results.tap | grep "not ok 1 fail" + python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- magic:pass + tail -n1 ~/avocado/job-results/latest/results.tap | grep "ok 1 magic:pass" + python3 -m avocado run --spawner=podman --spawner-podman-image=quay.io/avocado-framework/avocado-ci-magic -- magic:fail || true + tail -n1 ~/avocado/job-results/latest/results.tap | grep "not ok 1 magic:fail" fedora_develop_install_uninstall_task: name: Fedora develop install/uninstall task diff --git a/avocado/core/resolver.py b/avocado/core/resolver.py index 51a582950b..96632e3f17 100644 --- a/avocado/core/resolver.py +++ b/avocado/core/resolver.py @@ -22,6 +22,7 @@ from avocado.core.enabled_extension_manager import EnabledExtensionManager from avocado.core.exceptions import JobTestSuiteReferenceResolutionError +from avocado.core.output import LOG_UI class ReferenceResolutionAssetType(Enum): @@ -38,6 +39,8 @@ class ReferenceResolutionAssetType(Enum): class ReferenceResolutionResult(Enum): #: Given test reference was properly resolved SUCCESS = object() + #: Given test reference might be resolved, but it is corrupted. + CORRUPT = object() #: Given test reference was not properly resolved NOTFOUND = object() #: Internal error in the resolution process @@ -108,6 +111,7 @@ class Resolver(EnabledExtensionManager): DEFAULT_POLICY = { ReferenceResolutionResult.SUCCESS: ReferenceResolutionAction.RETURN, + ReferenceResolutionResult.CORRUPT: ReferenceResolutionAction.RETURN, ReferenceResolutionResult.NOTFOUND: ReferenceResolutionAction.CONTINUE, ReferenceResolutionResult.ERROR: ReferenceResolutionAction.CONTINUE, } @@ -267,6 +271,14 @@ def resolve(references, hint=None, ignore_missing=True, config=None): discoverer = Discoverer(config) resolutions.extend(discoverer.discover()) + for res in resolutions: + if res.result == ReferenceResolutionResult.CORRUPT: + LOG_UI.warning( + "Reference %s might be resolved by %s resolver, but the file is corrupted: %s", + res.reference, + res.origin, + res.info or "", + ) # This came up from a previous method and can be refactored to improve # performance since that we could merge with the loop above. if not ignore_missing: diff --git a/avocado/plugins/list.py b/avocado/plugins/list.py index 97ee84c753..ac48d56d67 100644 --- a/avocado/plugins/list.py +++ b/avocado/plugins/list.py @@ -95,6 +95,7 @@ def _display_extra(suite, verbose=True): mapping = { ReferenceResolutionResult.SUCCESS: TERM_SUPPORT.healthy_str, + ReferenceResolutionResult.CORRUPT: TERM_SUPPORT.warn_header_str, ReferenceResolutionResult.NOTFOUND: TERM_SUPPORT.fail_header_str, ReferenceResolutionResult.ERROR: TERM_SUPPORT.fail_header_str, } diff --git a/docs/source/guides/contributor/chapters/plugins.rst b/docs/source/guides/contributor/chapters/plugins.rst index 27f5845911..9947f41ef1 100644 --- a/docs/source/guides/contributor/chapters/plugins.rst +++ b/docs/source/guides/contributor/chapters/plugins.rst @@ -357,32 +357,35 @@ Resolving magic tests --------------------- Resolving the "pass" and "fail" references that the magic plugin knows about -can be seen by running ``avocado list pass fail``:: +can be seen by running ``avocado list magic:pass magic:fail``:: - magic pass - magic fail + magic magic:pass + magic magic:fail And you may get more insight into the resolution results, by adding a verbose parameter and another reference. Try running ``avocado -V -list pass fail something-else``:: +list magic:pass magic:fail magic:foo something-else``:: - Type Test Tag(s) - magic pass - magic fail + Reference magic:foo might be resolved by magic resolver, but the file is corrupted: Word "magic:foo" is magic type but the foo is not a valid magic word + Type Test Tag(s) + magic magic:pass + magic magic:fail Resolver Reference Info - avocado-instrumented pass File "pass" does not end with ".py" - exec-test pass File "pass" does not exist or is not a executable file - golang pass - avocado-instrumented fail File "fail" does not end with ".py" - exec-test fail File "fail" does not exist or is not a executable file - golang fail + avocado-instrumented magic:pass File "magic" does not end with ".py" + golang magic:pass go binary not found + avocado-instrumented magic:fail File "magic" does not end with ".py" + golang magic:fail go binary not found + avocado-instrumented magic:foo File "magic" does not end with ".py" + golang magic:foo go binary not found + magic magic:foo Word "magic:foo" is magic type but the foo is not a valid magic word avocado-instrumented something-else File "something-else" does not end with ".py" - exec-test something-else File "something-else" does not exist or is not a executable file - golang something-else + golang something-else go binary not found magic something-else Word "something-else" is not a valid magic word python-unittest something-else File "something-else" does not end with ".py" robot something-else File "something-else" does not end with ".robot" + rogue something-else Word "something-else" is not the magic word + exec-test something-else File "something-else" does not exist or is not a executable file tap something-else File "something-else" does not exist or is not a executable file TEST TYPES SUMMARY @@ -390,27 +393,34 @@ list pass fail something-else``:: magic: 2 It's worth realizing that magic (and other plugins) were asked to -resolve the ``something-else`` reference, but couldn't:: +resolve the ``magic:foo`` and ``something-else`` references, but couldn't:: Resolver Reference Info ... + magic magic:foo Word "magic:foo" is magic type but the foo is not a valid magic word + ... magic something-else Word "something-else" is not a valid magic word ... +We can see that the reference "magic:foo" resembles the magic words by type but it is not magic words ``pass`` or ``fail``. +Consequently, the resolver can provide the user with information about potentially corrupted references. +This can assist the user in identifying typos or reference mistakes. As the creator of the resolver, +you can use the "ReferenceResolutionResult.CORRUPT" variable to notify the user of such a situation. + Running magic tests ------------------- The common way of running Avocado tests is to run them through ``avocado run``. To run both the ``pass`` and ``fail`` magic tests, -you'd run ``avocado run -- pass fail``:: +you'd run ``avocado run -- magic:pass magic:fail``:: - $ avocado run -- pass fail + $ avocado run -- magic:pass magic:fail JOB ID : 86fd45f8c1f2fe766c252eefbcac2704c2106db9 JOB LOG : $HOME/avocado/job-results/job-2021-02-05T12.43-86fd45f/job.log - (1/2) pass: STARTED - (1/2) pass: PASS (0.00 s) - (2/2) fail: STARTED - (2/2) fail: FAIL (0.00 s) + (1/2) magic:pass: STARTED + (1/2) magic:pass: PASS (0.00 s) + (2/2) magic:fail: STARTED + (2/2) magic:fail: FAIL (0.00 s) RESULTS : PASS 1 | ERROR 0 | FAIL 1 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0 JOB HTML : $HOME/avocado/job-results/job-2021-02-05T12.43-86fd45f/results.html JOB TIME : 1.83 s diff --git a/examples/plugins/tests/magic/avocado_magic/resolver.py b/examples/plugins/tests/magic/avocado_magic/resolver.py index 7fede87b15..67f4e3d0a3 100644 --- a/examples/plugins/tests/magic/avocado_magic/resolver.py +++ b/examples/plugins/tests/magic/avocado_magic/resolver.py @@ -31,13 +31,26 @@ class MagicResolver(Resolver): @staticmethod def resolve(reference): # pylint: disable=W0221 - if reference not in VALID_MAGIC_WORDS: + try: + key_word, magic_word = reference.split(":", 1) + except (ValueError): + key_word = None + magic_word = reference + if key_word != "magic": return ReferenceResolution( reference, ReferenceResolutionResult.NOTFOUND, info=f'Word "{reference}" is not a valid magic word', ) + if magic_word not in VALID_MAGIC_WORDS: + return ReferenceResolution( + reference, + ReferenceResolutionResult.CORRUPT, + [Runnable("magic", reference)], + info=f'Word "{reference}" is magic type but the {magic_word} is not a valid magic word', + ) + return ReferenceResolution( reference, ReferenceResolutionResult.SUCCESS, [Runnable("magic", reference)] ) diff --git a/examples/plugins/tests/magic/avocado_magic/runner.py b/examples/plugins/tests/magic/avocado_magic/runner.py index e101a32522..b4448e9f77 100644 --- a/examples/plugins/tests/magic/avocado_magic/runner.py +++ b/examples/plugins/tests/magic/avocado_magic/runner.py @@ -26,8 +26,8 @@ class MagicRunner(BaseRunner): def run(self, runnable): yield StartedMessage.get() - if runnable.uri in ["pass", "fail"]: - result = runnable.uri + if runnable.uri in ["magic:pass", "magic:fail"]: + result = runnable.uri.split(":")[1] else: result = "error" yield FinishedMessage.get(result) diff --git a/selftests/functional/resolver.py b/selftests/functional/resolver.py index f50ffe4a27..a226670ce1 100644 --- a/selftests/functional/resolver.py +++ b/selftests/functional/resolver.py @@ -8,7 +8,7 @@ # is also the same from selftests.functional.list import AVOCADO_TEST_OK as AVOCADO_INSTRUMENTED_TEST from selftests.functional.list import EXEC_TEST -from selftests.utils import AVOCADO, BASEDIR +from selftests.utils import AVOCADO, BASEDIR, python_module_available class ResolverFunctional(unittest.TestCase): @@ -106,6 +106,23 @@ def test_recursive_by_default(self): ) self.assertEqual("avocado-instrumented: 10", lines[-1]) + @unittest.skipUnless( + python_module_available("magic"), "avocado-magic not available" + ) + def test_corrupted_reference(self): + cmd_line = f"{AVOCADO} list magic:foo" + result = process.run(cmd_line) + self.assertIn( + "Reference magic:foo might be resolved by magic resolver, but the file is corrupted:", + result.stderr_text, + ) + cmd_line = f"{AVOCADO} run magic:foo" + result = process.run(cmd_line, ignore_status=True) + self.assertIn( + "Reference magic:foo might be resolved by magic resolver, but the file is corrupted:", + result.stderr_text, + ) + if __name__ == "__main__": unittest.main()