From 2f31aaaca26b73aa81beb973bdbfd52820d528f9 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Mon, 26 Aug 2024 15:58:16 +0200 Subject: [PATCH] Tap runner hides failures fix This commit adds fixes for TAP runner to not hide failures when some tests are skipped. Also, it updates Avocado documentation to respect newest changes in TAP runner. Reference: #6007 Signed-off-by: Jan Richter --- avocado/plugins/runners/tap.py | 12 +++++++----- docs/source/guides/user/chapters/concepts.rst | 15 +++++++++++++++ docs/source/guides/user/chapters/logging.rst | 2 ++ selftests/unit/plugin/runners/tap.py | 2 +- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/avocado/plugins/runners/tap.py b/avocado/plugins/runners/tap.py index 239ab56674..867d74a0ec 100644 --- a/avocado/plugins/runners/tap.py +++ b/avocado/plugins/runners/tap.py @@ -38,7 +38,7 @@ class TAPRunner(ExecTestRunner): @staticmethod def _get_tap_result(stdout): parser = TapParser(io.StringIO(stdout.decode())) - result = "error" + result = "" fail_reason = None for event in parser.parse(): if isinstance(event, TapParser.Bailout): @@ -50,16 +50,18 @@ def _get_tap_result(stdout): break elif isinstance(event, TapParser.Plan): if event.skipped: - result = "skip" - break + if not result: + result = "skip" + continue elif isinstance(event, TapParser.Test): if event.result == TestResult.FAIL: result = "fail" fail_reason = event.explanation break elif event.result == TestResult.SKIP: - result = "skip" - break + if not result: + result = "skip" + continue elif event.result == TestResult.XPASS: result = "warn" if event.name: diff --git a/docs/source/guides/user/chapters/concepts.rst b/docs/source/guides/user/chapters/concepts.rst index b32048d3c3..7bbe19072f 100644 --- a/docs/source/guides/user/chapters/concepts.rst +++ b/docs/source/guides/user/chapters/concepts.rst @@ -207,6 +207,21 @@ difference is that the test result will be decided based on the produced output, that should be in `Test Anything Protocol `_ format. +Even though such executable can be seen as test suite from avocado point of view, +it will be considered as one standalone test. If you want to get result of each +test of such executable, you can get generated tap output in debug.log file. +:ref:`avocado-log-files` + +.. note:: + The result of Tap test is based on the importance of individual results types + like this: + + `SKIP -> PASS -> FAIL`. + + This means if one of the tests in TAP output + is `not ok` the TAP test result is `FAIL`. If all tests are `ok` or `skip` + the results is `PASS` and if all results are `skip` the result is `SKIP`. + Test statuses ------------- diff --git a/docs/source/guides/user/chapters/logging.rst b/docs/source/guides/user/chapters/logging.rst index 288c3becf1..817db67e7a 100644 --- a/docs/source/guides/user/chapters/logging.rst +++ b/docs/source/guides/user/chapters/logging.rst @@ -50,6 +50,8 @@ from all tests in one place. built-in streams in this function. +.. _avocado-log-files: + Avocado log files ----------------- diff --git a/selftests/unit/plugin/runners/tap.py b/selftests/unit/plugin/runners/tap.py index 877e9cb0bb..63dc240809 100644 --- a/selftests/unit/plugin/runners/tap.py +++ b/selftests/unit/plugin/runners/tap.py @@ -69,7 +69,7 @@ def test_skip(self): results = [status for status in runner.run(runnable)] last_result = results[-1] self.assertEqual(last_result["status"], "finished") - self.assertEqual(last_result["result"], "skip") + self.assertEqual(last_result["result"], "pass") self.assertEqual(last_result["returncode"], 0) @skipUnlessPathExists("/bin/sh")