Save the subresults for tmt-report-result

All the results generated by tmt-report-results become objects of
`tmt.result.SubResult`.

stories/features/report-result.fmf

@@ -12,14 +12,12 @@ description: |
     guest and overwrite any existing scripts with the same name.
 
     The command can be called multiple times for a single test,
-    the final result will be the most severe rating. Available
-    values ordered by severity are SKIP, PASS, WARN and FAIL.
-    Note, that the only value currently processed by ``tmt`` is
-    the test result, other options are silently ignored.
+    all these calls will be saved as tmt subresults. The final result
+    will be the most severe rating. Available values ordered by severity are
+    SKIP, PASS, WARN and FAIL.
 
     When ``tmt-report-result`` is called, the return value of the
-    test itself is ignored, and only result saved by
-    ``tmt-report-result`` is consumed by tmt.
+    test itself is saved as a tmt subresult.
 
     __ https://restraint.readthedocs.io/en/latest/commands.html#rstrnt-report-result
 

tmt/result.py

@@ -161,6 +161,12 @@ class SubResult(BaseResult):
             SubCheckResult.from_serialized(check) for check in serialized]
         )
 
+    data_path: Optional[Path] = field(
+        default=None,
+        serialize=lambda path: None if path is None else str(path),
+        unserialize=lambda value: None if value is None else Path(value)
+        )
+
 
 @dataclasses.dataclass
 class PhaseResult(BaseResult):
@@ -218,7 +224,9 @@ def from_test_invocation(
             result: ResultOutcome,
             note: Optional[str] = None,
             ids: Optional[ResultIds] = None,
-            log: Optional[list[Path]] = None) -> 'Result':
+            log: Optional[list[Path]] = None,
+            subresult: Optional[list[SubResult]] = None,
+            ) -> 'Result':
         """
         Create a result from a test invocation.
 
@@ -268,7 +276,9 @@ def from_test_invocation(
             ids=ids,
             log=log or [],
             guest=guest_data,
-            data_path=invocation.relative_test_data_path)
+            data_path=invocation.relative_test_data_path,
+            subresult=subresult or [],
+            )
 
         return _result.interpret_result(ResultInterpret(
             invocation.test.result) if invocation.test.result else ResultInterpret.RESPECT)

tmt/steps/execute/__init__.py

@@ -664,48 +664,77 @@ def _load_tmt_report_results_file(self, invocation: TestInvocation) -> ResultCol
 
         return collection
 
-    def _process_results_reduce(
+    @staticmethod
+    def _process_subresult_outcomes_reduce(outcomes: list[ResultOutcome]) -> ResultOutcome:
+        """
+
+        Reduce given subresult outcomes to one main outcome.
+
+        This is the default behavior applied to test subresults, all subresults will be reduced
+        to the worst outcome possible.
+
+        :param outcomes: subresult outcomes to reduce.
+        :returns: main result outcome.
+        """
+        hierarchy = [
+            ResultOutcome.SKIP,
+            ResultOutcome.PASS,
+            ResultOutcome.WARN,
+            ResultOutcome.FAIL]
+        outcome_indices = [hierarchy.index(outcome) for outcome in outcomes]
+
+        # The actual outcome we decided is the best representing the subresults.
+        return hierarchy[max(outcome_indices)]
+
+    def _process_subresults(
             self,
             invocation: TestInvocation,
             results: list['tmt.result.RawResult']) -> list['tmt.result.Result']:
         """
-        Reduce given results to one outcome.
 
-        This is the default behavior applied to test results, all
-        results will be reduced to the worst outcome possible.
+        Load the subresults and assign them to their respective main result defined by current
+        invocation.
+
+        Also, reduce given subresult outcomes to one which is assigned to a main result.
+
+        This is the default behavior applied to test subresults, all subresult outcomes will be
+        reduced to the worst outcome possible.
 
         :param invocation: test invocation to which the results belong to.
         :param results: results to reduce.
         :returns: list of results.
         """
 
-        # The worst result outcome we can find among loaded results...
+        # The worst result outcome we can find among loaded results.
         original_outcome: Optional[ResultOutcome] = None
-        # ... and the actual outcome we decided is the best representing
-        # the results.
-        # The original one may be left unset - malformed results file,
-        # for example, provides no usable original outcome.
+
+        # The actual outcome we decided is the best representing the results.
+        # The original one may be left unset - malformed results file, for example, provides no
+        # usable original outcome.
         actual_outcome: ResultOutcome
         note: Optional[str] = None
 
+        # Loaded tmt subresults
+        subresults: list[tmt.result.SubResult] = []
+
+        # Load the results as tmt SubResults
         try:
-            outcomes = [
-                ResultOutcome.from_spec(result.get('result'))
-                for result in results
-                ]
+            for result in results:
+                subresult = tmt.result.SubResult.from_serialized(result)
+
+                # TODO: Somehow deal with the subresult checks
+
+                # Change the name hierarchy so the subresult name is under the main result name
+                # TODO: Check if the `subresult.name` startswith('/'), if not, add it?
+                subresult.name = f'{invocation.test.name}{subresult.name}'
+                subresults.append(subresult)
 
         except tmt.utils.SpecificationError as exc:
             actual_outcome = ResultOutcome.ERROR
             note = exc.message
-
         else:
-            hierarchy = [
-                ResultOutcome.SKIP,
-                ResultOutcome.PASS,
-                ResultOutcome.WARN,
-                ResultOutcome.FAIL]
-            outcome_indices = [hierarchy.index(outcome) for outcome in outcomes]
-            actual_outcome = original_outcome = hierarchy[max(outcome_indices)]
+            actual_outcome = original_outcome = self._process_subresult_outcomes_reduce(
+                [r.result for r in subresults])
 
         # Find a usable log - the first one matching our "interim" outcome.
         # We cannot use the "actual" outcome, because that one may not even
@@ -715,22 +744,21 @@ def _process_results_reduce(
         test_logs = [invocation.relative_path / TEST_OUTPUT_FILENAME]
 
         if original_outcome is not None:
-            for result in results:
-                if result.get('result') != original_outcome.value:
+            for subresult in subresults:
+                if subresult.result != original_outcome:
                     continue
 
-                result_logs: list[str] = result.get('log', [])
-
-                if result_logs:
-                    test_logs.append(invocation.relative_test_data_path / result_logs[0])
+                if subresult.log:
+                    test_logs.append(invocation.relative_test_data_path / subresult.log[0])
 
                 break
 
         return [tmt.Result.from_test_invocation(
             invocation=invocation,
             result=actual_outcome,
             log=test_logs,
-            note=note)]
+            note=note,
+            subresult=subresults)]
 
     def _process_results_partials(
             self,
@@ -834,7 +862,8 @@ def extract_tmt_report_results(self, invocation: TestInvocation) -> list["tmt.Re
         """
         Extract results from the file generated by ``tmt-report-result`` script.
 
-        All recorded results are reduced to one result eventually.
+        All recorded results are saved as subresults and their outomes are reduced to one result
+        which is assigned to a main result.
         """
 
         collection = self._load_tmt_report_results_file(invocation)
@@ -854,7 +883,7 @@ def extract_tmt_report_results(self, invocation: TestInvocation) -> list["tmt.Re
 
         collection.validate()
 
-        return self._process_results_reduce(invocation, collection.results)
+        return self._process_subresults(invocation, collection.results)
 
     def extract_tmt_report_results_restraint(
             self,
@@ -904,7 +933,7 @@ def extract_results(
                 invocation=invocation,
                 default_log=invocation.relative_path / TEST_OUTPUT_FILENAME)
 
-        # Handle the 'tmt-report-result' command results as a single test
+        # Handle the 'tmt-report-result' command results and save them as tmt subresults
         if self._tmt_report_results_filepath(invocation).exists():
             return self.extract_tmt_report_results(invocation)