diff --git a/doc/Unittesting/DataModel.rst b/doc/Unittesting/DataModel.rst new file mode 100644 index 00000000..8bd53490 --- /dev/null +++ b/doc/Unittesting/DataModel.rst @@ -0,0 +1,796 @@ +.. _UNITTEST/DataModel: + +Unified data model +****************** + +The unified data model for test entities (test summary, test suite, test case) implements a super-set of all (so far +known) unit test result summary file formats. pyEDAA.Report's data model is a structural and functional cleanup of the +Ant JUnit data model. Naming has been cleaned up and missing features have been added. + +As some of the JUnit XML dialects are too divergent from the original Ant + JUnit4 format, these dialects have an +independent test entity inheritance hierarchy. Nonetheless, instances of each data format can be converted to and from +the unified data model. + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + .. grid:: 2 + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/DataModel/Testcase` + ^^^ + A :dfn:`test case` is the leaf-element in the test entity hierarchy and describes an individual test run. + Test cases are grouped by test suites. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/DataModel/Testsuite` + ^^^ + A :dfn:`test suite` is a group of test cases and/or test suites. Test suites itself can be grouped by test + suites. The test suite hierarchy's root element is a test suite summary. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/DataModel/TestsuiteSummary` + ^^^ + The :dfn:`test suite summary` is derived from test suite and defines the root of the test suite hierarchy. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/DataModel/Document` + ^^^ + The :dfn:`document` is derived from a test suite summary and represents a file containing a test suite + summary. + + .. grid-item:: + :columns: 6 + + .. mermaid:: + + graph TD; + doc[Document] + sum[Summary] + ts1[Testsuite] + ts2[Testsuite] + ts21[Testsuite] + tc11[Testcase] + tc12[Testcase] + tc13[Testcase] + tc21[Testcase] + tc22[Testcase] + tc211[Testcase] + tc212[Testcase] + tc213[Testcase] + + doc:::root -.-> sum:::summary + sum --> ts1:::suite + sum --> ts2:::suite + ts2 --> ts21:::suite + ts1 --> tc11:::case + ts1 --> tc12:::case + ts1 --> tc13:::case + ts2 --> tc21:::case + ts2 --> tc22:::case + ts21 --> tc211:::case + ts21 --> tc212:::case + ts21 --> tc213:::case + + classDef root fill:#4dc3ff + classDef summary fill:#80d4ff + classDef suite fill:#b3e6ff + classDef case fill:#eeccff + + +.. _UNITTEST/DataModel/TestcaseStatus: + +Testcase Status +=============== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + :class:`~pyEDAA.Reports.Unittesting.TestcaseStatus` and :class:`~pyEDAA.Reports.Unittesting.TestsuiteStatus` are + flag enumerations to describe the overall status of a test case or test suite. + + Unknown + tbd + + Excluded + tbd + + Skipped + tbd + + Weak + tbd + + Passed + tbd + + Failed + tbd + + Inverted + tbd + + Warned + tbd + + Errored + tbd + + Failed + tbd + + SetupError + tbd + + TearDownError + tbd + + Inconsistent + tbd + + + .. grid-item:: + :columns: 6 + + .. code-block:: Python + + @export + class TestcaseStatus(Flag): + """A flag enumeration describing the status of a test case.""" + Unknown = 0 #: Testcase status is uninitialized and therefore unknown. + Excluded = 1 #: Testcase was permanently excluded / disabled + Skipped = 2 #: Testcase was temporarily skipped (e.g. based on a condition) + Weak = 4 #: No assertions were recorded. + Passed = 8 #: A passed testcase, because all assertions were successful. + Failed = 16 #: A failed testcase due to at least one failed assertion. + + Mask = Excluded | Skipped | Weak | Passed | Failed + + Inverted = 128 #: To mark inverted results + UnexpectedPassed = Failed | Inverted + ExpectedFailed = Passed | Inverted + + Warned = 1024 #: Runtime warning + Errored = 2048 #: Runtime error (mostly caught exceptions) + Aborted = 4096 #: Uncaught runtime exception + + SetupError = 8192 #: Preparation / compilation error + TearDownError = 16384 #: Cleanup error / resource release error + Inconsistent = 32768 #: Dataset is inconsistent + + Flags = Warned | Errored | Aborted | SetupError | TearDownError | Inconsistent + + .. code-block:: Python + + @export + class TestsuiteStatus(Flag): + """A flag enumeration describing the status of a test suite.""" + Unknown = 0 + Excluded = 1 #: Testcase was permanently excluded / disabled + Skipped = 2 #: Testcase was temporarily skipped (e.g. based on a condition) + Empty = 4 #: No tests in suite + Passed = 8 #: Passed testcase, because all assertions succeeded + Failed = 16 #: Failed testcase due to failing assertions + + Mask = Excluded | Skipped | Empty | Passed | Failed + + Inverted = 128 #: To mark inverted results + UnexpectedPassed = Failed | Inverted + ExpectedFailed = Passed | Inverted + + Warned = 1024 #: Runtime warning + Errored = 2048 #: Runtime error (mostly caught exceptions) + Aborted = 4096 #: Uncaught runtime exception + + SetupError = 8192 #: Preparation / compilation error + TearDownError = 16384 #: Cleanup error / resource release error + + Flags = Warned | Errored | Aborted | SetupError | TearDownError + + +.. _UNITTEST/DataModel/Testcase: + +Testcase +======== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + A :class:`~pyEDAA.Reports.Unittesting.Testcase` is the leaf-element in the test entity hierarchy and describes an + individual test run. Besides a test case status, it also contains statistics like the start time or the test + duration. Test cases are grouped by test suites and need to be unique per parent test suite. + + A test case (or its base classes) implements the following properties and methods: + + :data:`~pyEDAA.Reports.Unittesting.Base.Parent` + The test case has a reference to it's parent test suite in the hierarchy. By iterating parent references, the + root element (test suite summary) be be found, which has no parent reference (``None``). + + :data:`~pyEDAA.Reports.Unittesting.Base.Name` + The test case has a name. This name must be unique per hierarchy parent, but can exist multiple times in the + overall test hierarchy. + + In case the data format uses hierarchical names like ``pyEDAA.Reports.CLI.Application``, the name is split at + the separator and multiple hierarchy levels (test suites) are created in the unified data model. To be able to + recreate such an hierarchical name, :class:`~pyEDAA.Reports.Unittesting.TestsuiteKind` is applied accordingly + to test suite's :data:`~pyEDAA.Reports.Unittesting.TestsuiteBase.Kind` field. + + :data:`~pyEDAA.Reports.Unittesting.Base.StartTime` + The test case stores a time when the individual test run was started. In combination with + :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration`, the end time can be calculated. If the start time is + unknown, set this value to ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.SetupDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TestDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TeardownDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration` + The test case has fields to capture the setup duration, test run duration and teardown duration. The sum of all + durations is provided by total duration. + + :pycode:`TotalDuration := SetupDuration + TestDuration + TeardownDuration` + + The :dfn:`setup duration` is the time spend on setting up a test run. If the setup duration can't be + distinguished from the test's runtime, set this value to ``None``. + + The test's runtime without setup and teardown portions is captured by :dfn:`test duration`. If the duration is + unknown, set this value to ``None``. + + The :dfn:`teardown duration` of a test run is the time spend on tearing down a test run. If the teardown + duration can't be distinguished from the test's runtime, set this value to ``None``. + + The test case has a field :dfn:`total duration` to sum up setup duration, test duration and teardown duration. + If the duration is unknown, this value will be ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.WarningCount`, :data:`~pyEDAA.Reports.Unittesting.Base.ErrorCount`, :data:`~pyEDAA.Reports.Unittesting.Base.FatalCount` + The test case counts for warnings, errors and fatal errors observed in a test run while the test was executed. + + :meth:`~pyEDAA.Reports.Unittesting.Base.__len__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__getitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__setitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__delitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__contains__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__iter__` + The test case implements a dictionary interface, so arbitrary key-value pairs can be annotated per test entity. + + :data:`~pyEDAA.Reports.Unittesting.Testcase.Status` + The overall status of a test case. + + See also: :ref:`UNITTEST/DataModel/TestcaseStatus`. + + :data:`~pyEDAA.Reports.Unittesting.Testcase.AssertionCount`, :data:`~pyEDAA.Reports.Unittesting.Testcase.PassedAssertionCount`, :data:`~pyEDAA.Reports.Unittesting.Testcase.FailedAssertionCount` + The :dfn:`assertion count` represents the overall number of assertions (checks) in a test case. It can be + distinguished into :dfn:`passed assertions` and :dfn:`failed assertions`. If it can't be distinguished, set + passed and failed assertions to ``None``. + + :pycode:`AssertionCount := PassedAssertionCount + FailedAssertionCount` + + :meth:`~pyEDAA.Reports.Unittesting.Testcase.Copy` + tbd + + :meth:`~pyEDAA.Reports.Unittesting.Testcase.Aggregate` + Aggregate (recalculate) all durations, warnings, errors, assertions, etc. + + :meth:`~pyEDAA.Reports.Unittesting.Testcase.__str__` + tbd + + .. grid-item:: + :columns: 6 + + .. code-block:: Python + + @export + class Testcase(Base): + def __init__( + self, + name: str, + startTime: Nullable[datetime] = None, + setupDuration: Nullable[timedelta] = None, + testDuration: Nullable[timedelta] = None, + teardownDuration: Nullable[timedelta] = None, + totalDuration: Nullable[timedelta] = None, + status: TestcaseStatus = TestcaseStatus.Unknown, + assertionCount: Nullable[int] = None, + failedAssertionCount: Nullable[int] = None, + passedAssertionCount: Nullable[int] = None, + warningCount: int = 0, + errorCount: int = 0, + fatalCount: int = 0, + parent: Nullable["Testsuite"] = None + ): + ... + + @readonly + def Parent(self) -> Nullable["Testsuite"]: + ... + + @readonly + def Name(self) -> str: + ... + + @readonly + def StartTime(self) -> Nullable[datetime]: + ... + + @readonly + def SetupDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TestDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TeardownDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TotalDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def WarningCount(self) -> int: + ... + + @readonly + def ErrorCount(self) -> int: + ... + + @readonly + def FatalCount(self) -> int: + ... + + def __len__(self) -> int: + ... + + def __getitem__(self, key: str) -> Any: + ... + + def __setitem__(self, key: str, value: Any) -> None: + ... + + def __delitem__(self, key: str) -> None: + ... + + def __contains__(self, key: str) -> bool: + ... + + def __iter__(self) -> Generator[Tuple[str, Any], None, None]: + ... + + @readonly + def Status(self) -> TestcaseStatus: + ... + + @readonly + def AssertionCount(self) -> int: + ... + + @readonly + def FailedAssertionCount(self) -> int: + ... + + @readonly + def PassedAssertionCount(self) -> int: + ... + + def Copy(self) -> "Testcase": + ... + + def Aggregate(self, strict: bool = True) -> TestcaseAggregateReturnType: + ... + + def __str__(self) -> str: + ... + + +.. _UNITTEST/DataModel/Testsuite: + +Testsuite +========= + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + A :class:`~pyEDAA.Reports.Unittesting.Testsuite` is a grouping element in the test entity hierarchy and describes + a group of test runs. Besides a list of test cases and a test suite status, it also contains statistics like the + start time or the test duration for the group of tests. Test suites are grouped by other test suites or a test + suite summary and need to be unique per parent test suite. + + A test suite (or its base classes) implements the following properties and methods: + + :data:`~pyEDAA.Reports.Unittesting.Base.Parent` + The test suite has a reference to it's parent test entity in the hierarchy. By iterating parent references, the + root element (test suite summary) be be found, which has no parent reference (``None``). + + :data:`~pyEDAA.Reports.Unittesting.Base.Name` + The test suite has a name. This name must be unique per hierarchy parent, but can exist multiple times in the + overall test hierarchy. + + In case the data format uses hierarchical names like ``pyEDAA.Reports.CLI.Application``, the name is split at + the separator and multiple hierarchy levels (test suites) are created in the unified data model. To be able to + recreate such an hierarchical name, :class:`~pyEDAA.Reports.Unittesting.TestsuiteKind` is applied accordingly + to test suite's :data:`~pyEDAA.Reports.Unittesting.TestsuiteBase.Kind` field. + + :data:`~pyEDAA.Reports.Unittesting.Base.StartTime` + The test suite stores a time when the first test run was started. In combination with + :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration`, the end time can be calculated. If the start time is + unknown, set this value to ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.SetupDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TestDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TeardownDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration` + The test suite has fields to capture the suite's setup duration, test group run duration and suite's + teardown duration. The sum of all durations is provided by total duration. + + :pycode:`TotalDuration := SetupDuration + TestDuration + TeardownDuration` + + The :dfn:`setup duration` is the time spend on setting up a test suite. If the setup duration can't be + distinguished from the test group's runtime, set this value to ``None``. + + The test group's runtime without setup and teardown portions is captured by :dfn:`test duration`. If the + duration is unknown, set this value to ``None``. + + The :dfn:`teardown duration` of a test suite is the time spend on tearing down a test suite. If the teardown + duration can't be distinguished from the test group's runtime, set this value to ``None``. + + The test suite has a field :dfn:`total duration` to sum up setup duration, test duration and teardown duration. + If the duration is unknown, this value will be ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.WarningCount`, :data:`~pyEDAA.Reports.Unittesting.Base.ErrorCount`, :data:`~pyEDAA.Reports.Unittesting.Base.FatalCount` + The test suite counts for warnings, errors and fatal errors observed in a test suite while the tests were + executed. + + :meth:`~pyEDAA.Reports.Unittesting.Base.__len__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__getitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__setitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__delitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__contains__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__iter__` + The test suite implements a dictionary interface, so arbitrary key-value pairs can be annotated. + + + .. todo:: TestsuiteBase APIs + + + :data:`~pyEDAA.Reports.Unittesting.Testsuite.Testcases` + tbd + + :data:`~pyEDAA.Reports.Unittesting.Testsuite.TestcaseCount` + tbd + + :data:`~pyEDAA.Reports.Unittesting.Testsuite.AssertionCount` + The overall number of assertions (checks) in a test case. + + :meth:`~pyEDAA.Reports.Unittesting.Testsuite.Aggregate` + Aggregate (recalculate) all durations, warnings, errors, assertions, etc. + + :meth:`~pyEDAA.Reports.Unittesting.Testsuite.Iterate` + tbd + + :meth:`~pyEDAA.Reports.Unittesting.Testsuite.__str__` + tbd + + .. grid-item:: + :columns: 6 + + .. code-block:: Python + + @export + class Testsuite(TestsuiteBase[TestsuiteType]): + def __init__( + self, + name: str, + kind: TestsuiteKind = TestsuiteKind.Logical, + startTime: Nullable[datetime] = None, + setupDuration: Nullable[timedelta] = None, + testDuration: Nullable[timedelta] = None, + teardownDuration: Nullable[timedelta] = None, + totalDuration: Nullable[timedelta] = None, + status: TestsuiteStatus = TestsuiteStatus.Unknown, + warningCount: int = 0, + errorCount: int = 0, + fatalCount: int = 0, + testsuites: Nullable[Iterable[TestsuiteType]] = None, + testcases: Nullable[Iterable["Testcase"]] = None, + parent: Nullable[TestsuiteType] = None + ): + ... + + @readonly + def Parent(self) -> Nullable["Testsuite"]: + ... + + @readonly + def Name(self) -> str: + ... + + @readonly + def StartTime(self) -> Nullable[datetime]: + ... + + @readonly + def SetupDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TestDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TeardownDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TotalDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def WarningCount(self) -> int: + ... + + @readonly + def ErrorCount(self) -> int: + ... + + @readonly + def FatalCount(self) -> int: + ... + + def __len__(self) -> int: + ... + + def __getitem__(self, key: str) -> Any: + ... + + def __setitem__(self, key: str, value: Any) -> None: + ... + + def __delitem__(self, key: str) -> None: + ... + + def __contains__(self, key: str) -> bool: + ... + + def __iter__(self) -> Generator[Tuple[str, Any], None, None]: + ... + + # TestsuiteBase API + + + @readonly + def Testcases(self) -> Dict[str, "Testcase"]: + ... + + @readonly + def TestcaseCount(self) -> int: + ... + + @readonly + def AssertionCount(self) -> int: + ... + + def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: + ... + + def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]: + ... + + def __str__(self) -> str: + ... + + +.. _UNITTEST/DataModel/TestsuiteSummary: + +TestsuiteSummary +================ + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + A :class:`~pyEDAA.Reports.Unittesting.TestsuiteSummary` is the root element in the test entity hierarchy and + describes a group of test suites as well as overall statistics for the whole set of test cases. A test suite + summary is derived for the same base-class as a test suite, thus they share almost all properties and methods. + + A test suite summary (or its base classes) implements the following properties and methods: + + :data:`~pyEDAA.Reports.Unittesting.Base.Parent` + The test suite summary has a parent reference, but as the root element in the test entity hierarchy, its always + ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.Name` + The test suite summary has a name. + + :data:`~pyEDAA.Reports.Unittesting.Base.StartTime` + The test suite summary stores a time when the first test runs was started. In combination with + :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration`, the end time can be calculated. If the start time is + unknown, set this value to ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.SetupDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TestDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TeardownDuration`, :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration` + The test suite summary has fields to capture the suite summary's setup duration, overall run duration and + suite summary's teardown duration. The sum of all durations is provided by total duration. + + :pycode:`TotalDuration := SetupDuration + TestDuration + TeardownDuration` + + The :dfn:`setup duration` is the time spend on setting up an overall test run. If the setup duration can't be + distinguished from the test's runtimes, set this value to ``None``. + + The test suite summary's runtime without setup and teardown portions is captured by :dfn:`test duration`. If + the duration is unknown, set this value to ``None``. + + The :dfn:`teardown duration` of a test suite summary is the time spend on tearing down a test suite summary. If + the teardown duration can't be distinguished from the test's runtimes, set this value to ``None``. + + The test suite summary has a field :dfn:`total duration` to sum up setup duration, overall run duration and + teardown duration. If the duration is unknown, this value will be ``None``. + + :data:`~pyEDAA.Reports.Unittesting.Base.WarningCount`, :data:`~pyEDAA.Reports.Unittesting.Base.ErrorCount`, :data:`~pyEDAA.Reports.Unittesting.Base.FatalCount` + The test suite summary counts for warnings, errors and fatal errors observed in a test suite summary while the + tests were executed. + + :meth:`~pyEDAA.Reports.Unittesting.Base.__len__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__getitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__setitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__delitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__contains__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__iter__` + The test suite summary implements a dictionary interface, so arbitrary key-value pairs can be annotated. + + .. todo:: TestsuiteBase APIs + + :meth:`~pyEDAA.Reports.Unittesting.TestsuiteSummary.Aggregate` + tbd + + :meth:`~pyEDAA.Reports.Unittesting.TestsuiteSummary.Iterate` + tbd + + :meth:`~pyEDAA.Reports.Unittesting.TestsuiteSummary.__str__` + tbd + + .. grid-item:: + :columns: 6 + + .. code-block:: Python + + @export + class TestsuiteSummary(TestsuiteBase[TestsuiteType]): + def __init__( + self, + name: str, + startTime: Nullable[datetime] = None, + setupDuration: Nullable[timedelta] = None, + testDuration: Nullable[timedelta] = None, + teardownDuration: Nullable[timedelta] = None, + totalDuration: Nullable[timedelta] = None, + status: TestsuiteStatus = TestsuiteStatus.Unknown, + warningCount: int = 0, + errorCount: int = 0, + fatalCount: int = 0, + testsuites: Nullable[Iterable[TestsuiteType]] = None, + parent: Nullable[TestsuiteType] = None + ): + ... + + @readonly + def Parent(self) -> Nullable["Testsuite"]: + ... + + @readonly + def Name(self) -> str: + ... + + @readonly + def StartTime(self) -> Nullable[datetime]: + ... + + @readonly + def SetupDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TestDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TeardownDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def TotalDuration(self) -> Nullable[timedelta]: + ... + + @readonly + def WarningCount(self) -> int: + ... + + @readonly + def ErrorCount(self) -> int: + ... + + @readonly + def FatalCount(self) -> int: + ... + + def __len__(self) -> int: + ... + + def __getitem__(self, key: str) -> Any: + ... + + def __setitem__(self, key: str, value: Any) -> None: + ... + + def __delitem__(self, key: str) -> None: + ... + + def __contains__(self, key: str) -> bool: + ... + + def __iter__(self) -> Generator[Tuple[str, Any], None, None]: + ... + + # TestsuiteBase API + + def Aggregate(self) -> TestsuiteAggregateReturnType: + ... + + def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]: + ... + + def __str__(self) -> str: + ... + + +.. _UNITTEST/DataModel/Document: + +Document +======== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + A :class:`~pyEDAA.Reports.Unittesting.Document` is a mixin-class ... + + :data:`~pyEDAA.Reports.Unittesting.Document.Path` + tbd + + :data:`~pyEDAA.Reports.Unittesting.Document.AnalysisDuration` + tbd + + :data:`~pyEDAA.Reports.Unittesting.Document.ModelConversionDuration` + tbd + + :meth:`~pyEDAA.Reports.Unittesting.Document.Analyze` + tbd + + :meth:`~pyEDAA.Reports.Unittesting.Document.Convert` + tbd + + .. grid-item:: + :columns: 6 + + .. code-block:: Python + + @export + class Document(metaclass=ExtendedType, mixin=True): + def __init__(self, path: Path): + ... + + @readonly + def Path(self) -> Path: + ... + + @readonly + def AnalysisDuration(self) -> timedelta: + ... + + @readonly + def ModelConversionDuration(self) -> timedelta: + ... + + @abstractmethod + def Analyze(self) -> None: + ... + + @abstractmethod + def Convert(self): + ... diff --git a/doc/Unittesting/Features.rst b/doc/Unittesting/Features.rst new file mode 100644 index 00000000..60f65bf9 --- /dev/null +++ b/doc/Unittesting/Features.rst @@ -0,0 +1,506 @@ +.. _UNITTEST/Features: + +Features +******** + + +.. _UNITTEST/Feature/Create: + +Create test entities +==================== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + The hierarchy of test entities (test cases, test suites and test summaries) can be constructed top-down or + bottom-up. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Test Case + :sync: Testcase + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting import Testsuite, Testcase + + # Top-down + ts1 = Testsuite("ts1") + + tc = Testcase("tc", parent=ts) + + # Bottom-up + tc1 = Testcase("tc1") + tc2 = Testcase("tc2") + + ts2 = Testsuite("ts2", testcases=(tc1, tc2)) + + # ts.AddTestcase(...) + tc3 = Testcase("tc3") + tc4 = Testcase("tc4") + + ts3 = Testsuite("ts3") + ts3.AddTestcase(tc3) + ts3.AddTestcase(tc4) + + # ts.AddTestcases(...) + tc3 = Testcase("tc3") + tc4 = Testcase("tc4") + + ts3 = Testsuite("ts3") + ts3.AddTestcases((tc3, tc4)) + + .. tab-item:: Test Suite + :sync: Testsuite + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting import Testsuite, TestsuiteSummary + + # Top-down + ts = Testsuite("ts") + + ts1 = Testsuite("ts1", parent=tss) + + # Bottom-up + ts2 = Testsuite("ts2") + ts3 = Testsuite("ts3") + + ts4 = Testsuite("ts4", testsuites=(ts2, ts3)) + + # ts.AddTestsuite(...) + ts5 = Testcase("ts5") + ts6 = Testcase("ts6") + + ts7 = Testsuite("ts7") + ts7.AddTestsuite(ts5) + ts7.AddTestsuite(ts6) + + # ts.AddTestsuites(...) + ts8 = Testcase("ts8") + ts9 = Testcase("ts9") + + ts10 = Testsuite("ts10") + ts10.AddTestsuites((ts8, ts9)) + + .. tab-item:: Test Suite Summary + :sync: TestsuiteSummary + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting import Testsuite, TestsuiteSummary + + # Top-down + + # Bottom-up + + +.. _UNITTEST/Feature/Read: + +Reading unittest reports +======================== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + A JUnit XML test report summary file can be read by creating an instance of the :class:`~pyEDAA.Reports.Unittesting.JUnit.Document` + class. Because JUnit has so many dialects, a derived subclass for the dialect might be required. By choosing the + right Document class, also the XML schema for XML schema validation gets pre-selected. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Any JUnit + :sync: AnyJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit import Document + + xmlReport = Path("AnyJUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: Ant + JUnit4 + :sync: AntJUnit4 + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.AntJUnit import Document + + xmlReport = Path("AntJUnit4-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: CTest JUnit + :sync: CTestJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.CTestJUnit import Document + + xmlReport = Path("CTest-JUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: GoogleTest JUnit + :sync: GTestJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.GoogleTestJUnit import Document + + xmlReport = Path("GoogleTest-JUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: pyTest JUnit + :sync: pyTestJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.PyTestJUnit import Document + + xmlReport = Path("pyTest-JUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + + +.. _UNITTEST/Feature/Convert: + +Converting unittest reports +=========================== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + Any JUnit dialect specific data model can be converted to the generic hierarchy of test entities. + + + .. note:: + + This conversion is identical for all derived dialects. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Document + :sync: Document + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit import Document + + # Read from XML file + xmlReport = Path("JUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + # Convert to unified test data model + summary = doc.ToTestsuiteSummary() + + # Convert to a tree + rootNode = doc.ToTree() + + # Convert back to a document + newXmlReport = Path("New JUnit-Report.xml") + newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) + + # Write to XML file + newDoc.Write() + + +.. _UNITTEST/Feature/Annotation: + +Annotations +=========== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + Every test entity can be annotated with arbitrary key-value pairs. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Testcase + :sync: Testcase + + .. code-block:: Python + + # Add annotate a key-value pair + testcase["key"] = value + + # Update existing annotation with new value + testcase["key"] = newValue + + # Check if key exists + if "key" in testcase: + pass + + # Access annoation by key + value = testcase["key"] + + # Get number of annotations + annotationCount = len(testcase) + + # Delete annotation + del testcase["key"] + + # Iterate annotations + for key, value in testcases: + pass + + .. tab-item:: Testsuite + :sync: Testsuite + + .. code-block:: Python + + # Add annotate a key-value pair + testsuite["key"] = value + + # Update existing annotation with new value + testsuite["key"] = newValue + + # Check if key exists + if "key" in testsuite: + pass + + # Access annoation by key + value = testsuite["key"] + + # Get number of annotations + annotationCount = len(testsuite) + + # Delete annotation + del testsuite["key"] + + # Iterate annotations + for key, value in testsuite: + pass + + .. tab-item:: TestsuiteSummary + :sync: TestsuiteSummary + + .. code-block:: Python + + # Add annotate a key-value pair + testsuiteSummary["key"] = value + + # Update existing annotation with new value + testsuiteSummary["key"] = newValue + + # Check if key exists + if "key" in testsuiteSummary: + pass + + # Access annoation by key + value = testsuiteSummary["key"] + + # Get number of annotations + annotationCount = len(testsuiteSummary) + + # Delete annotation + del testsuiteSummary["key"] + + # Iterate annotations + for key, value in testsuiteSummary: + pass + + + +.. _UNITTEST/Feature/Merge: + +Merging unittest reports +======================== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + add description here + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Testcase + :sync: Testcase + + .. code-block:: Python + + # add code here + +.. _UNITTEST/Feature/Concat: + +Concatenate unittest reports +============================ + +.. todo:: Planned feature. + +.. _UNITTEST/Feature/Transform: + +Transforming the reports' hierarchy +=================================== + +.. _UNITTEST/Feature/Transform/pytest: + +pytest specific transformations +------------------------------- + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + add description here + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Testcase + :sync: Testcase + + .. code-block:: Python + + # add code here + +.. _UNITTEST/Feature/Write: + +Writing unittest reports +======================== + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + A test suite summary can be converted to a document of any JUnit dialect. Internally a deep-copy is created to + convert from a hierarchy of the unified test entities to a hierarchy of specific test entities (e.g. JUnit + entities). + + When the document was created, it can be written to disk. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Any JUnit + :sync: AnyJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit import Document + + # Convert a TestsuiteSummary back to a Document + newXmlReport = Path("JUnit-Report.xml") + newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) + + # Write to XML file + try: + newDoc.Write() + except UnittestException as ex: + ... + + .. tab-item:: Ant + JUnit4 + :sync: AntJUnit4 + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.AntJUnit import Document + + # Convert a TestsuiteSummary back to a Document + newXmlReport = Path("JUnit-Report.xml") + newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) + + # Write to XML file + try: + newDoc.Write() + except UnittestException as ex: + ... + + .. tab-item:: CTest JUnit + :sync: CTestJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.CTestJUnit import Document + + # Convert a TestsuiteSummary back to a Document + newXmlReport = Path("JUnit-Report.xml") + newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) + + # Write to XML file + try: + newDoc.Write() + except UnittestException as ex: + ... + + .. tab-item:: GoogleTest JUnit + :sync: GTestJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.GoogleTestJUnit import Document + + # Convert a TestsuiteSummary back to a Document + newXmlReport = Path("JUnit-Report.xml") + newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) + + # Write to XML file + try: + newDoc.Write() + except UnittestException as ex: + ... + + .. tab-item:: pyTest JUnit + :sync: pyTestJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.PyTestJUnit import Document + + # Convert a TestsuiteSummary back to a Document + newXmlReport = Path("JUnit-Report.xml") + newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) + + # Write to XML file + try: + newDoc.Write() + except UnittestException as ex: + ... diff --git a/doc/Unittesting/JUnitDataModel.rst b/doc/Unittesting/JUnitDataModel.rst new file mode 100644 index 00000000..065fcc3c --- /dev/null +++ b/doc/Unittesting/JUnitDataModel.rst @@ -0,0 +1,375 @@ +.. _UNITTEST/SpecificDataModel/JUnit: + +JUnit Data Model +================ + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + .. grid:: 2 + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/SpecificDataModel/JUnit/Testcase` + ^^^ + A :dfn:`test case` is the leaf-element in the test entity hierarchy and describes an individual test run. + Test cases are grouped by test classes. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/SpecificDataModel/JUnit/Testclass` + ^^^ + A :dfn:`test class` is the mid-level element in the test entity hierarchy and describes a group of test + runs. Test classes are grouped by test suites. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/SpecificDataModel/JUnit/Testsuite` + ^^^ + A :dfn:`test suite` is a group of test classes. Test suites are grouped by a test suite summary. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/SpecificDataModel/JUnit/TestsuiteSummary` + ^^^ + The :dfn:`test suite summary` is derived from test suite and defines the root of the test suite hierarchy. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/SpecificDataModel/JUnit/Document` + ^^^ + The :dfn:`document` is derived from a test suite summary and represents a file containing a test suite + summary. + + .. grid-item-card:: + :columns: 6 + + :ref:`UNITTEST/SpecificDataModel/JUnit/Dialects` + ^^^ + The JUnit format is not well defined, thus multiple dialects developed over time. + + .. grid-item:: + :columns: 6 + + .. mermaid:: + + graph TD; + doc[Document] + sum[Summary] + ts1[Testsuite] + ts11[Testsuite] + ts2[Testsuite] + + tc111[Testclass] + tc112[Testclass] + tc23[Testclass] + + tc1111[Testcase] + tc1112[Testcase] + tc1113[Testcase] + tc1121[Testcase] + tc1122[Testcase] + tc231[Testcase] + tc232[Testcase] + tc233[Testcase] + + doc:::root -.-> sum:::summary + sum --> ts1:::suite + sum ---> ts2:::suite + ts1 --> ts11:::suite + + ts11 --> tc111:::cls + ts11 --> tc112:::cls + ts2 --> tc23:::cls + + tc111 --> tc1111:::case + tc111 --> tc1112:::case + tc111 --> tc1113:::case + tc112 --> tc1121:::case + tc112 --> tc1122:::case + tc23 --> tc231:::case + tc23 --> tc232:::case + tc23 --> tc233:::case + + classDef root fill:#4dc3ff + classDef summary fill:#80d4ff + classDef suite fill:#b3e6ff + classDef cls fill:#ff9966 + classDef case fill:#eeccff + +.. _UNITTEST/SpecificDataModel/JUnit/Testcase: + +Testcase +-------- + +.. _UNITTEST/SpecificDataModel/JUnit/Testclass: + +Testclass +--------- + +.. _UNITTEST/SpecificDataModel/JUnit/Testsuite: + +Testsuite +--------- + +.. _UNITTEST/SpecificDataModel/JUnit/TestsuiteSummary: + +TestsuiteSummary +---------------- + +.. _UNITTEST/SpecificDataModel/JUnit/Document: + +Document +-------- + +.. _UNITTEST/SpecificDataModel/JUnit/Dialects: + +JUnit Dialects +============== + +As the JUnit XML format was not well specified and no XML Schema Definition (XSD) was provided, many variants and +dialects (and simplifications) were created by the various frameworks emitting JUnit XML files. + +.. rubric:: JUnit Dialect Comparison + ++------------------------+--------------+--------------+--------------------+------------------+--------------+ +| Feature | Any JUnit | Ant + JUnit4 | CTest JUnit | GoogleTest JUnit | pyTest JUnit | ++========================+==============+==============+====================+==================+==============+ +| Root element | testsuites | testsuite | testsuite | testsuites | testsuites | ++------------------------+--------------+--------------+--------------------+------------------+--------------+ +| Supports properties | ☑ | ☑ | | ⸺ | | ++------------------------+--------------+--------------+--------------------+------------------+--------------+ +| Testcase status | ... | ... | more status values | | | ++------------------------+--------------+--------------+--------------------+------------------+--------------+ + +.. _UNITTEST/SpecificDataModel/JUnit/Dialect/AnyJUnit: + +Any JUnit +--------- + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + The Any JUnit format uses a relaxed XML schema definition aiming to parse many JUnit XML dialects, which use a + ```` root element. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Reading, analyzing, converting Any JUnit + :sync: ReadJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit import Document + + xmlReport = Path("AnyJUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: Writing Any JUnit + :sync: WriteJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit import Document + + xmlReport = Path("AnyJUnit-Report.xml") + try: + doc.Write(xmlReport) + except UnittestException as ex: + ... + + +.. _UNITTEST/SpecificDataModel/JUnit/Dialect/AntJUnit4: + +Ant + JUnit4 +------------ + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + The original JUnit format created by `Ant `__ for `JUnit4 `__ + uses ```` as a root element. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Reading, analyzing, converting Ant + JUnit4 + :sync: ReadJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.AntJUnit4 import Document + + xmlReport = Path("AntJUnit4-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: Writing Ant + JUnit4 + :sync: WriteJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.AntJUnit4 import Document + + xmlReport = Path("AntJUnit4-Report.xml") + try: + doc.Write(xmlReport) + except UnittestException as ex: + ... + + + +.. _UNITTEST/SpecificDataModel/JUnit/Dialect/CTest: + +CTest JUnit +----------- + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + The CTest JUnit format written by `CTest `__ uses ```` as a root + element. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Reading, analyzing, converting CTest JUnit + :sync: ReadJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.CTestJUnit import Document + + xmlReport = Path("CTestJUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: Writing CTest JUnit + :sync: WriteJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.CTestJUnit import Document + + xmlReport = Path("CTestJUnit-Report.xml") + try: + doc.Write(xmlReport) + except UnittestException as ex: + ... + + +.. _UNITTEST/SpecificDataModel/JUnit/Dialect/GoogleTest: + +GoogleTest JUnit +---------------- + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + The GoogleTest JUnit format written by `GoogleTest `__ (sometimes GTest) + uses ```` as a root element. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Reading, analyzing, converting GoogleTest JUnit + :sync: ReadJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.GoogleTestJUnit import Document + + xmlReport = Path("GoogleTestJUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: Writing GoogleTest JUnit + :sync: WriteJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.GoogleTestJUnit import Document + + xmlReport = Path("GoogleTestJUnit-Report.xml") + try: + doc.Write(xmlReport) + except UnittestException as ex: + ... + + +.. _UNITTEST/SpecificDataModel/JUnit/Dialect/pyTest: + +pyTest JUnit +------------ + +.. grid:: 2 + + .. grid-item:: + :columns: 6 + + The pyTest JUnit format written by `pyTest `__ uses ```` as a + root element. + + .. grid-item:: + :columns: 6 + + .. tab-set:: + + .. tab-item:: Reading, analyzing, converting pyTest JUnit + :sync: ReadJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.PyTestJUnit import Document + + xmlReport = Path("PyTestJUnit-Report.xml") + try: + doc = Document(xmlReport, parse=True) + except UnittestException as ex: + ... + + .. tab-item:: Writing pyTest JUnit + :sync: WriteJUnit + + .. code-block:: Python + + from pyEDAA.Reports.Unittesting.JUnit.PyTestJUnit import Document + + xmlReport = Path("PyTestJUnit-Report.xml") + try: + doc.Write(xmlReport) + except UnittestException as ex: + ... diff --git a/doc/Unittesting/OSVVMDataModel.rst b/doc/Unittesting/OSVVMDataModel.rst new file mode 100644 index 00000000..f11cf638 --- /dev/null +++ b/doc/Unittesting/OSVVMDataModel.rst @@ -0,0 +1,8 @@ +.. _UNITTEST/SpecificDataModel/OSVVM: + +OSVVM +===== + +`Open Source VHDL Verification Methodology `__ writes test results as YAML files for its +internal data model storage. Some YAML files are written by the VHDL code of the verification framework, others are +written by `OSVVM-Scripts `__ as a test runner. diff --git a/doc/Unittesting/index.rst b/doc/Unittesting/index.rst index 72d5dfe3..24f9fa21 100644 --- a/doc/Unittesting/index.rst +++ b/doc/Unittesting/index.rst @@ -21,1046 +21,19 @@ format. See below for supported formats and their variations (dialects). description and XML schemas, but unfortunately many are not even compatible to each other. -.. _UNITTEST/DataModel: - -Unified data model -****************** - -The unified data model for test entities (test summary, test suite, test case) implements a super-set of all (so far -known) unit test result summary file formats. pyEDAA.Report's data model is a structural and functional cleanup of the -Ant JUnit data model. Naming has been cleaned up and missing features have been added. - -As some of the JUnit XML dialects are too divergent from the original Ant + JUnit4 format, these dialects have an -independent test entity inheritance hierarchy. Nonetheless, instances of each data format can be converted to and from -the unified data model. - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - .. grid:: 2 - - .. grid-item-card:: - :columns: 6 - - :ref:`UNITTEST/DataModel/Testcase` - ^^^ - A :dfn:`test case` is the leaf-element in the test entity hierarchy and describes an individual test run. - Test cases are grouped by test suites. - - .. grid-item-card:: - :columns: 6 - - :ref:`UNITTEST/DataModel/Testsuite` - ^^^ - A :dfn:`test suite` is a group of test cases and/or test suites. Test suites itself can be grouped by test - suites. The test suite hierarchy's root element is a test suite summary. - - .. grid-item-card:: - :columns: 6 - - :ref:`UNITTEST/DataModel/TestsuiteSummary` - ^^^ - The :dfn:`test suite summary` is derived from test suite and defines the root of the test suite hierarchy. - - .. grid-item-card:: - :columns: 6 - - :ref:`UNITTEST/DataModel/Document` - ^^^ - The :dfn:`document` is derived from a test suite summary and represents a file containing a test suite - summary. - - .. grid-item:: - :columns: 6 - - .. mermaid:: - - graph TD; - doc[Document] - sum[Summary] - ts1[Testsuite] - ts2[Testsuite] - ts21[Testsuite] - tc11[Testcase] - tc12[Testcase] - tc13[Testcase] - tc21[Testcase] - tc22[Testcase] - tc211[Testcase] - tc212[Testcase] - tc213[Testcase] - - doc:::root -.-> sum:::summary - sum --> ts1:::suite - sum --> ts2:::suite - ts2 --> ts21:::suite - ts1 --> tc11:::case - ts1 --> tc12:::case - ts1 --> tc13:::case - ts2 --> tc21:::case - ts2 --> tc22:::case - ts21 --> tc211:::case - ts21 --> tc212:::case - ts21 --> tc213:::case - - classDef root fill:#4dc3ff - classDef summary fill:#80d4ff - classDef suite fill:#b3e6ff - classDef case fill:#eeccff - -.. _UNITTEST/DataModel/Common: - -Common -====== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - The base-class for all test entities is :class:`pyEDAA.Reports.Unittesting.Base`. It implements the following - properties and methods, which are common to all test entities: - - :data:`~pyEDAA.Reports.Unittesting.Base.Parent` - Every test entity has a reference to it's parent test entity in the hierarchy. - - :data:`~pyEDAA.Reports.Unittesting.Base.Name` - Every test entity has a name. This name must be unique per hierarchy parent, but can exist multiple times in the - overall test hierarchy. - - In case the data format uses hierarchical names like ``pyEDAA.Reports.CLI.Application``, the name is split at - the separator and multiple hierarchy levels (test suites) are created in the unified data model. To be able to - recreate such an hierarchical name, :class:`~pyEDAA.Reports.Unittesting.TestsuiteKind` is applied accordingly to - test suite's :data:`~pyEDAA.Reports.Unittesting.TestsuiteBase.Kind` field. - - :data:`~pyEDAA.Reports.Unittesting.Base.StartTime` - Every test entity has a time when it was started. In case of a test case, it's the time when an individual test - was run. In case of a test suite, it's the time when the first test within this test suite was started. In case - of a test suite summary, it's the time when the whole regression test was started. - - If the start time is unknown, set this value to ``None``. - - :data:`~pyEDAA.Reports.Unittesting.Base.SetupDuration` - Every test entity has a field to capture the setup duration of a test run. In case of a test case, it's the - time spend on setting up an individual test run. In case of a test suite, it's the duration spend on preparing - the group of tests for the first test run. - - If the setup duration can't be distinguished from the test's runtime, set this value to ``None``. - - :data:`~pyEDAA.Reports.Unittesting.Base.TestDuration` - Every test entity has a field to capture the test's runtime. - - If the duration in unknown, set this value to ``None``. - - :data:`~pyEDAA.Reports.Unittesting.Base.TeardownDuration` - Every test entity has a field to capture the teardown duration of a test run. In case of a test case, it's the - time spend on tearing down an individual test run. In case of a test suite, it's the duration spend on - finalizing the group of tests after the last test run. - - If the teardown duration can't be distinguished from the test's runtime, set this value to ``None``. - - :data:`~pyEDAA.Reports.Unittesting.Base.TotalDuration` - Every test entity has a field summing setup duration, test duration and teardown duration. - - If the duration in unknown, set this value to ``None``. - - .. math:: - - TotalDuration := SetupDuration + TestDuration + TeardownDuration - - :data:`~pyEDAA.Reports.Unittesting.Base.WarningCount` - Every test entity counts for warnings observed in a test run. In case of a test case, these are warnings while - the test was executed. In case of a test suite, these warnings are an aggregate of all warnings within that - group of test cases and test suites. - - .. todo:: Separate setup and teardown warnings from runtime warnings. - - :data:`~pyEDAA.Reports.Unittesting.Base.ErrorCount` - Every test entity counts for errors observed in a test run. In case of a test case, these are errors while the - test was executed. In case of a test suite, these errors are an aggregate of all errors within that group of - test cases and test suites. - - .. todo:: Separate setup and teardown errors from runtime errors. - - :data:`~pyEDAA.Reports.Unittesting.Base.FatalCount` - Every test entity counts for fatal errors observed in a test run. In case of a test case, these are fatal errors - while the test was executed. In case of a test suite, these fatal errors are an aggregate of all fatal errors - within that group of test cases and test suites. - - .. todo:: Separate setup and teardown fatal errors from runtime fatal errors. - - :meth:`~pyEDAA.Reports.Unittesting.Base.__len__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__getitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__setitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__delitem__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__contains__`, :meth:`~pyEDAA.Reports.Unittesting.Base.__iter__` - Every test entity implements a dictionary interface, so arbitrary key-value pairs can be annotated per test - entity. - - :meth:`~pyEDAA.Reports.Unittesting.Base.Aggregate` - Aggregate (recalculate) all durations, warnings, errors, assertions, etc. - - - .. grid-item:: - :columns: 6 - - .. code-block:: Python - - @export - class Base(metaclass=ExtendedType, slots=True): - def __init__( - self, - name: str, - startTime: Nullable[datetime] = None, - setupDuration: Nullable[timedelta] = None, - testDuration: Nullable[timedelta] = None, - teardownDuration: Nullable[timedelta] = None, - totalDuration: Nullable[timedelta] = None, - warningCount: int = 0, - errorCount: int = 0, - fatalCount: int = 0, - parent: Nullable["Testsuite"] = None - ): - ... - - @readonly - def Parent(self) -> Nullable["Testsuite"]: - ... - - @readonly - def Name(self) -> str: - ... - - @readonly - def StartTime(self) -> Nullable[datetime]: - ... - - @readonly - def SetupDuration(self) -> Nullable[timedelta]: - ... - - @readonly - def TestDuration(self) -> Nullable[timedelta]: - ... - - @readonly - def TeardownDuration(self) -> Nullable[timedelta]: - ... - - @readonly - def TotalDuration(self) -> Nullable[timedelta]: - ... - - @readonly - def WarningCount(self) -> int: - ... - - @readonly - def ErrorCount(self) -> int: - ... - - @readonly - def FatalCount(self) -> int: - ... - - def __len__(self) -> int: - ... - - def __getitem__(self, key: str) -> Any: - ... - - def __setitem__(self, key: str, value: Any) -> None: - ... - - def __delitem__(self, key: str) -> None: - ... - - def __contains__(self, key: str) -> bool: - ... - - def __iter__(self) -> Generator[Tuple[str, Any], None, None]: - ... - - @abstractmethod - def Aggregate(self): - ... - -.. _UNITTEST/DataModel/TestcaseStatus: - -Testcase Status -=============== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - :class:`~pyEDAA.Reports.Unittesting.TestcaseStatus` is a flag enumeration to describe the status of a test case. - - - - .. grid-item:: - :columns: 6 - - .. code-block:: Python - - @export - class TestcaseStatus(Flag): - Unknown = 0 #: Testcase status is uninitialized and therefore unknown. - Excluded = 1 #: Testcase was permanently excluded / disabled - Skipped = 2 #: Testcase was temporarily skipped (e.g. based on a condition) - Weak = 4 #: No assertions were recorded. - Passed = 8 #: A passed testcase, because all assertions were successful. - Failed = 16 #: A failed testcase due to at least one failed assertion. - - Mask = Excluded | Skipped | Weak | Passed | Failed - - Inverted = 128 #: To mark inverted results - UnexpectedPassed = Failed | Inverted - ExpectedFailed = Passed | Inverted - - Warned = 1024 #: Runtime warning - Errored = 2048 #: Runtime error (mostly caught exceptions) - Aborted = 4096 #: Uncaught runtime exception - - SetupError = 8192 #: Preparation / compilation error - TearDownError = 16384 #: Cleanup error / resource release error - Inconsistent = 32768 #: Dataset is inconsistent - - Flags = Warned | Errored | Aborted | SetupError | TearDownError | Inconsistent - - -.. _UNITTEST/DataModel/Testcase: - -Testcase -======== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - A :class:`~pyEDAA.Reports.Unittesting.Testcase` is the leaf-element in the test entity hierarchy and describes an - individual test run. Test cases are grouped by test suites. - - :data:`~pyEDAA.Reports.Unittesting.Testcase.Status` - The overall status of a test case. - - See also: :ref:`UNITTEST/DataModel/TestcaseStatus`. - - :data:`~pyEDAA.Reports.Unittesting.Testcase.AssertionCount` - The overall number of assertions (checks) in a test case. - - .. math:: - - AssertionCount := PassedAssertionCount + FailedAssertionCount - - :data:`~pyEDAA.Reports.Unittesting.Testcase.FailedAssertionCount` - The number of failed assertions in a test case. - - :data:`~pyEDAA.Reports.Unittesting.Testcase.PassedAssertionCount` - The number of passed assertions in a test case. - - :meth:`~pyEDAA.Reports.Unittesting.Testcase.Copy` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Testcase.Aggregate` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Testcase.__str__` - tbd - - .. grid-item:: - :columns: 6 - - .. code-block:: Python - - @export - class Testcase(Base): - def __init__( - self, - name: str, - startTime: Nullable[datetime] = None, - setupDuration: Nullable[timedelta] = None, - testDuration: Nullable[timedelta] = None, - teardownDuration: Nullable[timedelta] = None, - totalDuration: Nullable[timedelta] = None, - status: TestcaseStatus = TestcaseStatus.Unknown, - assertionCount: Nullable[int] = None, - failedAssertionCount: Nullable[int] = None, - passedAssertionCount: Nullable[int] = None, - warningCount: int = 0, - errorCount: int = 0, - fatalCount: int = 0, - parent: Nullable["Testsuite"] = None - ): - ... - - @readonly - def Status(self) -> TestcaseStatus: - ... - - @readonly - def AssertionCount(self) -> int: - ... - - @readonly - def FailedAssertionCount(self) -> int: - ... - - @readonly - def PassedAssertionCount(self) -> int: - ... - - def Copy(self) -> "Testcase": - ... - - def Aggregate(self, strict: bool = True) -> TestcaseAggregateReturnType: - ... - - def __str__(self) -> str: - ... - -.. _UNITTEST/DataModel/Testsuite: - -Testsuite -========= - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - :class:`~pyEDAA.Reports.Unittesting.TestsuiteStatus` - - :class:`~pyEDAA.Reports.Unittesting.TestsuiteKind` - - :class:`~pyEDAA.Reports.Unittesting.Testsuite` - - - :data:`~pyEDAA.Reports.Unittesting.Testsuite.Testcases` - tbd - - :data:`~pyEDAA.Reports.Unittesting.Testsuite.TestcaseCount` - tbd - - :data:`~pyEDAA.Reports.Unittesting.Testsuite.AssertionCount` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Testsuite.Aggregate` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Testsuite.Iterate` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Testsuite.__str__` - tbd - - .. grid-item:: - :columns: 6 - - .. code-block:: Python - - @export - class Testsuite(TestsuiteBase[TestsuiteType]): - def __init__( - self, - name: str, - kind: TestsuiteKind = TestsuiteKind.Logical, - startTime: Nullable[datetime] = None, - setupDuration: Nullable[timedelta] = None, - testDuration: Nullable[timedelta] = None, - teardownDuration: Nullable[timedelta] = None, - totalDuration: Nullable[timedelta] = None, - status: TestsuiteStatus = TestsuiteStatus.Unknown, - warningCount: int = 0, - errorCount: int = 0, - fatalCount: int = 0, - testsuites: Nullable[Iterable[TestsuiteType]] = None, - testcases: Nullable[Iterable["Testcase"]] = None, - parent: Nullable[TestsuiteType] = None - ): - - @readonly - def Testcases(self) -> Dict[str, "Testcase"]: - ... - - @readonly - def TestcaseCount(self) -> int: - ... - - @readonly - def AssertionCount(self) -> int: - ... - - def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: - ... - - def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]: - ... - - def __str__(self) -> str: - ... - - -.. _UNITTEST/DataModel/TestsuiteSummary: - -TestsuiteSummary -================ - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - :class:`~pyEDAA.Reports.Unittesting.TestsuiteSummary` - - :meth:`~pyEDAA.Reports.Unittesting.TestsuiteSummary.Aggregate` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.TestsuiteSummary.Iterate` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.TestsuiteSummary.__str__` - tbd - - .. grid-item:: - :columns: 6 - - .. code-block:: Python - - @export - class TestsuiteSummary(TestsuiteBase[TestsuiteType]): - def __init__( - self, - name: str, - startTime: Nullable[datetime] = None, - setupDuration: Nullable[timedelta] = None, - testDuration: Nullable[timedelta] = None, - teardownDuration: Nullable[timedelta] = None, - totalDuration: Nullable[timedelta] = None, - status: TestsuiteStatus = TestsuiteStatus.Unknown, - warningCount: int = 0, - errorCount: int = 0, - fatalCount: int = 0, - testsuites: Nullable[Iterable[TestsuiteType]] = None, - parent: Nullable[TestsuiteType] = None - ): - ... - - def Aggregate(self) -> TestsuiteAggregateReturnType: - ... - - def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]: - ... - - def __str__(self) -> str: - ... - - -.. _UNITTEST/DataModel/Document: - -Document -======== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - :class:`~pyEDAA.Reports.Unittesting.Document` - - :data:`~pyEDAA.Reports.Unittesting.Document.Path` - tbd - - :data:`~pyEDAA.Reports.Unittesting.Document.AnalysisDuration` - tbd - - :data:`~pyEDAA.Reports.Unittesting.Document.ModelConversionDuration` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Document.Analyze` - tbd - - :meth:`~pyEDAA.Reports.Unittesting.Document.Convert` - tbd - - .. grid-item:: - :columns: 6 - - .. code-block:: Python - - @export - class Document(metaclass=ExtendedType, mixin=True): - def __init__(self, path: Path): - ... - - @readonly - def Path(self) -> Path: - ... - - @readonly - def AnalysisDuration(self) -> timedelta: - ... - - @readonly - def ModelConversionDuration(self) -> timedelta: - ... - - @abstractmethod - def Analyze(self) -> None: - ... - - @abstractmethod - def Convert(self): - ... - +.. include:: DataModel.rst .. _UNITTEST/SpecificDataModels: Specific Data Models ******************** -.. _UNITTEST/SpecificDataModel/AnyJUnit4: - -Any JUnit4 -========== - - -.. _UNITTEST/SpecificDataModel/AntJUnit4: - -Ant + JUnit4 -============ - - -.. _UNITTEST/SpecificDataModel/CTest: +.. include:: JUnitDataModel.rst +.. include:: OSVVMDataModel.rst -CTest JUnit -=========== - - -.. _UNITTEST/SpecificDataModel/GoogleTest: - -GoogleTest JUnit -================ - - -.. _UNITTEST/SpecificDataModel/pyTest: - -pyTest JUnit -============ - - - - -.. _UNITTEST/Features: - -Features -******** - - -.. _UNITTEST/Feature/Read: - -Reading unittest reports -======================== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - A JUnit XML test report summary file can be read by creating an instance of the :class:`~pyEDAA.Reports.Unittesting.JUnit.Document` - class. Because JUnit has so many dialects, a derived subclass for the dialect might be required. By choosing the - right Document class, also the XML schema for XML schema validation gets pre-selected. - - .. grid-item:: - :columns: 6 - - .. tab-set:: - - .. tab-item:: Any JUnit - :sync: AnyJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit import Document - - xmlReport = Path("AnyJUnit-Report.xml") - try: - doc = Document(xmlReport, parse=True) - except UnittestException as ex: - ... - - .. tab-item:: Ant + JUnit4 - :sync: AntJUnit4 - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.AntJUnit import Document - - xmlReport = Path("AntJUnit4-Report.xml") - try: - doc = Document(xmlReport, parse=True) - except UnittestException as ex: - ... - - .. tab-item:: CTest JUnit - :sync: CTestJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.CTestJUnit import Document - - xmlReport = Path("CTest-JUnit-Report.xml") - try: - doc = Document(xmlReport, parse=True) - except UnittestException as ex: - ... - - .. tab-item:: GoogleTest JUnit - :sync: GTestJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.GoogleTestJUnit import Document - - xmlReport = Path("GoogleTest-JUnit-Report.xml") - try: - doc = Document(xmlReport, parse=True) - except UnittestException as ex: - ... - - .. tab-item:: pyTest JUnit - :sync: pyTestJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.PyTestJUnit import Document - - xmlReport = Path("pyTest-JUnit-Report.xml") - try: - doc = Document(xmlReport, parse=True) - except UnittestException as ex: - ... - - - -.. _UNITTEST/Feature/Convert: - -Converting unittest reports -=========================== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - Any JUnit dialect specific data model can be converted to the generic hierarchy of test entities. - - - .. note:: - - This conversion is identical for all derived dialects. - - .. grid-item:: - :columns: 6 - - .. tab-set:: - - .. tab-item:: Document - :sync: Document - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit import Document - - # Read from XML file - xmlReport = Path("JUnit-Report.xml") - try: - doc = Document(xmlReport, parse=True) - except UnittestException as ex: - ... - - # Convert to unified test data model - summary = doc.ToTestsuiteSummary() - - # Convert to a tree - rootNode = doc.ToTree() - - # Convert back to a document - newXmlReport = Path("New JUnit-Report.xml") - newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) - - # Write to XML file - newDoc.Write() - - -.. _UNITTEST/Feature/Annotation: - -Annotations -=========== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - Every test entity can be annotated with arbitrary key-value pairs. - - .. grid-item:: - :columns: 6 - - .. tab-set:: - - .. tab-item:: Testcase - :sync: Testcase - - .. code-block:: Python - - # Add annotate a key-value pair - testcase["key"] = value - - # Update existing annotation with new value - testcase["key"] = newValue - - # Check if key exists - if "key" in testcase: - pass - - # Access annoation by key - value = testcase["key"] - - # Get number of annotations - annotationCount = len(testcase) - - # Delete annotation - del testcase["key"] - - # Iterate annotations - for key, value in testcases: - pass - - .. tab-item:: Testsuite - :sync: Testsuite - .. code-block:: Python +.. include:: Features.rst - # Add annotate a key-value pair - testsuite["key"] = value - - # Update existing annotation with new value - testsuite["key"] = newValue - - # Check if key exists - if "key" in testsuite: - pass - - # Access annoation by key - value = testsuite["key"] - - # Get number of annotations - annotationCount = len(testsuite) - - # Delete annotation - del testsuite["key"] - - # Iterate annotations - for key, value in testsuite: - pass - - .. tab-item:: TestsuiteSummary - :sync: TestsuiteSummary - - .. code-block:: Python - - # Add annotate a key-value pair - testsuiteSummary["key"] = value - - # Update existing annotation with new value - testsuiteSummary["key"] = newValue - - # Check if key exists - if "key" in testsuiteSummary: - pass - - # Access annoation by key - value = testsuiteSummary["key"] - - # Get number of annotations - annotationCount = len(testsuiteSummary) - - # Delete annotation - del testsuiteSummary["key"] - - # Iterate annotations - for key, value in testsuiteSummary: - pass - - - -.. _UNITTEST/Feature/Merge: - -Merging unittest reports -======================== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - add description here - - .. grid-item:: - :columns: 6 - - .. tab-set:: - - .. tab-item:: Testcase - :sync: Testcase - - .. code-block:: Python - - # add code here - -.. _UNITTEST/Feature/Concat: - -Concatenate unittest reports -============================ - -.. todo:: Planned feature. - -.. _UNITTEST/Feature/Transform: - -Transforming the reports' hierarchy -=================================== - -.. _UNITTEST/Feature/Transform/pytest: - -pytest specific transformations -------------------------------- - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - add description here - - .. grid-item:: - :columns: 6 - - .. tab-set:: - - .. tab-item:: Testcase - :sync: Testcase - - .. code-block:: Python - - # add code here - -.. _UNITTEST/Feature/Write: - -Writing unittest reports -======================== - -.. grid:: 2 - - .. grid-item:: - :columns: 6 - - A test suite summary can be converted to a document of any JUnit dialect. Internally a deep-copy is created to - convert from a hierarchy of the unified test entities to a hierarchy of specific test entities (e.g. JUnit - entities). - - When the document was created, it can be written to disk. - - .. grid-item:: - :columns: 6 - - .. tab-set:: - - .. tab-item:: Any JUnit - :sync: AnyJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit import Document - - # Convert a TestsuiteSummary back to a Document - newXmlReport = Path("JUnit-Report.xml") - newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) - - # Write to XML file - try: - newDoc.Write() - except UnittestException as ex: - ... - - .. tab-item:: Ant + JUnit4 - :sync: AntJUnit4 - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.AntJUnit import Document - - # Convert a TestsuiteSummary back to a Document - newXmlReport = Path("JUnit-Report.xml") - newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) - - # Write to XML file - try: - newDoc.Write() - except UnittestException as ex: - ... - - .. tab-item:: CTest JUnit - :sync: CTestJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.CTestJUnit import Document - - # Convert a TestsuiteSummary back to a Document - newXmlReport = Path("JUnit-Report.xml") - newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) - - # Write to XML file - try: - newDoc.Write() - except UnittestException as ex: - ... - - .. tab-item:: GoogleTest JUnit - :sync: GTestJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.GoogleTestJUnit import Document - - # Convert a TestsuiteSummary back to a Document - newXmlReport = Path("JUnit-Report.xml") - newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) - - # Write to XML file - try: - newDoc.Write() - except UnittestException as ex: - ... - - .. tab-item:: pyTest JUnit - :sync: pyTestJUnit - - .. code-block:: Python - - from pyEDAA.Reports.Unittesting.JUnit.PyTestJUnit import Document - - # Convert a TestsuiteSummary back to a Document - newXmlReport = Path("JUnit-Report.xml") - newDoc = Document.FromTestsuiteSummary(newXmlReport, summary) - - # Write to XML file - try: - newDoc.Write() - except UnittestException as ex: - ... .. _UNITTEST/CLI: @@ -1093,27 +66,17 @@ by JUnit4, many dialects spread out. Many tools and test frameworks have minor o original format. While some modifications seam logical additions or adjustments to the needs of the respective framework, others undermine the ideas and intents of the data format. -Many issues arise because the Ant + JUnit4 format is specific to unit testing with Java. Other languages and frameworks -were lazy and didn't derive an own format, but rather stuffed there language into the limitations of the Ant + JUnit4 -XML format. +Many issues arise because the :ref:`Ant + JUnit4 ` format is +specific to unit testing with Java. Other languages and frameworks were lazy and didn't derive their own format, but +rather stuffed their language constructs into the concepts and limitations of the Ant + JUnit4 XML format. .. rubric:: JUnit Dialects * 🚧 Bamboo JUnit (planned) -* ✅ CTest JUnit format -* ✅ GoogleTest JUnit format +* ✅ :ref:`CTest JUnit format ` +* ✅ :ref:`GoogleTest JUnit format ` * 🚧 Jenkins JUnit (planned) -* ✅ pyTest JUnit format - -.. rubric:: JUnit Dialect Comparison - -+------------------------+--------------+--------------------+------------------+--------------+ -| Feature | Ant + JUnit4 | CTest JUnit | GoogleTest JUnit | pyTest JUnit | -+========================+==============+====================+==================+==============+ -| Root element | | testsuite | | testsuites | -+------------------------+--------------+--------------------+------------------+--------------+ -| Testcase status | ... | more status values | | | -+------------------------+--------------+--------------------+------------------+--------------+ +* ✅ :ref:`pyTest JUnit format ` .. _UNITTEST/FileFormats/JUnit5: @@ -1131,10 +94,13 @@ Java specifics is provided too. Open Test Reporting =================== +The `Open Test Alliance `__ created a new format called +`Open Test Reporting `__ (OTR) to overcome the shortcommings of a +missing file format for JUnit5 as well as the problems of Ant + JUnit4. -`Open Test Alliance `__ - -https://github.com/ota4j-team/open-test-reporting +OTR defines a structure of test groups and tests, but no specifics of a certain programming languge. The logical +structure of tests and test groups is decoupled from language specifics like namespaces, packages or classes hosting the +individual tests. .. _UNITTEST/FileFormats/OSVVM: diff --git a/doc/_static/css/override.css b/doc/_static/css/override.css index 2c29d15a..9dc1540d 100644 --- a/doc/_static/css/override.css +++ b/doc/_static/css/override.css @@ -35,7 +35,10 @@ footer p { section > p, .section p, -.simple li { +.simple li, +.admonition > p, +.sd-col > p, +dl > dd > p { text-align: justify } @@ -102,6 +105,10 @@ section > p, margin: 0 } +.sd-row>* { + margin-top: 1.5em; +} + .sd-tab-set > label { padding-top: .5em; padding-right: 1em; diff --git a/pyEDAA/Reports/Resources/Ant-JUnit.xsd b/pyEDAA/Reports/Resources/Ant-JUnit.xsd index dab0b2fe..98a05976 100644 --- a/pyEDAA/Reports/Resources/Ant-JUnit.xsd +++ b/pyEDAA/Reports/Resources/Ant-JUnit.xsd @@ -116,7 +116,7 @@ - Describes the overall result of all testcases in all testsuites. + Describes the overall result of all testcases in a single testsuite. diff --git a/pyEDAA/Reports/Resources/Generic-JUnit.xsd b/pyEDAA/Reports/Resources/Any-JUnit.xsd similarity index 100% rename from pyEDAA/Reports/Resources/Generic-JUnit.xsd rename to pyEDAA/Reports/Resources/Any-JUnit.xsd diff --git a/pyEDAA/Reports/Resources/__init__.py b/pyEDAA/Reports/Resources/__init__.py index 25aecb5d..35e4c374 100644 --- a/pyEDAA/Reports/Resources/__init__.py +++ b/pyEDAA/Reports/Resources/__init__.py @@ -5,7 +5,7 @@ * :file:`Ant-JUnit.xsd` * :file:`CTest-JUnit.xsd` -* :file:`Generic-JUnit.xsd` +* :file:`Any-JUnit.xsd` * :file:`GoogleTest-JUnit.xsd` * :file:`PyTest-JUnit.xsd` """ diff --git a/pyEDAA/Reports/Unittesting/JUnit/AntJUnit.py b/pyEDAA/Reports/Unittesting/JUnit/AntJUnit.py index 0de1cad1..f0463ab9 100644 --- a/pyEDAA/Reports/Unittesting/JUnit/AntJUnit.py +++ b/pyEDAA/Reports/Unittesting/JUnit/AntJUnit.py @@ -45,24 +45,37 @@ from pyEDAA.Reports.Unittesting.JUnit import Testcase as ju_Testcase, Testclass as ju_Testclass, Testsuite as ju_Testsuite from pyEDAA.Reports.Unittesting.JUnit import TestsuiteSummary as ju_TestsuiteSummary, Document as ju_Document - TestsuiteType = TypeVar("TestsuiteType", bound="Testsuite") TestcaseAggregateReturnType = Tuple[int, int, int] TestsuiteAggregateReturnType = Tuple[int, int, int, int, int] +from pyEDAA.Reports.helper import InheritDocumentation + + @export +@InheritDocumentation(ju_Testcase, merge=True) class Testcase(ju_Testcase): - pass + """ + This is a derived implementation for the Ant + JUnit4 dialect. + """ @export +@InheritDocumentation(ju_Testclass, merge=True) class Testclass(ju_Testclass): - pass + """ + This is a derived implementation for the Ant + JUnit4 dialect. + """ @export +@InheritDocumentation(ju_Testsuite, merge=True) class Testsuite(ju_Testsuite): + """ + This is a derived implementation for the Ant + JUnit4 dialect. + """ + def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: tests, skipped, errored, failed, passed = super().Aggregate() @@ -110,6 +123,13 @@ def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: @classmethod def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": + """ + Convert a test suite of the unified test entity data model to the JUnit specific data model's test suite object + adhering to the Ant + JUnit4 dialect. + + :param testsuite: Test suite from unified data model. + :return: Test suite from JUnit specific data model (Ant + JUnit4 dialect). + """ juTestsuite = cls( testsuite._name, startTime=testsuite._startTime, @@ -143,71 +163,23 @@ def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": return juTestsuite - def ToTestsuite(self) -> ut_Testsuite: - testsuite = ut_Testsuite( - self._name, - TestsuiteKind.Logical, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - ) - - for testclass in self._testclasses.values(): - suite = testsuite - classpath = testclass._name.split(".") - for element in classpath: - if element in suite._testsuites: - suite = suite._testsuites[element] - else: - suite = ut_Testsuite(element, kind=TestsuiteKind.Package, parent=suite) - - suite._kind = TestsuiteKind.Class - if suite._parent is not testsuite: - suite._parent._kind = TestsuiteKind.Module - - suite.AddTestcases(tc.ToTestcase() for tc in testclass._testcases.values()) - - return testsuite - @export +@InheritDocumentation(ju_TestsuiteSummary, merge=True) class TestsuiteSummary(ju_TestsuiteSummary): - def Aggregate(self) -> TestsuiteAggregateReturnType: - tests, skipped, errored, failed, passed = super().Aggregate() - - self._tests = tests - self._skipped = skipped - self._errored = errored - self._failed = failed - self._passed = passed - - if errored > 0: - self._status = TestsuiteStatus.Errored - elif failed > 0: - self._status = TestsuiteStatus.Failed - elif tests == 0: - self._status = TestsuiteStatus.Empty - elif tests - skipped == passed: - self._status = TestsuiteStatus.Passed - elif tests == skipped: - self._status = TestsuiteStatus.Skipped - else: - self._status = TestsuiteStatus.Unknown - - return tests, skipped, errored, failed, passed - - def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[Testsuite, Testcase], None, None]: - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PreOrder in scheme: - yield self - - for testsuite in self._testsuites.values(): - yield from testsuite.IterateTestsuites(scheme | IterationScheme.IncludeSelf) - - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PostOrder in scheme: - yield self + """ + This is a derived implementation for the Ant + JUnit4 dialect. + """ @classmethod def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "TestsuiteSummary": + """ + Convert a test suite summary of the unified test entity data model to the JUnit specific data model's test suite + summary object adhering to the Ant + JUnit4 dialect. + + :param testsuiteSummary: Test suite summary from unified data model. + :return: Test suite summary from JUnit specific data model (Ant + JUnit4 dialect). + """ return cls( testsuiteSummary._name, startTime=testsuiteSummary._startTime, @@ -216,15 +188,6 @@ def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "Testsui testsuites=(ut_Testsuite.FromTestsuite(testsuite) for testsuite in testsuiteSummary._testsuites.values()) ) - def ToTestsuiteSummary(self) -> ut_TestsuiteSummary: - return ut_TestsuiteSummary( - self._name, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - testsuites=(testsuite.ToTestsuite() for testsuite in self._testsuites.values()) - ) - @export class Document(ju_Document): @@ -274,11 +237,21 @@ def Analyze(self) -> None: self._Analyze(xmlSchemaFile) def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate: bool = False) -> None: + """ + Write the data model as XML into a file adhering to the Ant + JUnit4 dialect. + + :param path: Optional path to the XMl file, if internal path shouldn't be used. + :param overwrite: If true, overwrite an existing file. + :param regenerate: If true, regenerate the XML structure from data model. + :raises UnittestException: If the file cannot be overwritten. + :raises UnittestException: If the internal XML data structure wasn't generated. + :raises UnittestException: If the file cannot be opened or written. + """ if path is None: path = self._path if not overwrite and path.exists(): - raise UnittestException(f"JUnit XML file '{path}' can not be written.") \ + raise UnittestException(f"JUnit XML file '{path}' can not be overwritten.") \ from FileExistsError(f"File '{path}' already exists.") if regenerate: @@ -289,13 +262,18 @@ def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate ex.add_note(f"Call 'JUnitDocument.Generate()' or 'JUnitDocument.Write(..., regenerate=True)'.") raise ex - with path.open("wb") as file: - file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + try: + with path.open("wb") as file: + file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + except Exception as ex: + raise UnittestException(f"JUnit XML file '{path}' can not be written.") from ex def Convert(self) -> None: """ Convert the parsed and validated XML data structure into a JUnit test entity hierarchy. + This method converts the root element. + .. hint:: The time spend for model conversion will be made available via property :data:`ModelConversionDuration`. @@ -328,6 +306,14 @@ def Convert(self) -> None: self._modelConversion = (endConversation - startConversion) / 1e9 def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> None: + """ + Convert the XML data structure of a ```` to a test suite. + + This method uses private helper methods provided by the base-class. + + :param parent: The test suite summary as a parent element in the test entity hierarchy. + :param testsuitesNode: The current XML element node representing a test suite. + """ newTestsuite = Testsuite( self._ParseName(testsuitesNode, optional=False), self._ParseHostname(testsuitesNode, optional=False), @@ -339,7 +325,15 @@ def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> self._ParseTestsuiteChildren(testsuitesNode, newTestsuite) def Generate(self, overwrite: bool = False) -> None: - if self._xmlDocument is not None: + """ + Generate the internal XML data structure from test suites and test cases. + + This method generates the XML root element (````) and recursively calls other generated methods. + + :param overwrite: Overwrite the internal XML data structure. + :raises UnittestException: If overwrite is false and the internal XML data structure is not empty. + """ + if not overwrite and self._xmlDocument is not None: raise UnittestException(f"Internal XML document is populated with data.") rootElement = Element("testsuites") @@ -360,7 +354,16 @@ def Generate(self, overwrite: bool = False) -> None: for testsuite in self._testsuites.values(): self._GenerateTestsuite(testsuite, rootElement) - def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): + def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test suite. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testsuite: The test suite to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testsuiteElement = SubElement(parentElement, "testsuite") testsuiteElement.attrib["name"] = testsuite._name if testsuite._startTime is not None: @@ -380,7 +383,16 @@ def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): for tc in testclass._testcases.values(): self._GenerateTestcase(tc, testsuiteElement) - def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element): + def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test case. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testcase: The test case to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testcaseElement = SubElement(parentElement, "testcase") if testcase.Classname is not None: testcaseElement.attrib["classname"] = testcase.Classname diff --git a/pyEDAA/Reports/Unittesting/JUnit/CTestJUnit.py b/pyEDAA/Reports/Unittesting/JUnit/CTestJUnit.py index 8f920f32..97e8ce34 100644 --- a/pyEDAA/Reports/Unittesting/JUnit/CTestJUnit.py +++ b/pyEDAA/Reports/Unittesting/JUnit/CTestJUnit.py @@ -51,18 +51,32 @@ TestsuiteAggregateReturnType = Tuple[int, int, int, int, int] +from pyEDAA.Reports.helper import InheritDocumentation + + @export +@InheritDocumentation(ju_Testcase, merge=True) class Testcase(ju_Testcase): - pass + """ + This is a derived implementation for the CTest JUnit dialect. + """ @export +@InheritDocumentation(ju_Testclass, merge=True) class Testclass(ju_Testclass): - pass + """ + This is a derived implementation for the CTest JUnit dialect. + """ @export +@InheritDocumentation(ju_Testsuite, merge=True) class Testsuite(ju_Testsuite): + """ + This is a derived implementation for the CTest JUnit dialect. + """ + def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: tests, skipped, errored, failed, passed = super().Aggregate() @@ -110,6 +124,13 @@ def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: @classmethod def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": + """ + Convert a test suite of the unified test entity data model to the JUnit specific data model's test suite object + adhering to the CTest JUnit dialect. + + :param testsuite: Test suite from unified data model. + :return: Test suite from JUnit specific data model (CTest JUnit dialect). + """ juTestsuite = cls( testsuite._name, startTime=testsuite._startTime, @@ -143,71 +164,23 @@ def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": return juTestsuite - def ToTestsuite(self) -> ut_Testsuite: - testsuite = ut_Testsuite( - self._name, - TestsuiteKind.Logical, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - ) - - for testclass in self._testclasses.values(): - suite = testsuite - classpath = testclass._name.split(".") - for element in classpath: - if element in suite._testsuites: - suite = suite._testsuites[element] - else: - suite = ut_Testsuite(element, kind=TestsuiteKind.Package, parent=suite) - - suite._kind = TestsuiteKind.Class - if suite._parent is not testsuite: - suite._parent._kind = TestsuiteKind.Module - - suite.AddTestcases(tc.ToTestcase() for tc in testclass._testcases.values()) - - return testsuite - @export +@InheritDocumentation(ju_TestsuiteSummary, merge=True) class TestsuiteSummary(ju_TestsuiteSummary): - def Aggregate(self) -> TestsuiteAggregateReturnType: - tests, skipped, errored, failed, passed = super().Aggregate() - - self._tests = tests - self._skipped = skipped - self._errored = errored - self._failed = failed - self._passed = passed - - if errored > 0: - self._status = TestsuiteStatus.Errored - elif failed > 0: - self._status = TestsuiteStatus.Failed - elif tests == 0: - self._status = TestsuiteStatus.Empty - elif tests - skipped == passed: - self._status = TestsuiteStatus.Passed - elif tests == skipped: - self._status = TestsuiteStatus.Skipped - else: - self._status = TestsuiteStatus.Unknown - - return tests, skipped, errored, failed, passed - - def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[Testsuite, Testcase], None, None]: - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PreOrder in scheme: - yield self - - for testsuite in self._testsuites.values(): - yield from testsuite.IterateTestsuites(scheme | IterationScheme.IncludeSelf) - - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PostOrder in scheme: - yield self + """ + This is a derived implementation for the CTest JUnit dialect. + """ @classmethod def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "TestsuiteSummary": + """ + Convert a test suite summary of the unified test entity data model to the JUnit specific data model's test suite + summary object adhering to the CTest JUnit dialect. + + :param testsuiteSummary: Test suite summary from unified data model. + :return: Test suite summary from JUnit specific data model (CTest JUnit dialect). + """ return cls( testsuiteSummary._name, startTime=testsuiteSummary._startTime, @@ -216,15 +189,6 @@ def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "Testsui testsuites=(ut_Testsuite.FromTestsuite(testsuite) for testsuite in testsuiteSummary._testsuites.values()) ) - def ToTestsuiteSummary(self) -> ut_TestsuiteSummary: - return ut_TestsuiteSummary( - self._name, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - testsuites=(testsuite.ToTestsuite() for testsuite in self._testsuites.values()) - ) - @export class Document(ju_Document): @@ -274,11 +238,21 @@ def Analyze(self) -> None: self._Analyze(xmlSchemaFile) def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate: bool = False) -> None: + """ + Write the data model as XML into a file adhering to the CTest dialect. + + :param path: Optional path to the XMl file, if internal path shouldn't be used. + :param overwrite: If true, overwrite an existing file. + :param regenerate: If true, regenerate the XML structure from data model. + :raises UnittestException: If the file cannot be overwritten. + :raises UnittestException: If the internal XML data structure wasn't generated. + :raises UnittestException: If the file cannot be opened or written. + """ if path is None: path = self._path if not overwrite and path.exists(): - raise UnittestException(f"JUnit XML file '{path}' can not be written.") \ + raise UnittestException(f"JUnit XML file '{path}' can not be overwritten.") \ from FileExistsError(f"File '{path}' already exists.") if regenerate: @@ -289,13 +263,18 @@ def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate ex.add_note(f"Call 'JUnitDocument.Generate()' or 'JUnitDocument.Write(..., regenerate=True)'.") raise ex - with path.open("wb") as file: - file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + try: + with path.open("wb") as file: + file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + except Exception as ex: + raise UnittestException(f"JUnit XML file '{path}' can not be written.") from ex def Convert(self) -> None: """ Convert the parsed and validated XML data structure into a JUnit test entity hierarchy. + This method converts the root element. + .. hint:: The time spend for model conversion will be made available via property :data:`ModelConversionDuration`. @@ -328,6 +307,14 @@ def Convert(self) -> None: self._modelConversion = (endConversation - startConversion) / 1e9 def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> None: + """ + Convert the XML data structure of a ```` to a test suite. + + This method uses private helper methods provided by the base-class. + + :param parent: The test suite summary as a parent element in the test entity hierarchy. + :param testsuitesNode: The current XML element node representing a test suite. + """ newTestsuite = Testsuite( self._ParseName(testsuitesNode, optional=False), self._ParseHostname(testsuitesNode, optional=False), @@ -339,7 +326,15 @@ def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> self._ParseTestsuiteChildren(testsuitesNode, newTestsuite) def Generate(self, overwrite: bool = False) -> None: - if self._xmlDocument is not None: + """ + Generate the internal XML data structure from test suites and test cases. + + This method generates the XML root element (````) and recursively calls other generated methods. + + :param overwrite: Overwrite the internal XML data structure. + :raises UnittestException: If overwrite is false and the internal XML data structure is not empty. + """ + if not overwrite and self._xmlDocument is not None: raise UnittestException(f"Internal XML document is populated with data.") rootElement = Element("testsuites") @@ -360,7 +355,16 @@ def Generate(self, overwrite: bool = False) -> None: for testsuite in self._testsuites.values(): self._GenerateTestsuite(testsuite, rootElement) - def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): + def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test suite. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testsuite: The test suite to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testsuiteElement = SubElement(parentElement, "testsuite") testsuiteElement.attrib["name"] = testsuite._name if testsuite._startTime is not None: @@ -380,7 +384,16 @@ def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): for tc in testclass._testcases.values(): self._GenerateTestcase(tc, testsuiteElement) - def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element): + def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test case. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testcase: The test case to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testcaseElement = SubElement(parentElement, "testcase") if testcase.Classname is not None: testcaseElement.attrib["classname"] = testcase.Classname diff --git a/pyEDAA/Reports/Unittesting/JUnit/GoogleTestJUnit.py b/pyEDAA/Reports/Unittesting/JUnit/GoogleTestJUnit.py index 35718367..de8bcc58 100644 --- a/pyEDAA/Reports/Unittesting/JUnit/GoogleTestJUnit.py +++ b/pyEDAA/Reports/Unittesting/JUnit/GoogleTestJUnit.py @@ -51,18 +51,32 @@ TestsuiteAggregateReturnType = Tuple[int, int, int, int, int] +from pyEDAA.Reports.helper import InheritDocumentation + + @export +@InheritDocumentation(ju_Testcase, merge=True) class Testcase(ju_Testcase): - pass + """ + This is a derived implementation for the GoogleTest JUnit dialect. + """ @export +@InheritDocumentation(ju_Testclass, merge=True) class Testclass(ju_Testclass): - pass + """ + This is a derived implementation for the GoogleTest JUnit dialect. + """ @export +@InheritDocumentation(ju_Testsuite, merge=True) class Testsuite(ju_Testsuite): + """ + This is a derived implementation for the GoogleTest JUnit dialect. + """ + def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: tests, skipped, errored, failed, passed = super().Aggregate() @@ -110,6 +124,13 @@ def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: @classmethod def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": + """ + Convert a test suite of the unified test entity data model to the JUnit specific data model's test suite object + adhering to the GoogleTest JUnit dialect. + + :param testsuite: Test suite from unified data model. + :return: Test suite from JUnit specific data model (GoogleTest JUnit dialect). + """ juTestsuite = cls( testsuite._name, startTime=testsuite._startTime, @@ -143,71 +164,23 @@ def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": return juTestsuite - def ToTestsuite(self) -> ut_Testsuite: - testsuite = ut_Testsuite( - self._name, - TestsuiteKind.Logical, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - ) - - for testclass in self._testclasses.values(): - suite = testsuite - classpath = testclass._name.split(".") - for element in classpath: - if element in suite._testsuites: - suite = suite._testsuites[element] - else: - suite = ut_Testsuite(element, kind=TestsuiteKind.Package, parent=suite) - - suite._kind = TestsuiteKind.Class - if suite._parent is not testsuite: - suite._parent._kind = TestsuiteKind.Module - - suite.AddTestcases(tc.ToTestcase() for tc in testclass._testcases.values()) - - return testsuite - @export +@InheritDocumentation(ju_TestsuiteSummary, merge=True) class TestsuiteSummary(ju_TestsuiteSummary): - def Aggregate(self) -> TestsuiteAggregateReturnType: - tests, skipped, errored, failed, passed = super().Aggregate() - - self._tests = tests - self._skipped = skipped - self._errored = errored - self._failed = failed - self._passed = passed - - if errored > 0: - self._status = TestsuiteStatus.Errored - elif failed > 0: - self._status = TestsuiteStatus.Failed - elif tests == 0: - self._status = TestsuiteStatus.Empty - elif tests - skipped == passed: - self._status = TestsuiteStatus.Passed - elif tests == skipped: - self._status = TestsuiteStatus.Skipped - else: - self._status = TestsuiteStatus.Unknown - - return tests, skipped, errored, failed, passed - - def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[Testsuite, Testcase], None, None]: - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PreOrder in scheme: - yield self - - for testsuite in self._testsuites.values(): - yield from testsuite.IterateTestsuites(scheme | IterationScheme.IncludeSelf) - - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PostOrder in scheme: - yield self + """ + This is a derived implementation for the GoogleTest JUnit dialect. + """ @classmethod def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "TestsuiteSummary": + """ + Convert a test suite summary of the unified test entity data model to the JUnit specific data model's test suite + summary object adhering to the GoogleTest JUnit dialect. + + :param testsuiteSummary: Test suite summary from unified data model. + :return: Test suite summary from JUnit specific data model (GoogleTest JUnit dialect). + """ return cls( testsuiteSummary._name, startTime=testsuiteSummary._startTime, @@ -216,15 +189,6 @@ def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "Testsui testsuites=(ut_Testsuite.FromTestsuite(testsuite) for testsuite in testsuiteSummary._testsuites.values()) ) - def ToTestsuiteSummary(self) -> ut_TestsuiteSummary: - return ut_TestsuiteSummary( - self._name, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - testsuites=(testsuite.ToTestsuite() for testsuite in self._testsuites.values()) - ) - @export class Document(ju_Document): @@ -274,11 +238,21 @@ def Analyze(self) -> None: self._Analyze(xmlSchemaFile) def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate: bool = False) -> None: + """ + Write the data model as XML into a file adhering to the GoogleTest dialect. + + :param path: Optional path to the XMl file, if internal path shouldn't be used. + :param overwrite: If true, overwrite an existing file. + :param regenerate: If true, regenerate the XML structure from data model. + :raises UnittestException: If the file cannot be overwritten. + :raises UnittestException: If the internal XML data structure wasn't generated. + :raises UnittestException: If the file cannot be opened or written. + """ if path is None: path = self._path if not overwrite and path.exists(): - raise UnittestException(f"JUnit XML file '{path}' can not be written.") \ + raise UnittestException(f"JUnit XML file '{path}' can not be overwritten.") \ from FileExistsError(f"File '{path}' already exists.") if regenerate: @@ -289,13 +263,18 @@ def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate ex.add_note(f"Call 'JUnitDocument.Generate()' or 'JUnitDocument.Write(..., regenerate=True)'.") raise ex - with path.open("wb") as file: - file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + try: + with path.open("wb") as file: + file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + except Exception as ex: + raise UnittestException(f"JUnit XML file '{path}' can not be written.") from ex def Convert(self) -> None: """ Convert the parsed and validated XML data structure into a JUnit test entity hierarchy. + This method converts the root element. + .. hint:: The time spend for model conversion will be made available via property :data:`ModelConversionDuration`. @@ -328,6 +307,14 @@ def Convert(self) -> None: self._modelConversion = (endConversation - startConversion) / 1e9 def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> None: + """ + Convert the XML data structure of a ```` to a test suite. + + This method uses private helper methods provided by the base-class. + + :param parent: The test suite summary as a parent element in the test entity hierarchy. + :param testsuitesNode: The current XML element node representing a test suite. + """ newTestsuite = Testsuite( self._ParseName(testsuitesNode, optional=False), self._ParseHostname(testsuitesNode, optional=True), @@ -339,7 +326,15 @@ def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> self._ParseTestsuiteChildren(testsuitesNode, newTestsuite) def Generate(self, overwrite: bool = False) -> None: - if self._xmlDocument is not None: + """ + Generate the internal XML data structure from test suites and test cases. + + This method generates the XML root element (````) and recursively calls other generated methods. + + :param overwrite: Overwrite the internal XML data structure. + :raises UnittestException: If overwrite is false and the internal XML data structure is not empty. + """ + if not overwrite and self._xmlDocument is not None: raise UnittestException(f"Internal XML document is populated with data.") rootElement = Element("testsuites") @@ -360,7 +355,16 @@ def Generate(self, overwrite: bool = False) -> None: for testsuite in self._testsuites.values(): self._GenerateTestsuite(testsuite, rootElement) - def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): + def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test suite. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testsuite: The test suite to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testsuiteElement = SubElement(parentElement, "testsuite") testsuiteElement.attrib["name"] = testsuite._name if testsuite._startTime is not None: @@ -380,7 +384,16 @@ def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): for tc in testclass._testcases.values(): self._GenerateTestcase(tc, testsuiteElement) - def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element): + def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test case. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testcase: The test case to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testcaseElement = SubElement(parentElement, "testcase") if testcase.Classname is not None: testcaseElement.attrib["classname"] = testcase.Classname diff --git a/pyEDAA/Reports/Unittesting/JUnit/PyTestJUnit.py b/pyEDAA/Reports/Unittesting/JUnit/PyTestJUnit.py index 0b71ebf1..26acb184 100644 --- a/pyEDAA/Reports/Unittesting/JUnit/PyTestJUnit.py +++ b/pyEDAA/Reports/Unittesting/JUnit/PyTestJUnit.py @@ -51,18 +51,32 @@ TestsuiteAggregateReturnType = Tuple[int, int, int, int, int] +from pyEDAA.Reports.helper import InheritDocumentation + + @export +@InheritDocumentation(ju_Testcase, merge=True) class Testcase(ju_Testcase): - pass + """ + This is a derived implementation for the pyTest JUnit dialect. + """ @export +@InheritDocumentation(ju_Testclass, merge=True) class Testclass(ju_Testclass): - pass + """ + This is a derived implementation for the pyTest JUnit dialect. + """ @export +@InheritDocumentation(ju_Testsuite, merge=True) class Testsuite(ju_Testsuite): + """ + This is a derived implementation for the pyTest JUnit dialect. + """ + def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: tests, skipped, errored, failed, passed = super().Aggregate() @@ -110,6 +124,13 @@ def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType: @classmethod def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": + """ + Convert a test suite of the unified test entity data model to the JUnit specific data model's test suite object + adhering to the pyTest JUnit dialect. + + :param testsuite: Test suite from unified data model. + :return: Test suite from JUnit specific data model (pyTest JUnitdialect). + """ juTestsuite = cls( testsuite._name, startTime=testsuite._startTime, @@ -143,71 +164,23 @@ def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": return juTestsuite - def ToTestsuite(self) -> ut_Testsuite: - testsuite = ut_Testsuite( - self._name, - TestsuiteKind.Logical, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - ) - - for testclass in self._testclasses.values(): - suite = testsuite - classpath = testclass._name.split(".") - for element in classpath: - if element in suite._testsuites: - suite = suite._testsuites[element] - else: - suite = ut_Testsuite(element, kind=TestsuiteKind.Package, parent=suite) - - suite._kind = TestsuiteKind.Class - if suite._parent is not testsuite: - suite._parent._kind = TestsuiteKind.Module - - suite.AddTestcases(tc.ToTestcase() for tc in testclass._testcases.values()) - - return testsuite - @export +@InheritDocumentation(ju_TestsuiteSummary, merge=True) class TestsuiteSummary(ju_TestsuiteSummary): - def Aggregate(self) -> TestsuiteAggregateReturnType: - tests, skipped, errored, failed, passed = super().Aggregate() - - self._tests = tests - self._skipped = skipped - self._errored = errored - self._failed = failed - self._passed = passed - - if errored > 0: - self._status = TestsuiteStatus.Errored - elif failed > 0: - self._status = TestsuiteStatus.Failed - elif tests == 0: - self._status = TestsuiteStatus.Empty - elif tests - skipped == passed: - self._status = TestsuiteStatus.Passed - elif tests == skipped: - self._status = TestsuiteStatus.Skipped - else: - self._status = TestsuiteStatus.Unknown - - return tests, skipped, errored, failed, passed - - def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[Testsuite, Testcase], None, None]: - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PreOrder in scheme: - yield self - - for testsuite in self._testsuites.values(): - yield from testsuite.IterateTestsuites(scheme | IterationScheme.IncludeSelf) - - if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PostOrder in scheme: - yield self + """ + This is a derived implementation for the pyTest JUnit dialect. + """ @classmethod def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "TestsuiteSummary": + """ + Convert a test suite summary of the unified test entity data model to the JUnit specific data model's test suite + summary object adhering to the pyTest JUnit dialect. + + :param testsuiteSummary: Test suite summary from unified data model. + :return: Test suite summary from JUnit specific data model (pyTest JUnit dialect). + """ return cls( testsuiteSummary._name, startTime=testsuiteSummary._startTime, @@ -216,15 +189,6 @@ def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "Testsui testsuites=(ut_Testsuite.FromTestsuite(testsuite) for testsuite in testsuiteSummary._testsuites.values()) ) - def ToTestsuiteSummary(self) -> ut_TestsuiteSummary: - return ut_TestsuiteSummary( - self._name, - startTime=self._startTime, - totalDuration=self._duration, - status=self._status, - testsuites=(testsuite.ToTestsuite() for testsuite in self._testsuites.values()) - ) - @export class Document(ju_Document): @@ -274,11 +238,21 @@ def Analyze(self) -> None: self._Analyze(xmlSchemaFile) def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate: bool = False) -> None: + """ + Write the data model as XML into a file adhering to the pyTest dialect. + + :param path: Optional path to the XMl file, if internal path shouldn't be used. + :param overwrite: If true, overwrite an existing file. + :param regenerate: If true, regenerate the XML structure from data model. + :raises UnittestException: If the file cannot be overwritten. + :raises UnittestException: If the internal XML data structure wasn't generated. + :raises UnittestException: If the file cannot be opened or written. + """ if path is None: path = self._path if not overwrite and path.exists(): - raise UnittestException(f"JUnit XML file '{path}' can not be written.") \ + raise UnittestException(f"JUnit XML file '{path}' can not be overwritten.") \ from FileExistsError(f"File '{path}' already exists.") if regenerate: @@ -289,13 +263,18 @@ def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate ex.add_note(f"Call 'JUnitDocument.Generate()' or 'JUnitDocument.Write(..., regenerate=True)'.") raise ex - with path.open("wb") as file: - file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + try: + with path.open("wb") as file: + file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + except Exception as ex: + raise UnittestException(f"JUnit XML file '{path}' can not be written.") from ex def Convert(self) -> None: """ Convert the parsed and validated XML data structure into a JUnit test entity hierarchy. + This method converts the root element. + .. hint:: The time spend for model conversion will be made available via property :data:`ModelConversionDuration`. @@ -328,6 +307,14 @@ def Convert(self) -> None: self._modelConversion = (endConversation - startConversion) / 1e9 def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> None: + """ + Convert the XML data structure of a ```` to a test suite. + + This method uses private helper methods provided by the base-class. + + :param parent: The test suite summary as a parent element in the test entity hierarchy. + :param testsuitesNode: The current XML element node representing a test suite. + """ newTestsuite = Testsuite( self._ParseName(testsuitesNode, optional=False), self._ParseHostname(testsuitesNode, optional=False), @@ -339,7 +326,15 @@ def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> self._ParseTestsuiteChildren(testsuitesNode, newTestsuite) def Generate(self, overwrite: bool = False) -> None: - if self._xmlDocument is not None: + """ + Generate the internal XML data structure from test suites and test cases. + + This method generates the XML root element (````) and recursively calls other generated methods. + + :param overwrite: Overwrite the internal XML data structure. + :raises UnittestException: If overwrite is false and the internal XML data structure is not empty. + """ + if not overwrite and self._xmlDocument is not None: raise UnittestException(f"Internal XML document is populated with data.") rootElement = Element("testsuites") @@ -360,7 +355,16 @@ def Generate(self, overwrite: bool = False) -> None: for testsuite in self._testsuites.values(): self._GenerateTestsuite(testsuite, rootElement) - def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): + def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test suite. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testsuite: The test suite to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testsuiteElement = SubElement(parentElement, "testsuite") testsuiteElement.attrib["name"] = testsuite._name if testsuite._startTime is not None: @@ -380,7 +384,16 @@ def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): for tc in testclass._testcases.values(): self._GenerateTestcase(tc, testsuiteElement) - def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element): + def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test case. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testcase: The test case to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testcaseElement = SubElement(parentElement, "testcase") if testcase.Classname is not None: testcaseElement.attrib["classname"] = testcase.Classname diff --git a/pyEDAA/Reports/Unittesting/JUnit/__init__.py b/pyEDAA/Reports/Unittesting/JUnit/__init__.py index 105c7abb..c2af8019 100644 --- a/pyEDAA/Reports/Unittesting/JUnit/__init__.py +++ b/pyEDAA/Reports/Unittesting/JUnit/__init__.py @@ -489,6 +489,12 @@ def Aggregate(self) -> None: @classmethod def FromTestcase(cls, testcase: ut_Testcase) -> "Testcase": + """ + Convert a test case of the unified test entity data model to the JUnit specific data model's test case object. + + :param testcase: Test case from unified data model. + :return: Test case from JUnit specific data model. + """ return cls( testcase._name, duration=testcase._testDuration, @@ -942,6 +948,12 @@ def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generato @classmethod def FromTestsuite(cls, testsuite: ut_Testsuite) -> "Testsuite": + """ + Convert a test suite of the unified test entity data model to the JUnit specific data model's test suite object. + + :param testsuite: Test suite from unified data model. + :return: Test suite from JUnit specific data model. + """ juTestsuite = cls( testsuite._name, startTime=testsuite._startTime, @@ -1100,6 +1112,14 @@ def Aggregate(self) -> TestsuiteAggregateReturnType: return tests, skipped, errored, failed, passed def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[Testsuite, Testcase], None, None]: + """ + Iterate the test suite summary and its child elements according to the iteration scheme. + + If no scheme is given, use the default scheme. + + :param scheme: Scheme how to iterate the test suite summary and its child elements. + :returns: A generator for iterating the results filtered and in the order defined by the iteration scheme. + """ if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PreOrder in scheme: yield self @@ -1111,6 +1131,12 @@ def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generato @classmethod def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "TestsuiteSummary": + """ + Convert a test suite summary of the unified test entity data model to the JUnit specific data model's test suite. + + :param testsuiteSummary: Test suite summary from unified data model. + :return: Test suite summary from JUnit specific data model. + """ return cls( testsuiteSummary._name, startTime=testsuiteSummary._startTime, @@ -1120,6 +1146,13 @@ def FromTestsuiteSummary(cls, testsuiteSummary: ut_TestsuiteSummary) -> "Testsui ) def ToTestsuiteSummary(self) -> ut_TestsuiteSummary: + """ + Convert this test suite summary a new test suite summary of the unified data model. + + All fields are copied to the new instance. Child elements like test suites are copied recursively. + + :return: A test suite summary of the unified test entity data model. + """ return ut_TestsuiteSummary( self._name, startTime=self._startTime, @@ -1194,7 +1227,7 @@ def Analyze(self) -> None: The used XML schema definition is generic to support "any" dialect. """ - xmlSchemaFile = "Generic-JUnit.xsd" + xmlSchemaFile = "Any-JUnit.xsd" self._Analyze(xmlSchemaFile) def _Analyze(self, xmlSchemaFile: str) -> None: @@ -1236,11 +1269,21 @@ def _Analyze(self, xmlSchemaFile: str) -> None: self._analysisDuration = (endAnalysis - startAnalysis) / 1e9 def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate: bool = False) -> None: + """ + Write the data model as XML into a file adhering to the Any JUnit dialect. + + :param path: Optional path to the XMl file, if internal path shouldn't be used. + :param overwrite: If true, overwrite an existing file. + :param regenerate: If true, regenerate the XML structure from data model. + :raises UnittestException: If the file cannot be overwritten. + :raises UnittestException: If the internal XML data structure wasn't generated. + :raises UnittestException: If the file cannot be opened or written. + """ if path is None: path = self._path if not overwrite and path.exists(): - raise UnittestException(f"JUnit XML file '{path}' can not be written.") \ + raise UnittestException(f"JUnit XML file '{path}' can not be overwritten.") \ from FileExistsError(f"File '{path}' already exists.") if regenerate: @@ -1251,13 +1294,18 @@ def Write(self, path: Nullable[Path] = None, overwrite: bool = False, regenerate ex.add_note(f"Call 'JUnitDocument.Generate()' or 'JUnitDocument.Write(..., regenerate=True)'.") raise ex - with path.open("wb") as file: - file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + try: + with path.open("wb") as file: + file.write(tostring(self._xmlDocument, encoding="utf-8", xml_declaration=True, pretty_print=True)) + except Exception as ex: + raise UnittestException(f"JUnit XML file '{path}' can not be written.") from ex def Convert(self) -> None: """ Convert the parsed and validated XML data structure into a JUnit test entity hierarchy. + This method converts the root element. + .. hint:: The time spend for model conversion will be made available via property :data:`ModelConversionDuration`. @@ -1293,6 +1341,15 @@ def Convert(self) -> None: self._modelConversion = (endConversation - startConversion) / 1e9 def _ParseName(self, element: _Element, default: str = "root", optional: bool = True) -> str: + """ + Convert the ``name`` attribute from an XML element node to a string. + + :param element: The XML element node with a ``name`` attribute. + :param default: The default value, if no ``name`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``name`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``name`` attribute exists on the given element node. + """ if "name" in element.attrib: return element.attrib["name"] elif not optional: @@ -1301,6 +1358,14 @@ def _ParseName(self, element: _Element, default: str = "root", optional: bool = return default def _ParseTimestamp(self, element: _Element, optional: bool = True) -> Nullable[datetime]: + """ + Convert the ``timestamp`` attribute from an XML element node to a datetime. + + :param element: The XML element node with a ``timestamp`` attribute. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``timestamp`` attribute's content if found, otherwise ``None``. + :raises UnittestException: If optional is false and no ``timestamp`` attribute exists on the given element node. + """ if "timestamp" in element.attrib: timestamp = element.attrib["timestamp"] return datetime.fromisoformat(timestamp) @@ -1310,6 +1375,14 @@ def _ParseTimestamp(self, element: _Element, optional: bool = True) -> Nullable[ return None def _ParseTime(self, element: _Element, optional: bool = True) -> Nullable[timedelta]: + """ + Convert the ``time`` attribute from an XML element node to a timedelta. + + :param element: The XML element node with a ``time`` attribute. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``time`` attribute's content if found, otherwise ``None``. + :raises UnittestException: If optional is false and no ``time`` attribute exists on the given element node. + """ if "time" in element.attrib: time = element.attrib["time"] return timedelta(seconds=float(time)) @@ -1319,6 +1392,15 @@ def _ParseTime(self, element: _Element, optional: bool = True) -> Nullable[timed return None def _ParseHostname(self, element: _Element, default: str = "localhost", optional: bool = True) -> str: + """ + Convert the ``hostname`` attribute from an XML element node to a string. + + :param element: The XML element node with a ``hostname`` attribute. + :param default: The default value, if no ``hostname`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``hostname`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``hostname`` attribute exists on the given element node. + """ if "hostname" in element.attrib: return element.attrib["hostname"] elif not optional: @@ -1326,13 +1408,29 @@ def _ParseHostname(self, element: _Element, default: str = "localhost", optional else: return default - def _ParseClassname(self, element: _Element, optional: bool = True) -> str: + def _ParseClassname(self, element: _Element) -> str: + """ + Convert the ``classname`` attribute from an XML element node to a string. + + :param element: The XML element node with a ``classname`` attribute. + :return: The ``classname`` attribute's content. + :raises UnittestException: If no ``classname`` attribute exists on the given element node. + """ if "classname" in element.attrib: return element.attrib["classname"] - elif not optional: + else: raise UnittestException(f"Required parameter 'classname' not found in tag '{element.tag}'.") def _ParseTests(self, element: _Element, default: Nullable[int] = None, optional: bool = True) -> Nullable[int]: + """ + Convert the ``tests`` attribute from an XML element node to an integer. + + :param element: The XML element node with a ``tests`` attribute. + :param default: The default value, if no ``tests`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``tests`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``tests`` attribute exists on the given element node. + """ if "tests" in element.attrib: return int(element.attrib["tests"]) elif not optional: @@ -1341,6 +1439,15 @@ def _ParseTests(self, element: _Element, default: Nullable[int] = None, optional return default def _ParseSkipped(self, element: _Element, default: Nullable[int] = None, optional: bool = True) -> Nullable[int]: + """ + Convert the ``skipped`` attribute from an XML element node to an integer. + + :param element: The XML element node with a ``skipped`` attribute. + :param default: The default value, if no ``skipped`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``skipped`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``skipped`` attribute exists on the given element node. + """ if "skipped" in element.attrib: return int(element.attrib["skipped"]) elif not optional: @@ -1349,6 +1456,15 @@ def _ParseSkipped(self, element: _Element, default: Nullable[int] = None, option return default def _ParseErrors(self, element: _Element, default: Nullable[int] = None, optional: bool = True) -> Nullable[int]: + """ + Convert the ``errors`` attribute from an XML element node to an integer. + + :param element: The XML element node with a ``errors`` attribute. + :param default: The default value, if no ``errors`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``errors`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``errors`` attribute exists on the given element node. + """ if "errors" in element.attrib: return int(element.attrib["errors"]) elif not optional: @@ -1357,6 +1473,15 @@ def _ParseErrors(self, element: _Element, default: Nullable[int] = None, optiona return default def _ParseFailures(self, element: _Element, default: Nullable[int] = None, optional: bool = True) -> Nullable[int]: + """ + Convert the ``failures`` attribute from an XML element node to an integer. + + :param element: The XML element node with a ``failures`` attribute. + :param default: The default value, if no ``failures`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``failures`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``failures`` attribute exists on the given element node. + """ if "failures" in element.attrib: return int(element.attrib["failures"]) elif not optional: @@ -1365,6 +1490,15 @@ def _ParseFailures(self, element: _Element, default: Nullable[int] = None, optio return default def _ParseAssertions(self, element: _Element, default: Nullable[int] = None, optional: bool = True) -> Nullable[int]: + """ + Convert the ``assertions`` attribute from an XML element node to an integer. + + :param element: The XML element node with a ``assertions`` attribute. + :param default: The default value, if no ``assertions`` attribute was found. + :param optional: If false, an exception is raised for the missing attribute. + :return: The ``assertions`` attribute's content if found, otherwise the given default value. + :raises UnittestException: If optional is false and no ``assertions`` attribute exists on the given element node. + """ if "assertions" in element.attrib: return int(element.attrib["assertions"]) elif not optional: @@ -1373,6 +1507,14 @@ def _ParseAssertions(self, element: _Element, default: Nullable[int] = None, opt return default def _ParseTestsuite(self, parent: TestsuiteSummary, testsuitesNode: _Element) -> None: + """ + Convert the XML data structure of a ```` to a test suite. + + This method uses private helper methods provided by the base-class. + + :param parent: The test suite summary as a parent element in the test entity hierarchy. + :param testsuitesNode: The current XML element node representing a test suite. + """ newTestsuite = self._TESTSUITE( self._ParseName(testsuitesNode, optional=False), self._ParseHostname(testsuitesNode, optional=True), @@ -1399,7 +1541,15 @@ def _ParseTestsuiteChildren(self, testsuitesNode: _Element, newTestsuite: Testsu self._ParseTestcase(newTestsuite, node) def _ParseTestcase(self, parent: Testsuite, testcaseNode: _Element) -> None: - className = self._ParseClassname(testcaseNode, optional=False) + """ + Convert the XML data structure of a ```` to a test case. + + This method uses private helper methods provided by the base-class. + + :param parent: The test suite as a parent element in the test entity hierarchy. + :param testcaseNode: The current XML element node representing a test case. + """ + className = self._ParseClassname(testcaseNode) testclass = self._FindOrCreateTestclass(parent, className) newTestcase = self._TESTCASE( @@ -1443,7 +1593,15 @@ def _ParseTestcaseChildren(self, testcaseNode: _Element, newTestcase: Testcase) newTestcase._status = TestcaseStatus.Passed def Generate(self, overwrite: bool = False) -> None: - if self._xmlDocument is not None: + """ + Generate the internal XML data structure from test suites and test cases. + + This method generates the XML root element (````) and recursively calls other generated methods. + + :param overwrite: Overwrite the internal XML data structure. + :raises UnittestException: If overwrite is false and the internal XML data structure is not empty. + """ + if not overwrite and self._xmlDocument is not None: raise UnittestException(f"Internal XML document is populated with data.") rootElement = Element("testsuites") @@ -1464,7 +1622,16 @@ def Generate(self, overwrite: bool = False) -> None: for testsuite in self._testsuites.values(): self._GenerateTestsuite(testsuite, rootElement) - def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): + def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test suite. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testsuite: The test suite to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testsuiteElement = SubElement(parentElement, "testsuite") testsuiteElement.attrib["name"] = testsuite._name if testsuite._startTime is not None: @@ -1484,7 +1651,16 @@ def _GenerateTestsuite(self, testsuite: Testsuite, parentElement: _Element): for tc in testclass._testcases.values(): self._GenerateTestcase(tc, testsuiteElement) - def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element): + def _GenerateTestcase(self, testcase: Testcase, parentElement: _Element) -> None: + """ + Generate the internal XML data structure for a test case. + + This method generates the XML element (````) and recursively calls other generated methods. + + :param testcase: The test case to convert to an XML data structures. + :param parentElement: The parent XML data structure element, this data structure part will be added to. + :return: + """ testcaseElement = SubElement(parentElement, "testcase") if testcase.Classname is not None: testcaseElement.attrib["classname"] = testcase.Classname diff --git a/pyEDAA/Reports/Unittesting/OSVVM.py b/pyEDAA/Reports/Unittesting/OSVVM.py index f6ec0fd4..46260cef 100644 --- a/pyEDAA/Reports/Unittesting/OSVVM.py +++ b/pyEDAA/Reports/Unittesting/OSVVM.py @@ -42,29 +42,52 @@ from pyEDAA.Reports.Unittesting import Testcase as ut_Testcase +from typing import Callable + +@export +def InheritDocumentation(baseClass: type, merge: bool = False) -> Callable[[type], type]: + """xxx""" + def decorator(c: type) -> type: + """yyy""" + if merge: + if c.__doc__ is None: + c.__doc__ = baseClass.__doc__ + elif baseClass.__doc__ is not None: + c.__doc__ = baseClass.__doc__ + "\n\n" + c.__doc__ + else: + c.__doc__ = baseClass.__doc__ + return c + + return decorator + + @export class OsvvmException: pass @export +@InheritDocumentation(UnittestException) class UnittestException(UnittestException, OsvvmException): - pass + """@InheritDocumentation(UnittestException)""" @export +@InheritDocumentation(ut_Testcase) class Testcase(ut_Testcase): - pass + """@InheritDocumentation(ut_Testcase)""" @export +@InheritDocumentation(ut_Testsuite) class Testsuite(ut_Testsuite): - pass + """@InheritDocumentation(ut_Testsuite)""" @export +@InheritDocumentation(ut_TestsuiteSummary) class TestsuiteSummary(ut_TestsuiteSummary): - pass + """@InheritDocumentation(ut_TestsuiteSummary)""" @export @@ -82,6 +105,13 @@ def __init__(self, yamlReportFile: Path, parse: bool = False) -> None: self.Convert() def Analyze(self) -> None: + """ + Analyze the YAML file, parse the content into an YAML data structure. + + .. hint:: + + The time spend for analysis will be made available via property :data:`AnalysisDuration`.. + """ if not self._path.exists(): raise UnittestException(f"OSVVM YAML file '{self._path}' does not exist.") \ from FileNotFoundError(f"File '{self._path}' not found.") @@ -210,6 +240,15 @@ def _ParseDurationFieldFromYAML(node: CommentedMap, fieldName: str) -> Nullable[ return timedelta(seconds=value) def Convert(self) -> None: + """ + Convert the parsed YAML data structure into a test entity hierarchy. + + .. hint:: + + The time spend for model conversion will be made available via property :data:`ModelConversionDuration`. + + :raises UnittestException: If XML was not read and parsed before. + """ if self._yamlDocument is None: ex = UnittestException(f"OSVVM YAML file '{self._path}' needs to be read and analyzed by a YAML parser.") ex.add_note(f"Call 'Document.Analyze()' or create document using 'Document(path, parse=True)'.") diff --git a/pyEDAA/Reports/helper.py b/pyEDAA/Reports/helper.py new file mode 100644 index 00000000..732dc05e --- /dev/null +++ b/pyEDAA/Reports/helper.py @@ -0,0 +1,20 @@ +from typing import Callable + +from pyTooling.Decorators import export + + +@export +def InheritDocumentation(baseClass: type, merge: bool = False) -> Callable[[type], type]: + """xxx""" + def decorator(c: type) -> type: + """yyy""" + if merge: + if c.__doc__ is None: + c.__doc__ = baseClass.__doc__ + elif baseClass.__doc__ is not None: + c.__doc__ = baseClass.__doc__ + "\n\n" + c.__doc__ + else: + c.__doc__ = baseClass.__doc__ + return c + + return decorator