Issue253 coverage (#557) (#563)

* Issue253 coverage (#557)

* add coverage script from Mans with changes based on review in #315 #243


---------

Co-authored-by: FWuellhorst <[email protected]>

buildingspy/CHANGES.txt

@@ -10,6 +10,9 @@ Version 5.2.0, xxxx
 - In buildingspy/development/regressiontest.py, add option to create reference
   results in batch mode.
   (https://github.com/lbl-srg/BuildingsPy/issues/560)
+- In buildingspy/development/regressiontest.py, add option to get the coverage
+  rate, i.e., what percentage of examples are covered by regression tests.
+  (https://github.com/lbl-srg/BuildingsPy/issues/253)
 - For Optimica regression tests, added check for Integers that are too large to be represented
 - In buildingspy/development/refactor.py, corrected moving images to avoid creating
   a directory if the target directory already exists.

buildingspy/development/regressiontest.py

@@ -352,6 +352,9 @@ def __init__(
         self._data = []
         self._reporter = rep.Reporter(os.path.join(os.getcwd(), "unitTests-{}.log".format(tool)))
 
+        # List to store tested packages, used for coverage report
+        self._packages = []
+
         # By default, include export of FMUs.
         self._include_fmu_test = True
 
@@ -776,11 +779,12 @@ def setSinglePackage(self, packageName):
 
         # Set data dictionary as it may have been generated earlier for the whole library.
         self._data = []
-
+        self._packages = []
         for pac in packages:
             pacSep = pac.find('.')
             pacPat = pac[pacSep + 1:]
             pacPat = pacPat.replace('.', os.sep)
+            self._packages.append(pacPat)
             rooPat = os.path.join(self._libHome, 'Resources', 'Scripts', 'Dymola', pacPat)
             # Verify that the directory indeed exists
             if not os.path.isdir(rooPat):
@@ -4297,3 +4301,130 @@ def _model_from_mo(self, mo_file):
         model = '.'.join(splt[root:])
         # remove the '.mo' at the end
         return model[:-3]
+
+    def getCoverage(self):
+        """
+        Analyse how many examples are tested.
+        If ``setSinglePackage`` is called before this function,
+        only packages set will be included. Else, the whole library
+        will be checked.
+
+        Returns:
+            - The coverage rate in percent as float
+            - The number of examples tested as int
+            - The total number of examples as int
+            - The list of models not tested as List[str]
+            - The list of packages included in the analysis as List[str]
+
+        Example:
+            >>> from buildingspy.development.regressiontest import Tester
+            >>> import os
+            >>> ut = Tester(tool='dymola')
+            >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary")
+            >>> ut.setLibraryRoot(myMoLib)
+            >>> ut.setSinglePackage('Examples')
+            Regression tests are only run for the following package:
+              Examples
+            MyModelicaLibrary.Examples.NoSolution: Excluded from simulation. Model excluded from simulation as it has no solution.
+            >>> coverage_result = ut.getCoverage()
+        """
+        # first lines copy and paste from run function
+        if self.get_number_of_tests() == 0:
+            self.setDataDictionary(self._rootPackage)
+
+        # Remove all data that do not require a simulation or an FMU export.
+        # Otherwise, some processes may have no simulation to run and then
+        # the json output file would have an invalid syntax
+
+        # now we got clean _data to compare
+        # next step get all examples in the package (whether whole library or
+        # single package)
+        if self._packages:
+            packages = self._packages
+        else:
+            packages = list(dict.fromkeys(
+                [pac['ScriptFile'].split(os.sep)[0] for pac in self._data])
+            )
+
+        all_examples = []
+        for package in packages:
+            package_path = os.path.join(self._libHome, package)
+            for dirpath, dirnames, filenames in os.walk(package_path):
+                for filename in filenames:
+                    filepath = os.path.abspath(os.path.join(dirpath, filename))
+                    if any(
+                            xs in filepath for xs in ['Examples', 'Validation']
+                    ) and not filepath.endswith(('package.mo', '.order')):
+                        all_examples.append(filepath)
+
+        n_tested_examples = len(self._data)
+        n_examples = len(all_examples)
+        if n_examples > 0:
+            coverage = round(n_tested_examples / n_examples, 2) * 100
+        else:
+            coverage = 100
+
+        tested_model_names = [
+            nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data
+        ]
+
+        missing_examples = [
+            i for i in all_examples if not any(
+                xs in i for xs in tested_model_names)
+        ]
+
+        return coverage, n_tested_examples, n_examples, missing_examples, packages
+
+    def printCoverage(
+            self,
+            coverage: float,
+            n_tested_examples: int,
+            n_examples: int,
+            missing_examples: list,
+            packages: list,
+            printer: callable = None
+    ) -> None:
+        """
+        Print the output of getCoverage to inform about
+        coverage rate and missing models.
+        The default printer is the ``reporter.writeOutput``.
+        If another printing method is required, e.g. ``print`` or
+        ``logging.info``, it may be passed via the ``printer`` argument.
+
+        Example:
+            >>> from buildingspy.development.regressiontest import Tester
+            >>> import os
+            >>> ut = Tester(tool='dymola')
+            >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary")
+            >>> ut.setLibraryRoot(myMoLib)
+            >>> ut.setSinglePackage('Examples')
+            Regression tests are only run for the following package:
+              Examples
+            MyModelicaLibrary.Examples.NoSolution: Excluded from simulation. Model excluded from simulation as it has no solution.
+            >>> coverage_result = ut.getCoverage()
+            >>> ut.printCoverage(*coverage_result, printer=print)
+            ***
+            Model Coverage: 88 %
+            ***
+            You are testing: 7 out of 8 examples in package:
+            Examples
+            ***
+            The following examples are not tested
+            <BLANKLINE>
+            /Examples/ParameterEvaluation.mo
+
+        """
+        if printer is None:
+            printer = self._reporter.writeOutput
+        printer(f'***\nModel Coverage: {int(coverage)} %')
+        printer(
+            f'***\nYou are testing: {n_tested_examples} '
+            f'out of {n_examples} examples in package{"s" if len(packages) > 1 else ""}:',
+        )
+        for package in packages:
+            printer(package)
+
+        if missing_examples:
+            print('***\nThe following examples are not tested\n')
+            for i in missing_examples:
+                print(i.split(self._libHome)[1])

buildingspy/tests/test_development_regressiontest.py

@@ -326,6 +326,41 @@ def test_expand_packages(self):
         self.assertRaises(ValueError,
                           r.Tester.expand_packages, "AB}a{")
 
+    def test_get_coverage_single_package(self):
+        coverage_result = self._test_get_and_print_coverage(package="Examples")
+        self.assertEqual(coverage_result[0], 88)
+        self.assertEqual(coverage_result[1], 7)
+        self.assertEqual(coverage_result[2], 8)
+        self.assertTrue(coverage_result[3][0].endswith("ParameterEvaluation.mo"))
+        self.assertEqual(coverage_result[4], ["Examples"])
+
+    def test_get_coverage_all_packages(self):
+        coverage_result = self._test_get_and_print_coverage(package=None)
+        self.assertEqual(coverage_result[0], 89)
+        self.assertEqual(coverage_result[1], 8)
+        self.assertEqual(coverage_result[2], 9)
+        self.assertEqual(len(coverage_result[3]), 1)
+        self.assertEqual(len(coverage_result[4]), 2)
+
+    def _test_get_and_print_coverage(self, package: str = None):
+        import buildingspy.development.regressiontest as r
+        ut = r.Tester(tool='dymola')
+        myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary")
+        ut.setLibraryRoot(myMoLib)
+        if package is not None:
+            ut.setSinglePackage(package)
+        coverage_result = ut.getCoverage()
+        self.assertIsInstance(coverage_result, tuple)
+        self.assertIsInstance(coverage_result[0], float)
+        self.assertIsInstance(coverage_result[1], int)
+        self.assertIsInstance(coverage_result[2], int)
+        self.assertIsInstance(coverage_result[3], list)
+        self.assertIsInstance(coverage_result[4], list)
+        # Check print with both custom and standard printer
+        ut.printCoverage(*coverage_result, printer=print)
+        ut.printCoverage(*coverage_result)
+        return coverage_result
+
 
 if __name__ == '__main__':
     unittest.main()