From 585a47f6874179d47d791e17ce18e498b4c0160b Mon Sep 17 00:00:00 2001 From: Michael Wetter Date: Thu, 22 Aug 2024 11:23:35 -0700 Subject: [PATCH] Issue253 coverage (#557) (#563) * Issue253 coverage (#557) * add coverage script from Mans with changes based on review in #315 #243 --------- Co-authored-by: FWuellhorst --- buildingspy/CHANGES.txt | 3 + buildingspy/development/regressiontest.py | 133 +++++++++++++++++- .../tests/test_development_regressiontest.py | 35 +++++ 3 files changed, 170 insertions(+), 1 deletion(-) diff --git a/buildingspy/CHANGES.txt b/buildingspy/CHANGES.txt index 49f06300..2ca9d06f 100644 --- a/buildingspy/CHANGES.txt +++ b/buildingspy/CHANGES.txt @@ -10,6 +10,9 @@ Version 5.2.0, xxxx - In buildingspy/development/regressiontest.py, add option to create reference results in batch mode. (https://github.com/lbl-srg/BuildingsPy/issues/560) +- In buildingspy/development/regressiontest.py, add option to get the coverage + rate, i.e., what percentage of examples are covered by regression tests. + (https://github.com/lbl-srg/BuildingsPy/issues/253) - For Optimica regression tests, added check for Integers that are too large to be represented - In buildingspy/development/refactor.py, corrected moving images to avoid creating a directory if the target directory already exists. diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index e5276336..46b3b1b3 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -352,6 +352,9 @@ def __init__( self._data = [] self._reporter = rep.Reporter(os.path.join(os.getcwd(), "unitTests-{}.log".format(tool))) + # List to store tested packages, used for coverage report + self._packages = [] + # By default, include export of FMUs. self._include_fmu_test = True @@ -776,11 +779,12 @@ def setSinglePackage(self, packageName): # Set data dictionary as it may have been generated earlier for the whole library. self._data = [] - + self._packages = [] for pac in packages: pacSep = pac.find('.') pacPat = pac[pacSep + 1:] pacPat = pacPat.replace('.', os.sep) + self._packages.append(pacPat) rooPat = os.path.join(self._libHome, 'Resources', 'Scripts', 'Dymola', pacPat) # Verify that the directory indeed exists if not os.path.isdir(rooPat): @@ -4297,3 +4301,130 @@ def _model_from_mo(self, mo_file): model = '.'.join(splt[root:]) # remove the '.mo' at the end return model[:-3] + + def getCoverage(self): + """ + Analyse how many examples are tested. + If ``setSinglePackage`` is called before this function, + only packages set will be included. Else, the whole library + will be checked. + + Returns: + - The coverage rate in percent as float + - The number of examples tested as int + - The total number of examples as int + - The list of models not tested as List[str] + - The list of packages included in the analysis as List[str] + + Example: + >>> from buildingspy.development.regressiontest import Tester + >>> import os + >>> ut = Tester(tool='dymola') + >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + >>> ut.setLibraryRoot(myMoLib) + >>> ut.setSinglePackage('Examples') + Regression tests are only run for the following package: + Examples + MyModelicaLibrary.Examples.NoSolution: Excluded from simulation. Model excluded from simulation as it has no solution. + >>> coverage_result = ut.getCoverage() + """ + # first lines copy and paste from run function + if self.get_number_of_tests() == 0: + self.setDataDictionary(self._rootPackage) + + # Remove all data that do not require a simulation or an FMU export. + # Otherwise, some processes may have no simulation to run and then + # the json output file would have an invalid syntax + + # now we got clean _data to compare + # next step get all examples in the package (whether whole library or + # single package) + if self._packages: + packages = self._packages + else: + packages = list(dict.fromkeys( + [pac['ScriptFile'].split(os.sep)[0] for pac in self._data]) + ) + + all_examples = [] + for package in packages: + package_path = os.path.join(self._libHome, package) + for dirpath, dirnames, filenames in os.walk(package_path): + for filename in filenames: + filepath = os.path.abspath(os.path.join(dirpath, filename)) + if any( + xs in filepath for xs in ['Examples', 'Validation'] + ) and not filepath.endswith(('package.mo', '.order')): + all_examples.append(filepath) + + n_tested_examples = len(self._data) + n_examples = len(all_examples) + if n_examples > 0: + coverage = round(n_tested_examples / n_examples, 2) * 100 + else: + coverage = 100 + + tested_model_names = [ + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data + ] + + missing_examples = [ + i for i in all_examples if not any( + xs in i for xs in tested_model_names) + ] + + return coverage, n_tested_examples, n_examples, missing_examples, packages + + def printCoverage( + self, + coverage: float, + n_tested_examples: int, + n_examples: int, + missing_examples: list, + packages: list, + printer: callable = None + ) -> None: + """ + Print the output of getCoverage to inform about + coverage rate and missing models. + The default printer is the ``reporter.writeOutput``. + If another printing method is required, e.g. ``print`` or + ``logging.info``, it may be passed via the ``printer`` argument. + + Example: + >>> from buildingspy.development.regressiontest import Tester + >>> import os + >>> ut = Tester(tool='dymola') + >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + >>> ut.setLibraryRoot(myMoLib) + >>> ut.setSinglePackage('Examples') + Regression tests are only run for the following package: + Examples + MyModelicaLibrary.Examples.NoSolution: Excluded from simulation. Model excluded from simulation as it has no solution. + >>> coverage_result = ut.getCoverage() + >>> ut.printCoverage(*coverage_result, printer=print) + *** + Model Coverage: 88 % + *** + You are testing: 7 out of 8 examples in package: + Examples + *** + The following examples are not tested + + /Examples/ParameterEvaluation.mo + + """ + if printer is None: + printer = self._reporter.writeOutput + printer(f'***\nModel Coverage: {int(coverage)} %') + printer( + f'***\nYou are testing: {n_tested_examples} ' + f'out of {n_examples} examples in package{"s" if len(packages) > 1 else ""}:', + ) + for package in packages: + printer(package) + + if missing_examples: + print('***\nThe following examples are not tested\n') + for i in missing_examples: + print(i.split(self._libHome)[1]) diff --git a/buildingspy/tests/test_development_regressiontest.py b/buildingspy/tests/test_development_regressiontest.py index 6d48fbe3..bf34f718 100644 --- a/buildingspy/tests/test_development_regressiontest.py +++ b/buildingspy/tests/test_development_regressiontest.py @@ -326,6 +326,41 @@ def test_expand_packages(self): self.assertRaises(ValueError, r.Tester.expand_packages, "AB}a{") + def test_get_coverage_single_package(self): + coverage_result = self._test_get_and_print_coverage(package="Examples") + self.assertEqual(coverage_result[0], 88) + self.assertEqual(coverage_result[1], 7) + self.assertEqual(coverage_result[2], 8) + self.assertTrue(coverage_result[3][0].endswith("ParameterEvaluation.mo")) + self.assertEqual(coverage_result[4], ["Examples"]) + + def test_get_coverage_all_packages(self): + coverage_result = self._test_get_and_print_coverage(package=None) + self.assertEqual(coverage_result[0], 89) + self.assertEqual(coverage_result[1], 8) + self.assertEqual(coverage_result[2], 9) + self.assertEqual(len(coverage_result[3]), 1) + self.assertEqual(len(coverage_result[4]), 2) + + def _test_get_and_print_coverage(self, package: str = None): + import buildingspy.development.regressiontest as r + ut = r.Tester(tool='dymola') + myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + ut.setLibraryRoot(myMoLib) + if package is not None: + ut.setSinglePackage(package) + coverage_result = ut.getCoverage() + self.assertIsInstance(coverage_result, tuple) + self.assertIsInstance(coverage_result[0], float) + self.assertIsInstance(coverage_result[1], int) + self.assertIsInstance(coverage_result[2], int) + self.assertIsInstance(coverage_result[3], list) + self.assertIsInstance(coverage_result[4], list) + # Check print with both custom and standard printer + ut.printCoverage(*coverage_result, printer=print) + ut.printCoverage(*coverage_result) + return coverage_result + if __name__ == '__main__': unittest.main()