Enable coverage calculation

Added pytest --force

.github/workflows/build.yaml

@@ -103,6 +103,7 @@ jobs:
         run: |
           pip install -e .
           pip install --upgrade pytest-md-report
+          pip install gcovr
       - name: Run tests
         env:
           REPORT_OUTPUT: md_report.md
@@ -111,7 +112,7 @@ jobs:
           rm tests/sim/_tsdz2.cdef # make sure cdef is generated from the source to check testing framework
 
           echo "REPORT_FILE=${REPORT_OUTPUT}" >> "$GITHUB_ENV"
-          pytest --md-report --md-report-flavor gfm --md-report-output "$REPORT_OUTPUT"
+          pytest --coverage --md-report --md-report-flavor gfm --md-report-output "$REPORT_OUTPUT"
       - name: Output reports to the job summary
         if: always()
         shell: bash
@@ -131,6 +132,16 @@ jobs:
           header: test-report
           recreate: true
           path: ${{ env.REPORT_FILE }}
+      - name: Collect coverage data
+        run: |
+          echo "### Coverage Report" >> $GITHUB_STEP_SUMMARY
+          gcovr -r tests --print-summary >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage_report
+          path: |
+            tests/coverage_report.html
 
   Compare_builds:
     needs: [Build_Windows, Build_Linux]

README.md

@@ -75,6 +75,11 @@ Run tests:
 
 Any changes should have a corresponding unit test added, unless unfeasible.
 
+Calculate coverage and generate html report (probably will not work on Windows):
+`pytest --coverage`
+
+Tests with coverage are executed in the CI as well.
+
 ### Compile the firmware manually
 - `cd src/` and use `make` or `compile.bat` to compile the firmware.
 

tests/conftest.py

@@ -1,11 +1,19 @@
+import subprocess
+import importlib
 from load_c_code import load_code
 
+def pytest_addoption(parser):
+    parser.addoption("--coverage", action="store_true", help="Enable coverage analysis with gcovr")
+    parser.addoption("--force", action="store_true", help="Force recompile")
+
 def pytest_sessionstart(session):
     """
     Called after the Session object has been created and
     before performing collection and entering the run test loop.
     """
-    load_code('_tsdz2')
+    lib, _ = load_code('_tsdz2', coverage=session.config.option.coverage, force_recompile=session.config.option.force)
+    if session.config.option.coverage:
+        lib.__gcov_reset()
 
 
 def pytest_configure(config):
@@ -19,6 +27,25 @@ def pytest_sessionfinish(session, exitstatus):
     Called after whole test run finished, right before
     returning the exit status to the system.
     """
+
+    if session.config.option.coverage:
+        try:
+            module = importlib.import_module("sim._tsdz2")
+            module.lib.__gcov_dump()
+        except Exception as e:
+            # __gcov_dump() is stil hidden ion some compilers? Failing on CI Ubuntu 22.04, but worked on 20.04
+            print(f"Error dumping gcov: {e}. Try running gcovr manually after pytest process exits.")
+        try:
+            # Attempt to call gcovr with the specified arguments
+            subprocess.call(['gcovr', '-r', 'tests', '--print-summary'])
+        except FileNotFoundError:
+            # Handle the case where gcovr is not found (i.e., not installed)
+            print(" Install Gcovr to generate code coverage report.")
+        except Exception as e:
+            # E.g. gcov will fail if cffi compiled code with msvc
+            print(f"Error running gcovr: {e}")
+        # btw, if there is aerror(warning) "bgcov profiling error: ... overwriting an existing profile data with a different timestamp"
+        # it has to do with pytest without being called in the background by e.g. VScode without --coverage and recompiling the objects
 
 def pytest_unconfigure(config):
     """

tests/gcovr.cfg

@@ -0,0 +1,10 @@
+# Only show coverage for files in src/, lib/foo, or for main.cpp files.
+filter = sim/
+exclude-unreachable-branches=no
+exclude-noncode-lines=yes
+exclude-function-lines=yes
+
+
+html-details=coverage_report.html 
+html-self-contained=yes
+print-summary=yes

tests/load_c_code.py

@@ -150,12 +150,12 @@ def generate_cdef(module_name, src_file):
         fp.write(cdef)
     return cdef
 
-
-def load_code(module_name, force_recompile=False):
+def load_code(module_name, coverage=False, force_recompile=False):
     # Load previous combined hash
     hash_file_path = os.path.join(LIB_DIR, f"{module_name}.sha")
+    # Recalculate hash if code or arguments have changed
     with Checksum(hash_file_path, source_dirs, module_name+"".join(define_macros)) as skip:
-        if not skip or force_recompile:
+        if not skip or force_recompile or coverage: # also recompile if coverage is enabled because backround test runners are compiling without gcov
             print("Collecting source code..")
             source_content_list: List[str] = []
             source_files = [os.path.abspath(os.path.join(dir, file)) for dir in source_dirs for file in os.listdir(dir) if file.endswith('.c')]
@@ -167,7 +167,6 @@ def load_code(module_name, force_recompile=False):
             combined_source = fake_defines + combined_source
             combined_source = re.sub(r"#\s*include\s*<.*?>", r"//\g<0>", combined_source) # comment out standard includes
             combined_source_file_path = os.path.join(LIB_DIR, f"{module_name}.i")
-
             with open(combined_source_file_path, "w", encoding="utf8") as fp:
                 fp.write(combined_source)
             try:
@@ -176,20 +175,39 @@ def load_code(module_name, force_recompile=False):
                 print(f"{e}\n\033[93mFailed to generate cdef using your cpp standard headers!!!\nYou may have to edit it manually. Continuing...\033[0m")
                 with open(os.path.join(LIB_DIR, f"{module_name}.cdef"), "r", encoding="utf8") as fp:
                     cdef = fp.read()
-
+            extra_compile_args = compiler_args
+            extra_link_args = linker_args
+            # Coverage
+            if coverage:
+                # expose gcov api, (will not work with microsoft compiler)
+                cdef += "\n" + "extern void __gcov_reset(void);"
+                cdef += "\n" + "extern void __gcov_dump(void);"
+                # add inner coverage exclusion markers
+                combined_source = "extern void __gcov_reset(void);\n" + combined_source
+                combined_source = "extern void __gcov_dump(void);\n" + combined_source
+                combined_source =  "// GCOVR_EXCL_STOP\n" + combined_source + "\n// GCOVR_EXCL_START"
+                extra_compile_args += ["--coverage"]
+                extra_link_args +=    ["--coverage"]
+
             # Create a CFFI instance
             ffibuilder = cffi.FFI()
             print("Processing cdefs...")
             ffibuilder.cdef(cdef)
             ffibuilder.set_source(module_name, combined_source,
                                 include_dirs=[os.path.abspath(d) for d in include_dirs], 
                                 define_macros=[(macro, None) for macro in define_macros],
-                                extra_compile_args=compiler_args,
-                                extra_link_args=linker_args
+                                extra_compile_args=extra_compile_args,
+                                extra_link_args=extra_link_args
                                 )
             print("Compiling...")
             tmpdir = os.path.abspath(LIB_DIR)
             ffibuilder.compile(tmpdir=tmpdir)
+            if coverage: # Add outer coverage exclusion markers after generating ffi api c-code
+                with open(os.path.join(LIB_DIR, f"{module_name}.c"), "r+", encoding="utf8") as fp:
+                        c = fp.readlines() # add inline comments because it may matter for gcov to keep the number of lines, idk:
+                        c[0] = c[0].strip() + " // GCOVR_EXCL_START"
+                        c[-1] = c[-1].strip() + " // GCOVR_EXCL_STOP"
+                        fp.seek(0); fp.writelines(c); fp.truncate()
         else:
             print("No changes found. Skipping compilation")