Collect and check test coverage stats (#49)

Closes #9

.github/workflows/test.yml

@@ -32,7 +32,7 @@ permissions:
   contents: read
 
 jobs:
-  tox:
+  tests:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false  # Always report results for all targets
@@ -131,3 +131,52 @@ jobs:
         path: |
           export/tests
         retention-days: 3 # Just for debugging, don't need to keep these long term
+
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-data-${{ matrix.os }}-py${{ matrix.python-version }}
+        path: .coverage.*
+        include-hidden-files: true
+        if-no-files-found: ignore
+
+  # Coverage check based on https://hynek.me/articles/ditch-codecov-python/
+  coverage:
+    name: Combine & check coverage
+    if: always()
+    needs: tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          # Use latest Python, so it understands all syntax.
+          python-version: "3.13"
+      - uses: hynek/setup-cached-uv@v2
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+
+      - name: Combine coverage & fail if it's <100%
+        run: |
+          uv tool install 'coverage[toml]'
+
+          coverage combine
+          coverage html --skip-covered --skip-empty
+
+          # Report and write to summary.
+          coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+
+          # Report again and fail if under 92%.
+          # (threshold is based on 0.1.0rc1 CI statement coverage)
+          coverage report --fail-under=92
+
+      - name: Upload HTML report if check failed
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}

.gitignore

@@ -14,4 +14,4 @@ __pycache__
 *.egg-info
 *.dist-info
 /dist/
-
+.coverage*

ci-constraints.txt

@@ -52,6 +52,24 @@ click==8.1.7 \
 colorama==0.4.6 \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
     --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+coverage==7.6.4 \
+    --hash=sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9 \
+    --hash=sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08 \
+    --hash=sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2 \
+    --hash=sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963 \
+    --hash=sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73 \
+    --hash=sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117 \
+    --hash=sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25 \
+    --hash=sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52 \
+    --hash=sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b \
+    --hash=sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19 \
+    --hash=sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f \
+    --hash=sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a \
+    --hash=sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba \
+    --hash=sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e \
+    --hash=sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f \
+    --hash=sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806 \
+    --hash=sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1
 dep-logic==0.4.9 \
     --hash=sha256:06faa33814e5ff881922f644284a608d7da7946462760f710217d829ae864a0e \
     --hash=sha256:5d455ea2a3da4fea2be6186d886905c57eeeebe3ea7fa967f599cb8e0f01d5c9

docs/requirements.txt

@@ -13,6 +13,7 @@ chardet==5.2.0
 charset-normalizer==3.4.0
 click==8.1.7
 colorama==0.4.6
+coverage[toml]==7.6.4
 dep-logic==0.4.9
 distlib==0.3.9
 docutils==0.21.2

pyproject.toml

@@ -65,6 +65,7 @@ dev = [
     "ruff>=0.5.4",
     "mypy>=1.11.0",
     "pytest-subtests>=0.13.1",
+    "coverage[toml]>=7.6.4",
     # Use exact pin for dev as runtime environments are sensitive to the exact pbs version
     "pbs-installer==2024.10.10",
     # Uses exact pin for dev as lock file regeneration is sensitive to the exact uv version
@@ -91,6 +92,26 @@ verbosity_assertions = 2
 tmp_path_retention_policy = "failed"
 tmp_path_retention_count = 1
 
+[tool.coverage.run]
+relative_files = true
+source_pkgs = [
+    "venvstacks",
+]
+source = [
+    "tests/",
+]
+omit = [
+    # There is quite a bit of test support code that
+    # only runs to help diagnose failures in CI
+    "tests/support.py",
+]
+
+[tool.coverage.paths]
+source = [
+    "src/",
+    "**/.tox/**/site-packages/",
+]
+
 [tool.ruff]
 # Assume Python 3.11
 target-version = "py311"

tests/README.md

@@ -47,6 +47,22 @@ Tests which take more than a few seconds to run should be marked as slow:
     def test_locking_and_publishing(self) -> None:
         ...
 
+The slow tests are part of the test suite because the fast tests only
+get to just over 60% coverage of `venvstacks.stacks` and less than
+20% coverage of `venvstacks.pack_venv`. The combined fast coverage
+on a single platform (Linux for these numbers) is just over 60%.
+
+When the slow tests are included, even running on a single platform,
+statement coverages rises to nearly 90% coverage of `venvstacks.stacks`,
+nearly 70% coverage of `venvstacks.pack_venv`, and just under 90%
+combined coverage across the test suite and package source code.
+
+When the results across all platforms are combined, the overall
+coverage of `venvstacks.stacks` doesn't improve much, but
+`venvstacks.pack_venv` improves to more than 85%, and the overall
+test coverage exceeds 90% (as of 0.1.0, CI checks for at least 92%
+statement coverage).
+
 
 Marking tests with committed output files
 -----------------------------------------

tests/test_cli_invocation.py

@@ -186,6 +186,24 @@ def test_entry_point_help(self) -> None:
         assert result.returncode == 0
         assert result.stdout is not None
 
+    def test_module_execution(self) -> None:
+        # TODO: `coverage.py` isn't picking this up as executing `venvstacks/__main__.py`
+        #       (even an indirect invocation via the runpy module doesn't get detected)
+        command = [sys.executable, "-m", "venvstacks", "--help"]
+        result = run_python_command_unchecked(
+            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        if result.stdout is not None:
+            # Usage message should suggest indirect execution
+            assert "Usage: python -m venvstacks [" in result.stdout
+            # Top-level callback docstring is used as the overall CLI help text
+            cli_help = cli.handle_app_options.__doc__
+            assert cli_help is not None
+            assert cli_help.strip() in result.stdout
+        # Check operation result last to ensure test results are as informative as possible
+        assert result.returncode == 0
+        assert result.stdout is not None
+
 
 EXPECTED_USAGE_PREFIX = "Usage: python -m venvstacks "
 EXPECTED_SUBCOMMANDS = ["lock", "build", "local-export", "publish"]

tox.ini

@@ -10,27 +10,36 @@ labels =
      static = lint,typecheck
 
 [testenv]
+# Multi-env performance tweak based on https://hynek.me/articles/turbo-charge-tox/
+package = wheel
+wheel_build_env = .pkg
 groups = dev
 allowlist_externals = pytest
 passenv =
+    CI
     VENVSTACKS_*
 commands =
     pytest {posargs:-m "not slow"} tests/
 
+[testenv:coverage]
+# Subprocess coverage based on https://hynek.me/articles/turbo-charge-tox/
+allowlist_externals = coverage
+set_env = COVERAGE_PROCESS_START={toxinidir}/pyproject.toml
+commands_pre = python -c 'import pathlib; pathlib.Path("{env_site_packages_dir}/cov.pth").write_text("import coverage; coverage.process_startup()")'
+commands =
+    coverage run --parallel -m pytest {posargs} tests/
+
 [testenv:format]
-groups = dev
 allowlist_externals = ruff
 commands =
     ruff format {posargs} src/ tests/ misc/
 
 [testenv:lint]
-groups = dev
 allowlist_externals = ruff
 commands =
     ruff check --exclude 'tests/sample_project' {posargs} src/ tests/ misc/
 
 [testenv:typecheck]
-groups = dev
 allowlist_externals = mypy
 commands =
     mypy --strict --exclude 'tests/sample_project' {posargs} src/ tests/ misc/
@@ -60,4 +69,5 @@ commands =
 python =
     3.11 = py3.11
     3.12 = py3.12
-    3.13 = py3.13
+    # Collect coverage stats on the newest version
+    3.13 = coverage