diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c63eaad..57a5a95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ permissions: contents: read jobs: - tox: + tests: runs-on: ${{ matrix.os }} strategy: fail-fast: false # Always report results for all targets @@ -131,3 +131,52 @@ jobs: path: | export/tests retention-days: 3 # Just for debugging, don't need to keep these long term + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-data-${{ matrix.os }}-py${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: ignore + + # Coverage check based on https://hynek.me/articles/ditch-codecov-python/ + coverage: + name: Combine & check coverage + if: always() + needs: tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + # Use latest Python, so it understands all syntax. + python-version: "3.13" + - uses: hynek/setup-cached-uv@v2 + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: | + uv tool install 'coverage[toml]' + + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> $GITHUB_STEP_SUMMARY + + # Report again and fail if under 92%. + # (threshold is based on 0.1.0rc1 CI statement coverage) + coverage report --fail-under=92 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report + path: htmlcov + if: ${{ failure() }} diff --git a/.gitignore b/.gitignore index 6aede0b..063f8c9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ __pycache__ *.egg-info *.dist-info /dist/ - +.coverage* diff --git a/ci-constraints.txt b/ci-constraints.txt index c0bd57a..ce7f547 100644 --- a/ci-constraints.txt +++ b/ci-constraints.txt @@ -52,6 +52,24 @@ click==8.1.7 \ colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +coverage==7.6.4 \ + --hash=sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9 \ + --hash=sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08 \ + --hash=sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2 \ + --hash=sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963 \ + --hash=sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73 \ + --hash=sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117 \ + --hash=sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25 \ + --hash=sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52 \ + --hash=sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b \ + --hash=sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19 \ + --hash=sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f \ + --hash=sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a \ + --hash=sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba \ + --hash=sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e \ + --hash=sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f \ + --hash=sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806 \ + --hash=sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1 dep-logic==0.4.9 \ --hash=sha256:06faa33814e5ff881922f644284a608d7da7946462760f710217d829ae864a0e \ --hash=sha256:5d455ea2a3da4fea2be6186d886905c57eeeebe3ea7fa967f599cb8e0f01d5c9 diff --git a/docs/requirements.txt b/docs/requirements.txt index cb309a7..fba43bd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -13,6 +13,7 @@ chardet==5.2.0 charset-normalizer==3.4.0 click==8.1.7 colorama==0.4.6 +coverage[toml]==7.6.4 dep-logic==0.4.9 distlib==0.3.9 docutils==0.21.2 diff --git a/pdm.lock b/pdm.lock index e6b94d0..41bd8d7 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "bootstrap", "dev", "docs"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:6b3ff1f79f6ee9817518655d1bdb5179f333894b6bfd08a3088f48c683631aa0" +content_hash = "sha256:b87d49f37d3f661dbd8fc83049b4430eaa012c35b3de694fd1f309d371a7fd1e" [[metadata.targets]] requires_python = ">=3.11" @@ -197,6 +197,63 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coverage" +version = "7.6.4" +requires_python = ">=3.9" +summary = "Code coverage measurement for Python" +groups = ["dev"] +files = [ + {file = "coverage-7.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b"}, + {file = "coverage-7.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e"}, + {file = "coverage-7.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52"}, + {file = "coverage-7.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f"}, + {file = "coverage-7.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a"}, + {file = "coverage-7.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1"}, + {file = "coverage-7.6.4.tar.gz", hash = "sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73"}, +] + +[[package]] +name = "coverage" +version = "7.6.4" +extras = ["toml"] +requires_python = ">=3.9" +summary = "Code coverage measurement for Python" +groups = ["dev"] +dependencies = [ + "coverage==7.6.4", + "tomli; python_full_version <= \"3.11.0a6\"", +] +files = [ + {file = "coverage-7.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b"}, + {file = "coverage-7.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e"}, + {file = "coverage-7.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52"}, + {file = "coverage-7.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f"}, + {file = "coverage-7.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a"}, + {file = "coverage-7.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1"}, + {file = "coverage-7.6.4.tar.gz", hash = "sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73"}, +] + [[package]] name = "dep-logic" version = "0.4.9" diff --git a/pyproject.toml b/pyproject.toml index 36e2f35..27d26d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ dev = [ "ruff>=0.5.4", "mypy>=1.11.0", "pytest-subtests>=0.13.1", + "coverage[toml]>=7.6.4", # Use exact pin for dev as runtime environments are sensitive to the exact pbs version "pbs-installer==2024.10.10", # Uses exact pin for dev as lock file regeneration is sensitive to the exact uv version @@ -91,6 +92,26 @@ verbosity_assertions = 2 tmp_path_retention_policy = "failed" tmp_path_retention_count = 1 +[tool.coverage.run] +relative_files = true +source_pkgs = [ + "venvstacks", +] +source = [ + "tests/", +] +omit = [ + # There is quite a bit of test support code that + # only runs to help diagnose failures in CI + "tests/support.py", +] + +[tool.coverage.paths] +source = [ + "src/", + "**/.tox/**/site-packages/", +] + [tool.ruff] # Assume Python 3.11 target-version = "py311" diff --git a/tests/README.md b/tests/README.md index 8a6224d..57d0632 100644 --- a/tests/README.md +++ b/tests/README.md @@ -47,6 +47,22 @@ Tests which take more than a few seconds to run should be marked as slow: def test_locking_and_publishing(self) -> None: ... +The slow tests are part of the test suite because the fast tests only +get to just over 60% coverage of `venvstacks.stacks` and less than +20% coverage of `venvstacks.pack_venv`. The combined fast coverage +on a single platform (Linux for these numbers) is just over 60%. + +When the slow tests are included, even running on a single platform, +statement coverages rises to nearly 90% coverage of `venvstacks.stacks`, +nearly 70% coverage of `venvstacks.pack_venv`, and just under 90% +combined coverage across the test suite and package source code. + +When the results across all platforms are combined, the overall +coverage of `venvstacks.stacks` doesn't improve much, but +`venvstacks.pack_venv` improves to more than 85%, and the overall +test coverage exceeds 90% (as of 0.1.0, CI checks for at least 92% +statement coverage). + Marking tests with committed output files ----------------------------------------- diff --git a/tests/test_cli_invocation.py b/tests/test_cli_invocation.py index 144c598..3afa0a4 100644 --- a/tests/test_cli_invocation.py +++ b/tests/test_cli_invocation.py @@ -186,6 +186,24 @@ def test_entry_point_help(self) -> None: assert result.returncode == 0 assert result.stdout is not None + def test_module_execution(self) -> None: + # TODO: `coverage.py` isn't picking this up as executing `venvstacks/__main__.py` + # (even an indirect invocation via the runpy module doesn't get detected) + command = [sys.executable, "-m", "venvstacks", "--help"] + result = run_python_command_unchecked( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if result.stdout is not None: + # Usage message should suggest indirect execution + assert "Usage: python -m venvstacks [" in result.stdout + # Top-level callback docstring is used as the overall CLI help text + cli_help = cli.handle_app_options.__doc__ + assert cli_help is not None + assert cli_help.strip() in result.stdout + # Check operation result last to ensure test results are as informative as possible + assert result.returncode == 0 + assert result.stdout is not None + EXPECTED_USAGE_PREFIX = "Usage: python -m venvstacks " EXPECTED_SUBCOMMANDS = ["lock", "build", "local-export", "publish"] diff --git a/tox.ini b/tox.ini index 1d1e94b..f3c42cc 100644 --- a/tox.ini +++ b/tox.ini @@ -10,27 +10,36 @@ labels = static = lint,typecheck [testenv] +# Multi-env performance tweak based on https://hynek.me/articles/turbo-charge-tox/ +package = wheel +wheel_build_env = .pkg groups = dev allowlist_externals = pytest passenv = + CI VENVSTACKS_* commands = pytest {posargs:-m "not slow"} tests/ +[testenv:coverage] +# Subprocess coverage based on https://hynek.me/articles/turbo-charge-tox/ +allowlist_externals = coverage +set_env = COVERAGE_PROCESS_START={toxinidir}/pyproject.toml +commands_pre = python -c 'import pathlib; pathlib.Path("{env_site_packages_dir}/cov.pth").write_text("import coverage; coverage.process_startup()")' +commands = + coverage run --parallel -m pytest {posargs} tests/ + [testenv:format] -groups = dev allowlist_externals = ruff commands = ruff format {posargs} src/ tests/ misc/ [testenv:lint] -groups = dev allowlist_externals = ruff commands = ruff check --exclude 'tests/sample_project' {posargs} src/ tests/ misc/ [testenv:typecheck] -groups = dev allowlist_externals = mypy commands = mypy --strict --exclude 'tests/sample_project' {posargs} src/ tests/ misc/ @@ -60,4 +69,5 @@ commands = python = 3.11 = py3.11 3.12 = py3.12 - 3.13 = py3.13 + # Collect coverage stats on the newest version + 3.13 = coverage