diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04e80ef..d7bc2a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,7 +162,7 @@ jobs: pattern: coverage-data-* merge-multiple: true - - name: Combine coverage & fail if it's <100% + - name: Combine coverage & fail if it goes down run: | uv tool install 'coverage[toml]' @@ -172,9 +172,11 @@ jobs: # Report and write to summary. coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - # Report again and fail if under 92%. - # (threshold is based on 0.1.0rc1 CI statement coverage) - coverage report --fail-under=92 + # Report again and fail if under 91%. + # Highest historical coverage: 92% + # Subsequent proportional coverage reductions: + # - de-duplicated the deployment checking code + coverage report --fail-under=91 - name: Upload HTML report if check failed uses: actions/upload-artifact@v4 diff --git a/tests/support.py b/tests/support.py index 5513ca8..8494fbb 100644 --- a/tests/support.py +++ b/tests/support.py @@ -5,18 +5,22 @@ import subprocess import sys import tomllib +import unittest from dataclasses import dataclass, fields from pathlib import Path -from typing import Any, cast, Mapping +from typing import Any, Callable, cast, Mapping, Sequence, TypeVar from unittest.mock import create_autospec import pytest -from venvstacks._util import capture_python_output +from venvstacks._util import get_env_python, capture_python_output + from venvstacks.stacks import ( BuildEnvironment, EnvNameDeploy, + ExportedEnvironmentPaths, + ExportMetadata, LayerBaseName, PackageIndexConfig, ) @@ -218,3 +222,91 @@ def get_sys_path(env_python: Path) -> list[str]: def run_module(env_python: Path, module_name: str) -> subprocess.CompletedProcess[str]: command = [str(env_python), "-X", "utf8", "-Im", module_name] return capture_python_output(command) + + +########################################################### +# Checking deployed environments for the expected details +########################################################### + + +class DeploymentTestCase(unittest.TestCase): + """Native unittest test case with additional deployment validation checks""" + EXPECTED_APP_OUTPUT = "" + + + def assertPathExists(self, expected_path: Path) -> None: + self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") + + def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None: + self.assertTrue( + any(expected in path_entry for path_entry in env_sys_path), + f"No entry containing {expected!r} found in {env_sys_path}", + ) + + T = TypeVar("T", bound=Mapping[str, Any]) + + def check_deployed_environments( + self, + layered_metadata: dict[str, Sequence[T]], + get_exported_python: Callable[[T], tuple[str, Path, list[str]]], + ) -> None: + for rt_env in layered_metadata["runtimes"]: + env_name, _, env_sys_path = get_exported_python(rt_env) + self.assertTrue(env_sys_path) # Environment should have sys.path entries + # Runtime environment layer should be completely self-contained + self.assertTrue( + all(env_name in path_entry for path_entry in env_sys_path), + f"Path outside {env_name} in {env_sys_path}", + ) + for fw_env in layered_metadata["frameworks"]: + env_name, _, env_sys_path = get_exported_python(fw_env) + self.assertTrue(env_sys_path) # Environment should have sys.path entries + # Framework and runtime should both appear in sys.path + runtime_name = fw_env["runtime_name"] + short_runtime_name = ".".join(runtime_name.split(".")[:2]) + self.assertSysPathEntry(env_name, env_sys_path) + self.assertSysPathEntry(short_runtime_name, env_sys_path) + for app_env in layered_metadata["applications"]: + env_name, env_python, env_sys_path = get_exported_python(app_env) + self.assertTrue(env_sys_path) # Environment should have sys.path entries + # Application, frameworks and runtime should all appear in sys.path + runtime_name = app_env["runtime_name"] + short_runtime_name = ".".join(runtime_name.split(".")[:2]) + self.assertSysPathEntry(env_name, env_sys_path) + self.assertTrue( + any(env_name in path_entry for path_entry in env_sys_path), + f"No entry containing {env_name} found in {env_sys_path}", + ) + for fw_env_name in app_env["required_layers"]: + self.assertSysPathEntry(fw_env_name, env_sys_path) + self.assertSysPathEntry(short_runtime_name, env_sys_path) + # Launch module should be executable + launch_module = app_env["app_launch_module"] + launch_result = run_module(env_python, launch_module) + # Tolerate extra trailing whitespace on stdout + self.assertEqual(launch_result.stdout.rstrip(), self.EXPECTED_APP_OUTPUT) + # Nothing at all should be emitted on stderr + self.assertEqual(launch_result.stderr, "") + + def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> None: + metadata_path, snippet_paths, env_paths = export_paths + exported_manifests = ManifestData(metadata_path, snippet_paths) + env_name_to_path: dict[str, Path] = {} + for env_metadata, env_path in zip(exported_manifests.snippet_data, env_paths): + # TODO: Check more details regarding expected metadata contents + self.assertTrue(env_path.exists()) + env_name = EnvNameDeploy(env_metadata["install_target"]) + self.assertEqual(env_path.name, env_name) + env_name_to_path[env_name] = env_path + layered_metadata = exported_manifests.combined_data["layers"] + + def get_exported_python( + env: ExportMetadata, + ) -> tuple[EnvNameDeploy, Path, list[str]]: + env_name = env["install_target"] + env_path = env_name_to_path[env_name] + env_python = get_env_python(env_path) + env_sys_path = get_sys_path(env_python) + return env_name, env_python, env_sys_path + + self.check_deployed_environments(layered_metadata, get_exported_python) diff --git a/tests/test_minimal_project.py b/tests/test_minimal_project.py index f5860a2..64a90ee 100644 --- a/tests/test_minimal_project.py +++ b/tests/test_minimal_project.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable, cast, Mapping, Sequence, TypeVar +from typing import Any, cast # Use unittest for consistency with test_sample_project (which needs the better diff support) import unittest @@ -17,13 +17,13 @@ import pytest # To mark slow test cases from support import ( + ApplicationEnvSummary, + DeploymentTestCase, EnvSummary, LayeredEnvSummary, - ApplicationEnvSummary, ManifestData, make_mock_index_config, get_sys_path, - run_module, ) from venvstacks.stacks import ( @@ -34,8 +34,6 @@ EnvNameDeploy, LayerVariants, StackSpec, - ExportedEnvironmentPaths, - ExportMetadata, PackageIndexConfig, PublishedArchivePaths, get_build_platform, @@ -335,7 +333,7 @@ def test_custom_output_directory_absolute(self) -> None: self.assertFalse(expected_output_path.exists()) -class TestMinimalBuild(unittest.TestCase): +class TestMinimalBuild(DeploymentTestCase): # Test cases that actually create the build environment folders working_path: Path @@ -428,59 +426,8 @@ def check_publication_result( expected_archive_paths.sort() self.assertEqual(sorted(archive_paths), expected_archive_paths) - # TODO: Refactor to share the environment checking code with test_sample_project - def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None: - self.assertTrue( - any(expected in path_entry for path_entry in env_sys_path), - f"No entry containing {expected!r} found in {env_sys_path}", - ) - - T = TypeVar("T", bound=Mapping[str, Any]) - - def check_deployed_environments( - self, - layered_metadata: dict[str, Sequence[T]], - get_exported_python: Callable[[T], tuple[str, Path, list[str]]], - ) -> None: - for rt_env in layered_metadata["runtimes"]: - env_name, _, env_sys_path = get_exported_python(rt_env) - self.assertTrue(env_sys_path) # Environment should have sys.path entries - # Runtime environment layer should be completely self-contained - self.assertTrue( - all(env_name in path_entry for path_entry in env_sys_path), - f"Path outside {env_name} in {env_sys_path}", - ) - for fw_env in layered_metadata["frameworks"]: - env_name, _, env_sys_path = get_exported_python(fw_env) - self.assertTrue(env_sys_path) # Environment should have sys.path entries - # Framework and runtime should both appear in sys.path - runtime_name = fw_env["runtime_name"] - short_runtime_name = ".".join(runtime_name.split(".")[:2]) - self.assertSysPathEntry(env_name, env_sys_path) - self.assertSysPathEntry(short_runtime_name, env_sys_path) - for app_env in layered_metadata["applications"]: - env_name, env_python, env_sys_path = get_exported_python(app_env) - self.assertTrue(env_sys_path) # Environment should have sys.path entries - # Application, frameworks and runtime should all appear in sys.path - runtime_name = app_env["runtime_name"] - short_runtime_name = ".".join(runtime_name.split(".")[:2]) - self.assertSysPathEntry(env_name, env_sys_path) - self.assertTrue( - any(env_name in path_entry for path_entry in env_sys_path), - f"No entry containing {env_name} found in {env_sys_path}", - ) - for fw_env_name in app_env["required_layers"]: - self.assertSysPathEntry(fw_env_name, env_sys_path) - self.assertSysPathEntry(short_runtime_name, env_sys_path) - # Launch module should be executable - launch_module = app_env["app_launch_module"] - launch_result = run_module(env_python, launch_module) - self.assertEqual(launch_result.stdout, "") - self.assertEqual(launch_result.stderr, "") - - def _run_postinstall(self, env_path: Path) -> None: - config_path = env_path / DEPLOYED_LAYER_CONFIG - self.assertTrue(config_path.exists()) + @staticmethod + def _run_postinstall(env_path: Path) -> None: postinstall_script = env_path / "postinstall.py" if postinstall_script.exists(): # Post-installation scripts are required to work even when they're @@ -494,7 +441,10 @@ def check_archive_deployment(self, published_paths: PublishedArchivePaths) -> No published_manifests = ManifestData(metadata_path, snippet_paths) # TODO: read the base Python path for each environment from the metadata # https://github.com/lmstudio-ai/venvstacks/issues/19 - with tempfile.TemporaryDirectory() as deployment_dir: + # TODO: figure out a more robust way of handling Windows potentially still + # having the Python executables in the environment open when the + # parent process tries to clean up the deployment directory. + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as deployment_dir: # Extract archives deployment_path = Path(deployment_dir) env_name_to_path: dict[EnvNameDeploy, Path] = {} @@ -540,32 +490,6 @@ def get_exported_python( self.check_deployed_environments(layered_metadata, get_exported_python) - def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> None: - metadata_path, snippet_paths, env_paths = export_paths - exported_manifests = ManifestData(metadata_path, snippet_paths) - env_name_to_path: dict[str, Path] = {} - for env_metadata, env_path in zip(exported_manifests.snippet_data, env_paths): - # TODO: Check more details regarding expected metadata contents - self.assertTrue(env_path.exists()) - env_name = EnvNameDeploy(env_metadata["install_target"]) - self.assertEqual(env_path.name, env_name) - env_name_to_path[env_name] = env_path - layered_metadata = exported_manifests.combined_data["layers"] - - def get_exported_python( - env: ExportMetadata, - ) -> tuple[EnvNameDeploy, Path, list[str]]: - env_name = env["install_target"] - env_path = env_name_to_path[env_name] - env_python = get_env_python(env_path) - env_sys_path = get_sys_path(env_python) - return env_name, env_python, env_sys_path - - self.check_deployed_environments(layered_metadata, get_exported_python) - - def assertPathExists(self, expected_path: Path) -> None: - self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") - @pytest.mark.slow def test_locking_and_publishing(self) -> None: # This is organised as subtests in a monolothic test sequence to reduce CI overhead diff --git a/tests/test_sample_project.py b/tests/test_sample_project.py index a02da94..647aec7 100644 --- a/tests/test_sample_project.py +++ b/tests/test_sample_project.py @@ -7,7 +7,7 @@ from itertools import chain from pathlib import Path -from typing import Any, Callable, Mapping, Sequence, TypeVar +from typing import Any # Use unittest for the actual test implementations due to the diff-handling in pytest being @@ -18,6 +18,7 @@ import pytest # To mark slow test cases from support import ( + DeploymentTestCase, EnvSummary, LayeredEnvSummary, ApplicationEnvSummary, @@ -25,23 +26,17 @@ get_artifact_export_path, force_artifact_export, get_os_environ_settings, - get_sys_path, - run_module, ) from venvstacks.stacks import ( ArchiveBuildMetadata, ArchiveMetadata, BuildEnvironment, - EnvNameDeploy, StackSpec, LayerCategories, - ExportedEnvironmentPaths, - ExportMetadata, LayerVariants, ) from venvstacks._injected.postinstall import DEPLOYED_LAYER_CONFIG -from venvstacks._util import get_env_python ################################## # Sample project test helpers @@ -253,8 +248,9 @@ def test_spec_loading(self) -> None: self.assertEqual(app_env.env_name, app_summary.env_name) -class TestBuildEnvironment(unittest.TestCase): +class TestBuildEnvironment(DeploymentTestCase): # Test cases that need the full build environment to exist + EXPECTED_APP_OUTPUT = "Environment launch module executed successfully" working_path: Path build_env: BuildEnvironment @@ -272,84 +268,6 @@ def setUp(self) -> None: self.artifact_export_path = get_artifact_export_path() self.export_on_success = force_artifact_export() - # TODO: Refactor to share the environment checking code with test_minimal_project - def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None: - self.assertTrue( - any(expected in path_entry for path_entry in env_sys_path), - f"No entry containing {expected!r} found in {env_sys_path}", - ) - - T = TypeVar("T", bound=Mapping[str, Any]) - - def check_deployed_environments( - self, - layered_metadata: dict[str, Sequence[T]], - get_exported_python: Callable[[T], tuple[str, Path, list[str]]], - ) -> None: - for rt_env in layered_metadata["runtimes"]: - deployed_name, _, env_sys_path = get_exported_python(rt_env) - self.assertTrue(env_sys_path) # Environment should have sys.path entries - # Runtime environment layer should be completely self-contained - self.assertTrue( - all(deployed_name in path_entry for path_entry in env_sys_path), - f"Path outside {deployed_name} in {env_sys_path}", - ) - for fw_env in layered_metadata["frameworks"]: - deployed_name, _, env_sys_path = get_exported_python(fw_env) - self.assertTrue(env_sys_path) # Environment should have sys.path entries - # Framework and runtime should both appear in sys.path - runtime_name = fw_env["runtime_name"] - short_runtime_name = ".".join(runtime_name.split(".")[:2]) - self.assertSysPathEntry(deployed_name, env_sys_path) - self.assertSysPathEntry(short_runtime_name, env_sys_path) - for app_env in layered_metadata["applications"]: - deployed_name, env_python, env_sys_path = get_exported_python(app_env) - self.assertTrue(env_sys_path) # Environment should have sys.path entries - # Application, frameworks and runtime should all appear in sys.path - runtime_name = app_env["runtime_name"] - short_runtime_name = ".".join(runtime_name.split(".")[:2]) - self.assertSysPathEntry(deployed_name, env_sys_path) - self.assertTrue( - any(deployed_name in path_entry for path_entry in env_sys_path), - f"No entry containing {deployed_name} found in {env_sys_path}", - ) - for fw_env_name in app_env["required_layers"]: - self.assertSysPathEntry(fw_env_name, env_sys_path) - self.assertSysPathEntry(short_runtime_name, env_sys_path) - # Launch module should be executable - launch_module = app_env["app_launch_module"] - launch_result = run_module(env_python, launch_module) - self.assertEqual( - launch_result.stdout.strip(), - "Environment launch module executed successfully", - ) - self.assertEqual(launch_result.stderr, "") - - def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> None: - metadata_path, snippet_paths, env_paths = export_paths - exported_manifests = ManifestData(metadata_path, snippet_paths) - deployed_name_to_path: dict[str, Path] = {} - for env_metadata, env_path in zip(exported_manifests.snippet_data, env_paths): - self.assertTrue(env_path.exists()) - deployed_name = EnvNameDeploy(env_metadata["install_target"]) - self.assertEqual(env_path.name, deployed_name) - deployed_name_to_path[deployed_name] = env_path - layered_metadata = exported_manifests.combined_data["layers"] - - def get_exported_python( - env: ExportMetadata, - ) -> tuple[EnvNameDeploy, Path, list[str]]: - deployed_name = env["install_target"] - env_path = deployed_name_to_path[deployed_name] - env_python = get_env_python(env_path) - env_sys_path = get_sys_path(env_python) - return deployed_name, env_python, env_sys_path - - self.check_deployed_environments(layered_metadata, get_exported_python) - - def assertPathExists(self, expected_path: Path) -> None: - self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") - @pytest.mark.slow @pytest.mark.expected_output def test_build_is_reproducible(self) -> None: