From b960f32a087ea6f8d1cdf1d44bfbcaf48be98f3f Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Thu, 7 Nov 2024 21:21:22 +1000 Subject: [PATCH] Improve postinstall script resilience (#69) * Allow postinstall scripts to be executed with any Python interpreter (not just the deployed base runtime interpreter) * Generate layer config file as part of layers * Use a common postinstall script in all layers * Generate the deployed `sitecustomize.py` file from the layer config in the postinstall script * Add unit tests for the common postinstall script * Add build env creation test cases (separate from the slow lock-and-publish/export test cases) Closes #66. Implements initial steps towards #19. --- .github/workflows/update-expected-output.yml | 3 + ...35_ncoghlan_more_resilient_postinstall.rst | 7 + src/venvstacks/_injected/README.md | 11 + src/venvstacks/_injected/postinstall.py | 180 +++++++++++++ src/venvstacks/_util.py | 4 + src/venvstacks/pack_venv.py | 80 ++---- src/venvstacks/stacks.py | 247 +++++++++++------- .../env_metadata/app-scipy-client.json | 4 +- .../env_metadata/app-scipy-import.json | 4 +- .../env_metadata/app-sklearn-import.json | 4 +- .../env_metadata/cpython@3.11.json | 4 +- .../env_metadata/cpython@3.12.json | 4 +- .../env_metadata/framework-http-client.json | 4 +- .../env_metadata/framework-scipy.json | 4 +- .../env_metadata/framework-sklearn.json | 4 +- .../linux_x86_64/venvstacks.json | 32 +-- .../env_metadata/app-scipy-client.json | 4 +- .../env_metadata/app-scipy-import.json | 4 +- .../env_metadata/cpython@3.11.json | 4 +- .../env_metadata/cpython@3.12.json | 4 +- .../env_metadata/framework-http-client.json | 4 +- .../env_metadata/framework-scipy.json | 4 +- .../env_metadata/framework-sklearn.json | 4 +- .../macosx_arm64/venvstacks.json | 28 +- .../env_metadata/app-scipy-client.json | 4 +- .../env_metadata/app-scipy-import.json | 4 +- .../win_amd64/env_metadata/cpython@3.11.json | 4 +- .../win_amd64/env_metadata/cpython@3.12.json | 4 +- .../env_metadata/framework-http-client.json | 4 +- .../env_metadata/framework-scipy.json | 4 +- .../env_metadata/framework-sklearn.json | 4 +- .../win_amd64/venvstacks.json | 28 +- tests/support.py | 130 +++++++-- tests/test_minimal_project.py | 38 ++- tests/test_postinstall.py | 98 +++++++ tests/test_sample_project.py | 16 +- 36 files changed, 700 insertions(+), 290 deletions(-) create mode 100644 changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst create mode 100644 src/venvstacks/_injected/README.md create mode 100644 src/venvstacks/_injected/postinstall.py create mode 100644 tests/test_postinstall.py diff --git a/.github/workflows/update-expected-output.yml b/.github/workflows/update-expected-output.yml index b2b002e..b3b2672 100644 --- a/.github/workflows/update-expected-output.yml +++ b/.github/workflows/update-expected-output.yml @@ -10,6 +10,9 @@ on: paths: # Run for changes to *this* workflow file, but not for other workflows - ".github/workflows/update-expected-output.yml" + # Check PRs that update the files injected into deployed environments + # (the layer config metadata format is also specified in these files) + - "src/venvstacks/_injected/**/*.py" # Check PRs that update the expected test suite output results - "tests/expected-output-config.toml" - "tests/sample_project/venvstacks.toml" diff --git a/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst b/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst new file mode 100644 index 0000000..c9da85f --- /dev/null +++ b/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst @@ -0,0 +1,7 @@ +Fixed +----- + +- Post-installation scripts for layered environments now work + correctly even when run with a Python installation other + than the expected base runtime (resolved in :issue:`66`) + diff --git a/src/venvstacks/_injected/README.md b/src/venvstacks/_injected/README.md new file mode 100644 index 0000000..f0b56c1 --- /dev/null +++ b/src/venvstacks/_injected/README.md @@ -0,0 +1,11 @@ +Files injected into deployed environments +========================================= + +Files in this folder are injected into the deployed +environments when publishing artifacts or locally +exporting environments. + +They are also designed to be importable so that +the build process can access their functionality +without needing to duplicate the implementation, +and to make them more amenable to unit testing. diff --git a/src/venvstacks/_injected/postinstall.py b/src/venvstacks/_injected/postinstall.py new file mode 100644 index 0000000..9823673 --- /dev/null +++ b/src/venvstacks/_injected/postinstall.py @@ -0,0 +1,180 @@ +"""venvstacks layer post-installation script + +* Loads `./share/venv/metadata/venvstacks_layer.json` +* Generates `pyvenv.cfg` for layered environments +* Generates `sitecustomize.py` for layered environments +* Precompiles all Python files in the library folder + +This post-installation script is automatically injected when packing environments. +""" + +import json +import os + +from compileall import compile_dir +from os.path import abspath +from pathlib import Path +from typing import cast, NotRequired, Sequence, TypedDict + +DEPLOYED_LAYER_CONFIG = "share/venv/metadata/venvstacks_layer.json" + + +class LayerConfig(TypedDict): + """Additional details needed to fully configure deployed environments""" + + # fmt: off + python: str # Relative path to this layer's Python executable + py_version: str # Expected X.Y.Z Python version for this environment + base_python: str # Relative path from layer dir to base Python executable + site_dir: str # Relative path to site-packages within this layer + pylib_dirs: Sequence[str] # Relative paths to additional sys.path entries + dynlib_dirs: Sequence[str] # Relative paths to additional Windows DLL directories + launch_module: NotRequired[str] # Module to run with `-m` to launch the application + # fmt: on + + # All relative paths are relative to the layer folder (and may refer to peer folders) + # Base runtime layers will have "python" and "base_python" set to the same value + # Application layers will have "launch_module" set + + +class ResolvedLayerConfig(TypedDict): + """LayerConfig with relative paths resolved for a specific layer location""" + + # fmt: off + layer_path: Path # Absolute path to layer environment + python_path: Path # Absolute path to this layer's Python executable + py_version: str # Expected X.Y.Z Python version for this environment + base_python_path: Path # Absolute path from layer dir to base Python executable + site_path: Path # Absolute path to site-packages within this layer + pylib_paths: Sequence[Path] # Absolute paths to additional sys.path entries + dynlib_paths: Sequence[Path] # Absolute paths to additional Windows DLL directories + launch_module: str|None # Module to run with `-m` to launch the application + # fmt: on + + +def load_layer_config(layer_path: Path) -> ResolvedLayerConfig: + """Read and resolve config for the specified layer environment""" + + def deployed_path(relative_path: str) -> Path: + """Normalize path and make it absolute, *without* resolving symlinks""" + return Path(abspath(layer_path / relative_path)) + + config_path = layer_path / DEPLOYED_LAYER_CONFIG + config_text = config_path.read_text(encoding="utf-8") + # Tolerate runtime errors for incorrectly generated config files + config = cast(LayerConfig, json.loads(config_text)) + return ResolvedLayerConfig( + layer_path=layer_path, + python_path=deployed_path(config["python"]), + py_version=config["py_version"], + base_python_path=deployed_path(config["base_python"]), + site_path=deployed_path(config["site_dir"]), + pylib_paths=[deployed_path(d) for d in config["pylib_dirs"]], + dynlib_paths=[deployed_path(d) for d in config["dynlib_dirs"]], + launch_module=config.get("launch_module", None), + ) + + +def generate_pyvenv_cfg(base_python_path: Path, py_version: str) -> str: + """Generate `pyvenv.cfg` contents for given base Python path and version""" + if not base_python_path.is_absolute(): + raise RuntimeError("Post-installation must use absolute environment paths") + venv_config_lines = [ + f"home = {base_python_path.parent}", + "include-system-site-packages = false", + f"version = {py_version}", + f"executable = {base_python_path}", + "", + ] + return "\n".join(venv_config_lines) + + +_SITE_CUSTOMIZE_HEADER = '''\ +"""venvstacks layered environment site customization script + +* Calls `site.addsitedir` for any configured Python path entries +* Calls `os.add_dll_directory` for any configured Windows dynlib paths + +This venv module is automatically generated by the post-installation script. +""" + +''' + + +def generate_sitecustomize( + pylib_paths: Sequence[Path], + dynlib_paths: Sequence[Path], + *, + skip_missing_dynlib_paths: bool = True, +) -> str | None: + """Generate `sitecustomize.py` contents for given linked environment directories""" + sc_contents = [_SITE_CUSTOMIZE_HEADER] + if pylib_paths: + pylib_contents = [ + "# Allow loading modules and packages from linked environments", + "from site import addsitedir", + ] + for path_entry in pylib_paths: + if not path_entry.is_absolute(): + raise RuntimeError( + "Post-installation must use absolute environment paths" + ) + pylib_contents.append(f"addsitedir({str(path_entry)!r})") + pylib_contents.append("") + sc_contents.extend(pylib_contents) + if dynlib_paths and hasattr(os, "add_dll_directory"): + dynlib_contents = [ + "# Allow loading misplaced DLLs on Windows", + "from os import add_dll_directory", + ] + for dynlib_path in dynlib_paths: + if not dynlib_path.is_absolute(): + raise RuntimeError( + "Post-installation must use absolute environment paths" + ) + if skip_missing_dynlib_paths and not dynlib_path.exists(): + # Nothing added DLLs to this folder at build time, so skip it + # (add_dll_directory fails if the specified folder doesn't exist) + dynlib_entry = f"# Skipping {str(dynlib_path)!r} (no such directory)" + else: + dynlib_entry = f"add_dll_directory({str(dynlib_path)!r})" + dynlib_contents.append(dynlib_entry) + dynlib_contents.append("") + sc_contents.extend(dynlib_contents) + if len(sc_contents) == 1: + # Environment layer doesn't actually need customizing + return None + return "\n".join(sc_contents) + + +def _run_postinstall(layer_path: Path) -> None: + """Run the required post-installation steps in a deployed environment""" + + # Read the layer config file + config = load_layer_config(layer_path) + + base_python_path = config["base_python_path"] + if base_python_path != config["python_path"]: + # Generate `pyvenv.cfg` for layered environments + venv_config = generate_pyvenv_cfg(base_python_path, config["py_version"]) + venv_config_path = layer_path / "pyvenv.cfg" + venv_config_path.write_text(venv_config, encoding="utf-8") + + # Generate `sitecustomize.py` for layered environments + sc_contents = generate_sitecustomize( + config["pylib_paths"], config["dynlib_paths"] + ) + if sc_contents is not None: + sc_path = config["site_path"] / "sitecustomize.py" + sc_path.write_text(sc_contents, encoding="utf-8") + + # Precompile Python library modules + pylib_path = ( + layer_path / "lib" + ) # "Lib" on Windows, but Windows is not case sensitive + compile_dir(pylib_path, optimize=0, quiet=True) + + +if __name__ == "__main__": + # Actually executing the post-installation step in a deployed environment + _run_postinstall(Path(__file__).parent) diff --git a/src/venvstacks/_util.py b/src/venvstacks/_util.py index ea06074..0c3b93d 100644 --- a/src/venvstacks/_util.py +++ b/src/venvstacks/_util.py @@ -125,3 +125,7 @@ def run_python_command( result = run_python_command_unchecked(command, text=True, **kwds) result.check_returncode() return result + + +def capture_python_output(command: list[str]) -> subprocess.CompletedProcess[str]: + return run_python_command(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/src/venvstacks/pack_venv.py b/src/venvstacks/pack_venv.py index d60c282..4f2e557 100755 --- a/src/venvstacks/pack_venv.py +++ b/src/venvstacks/pack_venv.py @@ -43,61 +43,9 @@ from pathlib import Path from typing import cast, Any, Callable, TextIO +from ._injected import postinstall as _default_postinstall from ._util import as_normalized_path, StrPath, WINDOWS_BUILD as _WINDOWS_BUILD -_PRECOMPILATION_COMMANDS = """\ -# Precompile Python library modules -from compileall import compile_dir -venv_pylib_path = venv_path / "lib" # "Lib" on Windows, but Windows is not case sensitive -compile_dir(venv_pylib_path, optimize=0, quiet=True) -""" - -_BASE_RUNTIME_POST_INSTALL_SCRIPT = ( - '''\ -"""Base runtime post-installation script - -* Precompiles all Python files in the library folder - -This post-installation script is automatically injected when packing environments that -do NOT include a `pyvenv.cfg` file (i.e. base runtime environments) -""" -from pathlib import Path -venv_path = Path(__file__).parent - -''' - + _PRECOMPILATION_COMMANDS -) - -_LAYERED_ENV_POST_INSTALL_SCRIPT = ( - '''\ -"""Layered environment post-installation script - -* Generates pyvenv.cfg based on the Python runtime executing this script -* Precompiles all Python files in the library folder - -This post-installation script is automatically injected when packing environments that -would otherwise include a `pyvenv.cfg` file (as `pyvenv.cfg` files are not relocatable) -""" -from pathlib import Path -venv_path = Path(__file__).parent - -# Generate `pyvenv.cfg` based on the deployed runtime location -import sys -venv_config_path = venv_path / "pyvenv.cfg" -runtime_executable_path = Path(sys.executable).resolve() -runtime_version = ".".join(map(str, sys.version_info[:3])) -venv_config = f"""\ -home = {runtime_executable_path.parent} -include-system-site-packages = false -version = {runtime_version} -executable = {runtime_executable_path} -""" -venv_config_path.write_text(venv_config, encoding="utf-8") - -''' - + _PRECOMPILATION_COMMANDS -) - SymlinkInfo = tuple[Path, Path] @@ -165,18 +113,20 @@ def get_archive_path(archive_base_name: StrPath) -> Path: def _inject_postinstall_script( - env_path: Path, script_name: str = "postinstall.py" + env_path: Path, + script_name: str = "postinstall.py", + script_source: StrPath | None = None, ) -> Path: venv_config_path = env_path / "pyvenv.cfg" if venv_config_path.exists(): # The venv config contains absolute paths referencing the base runtime environment # Remove it here, let the post-install script recreate it venv_config_path.unlink() - script_contents = _LAYERED_ENV_POST_INSTALL_SCRIPT - else: - script_contents = _BASE_RUNTIME_POST_INSTALL_SCRIPT + if script_source is None: + # Nothing specified, inject the default postinstall script + script_source = _default_postinstall.__file__ script_path = env_path / script_name - script_path.write_text(script_contents, encoding="utf-8") + shutil.copy2(script_source, script_path) return script_path @@ -201,7 +151,9 @@ def export_venv( """Export the given build environment, skipping archive creation and unpacking * injects a suitable `postinstall.py` script for the environment being exported - * excludes __pycache__ folders and package metadata RECORD files + * excludes __pycache__ folders (for consistency with archive publication) + * excludes package metadata RECORD files (for consistency with archive publication) + * excludes `sitecustomize.py` files (generated by the post-installation script) * replaces symlinks with copies on Windows or if the target doesn't support symlinks If supplied, *run_postinstall* is called with the path to the environment's Python @@ -213,7 +165,7 @@ def export_venv( """ source_path = as_normalized_path(source_dir) target_path = as_normalized_path(target_dir) - excluded = shutil.ignore_patterns("__pycache__", "RECORD") + excluded = shutil.ignore_patterns("__pycache__", "RECORD", "sitecustomize.py") # Avoid symlinks on Windows, as they need elevated privileges to create # Also avoid them if the target folder doesn't support symlink creation # (that way exports to FAT/FAT32/VFAT file systems should work, even if @@ -247,9 +199,13 @@ def create_archive( * injects a suitable `postinstall.py` script for the environment being archived * always creates zipfile archives on Windows and xztar archives elsewhere - * excludes __pycache__ folders and package metadata RECORD files + * excludes __pycache__ folders (to reduce archive size and improve reproducibility) + * excludes package metadata RECORD files (to improve reproducibility) + * excludes `sitecustomize.py` files (generated by the post-installation script) * replaces symlinks with copies on Windows and allows external symlinks elsewhere - * discards owner and group information for tar archives + * discards tar entry owner and group information + * clears tar entry high mode bits (setuid, setgid, sticky) + * clears tar entry group/other write mode bits * clamps mtime of archived files to the given clamp mtime at the latest * shows progress reporting by default (archiving built ML/AI libs is *slooooow*) diff --git a/src/venvstacks/stacks.py b/src/venvstacks/stacks.py index 5ba244b..8c02313 100755 --- a/src/venvstacks/stacks.py +++ b/src/venvstacks/stacks.py @@ -32,6 +32,7 @@ MutableMapping, NamedTuple, NewType, + NoReturn, NotRequired, Self, Sequence, @@ -40,12 +41,12 @@ ) from . import pack_venv +from ._injected import postinstall from ._util import ( as_normalized_path, + capture_python_output, default_tarfile_filter, - get_env_python, run_python_command, - run_python_command_unchecked, StrPath, WINDOWS_BUILD as _WINDOWS_BUILD, ) @@ -207,10 +208,14 @@ def get_deployed_name(self, env_name: EnvNameBuild) -> EnvNameDeploy: return EnvNameDeploy(f"{env_name}@{self.lock_version}") return EnvNameDeploy(env_name) - @staticmethod - def _raise_if_none(value: _T | None) -> _T: + def _fail_lock(self, message: str) -> NoReturn: + missing = self.requirements_path.name + msg = f"Environment has not been locked ({missing} not found)" + raise BuildEnvError(msg) + + def _raise_if_none(self, value: _T | None) -> _T: if value is None: - raise BuildEnvError("Environment has not been locked") + self._fail_lock("Environment has not been locked") return value @property @@ -303,9 +308,7 @@ def _purge_lock(self) -> bool: def _write_lock_metadata(self) -> None: requirements_hash = self._requirements_hash if requirements_hash is None: - raise BuildEnvError( - "Environment must be locked before writing lock metadata" - ) + self._fail_lock("Environment must be locked before writing lock metadata") last_version = self._lock_version if last_version is None: lock_version = 1 @@ -324,9 +327,7 @@ def update_lock_metadata(self) -> bool: # Calculate current requirements hash requirements_hash = self._hash_requirements() if requirements_hash is None: - raise BuildEnvError( - "Environment must be locked before updating lock metadata" - ) + self._fail_lock("Environment must be locked before updating lock metadata") self._requirements_hash = requirements_hash # Only update and save the last locked time if # the lockfile contents have changed or if @@ -774,25 +775,11 @@ def define_export( return cls(env_name, env_lock, export_path, export_metadata, needs_export) @staticmethod - def _run_postinstall( - src_path: Path, export_path: Path, postinstall_path: Path - ) -> None: - exported_env_python_path = get_env_python(export_path) - command = [str(exported_env_python_path), "-I", str(postinstall_path)] - result = run_python_command_unchecked(command) - if result.returncode == 0: - # All good, nothing else to check - return - # Running with the Python inside the exported environment didn't work - # This can happen on Windows when "pyvenv.cfg" doesn't exist yet - # If that is what has happened, the reported return code will be 106 - if result.returncode != 106: - result.check_returncode() - # Self-generating the venv config failed, retry with the build venv - # rather than finding and using the exported base runtime environment - src_env_python_path = get_env_python(src_path) - command[0] = str(src_env_python_path) - run_python_command(command) + def _run_postinstall(postinstall_path: Path) -> None: + # Post-installation scripts are required to work even when they're + # executed with an entirely unrelated Python installation + command = [sys.executable, "-X", "utf8", "-I", str(postinstall_path)] + capture_python_output(command) def export_environment( self, @@ -816,8 +803,8 @@ def export_environment( export_path.unlink() print(f"Exporting {str(env_path)!r} to {str(export_path)!r}") - def _run_postinstall(export_path: Path, postinstall_path: Path) -> None: - self._run_postinstall(env_path, export_path, postinstall_path) + def _run_postinstall(_export_path: Path, postinstall_path: Path) -> None: + self._run_postinstall(postinstall_path) exported_path = pack_venv.export_venv( env_path, @@ -1047,6 +1034,80 @@ def env_spec(self) -> _PythonEnvironmentSpec: # Define property to allow covariance of the declared type of `env_spec` return self._env_spec + @abstractmethod + def get_deployed_config(self) -> postinstall.LayerConfig: + """Layer config to be published in `venvstacks_layer.json`""" + raise NotImplementedError + + def _get_deployed_config( + self, + pylib_paths: Iterable[Path], + dynlib_paths: Iterable[Path], + link_external_base: bool = True, + ) -> postinstall.LayerConfig: + # Helper for subclass get_deployed_config implementations + base_python_path = self.base_python_path + if base_python_path is None: + self._fail_build("Cannot get deployment config for unlinked layer") + build_env_path = self.env_path + build_env_name = build_env_path.name + build_path = build_env_path.parent + + def from_internal_path(target_build_path: Path) -> str: + # Input path is an absolute path inside the environment + # Output path is relative to the base of the environment + return str(target_build_path.relative_to(build_env_path)) + + def from_relative_path(relative_build_path: Path) -> str: + # Input path is relative to the base of the build directory + # Output path is relative to the base of the environment + # Note: we avoid `walk_up=True` here, firstly to maintain + # Python 3.11 compatibility, but also to limit the + # the relative paths to *peer* environments, rather + # than all potentially value relative path calculations + if relative_build_path.is_absolute(): + self._fail_build(f"{relative_build_path} is not a relative path") + if relative_build_path.parts[0] == build_env_name: + # Emit internally relative path + return str(Path(*relative_build_path.parts[1:])) + # Emit relative reference to peer environment + return str(Path("..", *relative_build_path.parts)) + + def from_external_path(target_build_path: Path) -> str: + # Input path is an absolute path, potentially from a peer environment + # Output path is relative to the base of the environment + relative_build_path = target_build_path.relative_to(build_path) + return from_relative_path(relative_build_path) + + layer_python = from_internal_path(self.python_path) + if link_external_base: + base_python = from_external_path(base_python_path) + else: + # "base_python" in the runtime layer refers solely to + # the external environment used to set up the base + # runtime layer, rather than being a linked environment + base_python = layer_python + + return postinstall.LayerConfig( + python=layer_python, + py_version=self.py_version, + base_python=base_python, + site_dir=from_internal_path(self.pylib_path), + pylib_dirs=[from_relative_path(p) for p in pylib_paths], + dynlib_dirs=[from_relative_path(p) for p in dynlib_paths], + ) + + def _write_deployed_config(self) -> None: + # This is written as part of creating/updating the build environments + config_path = self.env_path / postinstall.DEPLOYED_LAYER_CONFIG + print(f"Generating {str(config_path)!r}...") + config_path.parent.mkdir(parents=True, exist_ok=True) + _write_json(config_path, self.get_deployed_config()) + + def _fail_build(self, message: str) -> NoReturn: + attributed_message = f"Layer {self.env_name}: {message}" + raise BuildEnvError(attributed_message) + def select_operations( self, lock: bool | None = False, @@ -1083,6 +1144,7 @@ def _create_environment( create_env = False if create_env: self._create_new_environment(lock_only=lock_only) + self._write_deployed_config() self.was_created = create_env self.was_built = create_env or env_updated @@ -1222,7 +1284,7 @@ def lock_requirements(self) -> EnvironmentLock: requirements_path, requirements_input_path, constraints ) if not requirements_path.exists(): - raise BuildEnvError(f"Failed to generate {str(requirements_path)!r}") + self._fail_build(f"Failed to generate {str(requirements_path)!r}") if self.env_lock.update_lock_metadata(): print(f" Environment lock time set: {self.env_lock.locked_at!r}") return self.env_lock @@ -1230,7 +1292,7 @@ def lock_requirements(self) -> EnvironmentLock: def install_requirements(self) -> subprocess.CompletedProcess[str]: # Run a pip dependency upgrade inside the target environment if not self.env_lock.is_locked: - raise BuildEnvError( + self._fail_build( "Environment must be locked before installing dependencies" ) return self._run_pip_install( @@ -1376,6 +1438,10 @@ def env_spec(self) -> RuntimeSpec: assert isinstance(self._env_spec, RuntimeSpec) return self._env_spec + def get_deployed_config(self) -> postinstall.LayerConfig: + """Layer config to be published in `venvstacks_layer.json`""" + return self._get_deployed_config([], [], link_external_base=False) + def _remove_pip(self) -> subprocess.CompletedProcess[str] | None: to_be_checked = ["pip", "wheel", "setuptools"] to_be_removed = [] @@ -1391,7 +1457,7 @@ def _create_new_environment(self, *, lock_only: bool = False) -> None: python_runtime = self.env_spec.fully_versioned_name install_path = _pdm_python_install(self.build_path, python_runtime) if install_path is None: - raise BuildEnvError(f"Failed to install {python_runtime}") + self._fail_build(f"Failed to install {python_runtime}") shutil.move(install_path, self.env_path) # No build step needs `pip` to be installed in the target environment, # and we don't want to ship it unless explicitly requested to do so @@ -1415,14 +1481,16 @@ def create_build_environment(self, *, clean: bool = False) -> None: class _VirtualEnvironment(_PythonEnvironment): - _include_system_site_packages = False - linked_constraints_paths: list[Path] = field(init=False, repr=False) + linked_pylib_paths: list[Path] = field(init=False, repr=False) + linked_dynlib_paths: list[Path] = field(init=False, repr=False) def __post_init__(self) -> None: self.py_version = self.env_spec.runtime.py_version super().__post_init__() self.linked_constraints_paths = [] + self.linked_pylib_paths = [] + self.linked_dynlib_paths = [] @property def env_spec(self) -> _VirtualEnvironmentSpec: @@ -1435,19 +1503,23 @@ def link_base_runtime_paths(self, runtime: RuntimeEnv) -> None: self.base_python_path = runtime.python_path self.tools_python_path = runtime.tools_python_path if self.linked_constraints_paths: - raise BuildEnvError("Layered environment base runtime already linked") + self._fail_build("Layered environment base runtime already linked") self.linked_constraints_paths[:] = [runtime.requirements_path] + def get_deployed_config(self) -> postinstall.LayerConfig: + """Layer config to be published in `venvstacks_layer.json`""" + return self._get_deployed_config( + self.linked_pylib_paths, self.linked_dynlib_paths + ) + def get_constraint_paths(self) -> list[Path]: return self.linked_constraints_paths def _ensure_virtual_environment(self) -> subprocess.CompletedProcess[str]: # Use the base Python installation to create a new virtual environment if self.base_python_path is None: - raise RuntimeError("Base Python path not set in {self!r}") + self._fail_build("Base Python path not set") options = ["--without-pip"] - if self._include_system_site_packages: - options.append("--system-site-packages") if self.env_path.exists(): options.append("--upgrade") if _WINDOWS_BUILD: @@ -1464,13 +1536,26 @@ def _ensure_virtual_environment(self) -> subprocess.CompletedProcess[str]: str(self.env_path), ] result = run_python_command(command) - self._link_layered_environment() + self._link_build_environment() fs_sync() print(f"Virtual environment configured in {str(self.env_path)!r}") return result - def _link_layered_environment(self) -> None: - pass # Nothing to do by default, subclasses override if necessary + def _link_build_environment(self) -> None: + # Create sitecustomize file for the build environment + build_path = self.build_path + build_pylib_paths = [build_path / p for p in self.linked_pylib_paths] + build_dynlib_paths = [build_path / p for p in self.linked_dynlib_paths] + sc_contents = postinstall.generate_sitecustomize( + build_pylib_paths, build_dynlib_paths + ) + if sc_contents is None: + self._fail_build( + "Layered environments must at least link a base runtime environment" + ) + sc_path = self.pylib_path / "sitecustomize.py" + print(f"Generating {str(sc_path)!r}...") + sc_path.write_text(sc_contents, encoding="utf-8") def _update_existing_environment(self, *, lock_only: bool = False) -> None: if lock_only: @@ -1499,7 +1584,6 @@ class FrameworkEnv(_VirtualEnvironment): kind = LayerVariants.FRAMEWORK category = LayerCategories.FRAMEWORKS - _include_system_site_packages = True @property def env_spec(self) -> FrameworkSpec: @@ -1507,6 +1591,21 @@ def env_spec(self) -> FrameworkSpec: assert isinstance(self._env_spec, FrameworkSpec) return self._env_spec + def link_base_runtime_paths(self, runtime: RuntimeEnv) -> None: + super().link_base_runtime_paths(runtime) + # TODO: Reduce code duplication with ApplicationEnv + runtime_target_path = Path(runtime.install_target) + + def _runtime_path(build_path: Path) -> Path: + relative_path = build_path.relative_to(runtime.env_path) + return runtime_target_path / relative_path + + pylib_paths = self.linked_pylib_paths + dynlib_paths = self.linked_dynlib_paths + pylib_paths.append(_runtime_path(runtime.pylib_path)) + if runtime.dynlib_path is not None: + dynlib_paths.append(_runtime_path(runtime.dynlib_path)) + class ApplicationEnv(_VirtualEnvironment): """Application layer build environment""" @@ -1515,8 +1614,6 @@ class ApplicationEnv(_VirtualEnvironment): category = LayerCategories.APPLICATIONS launch_module_name: str = field(init=False, repr=False) - linked_pylib_paths: list[Path] = field(init=False, repr=False) - linked_dynlib_paths: list[Path] = field(init=False, repr=False) linked_frameworks: list[FrameworkEnv] = field(init=False, repr=False) @property @@ -1528,8 +1625,6 @@ def env_spec(self) -> ApplicationSpec: def __post_init__(self) -> None: super().__post_init__() self.launch_module_name = self.env_spec.launch_module_path.stem - self.linked_pylib_paths = [] - self.linked_dynlib_paths = [] self.linked_frameworks = [] def link_layered_environments( @@ -1538,14 +1633,14 @@ def link_layered_environments( self.link_base_runtime_paths(runtime) constraints_paths = self.linked_constraints_paths if not constraints_paths: - raise BuildEnvError("Failed to add base environment constraints path") + self._fail_build("Failed to add base environment constraints path") # The runtime site-packages folder is added here rather than via pyvenv.cfg # to ensure it appears in sys.path after the framework site-packages folders pylib_paths = self.linked_pylib_paths dynlib_paths = self.linked_dynlib_paths fw_envs = self.linked_frameworks if pylib_paths or dynlib_paths or fw_envs: - raise BuildEnvError("Layered application environment already linked") + self._fail_build("Layered application environment already linked") for env_spec in self.env_spec.frameworks: env = frameworks[env_spec.name] fw_envs.append(env) @@ -1569,56 +1664,6 @@ def _runtime_path(build_path: Path) -> Path: if runtime.dynlib_path is not None: dynlib_paths.append(_runtime_path(runtime.dynlib_path)) - def _link_layered_environment(self) -> None: - # Create sitecustomize file - sc_dir_path = self.pylib_path - sc_contents = [ - "# Automatically generated by venvstacks", - "import site", - "import os", - "from os.path import abspath, dirname, join as joinpath", - "# Allow loading modules and packages from framework environments", - "this_dir = dirname(abspath(__file__))", - ] - # Add framework and runtime folders to sys.path - parent_path = self.env_path.parent - relative_prefix = Path( - os.path.relpath(str(parent_path), start=str(sc_dir_path)) - ) - for pylib_path in self.linked_pylib_paths: - relative_path = relative_prefix / pylib_path - sc_contents.extend( - [ - f"path_entry = abspath(joinpath(this_dir, {str(relative_path)!r}))", - "site.addsitedir(path_entry)", - ] - ) - # Add DLL search folders if needed - dynlib_paths = self.linked_dynlib_paths - if _WINDOWS_BUILD and dynlib_paths: - sc_contents.extend( - [ - "", - "# Allow loading misplaced DLLs on Windows", - ] - ) - for dynlib_path in dynlib_paths: - if not dynlib_path.exists(): - # Nothing added DLLs to this folder at build time, so skip it - continue - relative_path = relative_prefix / dynlib_path - sc_contents.extend( - [ - f"dll_dir = abspath(joinpath(this_dir, {str(relative_path)!r}))", - "os.add_dll_directory(dll_dir)", - ] - ) - sc_contents.append("") - sc_path = self.pylib_path / "sitecustomize.py" - print(f"Generating {sc_path!r}...") - with open(sc_path, "w", encoding="utf-8") as f: - f.write("\n".join(sc_contents)) - def _update_existing_environment(self, *, lock_only: bool = False) -> None: super()._update_existing_environment(lock_only=lock_only) # Also publish the specified launch module as an importable top level module diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json index 6636940..8a70e77 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "84c38c5717c3ff4e5fc9a13b22045b8c7fa2a96682648e419e22983e9023f554" + "sha256": "bb594ea66705fe5a122e930e2c48f1aaf6f720d568e37e0b590db95fb7261dea" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 1504, + "archive_size": 3004, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json index 36b7bde..64b962f 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "4c30fb1472a1905d0369b700bd922f317f8ab4875e7b02a9c997dedf5cb0b175" + "sha256": "8d69cfe0f408ba396dde664cd58c18b4541e6937f470e6e4f06a03d6ec069e46" }, "archive_name": "app-scipy-import.tar.xz", - "archive_size": 1412, + "archive_size": 2912, "install_target": "app-scipy-import", "layer_name": "app-scipy-import", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-sklearn-import.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-sklearn-import.json index 76a2b85..cdff627 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-sklearn-import.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-sklearn-import.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256:f66c01bbcca47cd31d79d2fb5377de0de18631ffc3c904629d46f6cad2918694", "archive_build": 1, "archive_hashes": { - "sha256": "ba0b38bb3c8539b9882bdfd752f1407961f26fe8a2c1af3d1bed62d83478b8e7" + "sha256": "1ff13c6ae6146bf0e028b56d68f3a6ea63f0c33b1c1cf891db5580112e0f09ef" }, "archive_name": "app-sklearn-import.tar.xz", - "archive_size": 1420, + "archive_size": 2916, "install_target": "app-sklearn-import", "layer_name": "app-sklearn-import", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.11.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.11.json index 82dc71f..d254417 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.11.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.11.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "552dbdf8a29c65ae31bc4786a6e754b146ed8965e5cf4d360f29b41d2f880dbb" + "sha256": "0628d08555e421d3c3b4ae7fe141cd368f6cbe9aac761c2aeb0c0cf956ab1583" }, "archive_name": "cpython@3.11.tar.xz", - "archive_size": 29721908, + "archive_size": 29723440, "install_target": "cpython@3.11", "layer_name": "cpython@3.11", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.12.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.12.json index b9844be..bd36aaa 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.12.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.12.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "d41dc43405cd5c1e96ce81396075f25bd1aec731827de6c1d5ead264d45f02b0" + "sha256": "681c4d6f77de9745af858422ce080e483e6d9dee81114f4758cf8c07f0ca9efb" }, "archive_name": "cpython@3.12.tar.xz", - "archive_size": 42726560, + "archive_size": 42728260, "install_target": "cpython@3.12", "layer_name": "cpython@3.12", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-http-client.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-http-client.json index 7bd5a37..571db7e 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-http-client.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-http-client.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "a68b0063f9149b0475faacf7bc6987db30e6ad1171438d6eac6a8d1c22cd8db4" + "sha256": "71870e0a056dff917dd4d97ffb61d184a6cd5a2f1cf98e1c305c12debaf57026" }, "archive_name": "framework-http-client.tar.xz", - "archive_size": 362568, + "archive_size": 364020, "install_target": "framework-http-client", "layer_name": "framework-http-client", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json index b22ca25..ae73caf 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "bd8bf5b409d03d78e878f3a5d7e350daea0e94cc983a6146997a1bd0d1834bf5" + "sha256": "e8ef68225cfa6a016539a4db3c5dc1b984f53de0df51181eefeb0e9449771ea9" }, "archive_name": "framework-scipy.tar.xz", - "archive_size": 23956040, + "archive_size": 23957800, "install_target": "framework-scipy", "layer_name": "framework-scipy", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-sklearn.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-sklearn.json index 0ae0993..7372d23 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-sklearn.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-sklearn.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "0faa4c3e0709d4ebb19e8f9a81c36d0d422f6ef77d31af7496738f9bb937da4b" + "sha256": "8087f86b0196ae91b13d78f738bb3d0266ee76e2cfcfb3f20b76dd7f103bdace" }, "archive_name": "framework-sklearn.tar.xz", - "archive_size": 30370864, + "archive_size": 30372332, "install_target": "framework-sklearn", "layer_name": "framework-sklearn", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json b/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json index f1bd931..c9818e3 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json @@ -6,10 +6,10 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "4c30fb1472a1905d0369b700bd922f317f8ab4875e7b02a9c997dedf5cb0b175" + "sha256": "8d69cfe0f408ba396dde664cd58c18b4541e6937f470e6e4f06a03d6ec069e46" }, "archive_name": "app-scipy-import.tar.xz", - "archive_size": 1412, + "archive_size": 2912, "install_target": "app-scipy-import", "layer_name": "app-scipy-import", "lock_version": 1, @@ -26,10 +26,10 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "84c38c5717c3ff4e5fc9a13b22045b8c7fa2a96682648e419e22983e9023f554" + "sha256": "bb594ea66705fe5a122e930e2c48f1aaf6f720d568e37e0b590db95fb7261dea" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 1504, + "archive_size": 3004, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, @@ -47,10 +47,10 @@ "app_launch_module_hash": "sha256:f66c01bbcca47cd31d79d2fb5377de0de18631ffc3c904629d46f6cad2918694", "archive_build": 1, "archive_hashes": { - "sha256": "ba0b38bb3c8539b9882bdfd752f1407961f26fe8a2c1af3d1bed62d83478b8e7" + "sha256": "1ff13c6ae6146bf0e028b56d68f3a6ea63f0c33b1c1cf891db5580112e0f09ef" }, "archive_name": "app-sklearn-import.tar.xz", - "archive_size": 1420, + "archive_size": 2916, "install_target": "app-sklearn-import", "layer_name": "app-sklearn-import", "lock_version": 1, @@ -67,10 +67,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "bd8bf5b409d03d78e878f3a5d7e350daea0e94cc983a6146997a1bd0d1834bf5" + "sha256": "e8ef68225cfa6a016539a4db3c5dc1b984f53de0df51181eefeb0e9449771ea9" }, "archive_name": "framework-scipy.tar.xz", - "archive_size": 23956040, + "archive_size": 23957800, "install_target": "framework-scipy", "layer_name": "framework-scipy", "lock_version": 1, @@ -82,10 +82,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "0faa4c3e0709d4ebb19e8f9a81c36d0d422f6ef77d31af7496738f9bb937da4b" + "sha256": "8087f86b0196ae91b13d78f738bb3d0266ee76e2cfcfb3f20b76dd7f103bdace" }, "archive_name": "framework-sklearn.tar.xz", - "archive_size": 30370864, + "archive_size": 30372332, "install_target": "framework-sklearn", "layer_name": "framework-sklearn", "lock_version": 1, @@ -97,10 +97,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "a68b0063f9149b0475faacf7bc6987db30e6ad1171438d6eac6a8d1c22cd8db4" + "sha256": "71870e0a056dff917dd4d97ffb61d184a6cd5a2f1cf98e1c305c12debaf57026" }, "archive_name": "framework-http-client.tar.xz", - "archive_size": 362568, + "archive_size": 364020, "install_target": "framework-http-client", "layer_name": "framework-http-client", "lock_version": 1, @@ -114,10 +114,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "552dbdf8a29c65ae31bc4786a6e754b146ed8965e5cf4d360f29b41d2f880dbb" + "sha256": "0628d08555e421d3c3b4ae7fe141cd368f6cbe9aac761c2aeb0c0cf956ab1583" }, "archive_name": "cpython@3.11.tar.xz", - "archive_size": 29721908, + "archive_size": 29723440, "install_target": "cpython@3.11", "layer_name": "cpython@3.11", "lock_version": 1, @@ -129,10 +129,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "d41dc43405cd5c1e96ce81396075f25bd1aec731827de6c1d5ead264d45f02b0" + "sha256": "681c4d6f77de9745af858422ce080e483e6d9dee81114f4758cf8c07f0ca9efb" }, "archive_name": "cpython@3.12.tar.xz", - "archive_size": 42726560, + "archive_size": 42728260, "install_target": "cpython@3.12", "layer_name": "cpython@3.12", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json index a9a8af9..2282818 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "2cdf88d7a5ed2dca88d7a59e2b5d9744503cf824218371948000f1b08486df4e" + "sha256": "bf065fe724e53e03886117ff0a7d61e29910da9c46b4a14799581cfc1238c77f" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 1484, + "archive_size": 2980, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json index 2cbf816..40a1486 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "5034f1fa0a0731af9e88699519a508e72662b003c30e14814ce72e00ef8a6223" + "sha256": "3ab7e8458a233e96790809ad0437209bcbf1823d7883220a44554b1c3fd51afa" }, "archive_name": "app-scipy-import.tar.xz", - "archive_size": 1392, + "archive_size": 2896, "install_target": "app-scipy-import", "layer_name": "app-scipy-import", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.11.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.11.json index 8019d4e..38a2217 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.11.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.11.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "671fbe66582efd0a4cfc0c837b303ef1e267b8929ff0b87aec648471f2c6392c" + "sha256": "b23a9bda7297579207664e52f657ffb59a378f47cf2ae7c336f1ad3a56549ad1" }, "archive_name": "cpython@3.11.tar.xz", - "archive_size": 14965684, + "archive_size": 14967236, "install_target": "cpython@3.11", "layer_name": "cpython@3.11", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.12.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.12.json index c0a4652..12e0aa4 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.12.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.12.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "f3e6749b5c07afde8468ca03457e1ea4bdda2adab8e510f1d353f21ee0f71981" + "sha256": "2c598215a16b22911bc8ae79fcb79a611d56d9f02076fa9c716f4217e29cbe48" }, "archive_name": "cpython@3.12.tar.xz", - "archive_size": 13599424, + "archive_size": 13600984, "install_target": "cpython@3.12", "layer_name": "cpython@3.12", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-http-client.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-http-client.json index f6aa78d..41828bf 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-http-client.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-http-client.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "ccaa6c54492390af869045894977bf7398b2922aad3622b6a6801ea2e02b1d23" + "sha256": "37f1d87e0e1a06f0ff86b288565cc1fc4d8d6b33f53e229a3f87d163b4b2d793" }, "archive_name": "framework-http-client.tar.xz", - "archive_size": 362476, + "archive_size": 363928, "install_target": "framework-http-client", "layer_name": "framework-http-client", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json index 331a486..996c5bd 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "45d6b5abeddedc12978b54d8ff4a29db76495bd5ab7b7a051ed45b8aa8fc76b9" + "sha256": "6872599275f0b5448926033968ba5f3467b07bf09ac19af8538975a39bf7b712" }, "archive_name": "framework-scipy.tar.xz", - "archive_size": 15077296, + "archive_size": 15078848, "install_target": "framework-scipy", "layer_name": "framework-scipy", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-sklearn.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-sklearn.json index 3acfd22..da1848a 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-sklearn.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-sklearn.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "d7ec74fe1f72988bba8b6342e561037a01eee26b07039439b1fe92ad0f3594a1" + "sha256": "a51ee3b3a25b0cb03044f2f9d23078f1c8982a1438a5510d655c628f71bd4f03" }, "archive_name": "framework-sklearn.tar.xz", - "archive_size": 20687556, + "archive_size": 20689024, "install_target": "framework-sklearn", "layer_name": "framework-sklearn", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json b/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json index 86778c7..1006d12 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json @@ -6,10 +6,10 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "5034f1fa0a0731af9e88699519a508e72662b003c30e14814ce72e00ef8a6223" + "sha256": "3ab7e8458a233e96790809ad0437209bcbf1823d7883220a44554b1c3fd51afa" }, "archive_name": "app-scipy-import.tar.xz", - "archive_size": 1392, + "archive_size": 2896, "install_target": "app-scipy-import", "layer_name": "app-scipy-import", "lock_version": 1, @@ -26,10 +26,10 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "2cdf88d7a5ed2dca88d7a59e2b5d9744503cf824218371948000f1b08486df4e" + "sha256": "bf065fe724e53e03886117ff0a7d61e29910da9c46b4a14799581cfc1238c77f" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 1484, + "archive_size": 2980, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, @@ -47,10 +47,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "45d6b5abeddedc12978b54d8ff4a29db76495bd5ab7b7a051ed45b8aa8fc76b9" + "sha256": "6872599275f0b5448926033968ba5f3467b07bf09ac19af8538975a39bf7b712" }, "archive_name": "framework-scipy.tar.xz", - "archive_size": 15077296, + "archive_size": 15078848, "install_target": "framework-scipy", "layer_name": "framework-scipy", "lock_version": 1, @@ -62,10 +62,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "d7ec74fe1f72988bba8b6342e561037a01eee26b07039439b1fe92ad0f3594a1" + "sha256": "a51ee3b3a25b0cb03044f2f9d23078f1c8982a1438a5510d655c628f71bd4f03" }, "archive_name": "framework-sklearn.tar.xz", - "archive_size": 20687556, + "archive_size": 20689024, "install_target": "framework-sklearn", "layer_name": "framework-sklearn", "lock_version": 1, @@ -77,10 +77,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "ccaa6c54492390af869045894977bf7398b2922aad3622b6a6801ea2e02b1d23" + "sha256": "37f1d87e0e1a06f0ff86b288565cc1fc4d8d6b33f53e229a3f87d163b4b2d793" }, "archive_name": "framework-http-client.tar.xz", - "archive_size": 362476, + "archive_size": 363928, "install_target": "framework-http-client", "layer_name": "framework-http-client", "lock_version": 1, @@ -94,10 +94,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "671fbe66582efd0a4cfc0c837b303ef1e267b8929ff0b87aec648471f2c6392c" + "sha256": "b23a9bda7297579207664e52f657ffb59a378f47cf2ae7c336f1ad3a56549ad1" }, "archive_name": "cpython@3.11.tar.xz", - "archive_size": 14965684, + "archive_size": 14967236, "install_target": "cpython@3.11", "layer_name": "cpython@3.11", "lock_version": 1, @@ -109,10 +109,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "f3e6749b5c07afde8468ca03457e1ea4bdda2adab8e510f1d353f21ee0f71981" + "sha256": "2c598215a16b22911bc8ae79fcb79a611d56d9f02076fa9c716f4217e29cbe48" }, "archive_name": "cpython@3.12.tar.xz", - "archive_size": 13599424, + "archive_size": 13600984, "install_target": "cpython@3.12", "layer_name": "cpython@3.12", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json index a32e876..0c350fb 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "69060c0e0a74290723c5e65a343e463112a249da960d8bf29997933cd1565787" + "sha256": "9f3de2bf483797a9a93629feca94756de57d82e62b157a4836b96fa14a289180" }, "archive_name": "app-scipy-client.zip", - "archive_size": 255147, + "archive_size": 257112, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json index be1e7d7..f679796 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json @@ -3,10 +3,10 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "1907700fc74c6a0bc62d851dce87bfed6a0ad52d0fec5863c9eadd1e26c029ff" + "sha256": "3f4e2a19a1611db1139f9e68a268a963dd30139690ee6d8325896265007bf823" }, "archive_name": "app-scipy-import.zip", - "archive_size": 254658, + "archive_size": 256621, "install_target": "app-scipy-import", "layer_name": "app-scipy-import", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.11.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.11.json index 5346397..97159aa 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.11.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.11.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "70a00f1e469d6c4ab82e0d74946ca6da0fb97ee480b294d93e6b217d8a9f3cd0" + "sha256": "66095910a186e59d023ca357ccd0fbe7c464722f191786f6d090af6769bb5dd9" }, "archive_name": "cpython@3.11.zip", - "archive_size": 46592259, + "archive_size": 46594787, "install_target": "cpython@3.11", "layer_name": "cpython@3.11", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.12.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.12.json index 508e5a9..fb3e2cd 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.12.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/cpython@3.12.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "ab8860e56b834ff4b9eacd11e8a0c705c83edac6b1627990f5317e1514314d76" + "sha256": "85017932199a87b59bf17432f21c9ddf08361663a1cdf363c5d014146c3c754c" }, "archive_name": "cpython@3.12.zip", - "archive_size": 45864491, + "archive_size": 45867018, "install_target": "cpython@3.12", "layer_name": "cpython@3.12", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-http-client.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-http-client.json index 6af3112..11b6f70 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-http-client.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-http-client.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "b0193afc9d1f4a51b9fcb60903d335696804d2b35115986c68e4ebf5aaf1b621" + "sha256": "8b5f396783d7c583cb2202a7bf791b46257b59473be4f12b1583907f4bf931b2" }, "archive_name": "framework-http-client.zip", - "archive_size": 817522, + "archive_size": 819935, "install_target": "framework-http-client", "layer_name": "framework-http-client", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json index c86bc68..34fa1a6 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "132cd5e29dae8d082c453b276e5e207e5f82f29458db62d6c7df2a01b8225f92" + "sha256": "71146182f2bcf36ed9af674ac146ee32228e8d0a2890fb9c51ebd87e7fe58b45" }, "archive_name": "framework-scipy.zip", - "archive_size": 45078361, + "archive_size": 45080726, "install_target": "framework-scipy", "layer_name": "framework-scipy", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-sklearn.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-sklearn.json index 6bb7c4f..e6dacc5 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-sklearn.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-sklearn.json @@ -1,10 +1,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "eaaa57b8f636eb0258bf4b4ac11a7a76a4691b1659d9a14bd8b908129f75e936" + "sha256": "74549eb9c5252b7f91cf573811b71d8d703b6cd840932f7dbdddb9e3d03ccaee" }, "archive_name": "framework-sklearn.zip", - "archive_size": 56185753, + "archive_size": 56188134, "install_target": "framework-sklearn", "layer_name": "framework-sklearn", "lock_version": 1, diff --git a/tests/sample_project/expected_manifests/win_amd64/venvstacks.json b/tests/sample_project/expected_manifests/win_amd64/venvstacks.json index d06983c..b21d903 100644 --- a/tests/sample_project/expected_manifests/win_amd64/venvstacks.json +++ b/tests/sample_project/expected_manifests/win_amd64/venvstacks.json @@ -6,10 +6,10 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "1907700fc74c6a0bc62d851dce87bfed6a0ad52d0fec5863c9eadd1e26c029ff" + "sha256": "3f4e2a19a1611db1139f9e68a268a963dd30139690ee6d8325896265007bf823" }, "archive_name": "app-scipy-import.zip", - "archive_size": 254658, + "archive_size": 256621, "install_target": "app-scipy-import", "layer_name": "app-scipy-import", "lock_version": 1, @@ -26,10 +26,10 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "69060c0e0a74290723c5e65a343e463112a249da960d8bf29997933cd1565787" + "sha256": "9f3de2bf483797a9a93629feca94756de57d82e62b157a4836b96fa14a289180" }, "archive_name": "app-scipy-client.zip", - "archive_size": 255147, + "archive_size": 257112, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, @@ -47,10 +47,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "132cd5e29dae8d082c453b276e5e207e5f82f29458db62d6c7df2a01b8225f92" + "sha256": "71146182f2bcf36ed9af674ac146ee32228e8d0a2890fb9c51ebd87e7fe58b45" }, "archive_name": "framework-scipy.zip", - "archive_size": 45078361, + "archive_size": 45080726, "install_target": "framework-scipy", "layer_name": "framework-scipy", "lock_version": 1, @@ -62,10 +62,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "eaaa57b8f636eb0258bf4b4ac11a7a76a4691b1659d9a14bd8b908129f75e936" + "sha256": "74549eb9c5252b7f91cf573811b71d8d703b6cd840932f7dbdddb9e3d03ccaee" }, "archive_name": "framework-sklearn.zip", - "archive_size": 56185753, + "archive_size": 56188134, "install_target": "framework-sklearn", "layer_name": "framework-sklearn", "lock_version": 1, @@ -77,10 +77,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "b0193afc9d1f4a51b9fcb60903d335696804d2b35115986c68e4ebf5aaf1b621" + "sha256": "8b5f396783d7c583cb2202a7bf791b46257b59473be4f12b1583907f4bf931b2" }, "archive_name": "framework-http-client.zip", - "archive_size": 817522, + "archive_size": 819935, "install_target": "framework-http-client", "layer_name": "framework-http-client", "lock_version": 1, @@ -94,10 +94,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "70a00f1e469d6c4ab82e0d74946ca6da0fb97ee480b294d93e6b217d8a9f3cd0" + "sha256": "66095910a186e59d023ca357ccd0fbe7c464722f191786f6d090af6769bb5dd9" }, "archive_name": "cpython@3.11.zip", - "archive_size": 46592259, + "archive_size": 46594787, "install_target": "cpython@3.11", "layer_name": "cpython@3.11", "lock_version": 1, @@ -109,10 +109,10 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "ab8860e56b834ff4b9eacd11e8a0c705c83edac6b1627990f5317e1514314d76" + "sha256": "85017932199a87b59bf17432f21c9ddf08361663a1cdf363c5d014146c3c754c" }, "archive_name": "cpython@3.12.zip", - "archive_size": 45864491, + "archive_size": 45867018, "install_target": "cpython@3.12", "layer_name": "cpython@3.12", "lock_version": 1, diff --git a/tests/support.py b/tests/support.py index 5686a0a..8b790f6 100644 --- a/tests/support.py +++ b/tests/support.py @@ -9,12 +9,13 @@ from dataclasses import dataclass, fields from pathlib import Path -from typing import Any, Callable, cast, Mapping, Sequence, TypeVar +from typing import Any, Callable, cast, Iterable, Mapping, Sequence, TypeVar from unittest.mock import create_autospec import pytest -from venvstacks._util import get_env_python, run_python_command +from venvstacks._util import get_env_python, capture_python_output +from venvstacks._injected.postinstall import DEPLOYED_LAYER_CONFIG from venvstacks.stacks import ( BuildEnvironment, @@ -22,7 +23,9 @@ ExportedEnvironmentPaths, ExportMetadata, LayerBaseName, + LayerVariants, PackageIndexConfig, + _PythonEnvironment, ) _THIS_DIR = Path(__file__).parent @@ -207,62 +210,138 @@ def make_mock_index_config(reference_config: PackageIndexConfig | None = None) - ############################################## -def capture_python_output(command: list[str]) -> subprocess.CompletedProcess[str]: - return run_python_command(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - def get_sys_path(env_python: Path) -> list[str]: - command = [str(env_python), "-Ic", "import json, sys; print(json.dumps(sys.path))"] + command = [ + str(env_python), + "-X", + "utf8", + "-Ic", + "import json, sys; print(json.dumps(sys.path))", + ] result = capture_python_output(command) return cast(list[str], json.loads(result.stdout)) def run_module(env_python: Path, module_name: str) -> subprocess.CompletedProcess[str]: - command = [str(env_python), "-Im", module_name] + command = [str(env_python), "-X", "utf8", "-Im", module_name] return capture_python_output(command) -########################################################### -# Checking deployed environments for the expected details -########################################################### +####################################################### +# Checking deployed environments for expected details +####################################################### + + +_T = TypeVar("_T", bound=Mapping[str, Any]) class DeploymentTestCase(unittest.TestCase): """Native unittest test case with additional deployment validation checks""" + EXPECTED_APP_OUTPUT = "" + def assertPathExists(self, expected_path: Path) -> None: + self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") + def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None: self.assertTrue( any(expected in path_entry for path_entry in env_sys_path), f"No entry containing {expected!r} found in {env_sys_path}", ) - T = TypeVar("T", bound=Mapping[str, Any]) + def check_env_sys_path( + self, + env_path: Path, + env_sys_path: Sequence[str], + *, + self_contained: bool = False, + ) -> None: + sys_path_entries = [Path(path_entry) for path_entry in env_sys_path] + # Regardless of env type, sys.path entries must be absolute + self.assertTrue( + all(p.is_absolute() for p in sys_path_entries), + f"Relative path entry found in {env_sys_path}", + ) + # Regardless of env type, sys.path entries must exist + # (except the stdlib's optional zip archive entry) + for path_entry in sys_path_entries: + if path_entry.suffix: + continue + self.assertPathExists(path_entry) + # Check for sys.path references outside this environment + if self_contained: + # All sys.path entries should be inside the environment + self.assertTrue( + all(p.is_relative_to(env_path) for p in sys_path_entries), + f"Path outside deployed {env_path} in {env_sys_path}", + ) + else: + # All sys.path entries should be inside the environment's parent, + # but at least one sys.path entry should refer to a peer environment + peer_env_path = env_path.parent + self.assertTrue( + all(p.is_relative_to(peer_env_path) for p in sys_path_entries), + f"Path outside deployed {peer_env_path} in {env_sys_path}", + ) + self.assertFalse( + all(p.is_relative_to(env_path) for p in sys_path_entries), + f"No path outside deployed {env_path} in {env_sys_path}", + ) + + def check_build_environments( + self, build_envs: Iterable[_PythonEnvironment] + ) -> None: + for env in build_envs: + env_path = env.env_path + config_path = env_path / DEPLOYED_LAYER_CONFIG + self.assertPathExists(config_path) + layer_config = json.loads(config_path.read_text(encoding="utf-8")) + env_python = env_path / layer_config["python"] + expected_python_path = env.python_path + self.assertEqual(str(env_python), str(expected_python_path)) + base_python_path = env_path / layer_config["base_python"] + is_runtime_env = env.kind == LayerVariants.RUNTIME + if is_runtime_env: + # base_python should refer to the runtime layer itself + expected_base_python_path = expected_python_path + else: + # base_python should refer to the venv's base Python runtime + self.assertIsNotNone(env.base_python_path) + assert env.base_python_path is not None + base_python_path = Path(os.path.normpath(base_python_path)) + expected_base_python_path = env.base_python_path + self.assertEqual(str(base_python_path), str(expected_base_python_path)) + env_sys_path = get_sys_path(env_python) + # Base runtime environments are expected to be self-contained + self.check_env_sys_path( + env_path, env_sys_path, self_contained=is_runtime_env + ) def check_deployed_environments( self, - layered_metadata: dict[str, Sequence[T]], - get_exported_python: Callable[[T], tuple[str, Path, list[str]]], + layered_metadata: dict[str, Sequence[_T]], + get_env_details: Callable[[_T], tuple[str, Path, list[str]]], ) -> None: for rt_env in layered_metadata["runtimes"]: - env_name, _, env_sys_path = get_exported_python(rt_env) + env_name, env_path, env_sys_path = get_env_details(rt_env) self.assertTrue(env_sys_path) # Environment should have sys.path entries # Runtime environment layer should be completely self-contained - self.assertTrue( - all(env_name in path_entry for path_entry in env_sys_path), - f"Path outside {env_name} in {env_sys_path}", - ) + self.check_env_sys_path(env_path, env_sys_path, self_contained=True) for fw_env in layered_metadata["frameworks"]: - env_name, _, env_sys_path = get_exported_python(fw_env) + env_name, env_path, env_sys_path = get_env_details(fw_env) self.assertTrue(env_sys_path) # Environment should have sys.path entries + # Frameworks are expected to reference *at least* their base runtime environment + self.check_env_sys_path(env_path, env_sys_path) # Framework and runtime should both appear in sys.path runtime_name = fw_env["runtime_name"] short_runtime_name = ".".join(runtime_name.split(".")[:2]) self.assertSysPathEntry(env_name, env_sys_path) self.assertSysPathEntry(short_runtime_name, env_sys_path) for app_env in layered_metadata["applications"]: - env_name, env_python, env_sys_path = get_exported_python(app_env) + env_name, env_path, env_sys_path = get_env_details(app_env) self.assertTrue(env_sys_path) # Environment should have sys.path entries + # Applications are expected to reference *at least* their base runtime environment + self.check_env_sys_path(env_path, env_sys_path) # Application, frameworks and runtime should all appear in sys.path runtime_name = app_env["runtime_name"] short_runtime_name = ".".join(runtime_name.split(".")[:2]) @@ -275,6 +354,9 @@ def check_deployed_environments( self.assertSysPathEntry(fw_env_name, env_sys_path) self.assertSysPathEntry(short_runtime_name, env_sys_path) # Launch module should be executable + env_config_path = env_path / DEPLOYED_LAYER_CONFIG + env_config = json.loads(env_config_path.read_text(encoding="utf-8")) + env_python = env_path / env_config["python"] launch_module = app_env["app_launch_module"] launch_result = run_module(env_python, launch_module) # Tolerate extra trailing whitespace on stdout @@ -294,13 +376,13 @@ def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> N env_name_to_path[env_name] = env_path layered_metadata = exported_manifests.combined_data["layers"] - def get_exported_python( + def get_exported_env_details( env: ExportMetadata, ) -> tuple[EnvNameDeploy, Path, list[str]]: env_name = env["install_target"] env_path = env_name_to_path[env_name] env_python = get_env_python(env_path) env_sys_path = get_sys_path(env_python) - return env_name, env_python, env_sys_path + return env_name, env_path, env_sys_path - self.check_deployed_environments(layered_metadata, get_exported_python) + self.check_deployed_environments(layered_metadata, get_exported_env_details) diff --git a/tests/test_minimal_project.py b/tests/test_minimal_project.py index d145d8d..98f1f70 100644 --- a/tests/test_minimal_project.py +++ b/tests/test_minimal_project.py @@ -2,6 +2,7 @@ import json import shutil +import sys import tempfile from datetime import datetime, timezone @@ -35,7 +36,7 @@ PublishedArchivePaths, get_build_platform, ) -from venvstacks._util import get_env_python, run_python_command, WINDOWS_BUILD +from venvstacks._util import get_env_python, capture_python_output, WINDOWS_BUILD ################################## # Minimal project test helpers @@ -423,10 +424,14 @@ def check_publication_result( self.assertEqual(sorted(archive_paths), expected_archive_paths) @staticmethod - def _run_postinstall(base_python_path: Path, env_path: Path) -> None: + def _run_postinstall(env_path: Path) -> None: postinstall_script = env_path / "postinstall.py" if postinstall_script.exists(): - run_python_command([str(base_python_path), str(postinstall_script)]) + # Post-installation scripts are required to work even when they're + # executed with an entirely unrelated Python installation + capture_python_output( + [sys.executable, "-X", "utf8", "-I", str(postinstall_script)] + ) def check_archive_deployment(self, published_paths: PublishedArchivePaths) -> None: metadata_path, snippet_paths, archive_paths = published_paths @@ -463,25 +468,31 @@ def check_archive_deployment(self, published_paths: PublishedArchivePaths) -> No self.assertTrue(published_manifests.combined_data) layered_metadata = published_manifests.combined_data["layers"] base_runtime_env_name = layered_metadata["runtimes"][0]["install_target"] - base_runtime_env_path = env_name_to_path[base_runtime_env_name] - base_python_path = get_env_python(base_runtime_env_path) - self._run_postinstall(base_python_path, env_path) + env_path = env_name_to_path[base_runtime_env_name] + self._run_postinstall(env_path) for env_name, env_path in env_name_to_path.items(): if env_name == base_runtime_env_name: # Already configured continue - self._run_postinstall(base_python_path, env_path) + self._run_postinstall(env_path) - def get_exported_python( + def get_deployed_env_details( env: ArchiveMetadata, ) -> tuple[EnvNameDeploy, Path, list[str]]: env_name = env["install_target"] env_path = env_name_to_path[env_name] env_python = get_env_python(env_path) env_sys_path = get_sys_path(env_python) - return env_name, env_python, env_sys_path + return env_name, env_path, env_sys_path + + self.check_deployed_environments(layered_metadata, get_deployed_env_details) - self.check_deployed_environments(layered_metadata, get_exported_python) + def test_create_environments(self) -> None: + # Fast test to check the links between build envs are set up correctly + # (if this fails, there's no point even trying to full slow test case) + build_env = self.build_env + build_env.create_environments() + self.check_build_environments(self.build_env.all_environments()) @pytest.mark.slow def test_locking_and_publishing(self) -> None: @@ -500,14 +511,17 @@ def test_locking_and_publishing(self) -> None: ) expected_dry_run_result = EXPECTED_MANIFEST expected_tagged_dry_run_result = _tag_manifest(EXPECTED_MANIFEST, versioned_tag) + minimum_lock_time = datetime.now(timezone.utc) # Ensure the locking and publication steps always run for all environments build_env.select_operations(lock=True, build=True, publish=True) # Handle running this test case repeatedly in a local checkout for env in build_env.all_environments(): env.env_lock._purge_lock() - # Test stage: check dry run metadata results are as expected - minimum_lock_time = datetime.now(timezone.utc) + # Create and link the layer build environments build_env.create_environments() + # Don't even try to continue if the environments aren't properly linked + self.check_build_environments(self.build_env.all_environments()) + # Test stage: check dry run metadata results are as expected subtests_started += 1 with self.subTest("Check untagged dry run"): dry_run_result, dry_run_last_locked_times = _filter_manifest( diff --git a/tests/test_postinstall.py b/tests/test_postinstall.py new file mode 100644 index 0000000..2c85b10 --- /dev/null +++ b/tests/test_postinstall.py @@ -0,0 +1,98 @@ +"""Tests for venvstacks post-install script generation""" + +import os + +from pathlib import Path + +from venvstacks._injected import postinstall + +_EXPECTED_PYVENV_CFG = """\ +home = {python_home} +include-system-site-packages = false +version = {py_version} +executable = {python_bin} +""" + + +def test_pyvenv_cfg() -> None: + example_path = Path("/example/python/bin/python").absolute() + example_version = "6.28" + expected_pyvenv_cfg = _EXPECTED_PYVENV_CFG.format( + python_home=str(example_path.parent), + py_version=example_version, + python_bin=str(example_path), + ) + pyvenv_cfg = postinstall.generate_pyvenv_cfg( + example_path, + example_version, + ) + assert pyvenv_cfg == expected_pyvenv_cfg + + +def test_sitecustomize_empty() -> None: + assert postinstall.generate_sitecustomize([], []) is None + + +def _make_fake_paths(prefix: str, expected_line_fmt: str) -> tuple[list[Path], str]: + # Ensure fake paths are absolute (regardless of platform) + anchor = Path.cwd().anchor + fake_dirs = [f"{anchor}{prefix}{n}" for n in range(5)] + fake_paths = [Path(d) for d in fake_dirs] + # Also report the corresponding block of expected `sitecustomize.py` lines + expected_lines = "\n".join(expected_line_fmt.format(d) for d in fake_dirs) + return fake_paths, expected_lines + + +def _make_pylib_paths() -> tuple[list[Path], str]: + return _make_fake_paths("pylib", "addsitedir({!r})") + + +def _make_dynlib_paths() -> tuple[list[Path], str]: + return _make_fake_paths("dynlib", "add_dll_directory({!r})") + + +def _make_missing_dynlib_paths() -> tuple[list[Path], str]: + return _make_fake_paths("dynlib", "# Skipping {!r} (no such directory)") + + +def test_sitecustomize() -> None: + pylib_paths, expected_lines = _make_pylib_paths() + sc_text = postinstall.generate_sitecustomize(pylib_paths, []) + assert sc_text is not None + assert sc_text.startswith(postinstall._SITE_CUSTOMIZE_HEADER) + assert expected_lines in sc_text + assert "add_dll_directory(" not in sc_text + assert "# Skipping" not in sc_text + assert compile(sc_text, "_sitecustomize.py", "exec") is not None + + +def test_sitecustomize_with_dynlib() -> None: + pylib_paths, expected_pylib_lines = _make_pylib_paths() + dynlib_paths, expected_dynlib_lines = _make_dynlib_paths() + sc_text = postinstall.generate_sitecustomize( + pylib_paths, dynlib_paths, skip_missing_dynlib_paths=False + ) + assert sc_text is not None + assert sc_text.startswith(postinstall._SITE_CUSTOMIZE_HEADER) + assert expected_pylib_lines in sc_text + if hasattr(os, "add_dll_directory"): + assert expected_dynlib_lines in sc_text + else: + assert "add_dll_directory(" not in sc_text + assert "# Skipping" not in sc_text + assert compile(sc_text, "_sitecustomize.py", "exec") is not None + + +def test_sitecustomize_with_missing_dynlib() -> None: + pylib_paths, expected_pylib_lines = _make_pylib_paths() + dynlib_paths, expected_dynlib_lines = _make_missing_dynlib_paths() + sc_text = postinstall.generate_sitecustomize(pylib_paths, dynlib_paths) + assert sc_text is not None + assert sc_text.startswith(postinstall._SITE_CUSTOMIZE_HEADER) + assert expected_pylib_lines in sc_text + if hasattr(os, "add_dll_directory"): + assert expected_dynlib_lines in sc_text + else: + assert "add_dll_directory(" not in sc_text + assert "# Skipping" not in sc_text + assert compile(sc_text, "_sitecustomize.py", "exec") is not None diff --git a/tests/test_sample_project.py b/tests/test_sample_project.py index 9c32c7a..94a1b93 100644 --- a/tests/test_sample_project.py +++ b/tests/test_sample_project.py @@ -265,6 +265,13 @@ def setUp(self) -> None: self.artifact_export_path = get_artifact_export_path() self.export_on_success = force_artifact_export() + def test_create_environments(self) -> None: + # Fast test to check the links between build envs are set up correctly + # (if this fails, there's no point even trying to full slow test case) + build_env = self.build_env + build_env.create_environments() + self.check_build_environments(self.build_env.all_environments()) + @pytest.mark.slow @pytest.mark.expected_output def test_build_is_reproducible(self) -> None: @@ -284,9 +291,12 @@ def test_build_is_reproducible(self) -> None: expected_tagged_dry_run_result = _get_expected_dry_run_result( build_env, expect_tagged_outputs=True ) - # Test stage 1: ensure lock files can be regenerated without alteration committed_locked_requirements = _collect_locked_requirements(build_env) + # Create and link the layer build environments build_env.create_environments(lock=True) + # Don't even try to continue if the environments aren't properly linked + self.check_build_environments(self.build_env.all_environments()) + # Test stage: ensure lock files can be regenerated without alteration generated_locked_requirements = _collect_locked_requirements(build_env) export_locked_requirements = True subtests_started += 1 @@ -304,7 +314,7 @@ def test_build_is_reproducible(self) -> None: build_env, list(generated_locked_requirements.keys()), ) - # Test stage 2: ensure environments can be populated without building the artifacts + # Test stage: ensure environments can be populated without building the artifacts build_env.create_environments() # Use committed lock files subtests_started += 1 with self.subTest("Ensure archive publication requests are reproducible"): @@ -329,7 +339,7 @@ def test_build_is_reproducible(self) -> None: post_rebuild_locked_requirements, generated_locked_requirements ) subtests_passed += 1 - # Test stage 3: ensure built artifacts have the expected manifest contents + # Test stage: ensure built artifacts have the expected manifest contents manifest_path, snippet_paths, archive_paths = build_env.publish_artifacts() export_published_archives = True subtests_started += 1