From 78b79c6a15629a1f0aecd299368ac7c5301a65b4 Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Mon, 4 Nov 2024 19:28:35 +1000 Subject: [PATCH] Improve postinstall script resilience * Allow postinstall scripts to be executed with any Python interpreter (not just the deployed base runtime interpreter) * Generate layer config file as part of layers * Use a common postinstall script in all layers * Generate the deployed `sitecustomize.py` file from the layer config in the postinstall script * Add unit tests for the common postinstall script Closes #66. Implements initial steps towards #19. --- ...35_ncoghlan_more_resilient_postinstall.rst | 7 + src/venvstacks/_injected/README.md | 11 + src/venvstacks/_injected/postinstall.py | 164 ++++++++++++ src/venvstacks/_util.py | 3 + src/venvstacks/pack_venv.py | 80 ++---- src/venvstacks/stacks.py | 241 ++++++++++-------- tests/support.py | 23 +- tests/test_minimal_project.py | 55 +++- tests/test_postinstall.py | 63 +++++ tests/test_sample_project.py | 34 ++- 10 files changed, 496 insertions(+), 185 deletions(-) create mode 100644 changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst create mode 100644 src/venvstacks/_injected/README.md create mode 100644 src/venvstacks/_injected/postinstall.py create mode 100644 tests/test_postinstall.py diff --git a/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst b/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst new file mode 100644 index 0000000..c9da85f --- /dev/null +++ b/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst @@ -0,0 +1,7 @@ +Fixed +----- + +- Post-installation scripts for layered environments now work + correctly even when run with a Python installation other + than the expected base runtime (resolved in :issue:`66`) + diff --git a/src/venvstacks/_injected/README.md b/src/venvstacks/_injected/README.md new file mode 100644 index 0000000..3fe64a7 --- /dev/null +++ b/src/venvstacks/_injected/README.md @@ -0,0 +1,11 @@ +Files injected into deployed environments +========================================= + +Files in this folder are injected into the deployed +environments when publishing artifacts or locally +exporting environments. + +They are also designed to be importable so that +the build process can access their functionality +without need to duplicate the implementation, +and to make them more amenable to unit testing. diff --git a/src/venvstacks/_injected/postinstall.py b/src/venvstacks/_injected/postinstall.py new file mode 100644 index 0000000..14c5cee --- /dev/null +++ b/src/venvstacks/_injected/postinstall.py @@ -0,0 +1,164 @@ +"""venvstacks layer post-installation script + +* Loads `./share/venv/metadata/venvstacks_layer.json` +* Generates `pyvenv.cfg` for layered environments +* Generates `sitecustomize.py` for layered environments +* Precompiles all Python files in the library folder + +This post-installation script is automatically injected when packing environments. +""" + +import json +import os + +from compileall import compile_dir +from os.path import abspath +from pathlib import Path +from typing import cast, NotRequired, Sequence, TypedDict + +DEPLOYED_LAYER_CONFIG = "share/venv/metadata/venvstacks_layer.json" + + +class LayerConfig(TypedDict): + """Additional details needed to fully configure deployed environments""" + + # fmt: off + python: str # Relative path to this layer's Python executable + py_version: str # Expected X.Y.Z Python version for this environment + base_python: str # Relative path from layer dir to base Python executable + site_dir: str # Relative path to site-packages within this layer + pylib_dirs: Sequence[str] # Relative paths to additional sys.path entries + dynlib_dirs: Sequence[str] # Relative paths to additional Windows DLL directories + launch_module: NotRequired[str] # Module to run with `-m` to launch the application + # fmt: on + + # All relative paths are relative to the layer folder (and may refer to peer folders) + # Base runtime layers will have "python" and "base_python" set to the same value + # Application layers will have "launch_module" set + + +class ResolvedLayerConfig(TypedDict): + """LayerConfig with relative paths resolved for a specific layer location""" + + # fmt: off + layer_path: Path # Absolute path to layer environment + python_path: Path # Absolute path to this layer's Python executable + py_version: str # Expected X.Y.Z Python version for this environment + base_python_path: Path # Absolute path from layer dir to base Python executable + site_path: Path # Absolute path to site-packages within this layer + pylib_paths: Sequence[Path] # Absolute paths to additional sys.path entries + dynlib_paths: Sequence[Path] # Absolute paths to additional Windows DLL directories + launch_module: str|None # Module to run with `-m` to launch the application + # fmt: on + + +def load_layer_config(layer_path: Path) -> ResolvedLayerConfig: + """Read and resolve config for the specified layer environment""" + + def deployed_path(relative_path: str) -> Path: + """Normalize path and make it absolute, *without* resolving symlinks""" + return Path(abspath(layer_path / relative_path)) + + config_path = layer_path / DEPLOYED_LAYER_CONFIG + config_text = config_path.read_text(encoding="utf-8") + # Tolerate runtime errors for incorrectly generated config files + config = cast(LayerConfig, json.loads(config_text)) + return ResolvedLayerConfig( + layer_path=layer_path, + python_path=deployed_path(config["python"]), + py_version=config["py_version"], + base_python_path=deployed_path(config["base_python"]), + site_path=deployed_path(config["site_dir"]), + pylib_paths=[deployed_path(d) for d in config["pylib_dirs"]], + dynlib_paths=[deployed_path(d) for d in config["dynlib_dirs"]], + launch_module=config.get("launch_module", None), + ) + + +def generate_pyvenv_cfg(base_python_path: Path, py_version: str) -> str: + """Generate `pyvenv.cfg` contents for given base Python path and version""" + venv_config_lines = [ + f"home = {base_python_path.parent}", + "include-system-site-packages = false", + f"version = {py_version}", + f"executable = {base_python_path}", + "", + ] + return "\n".join(venv_config_lines) + + +_SITE_CUSTOMIZE_HEADER = '''\ +"""venvstacks layered environment site customization script + +* Calls `site.addsitedir` for any configured Python path entries +* Calls `os.add_dll_directory` for any configured Windows dynlib paths + +This venv module is automatically generated by the post-installation script. +""" + +''' + + +def generate_sitecustomize( + pylib_paths: Sequence[Path], dynlib_paths: Sequence[Path] +) -> str | None: + """Generate `sitecustomize.py` contents for given linked environment directories""" + sc_contents = [_SITE_CUSTOMIZE_HEADER] + if pylib_paths: + pylib_contents = [ + "# Allow loading modules and packages from linked environments", + "from site import addsitedir", + ] + for path_entry in pylib_paths: + pylib_contents.append(f"addsitedir({str(path_entry)!r})") + pylib_contents.append("") + sc_contents.extend(pylib_contents) + if dynlib_paths and hasattr(os, "add_dll_directory"): + dynlib_contents = [ + "# Allow loading misplaced DLLs on Windows", + "from os import add_dll_directory", + ] + for dynlib_path in dynlib_paths: + if not dynlib_path.exists(): + # Nothing added DLLs to this folder at build time, so skip it + continue + dynlib_contents.append(f"add_dll_directory({str(dynlib_path)!r})") + dynlib_contents.append("") + sc_contents.extend(dynlib_contents) + if len(sc_contents) == 1: + # Environment layer doesn't actually need customizing + return None + return "\n".join(sc_contents) + + +def _run_postinstall(layer_path: Path) -> None: + """Run the required post-installation steps in a deployed environment""" + + # Read the layer config file + config = load_layer_config(layer_path) + + base_python_path = config["base_python_path"] + if base_python_path != config["python_path"]: + # Generate `pyvenv.cfg` for layered environments + venv_config = generate_pyvenv_cfg(base_python_path, config["py_version"]) + venv_config_path = layer_path / "pyvenv.cfg" + venv_config_path.write_text(venv_config, encoding="utf-8") + + # Generate `sitecustomize.py` for layered environments + sc_contents = generate_sitecustomize( + config["pylib_paths"], config["dynlib_paths"] + ) + if sc_contents is not None: + sc_path = config["site_path"] / "sitecustomize.py" + sc_path.write_text(sc_contents, encoding="utf-8") + + # Precompile Python library modules + pylib_path = ( + layer_path / "lib" + ) # "Lib" on Windows, but Windows is not case sensitive + compile_dir(pylib_path, optimize=0, quiet=True) + + +if __name__ == "__main__": + # Actually executing the post-installation step in a deployed environment + _run_postinstall(Path(__file__).parent) diff --git a/src/venvstacks/_util.py b/src/venvstacks/_util.py index ea06074..d55848e 100644 --- a/src/venvstacks/_util.py +++ b/src/venvstacks/_util.py @@ -125,3 +125,6 @@ def run_python_command( result = run_python_command_unchecked(command, text=True, **kwds) result.check_returncode() return result + +def capture_python_output(command: list[str]) -> subprocess.CompletedProcess[str]: + return run_python_command(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/src/venvstacks/pack_venv.py b/src/venvstacks/pack_venv.py index d60c282..4f2e557 100755 --- a/src/venvstacks/pack_venv.py +++ b/src/venvstacks/pack_venv.py @@ -43,61 +43,9 @@ from pathlib import Path from typing import cast, Any, Callable, TextIO +from ._injected import postinstall as _default_postinstall from ._util import as_normalized_path, StrPath, WINDOWS_BUILD as _WINDOWS_BUILD -_PRECOMPILATION_COMMANDS = """\ -# Precompile Python library modules -from compileall import compile_dir -venv_pylib_path = venv_path / "lib" # "Lib" on Windows, but Windows is not case sensitive -compile_dir(venv_pylib_path, optimize=0, quiet=True) -""" - -_BASE_RUNTIME_POST_INSTALL_SCRIPT = ( - '''\ -"""Base runtime post-installation script - -* Precompiles all Python files in the library folder - -This post-installation script is automatically injected when packing environments that -do NOT include a `pyvenv.cfg` file (i.e. base runtime environments) -""" -from pathlib import Path -venv_path = Path(__file__).parent - -''' - + _PRECOMPILATION_COMMANDS -) - -_LAYERED_ENV_POST_INSTALL_SCRIPT = ( - '''\ -"""Layered environment post-installation script - -* Generates pyvenv.cfg based on the Python runtime executing this script -* Precompiles all Python files in the library folder - -This post-installation script is automatically injected when packing environments that -would otherwise include a `pyvenv.cfg` file (as `pyvenv.cfg` files are not relocatable) -""" -from pathlib import Path -venv_path = Path(__file__).parent - -# Generate `pyvenv.cfg` based on the deployed runtime location -import sys -venv_config_path = venv_path / "pyvenv.cfg" -runtime_executable_path = Path(sys.executable).resolve() -runtime_version = ".".join(map(str, sys.version_info[:3])) -venv_config = f"""\ -home = {runtime_executable_path.parent} -include-system-site-packages = false -version = {runtime_version} -executable = {runtime_executable_path} -""" -venv_config_path.write_text(venv_config, encoding="utf-8") - -''' - + _PRECOMPILATION_COMMANDS -) - SymlinkInfo = tuple[Path, Path] @@ -165,18 +113,20 @@ def get_archive_path(archive_base_name: StrPath) -> Path: def _inject_postinstall_script( - env_path: Path, script_name: str = "postinstall.py" + env_path: Path, + script_name: str = "postinstall.py", + script_source: StrPath | None = None, ) -> Path: venv_config_path = env_path / "pyvenv.cfg" if venv_config_path.exists(): # The venv config contains absolute paths referencing the base runtime environment # Remove it here, let the post-install script recreate it venv_config_path.unlink() - script_contents = _LAYERED_ENV_POST_INSTALL_SCRIPT - else: - script_contents = _BASE_RUNTIME_POST_INSTALL_SCRIPT + if script_source is None: + # Nothing specified, inject the default postinstall script + script_source = _default_postinstall.__file__ script_path = env_path / script_name - script_path.write_text(script_contents, encoding="utf-8") + shutil.copy2(script_source, script_path) return script_path @@ -201,7 +151,9 @@ def export_venv( """Export the given build environment, skipping archive creation and unpacking * injects a suitable `postinstall.py` script for the environment being exported - * excludes __pycache__ folders and package metadata RECORD files + * excludes __pycache__ folders (for consistency with archive publication) + * excludes package metadata RECORD files (for consistency with archive publication) + * excludes `sitecustomize.py` files (generated by the post-installation script) * replaces symlinks with copies on Windows or if the target doesn't support symlinks If supplied, *run_postinstall* is called with the path to the environment's Python @@ -213,7 +165,7 @@ def export_venv( """ source_path = as_normalized_path(source_dir) target_path = as_normalized_path(target_dir) - excluded = shutil.ignore_patterns("__pycache__", "RECORD") + excluded = shutil.ignore_patterns("__pycache__", "RECORD", "sitecustomize.py") # Avoid symlinks on Windows, as they need elevated privileges to create # Also avoid them if the target folder doesn't support symlink creation # (that way exports to FAT/FAT32/VFAT file systems should work, even if @@ -247,9 +199,13 @@ def create_archive( * injects a suitable `postinstall.py` script for the environment being archived * always creates zipfile archives on Windows and xztar archives elsewhere - * excludes __pycache__ folders and package metadata RECORD files + * excludes __pycache__ folders (to reduce archive size and improve reproducibility) + * excludes package metadata RECORD files (to improve reproducibility) + * excludes `sitecustomize.py` files (generated by the post-installation script) * replaces symlinks with copies on Windows and allows external symlinks elsewhere - * discards owner and group information for tar archives + * discards tar entry owner and group information + * clears tar entry high mode bits (setuid, setgid, sticky) + * clears tar entry group/other write mode bits * clamps mtime of archived files to the given clamp mtime at the latest * shows progress reporting by default (archiving built ML/AI libs is *slooooow*) diff --git a/src/venvstacks/stacks.py b/src/venvstacks/stacks.py index 5ba244b..d31bbeb 100755 --- a/src/venvstacks/stacks.py +++ b/src/venvstacks/stacks.py @@ -32,6 +32,7 @@ MutableMapping, NamedTuple, NewType, + NoReturn, NotRequired, Self, Sequence, @@ -40,12 +41,12 @@ ) from . import pack_venv +from ._injected import postinstall from ._util import ( as_normalized_path, + capture_python_output, default_tarfile_filter, - get_env_python, run_python_command, - run_python_command_unchecked, StrPath, WINDOWS_BUILD as _WINDOWS_BUILD, ) @@ -207,10 +208,14 @@ def get_deployed_name(self, env_name: EnvNameBuild) -> EnvNameDeploy: return EnvNameDeploy(f"{env_name}@{self.lock_version}") return EnvNameDeploy(env_name) - @staticmethod - def _raise_if_none(value: _T | None) -> _T: + def _fail_lock(self, message: str) -> NoReturn: + missing = self.requirements_path.name + msg = f"Environment has not been locked ({missing} not found)" + raise BuildEnvError(msg) + + def _raise_if_none(self, value: _T | None) -> _T: if value is None: - raise BuildEnvError("Environment has not been locked") + self._fail_lock("Environment has not been locked") return value @property @@ -303,9 +308,7 @@ def _purge_lock(self) -> bool: def _write_lock_metadata(self) -> None: requirements_hash = self._requirements_hash if requirements_hash is None: - raise BuildEnvError( - "Environment must be locked before writing lock metadata" - ) + self._fail_lock("Environment must be locked before writing lock metadata") last_version = self._lock_version if last_version is None: lock_version = 1 @@ -324,9 +327,7 @@ def update_lock_metadata(self) -> bool: # Calculate current requirements hash requirements_hash = self._hash_requirements() if requirements_hash is None: - raise BuildEnvError( - "Environment must be locked before updating lock metadata" - ) + self._fail_lock("Environment must be locked before updating lock metadata") self._requirements_hash = requirements_hash # Only update and save the last locked time if # the lockfile contents have changed or if @@ -774,25 +775,17 @@ def define_export( return cls(env_name, env_lock, export_path, export_metadata, needs_export) @staticmethod - def _run_postinstall( - src_path: Path, export_path: Path, postinstall_path: Path - ) -> None: - exported_env_python_path = get_env_python(export_path) - command = [str(exported_env_python_path), "-I", str(postinstall_path)] - result = run_python_command_unchecked(command) - if result.returncode == 0: - # All good, nothing else to check - return - # Running with the Python inside the exported environment didn't work - # This can happen on Windows when "pyvenv.cfg" doesn't exist yet - # If that is what has happened, the reported return code will be 106 - if result.returncode != 106: - result.check_returncode() - # Self-generating the venv config failed, retry with the build venv - # rather than finding and using the exported base runtime environment - src_env_python_path = get_env_python(src_path) - command[0] = str(src_env_python_path) - run_python_command(command) + def _run_postinstall(postinstall_path: Path) -> None: + # Post-installation scripts are required to work even when they're + # executed with an entirely unrelated Python installation + command = [ + sys.executable, + "-X", + "utf8", + "-I", + str(postinstall_path) + ] + capture_python_output(command) def export_environment( self, @@ -816,8 +809,8 @@ def export_environment( export_path.unlink() print(f"Exporting {str(env_path)!r} to {str(export_path)!r}") - def _run_postinstall(export_path: Path, postinstall_path: Path) -> None: - self._run_postinstall(env_path, export_path, postinstall_path) + def _run_postinstall(_export_path: Path, postinstall_path: Path) -> None: + self._run_postinstall(postinstall_path) exported_path = pack_venv.export_venv( env_path, @@ -1047,6 +1040,65 @@ def env_spec(self) -> _PythonEnvironmentSpec: # Define property to allow covariance of the declared type of `env_spec` return self._env_spec + @abstractmethod + def get_deployed_config(self) -> postinstall.LayerConfig: + """Layer config to be published in `venvstacks_layer.json`""" + raise NotImplementedError + + def _get_deployed_config( + self, + pylib_paths: Iterable[Path] | None, + dynlib_paths: Iterable[Path] | None, + link_external_base: bool = True + ) -> postinstall.LayerConfig: + # Helper for subclass get_deployed_config implementations + base_python_path = self.base_python_path + if base_python_path is None or pylib_paths is None or dynlib_paths is None: + self._fail_build("Cannot get deployment config for unlinked layer") + build_env_path = self.env_path + + def from_internal_path(build_path: Path) -> str: + # Absolute path, inside the environment + return str(build_path.relative_to(build_env_path)) + def from_external_path(build_path: Path) -> str: + # Absolute path, potentially outside the environment + return str(build_path.relative_to(build_env_path, walk_up=True)) + def from_relative_path(relative_build_path: Path) -> str: + # Path relative to the base of the build directory + build_path = self.build_path / relative_build_path + return str(build_path.relative_to(build_env_path, walk_up=True)) + + layer_python = from_internal_path(self.python_path) + if link_external_base: + base_python = from_external_path(base_python_path) + else: + # "base_python" in the runtime layer refers solely to + # the external environment used to set up the base + # runtime layer, rather than being a linked environment + base_python = layer_python + + return postinstall.LayerConfig( + python=layer_python, + py_version=self.py_version, + base_python=base_python, + site_dir=from_internal_path(self.pylib_path), + pylib_dirs=[from_relative_path(p) for p in pylib_paths], + dynlib_dirs=[from_relative_path(p) for p in dynlib_paths], + ) + + def _write_deployed_config(self) -> None: + # Subclasses call this when they have enough info to + # populate it correctly (on creation for base runtime + # environments, when linked for layered environments) + config_path = self.env_path / postinstall.DEPLOYED_LAYER_CONFIG + print(f"Generating {config_path!r}...") + config_path.parent.mkdir(parents=True, exist_ok=True) + _write_json(config_path, self.get_deployed_config()) + + def _fail_build(self, message: str) -> NoReturn: + attributed_message = f"Layer {self.env_name}: {message}" + raise BuildEnvError(attributed_message) + def select_operations( self, lock: bool | None = False, @@ -1095,8 +1147,6 @@ def report_python_site_details(self) -> subprocess.CompletedProcess[str]: print(f"Reporting environment details for {str(self.env_path)!r}") command = [ str(self.python_path), - "-X", - "utf8", "-Im", "site", ] @@ -1222,7 +1272,7 @@ def lock_requirements(self) -> EnvironmentLock: requirements_path, requirements_input_path, constraints ) if not requirements_path.exists(): - raise BuildEnvError(f"Failed to generate {str(requirements_path)!r}") + self._fail_build(f"Failed to generate {str(requirements_path)!r}") if self.env_lock.update_lock_metadata(): print(f" Environment lock time set: {self.env_lock.locked_at!r}") return self.env_lock @@ -1230,7 +1280,7 @@ def lock_requirements(self) -> EnvironmentLock: def install_requirements(self) -> subprocess.CompletedProcess[str]: # Run a pip dependency upgrade inside the target environment if not self.env_lock.is_locked: - raise BuildEnvError( + self._fail_build( "Environment must be locked before installing dependencies" ) return self._run_pip_install( @@ -1376,6 +1426,10 @@ def env_spec(self) -> RuntimeSpec: assert isinstance(self._env_spec, RuntimeSpec) return self._env_spec + def get_deployed_config(self) -> postinstall.LayerConfig: + """Layer config to be published in `venvstacks_layer.json`""" + return self._get_deployed_config([], [], link_external_base=False) + def _remove_pip(self) -> subprocess.CompletedProcess[str] | None: to_be_checked = ["pip", "wheel", "setuptools"] to_be_removed = [] @@ -1391,12 +1445,14 @@ def _create_new_environment(self, *, lock_only: bool = False) -> None: python_runtime = self.env_spec.fully_versioned_name install_path = _pdm_python_install(self.build_path, python_runtime) if install_path is None: - raise BuildEnvError(f"Failed to install {python_runtime}") + self._fail_build(f"Failed to install {python_runtime}") shutil.move(install_path, self.env_path) # No build step needs `pip` to be installed in the target environment, # and we don't want to ship it unless explicitly requested to do so # as a declared dependency of an included component self._remove_pip() + # No layer linking details needed for base runtime environments + self._write_deployed_config() fs_sync() if not lock_only: print( @@ -1415,14 +1471,16 @@ def create_build_environment(self, *, clean: bool = False) -> None: class _VirtualEnvironment(_PythonEnvironment): - _include_system_site_packages = False - linked_constraints_paths: list[Path] = field(init=False, repr=False) + linked_pylib_paths: list[Path] = field(init=False, repr=False) + linked_dynlib_paths: list[Path] = field(init=False, repr=False) def __post_init__(self) -> None: self.py_version = self.env_spec.runtime.py_version super().__post_init__() self.linked_constraints_paths = [] + self.linked_pylib_paths = [] + self.linked_dynlib_paths = [] @property def env_spec(self) -> _VirtualEnvironmentSpec: @@ -1435,19 +1493,23 @@ def link_base_runtime_paths(self, runtime: RuntimeEnv) -> None: self.base_python_path = runtime.python_path self.tools_python_path = runtime.tools_python_path if self.linked_constraints_paths: - raise BuildEnvError("Layered environment base runtime already linked") + self._fail_build("Layered environment base runtime already linked") self.linked_constraints_paths[:] = [runtime.requirements_path] + def get_deployed_config(self) -> postinstall.LayerConfig: + """Layer config to be published in `venvstacks_layer.json`""" + return self._get_deployed_config( + self.linked_pylib_paths, self.linked_dynlib_paths + ) + def get_constraint_paths(self) -> list[Path]: return self.linked_constraints_paths def _ensure_virtual_environment(self) -> subprocess.CompletedProcess[str]: # Use the base Python installation to create a new virtual environment if self.base_python_path is None: - raise RuntimeError("Base Python path not set in {self!r}") + self._fail_build("Base Python path not set") options = ["--without-pip"] - if self._include_system_site_packages: - options.append("--system-site-packages") if self.env_path.exists(): options.append("--upgrade") if _WINDOWS_BUILD: @@ -1464,13 +1526,26 @@ def _ensure_virtual_environment(self) -> subprocess.CompletedProcess[str]: str(self.env_path), ] result = run_python_command(command) - self._link_layered_environment() + self._link_build_environment() fs_sync() print(f"Virtual environment configured in {str(self.env_path)!r}") return result - def _link_layered_environment(self) -> None: - pass # Nothing to do by default, subclasses override if necessary + def _link_build_environment(self) -> None: + # Linking the build environments indicates all the required + # deployment config settings have been populated + self._write_deployed_config() + # Create sitecustomize file for the build environment + sc_contents = postinstall.generate_sitecustomize( + self.linked_pylib_paths, self.linked_dynlib_paths + ) + if sc_contents is None: + self._fail_build( + "Layered environments must at least link a base runtime environment" + ) + sc_path = self.pylib_path / "sitecustomize.py" + print(f"Generating {sc_path!r}...") + sc_path.write_text(sc_contents, encoding="utf-8") def _update_existing_environment(self, *, lock_only: bool = False) -> None: if lock_only: @@ -1499,7 +1574,6 @@ class FrameworkEnv(_VirtualEnvironment): kind = LayerVariants.FRAMEWORK category = LayerCategories.FRAMEWORKS - _include_system_site_packages = True @property def env_spec(self) -> FrameworkSpec: @@ -1507,6 +1581,21 @@ def env_spec(self) -> FrameworkSpec: assert isinstance(self._env_spec, FrameworkSpec) return self._env_spec + def link_base_runtime_paths(self, runtime: RuntimeEnv) -> None: + super().link_base_runtime_paths(runtime) + # TODO: Reduce code duplication with ApplicationEnv + runtime_target_path = Path(runtime.install_target) + + def _runtime_path(build_path: Path) -> Path: + relative_path = build_path.relative_to(runtime.env_path) + return runtime_target_path / relative_path + + pylib_paths = self.linked_pylib_paths + dynlib_paths = self.linked_dynlib_paths + pylib_paths.append(_runtime_path(runtime.pylib_path)) + if runtime.dynlib_path is not None: + dynlib_paths.append(_runtime_path(runtime.dynlib_path)) + class ApplicationEnv(_VirtualEnvironment): """Application layer build environment""" @@ -1515,8 +1604,6 @@ class ApplicationEnv(_VirtualEnvironment): category = LayerCategories.APPLICATIONS launch_module_name: str = field(init=False, repr=False) - linked_pylib_paths: list[Path] = field(init=False, repr=False) - linked_dynlib_paths: list[Path] = field(init=False, repr=False) linked_frameworks: list[FrameworkEnv] = field(init=False, repr=False) @property @@ -1528,8 +1615,6 @@ def env_spec(self) -> ApplicationSpec: def __post_init__(self) -> None: super().__post_init__() self.launch_module_name = self.env_spec.launch_module_path.stem - self.linked_pylib_paths = [] - self.linked_dynlib_paths = [] self.linked_frameworks = [] def link_layered_environments( @@ -1538,14 +1623,14 @@ def link_layered_environments( self.link_base_runtime_paths(runtime) constraints_paths = self.linked_constraints_paths if not constraints_paths: - raise BuildEnvError("Failed to add base environment constraints path") + self._fail_build("Failed to add base environment constraints path") # The runtime site-packages folder is added here rather than via pyvenv.cfg # to ensure it appears in sys.path after the framework site-packages folders pylib_paths = self.linked_pylib_paths dynlib_paths = self.linked_dynlib_paths fw_envs = self.linked_frameworks if pylib_paths or dynlib_paths or fw_envs: - raise BuildEnvError("Layered application environment already linked") + self._fail_build("Layered application environment already linked") for env_spec in self.env_spec.frameworks: env = frameworks[env_spec.name] fw_envs.append(env) @@ -1569,56 +1654,6 @@ def _runtime_path(build_path: Path) -> Path: if runtime.dynlib_path is not None: dynlib_paths.append(_runtime_path(runtime.dynlib_path)) - def _link_layered_environment(self) -> None: - # Create sitecustomize file - sc_dir_path = self.pylib_path - sc_contents = [ - "# Automatically generated by venvstacks", - "import site", - "import os", - "from os.path import abspath, dirname, join as joinpath", - "# Allow loading modules and packages from framework environments", - "this_dir = dirname(abspath(__file__))", - ] - # Add framework and runtime folders to sys.path - parent_path = self.env_path.parent - relative_prefix = Path( - os.path.relpath(str(parent_path), start=str(sc_dir_path)) - ) - for pylib_path in self.linked_pylib_paths: - relative_path = relative_prefix / pylib_path - sc_contents.extend( - [ - f"path_entry = abspath(joinpath(this_dir, {str(relative_path)!r}))", - "site.addsitedir(path_entry)", - ] - ) - # Add DLL search folders if needed - dynlib_paths = self.linked_dynlib_paths - if _WINDOWS_BUILD and dynlib_paths: - sc_contents.extend( - [ - "", - "# Allow loading misplaced DLLs on Windows", - ] - ) - for dynlib_path in dynlib_paths: - if not dynlib_path.exists(): - # Nothing added DLLs to this folder at build time, so skip it - continue - relative_path = relative_prefix / dynlib_path - sc_contents.extend( - [ - f"dll_dir = abspath(joinpath(this_dir, {str(relative_path)!r}))", - "os.add_dll_directory(dll_dir)", - ] - ) - sc_contents.append("") - sc_path = self.pylib_path / "sitecustomize.py" - print(f"Generating {sc_path!r}...") - with open(sc_path, "w", encoding="utf-8") as f: - f.write("\n".join(sc_contents)) - def _update_existing_environment(self, *, lock_only: bool = False) -> None: super()._update_existing_environment(lock_only=lock_only) # Also publish the specified launch module as an importable top level module diff --git a/tests/support.py b/tests/support.py index 89a6c95..d74fa02 100644 --- a/tests/support.py +++ b/tests/support.py @@ -13,7 +13,7 @@ import pytest -from venvstacks._util import run_python_command +from venvstacks._util import capture_python_output from venvstacks.stacks import ( BuildEnvironment, EnvNameDeploy, @@ -202,17 +202,24 @@ def make_mock_index_config(reference_config: PackageIndexConfig | None = None) - # Running commands in a deployed environment ############################################## - -def capture_python_output(command: list[str]) -> subprocess.CompletedProcess[str]: - return run_python_command(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - def get_sys_path(env_python: Path) -> list[str]: - command = [str(env_python), "-Ic", "import json, sys; print(json.dumps(sys.path))"] + command = [ + str(env_python), + "-X", + "utf8", + "-Ic", + "import json, sys; print(json.dumps(sys.path))" + ] result = capture_python_output(command) return cast(list[str], json.loads(result.stdout)) def run_module(env_python: Path, module_name: str) -> subprocess.CompletedProcess[str]: - command = [str(env_python), "-Im", module_name] + command = [ + str(env_python), + "-X", + "utf8", + "-Im", + module_name + ] return capture_python_output(command) diff --git a/tests/test_minimal_project.py b/tests/test_minimal_project.py index 3aefa28..10b524b 100644 --- a/tests/test_minimal_project.py +++ b/tests/test_minimal_project.py @@ -1,7 +1,9 @@ """Test building the minimal project produces the expected results""" import json +import os.path import shutil +import sys import tempfile from datetime import datetime, timezone @@ -30,6 +32,7 @@ StackPublishingRequest, BuildEnvironment, EnvNameDeploy, + LayerVariants, StackSpec, ExportedEnvironmentPaths, ExportMetadata, @@ -37,7 +40,8 @@ PublishedArchivePaths, get_build_platform, ) -from venvstacks._util import get_env_python, run_python_command, WINDOWS_BUILD +from venvstacks._injected.postinstall import DEPLOYED_LAYER_CONFIG +from venvstacks._util import get_env_python, capture_python_output, WINDOWS_BUILD ################################## # Minimal project test helpers @@ -474,11 +478,20 @@ def check_deployed_environments( self.assertEqual(launch_result.stdout, "") self.assertEqual(launch_result.stderr, "") - @staticmethod - def _run_postinstall(base_python_path: Path, env_path: Path) -> None: + def _run_postinstall(self, env_path: Path) -> None: + config_path = env_path / DEPLOYED_LAYER_CONFIG + self.assertTrue(config_path.exists()) postinstall_script = env_path / "postinstall.py" if postinstall_script.exists(): - run_python_command([str(base_python_path), str(postinstall_script)]) + # Post-installation scripts are required to work even when they're + # executed with an entirely unrelated Python installation + capture_python_output([ + sys.executable, + "-X", + "utf8", + "-I", + str(postinstall_script) + ]) def check_archive_deployment(self, published_paths: PublishedArchivePaths) -> None: metadata_path, snippet_paths, archive_paths = published_paths @@ -512,14 +525,13 @@ def check_archive_deployment(self, published_paths: PublishedArchivePaths) -> No self.assertTrue(published_manifests.combined_data) layered_metadata = published_manifests.combined_data["layers"] base_runtime_env_name = layered_metadata["runtimes"][0]["install_target"] - base_runtime_env_path = env_name_to_path[base_runtime_env_name] - base_python_path = get_env_python(base_runtime_env_path) - self._run_postinstall(base_python_path, env_path) + env_path = env_name_to_path[base_runtime_env_name] + self._run_postinstall(env_path) for env_name, env_path in env_name_to_path.items(): if env_name == base_runtime_env_name: # Already configured continue - self._run_postinstall(base_python_path, env_path) + self._run_postinstall(env_path) def get_exported_python( env: ArchiveMetadata, @@ -555,6 +567,9 @@ def get_exported_python( self.check_deployed_environments(layered_metadata, get_exported_python) + def assertPathExists(self, expected_path: Path) -> None: + self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") + @pytest.mark.slow def test_locking_and_publishing(self) -> None: # This is organised as subtests in a monolothic test sequence to reduce CI overhead @@ -577,10 +592,32 @@ def test_locking_and_publishing(self) -> None: # Handle running this test case repeatedly in a local checkout for env in build_env.all_environments(): env.env_lock._purge_lock() - # Test stage: check dry run metadata results are as expected + # Test stage: create and link build environments minimum_lock_time = datetime.now(timezone.utc) build_env.create_environments() subtests_started += 1 + with self.subTest("Check build environments have been linked"): + for env in self.build_env.all_environments(): + config_path = env.env_path / DEPLOYED_LAYER_CONFIG + self.assertPathExists(config_path) + layer_config = json.loads(config_path.read_text(encoding="utf-8")) + python_path = env.env_path / layer_config["python"] + expected_python_path = env.python_path + self.assertEqual(str(python_path), str(expected_python_path)) + base_python_path = env.env_path / layer_config["base_python"] + if env.kind == LayerVariants.RUNTIME: + # base_python should refer to the runtime layer itself + expected_base_python_path = expected_python_path + else: + # base_python should refer to the venv's base Python runtime + self.assertIsNotNone(env.base_python_path) + assert env.base_python_path is not None + base_python_path = Path(os.path.normpath(base_python_path)) + expected_base_python_path = env.base_python_path + self.assertEqual(str(base_python_path), str(expected_base_python_path)) + subtests_passed += 1 + # Test stage: check dry run metadata results are as expected + subtests_started += 1 with self.subTest("Check untagged dry run"): dry_run_result, dry_run_last_locked_times = _filter_manifest( build_env.publish_artifacts(dry_run=True)[1] diff --git a/tests/test_postinstall.py b/tests/test_postinstall.py new file mode 100644 index 0000000..8da0d9d --- /dev/null +++ b/tests/test_postinstall.py @@ -0,0 +1,63 @@ +"""Tests for venvstacks post-install script generation""" +import os + +from pathlib import Path + +from venvstacks._injected import postinstall + +_EXPECTED_PYVENV_CFG = """\ +home = {python_home} +include-system-site-packages = false +version = {py_version} +executable = {python_bin} +""" + +def test_pyvenv_cfg() -> None: + example_path = Path("/example/python/bin/python") + example_version = "6.28" + expected_pyvenv_cfg = _EXPECTED_PYVENV_CFG.format( + python_home=str(example_path.parent), + py_version=example_version, + python_bin=str(example_path), + ) + pyvenv_cfg = postinstall.generate_pyvenv_cfg( + example_path, example_version, + ) + assert pyvenv_cfg == expected_pyvenv_cfg + +def test_sitecustomize_empty() -> None: + assert postinstall.generate_sitecustomize([], []) is None + +def _make_pylib_paths() -> tuple[list[Path], str]: + pylib_dirs = [f"pylib{n}" for n in range(5)] + pylib_paths = [Path(d) for d in pylib_dirs] + expected_lines = "\n".join(f"addsitedir({d!r})" for d in pylib_dirs) + return pylib_paths, expected_lines + +def _make_dynlib_paths() -> tuple[list[Path], str]: + dynlib_dirs = [f"dynlib{n}" for n in range(5)] + dynlib_paths = [Path(d) for d in dynlib_dirs] + expected_lines = "\n".join(f"add_dll_directory({d!r})" for d in dynlib_dirs) + return dynlib_paths, expected_lines + +def test_sitecustomize() -> None: + pylib_paths, expected_lines = _make_pylib_paths() + sc_text = postinstall.generate_sitecustomize(pylib_paths, []) + assert sc_text is not None + assert sc_text.startswith(postinstall._SITE_CUSTOMIZE_HEADER) + assert expected_lines in sc_text + assert "add_dll_directory(" not in sc_text + assert compile(sc_text, "_sitecustomize.py", "exec") is not None + +def test_sitecustomize_with_dynlib() -> None: + pylib_paths, expected_pylib_lines = _make_pylib_paths() + dynlib_paths, expected_dynlib_lines = _make_dynlib_paths() + sc_text = postinstall.generate_sitecustomize(pylib_paths, dynlib_paths) + assert sc_text is not None + assert sc_text.startswith(postinstall._SITE_CUSTOMIZE_HEADER) + assert expected_pylib_lines in sc_text + if hasattr(os, "add_dll_directory"): + assert expected_dynlib_lines in sc_text + else: + assert "add_dll_directory(" not in sc_text + assert compile(sc_text, "_sitecustomize.py", "exec") is not None diff --git a/tests/test_sample_project.py b/tests/test_sample_project.py index ff4487d..a02da94 100644 --- a/tests/test_sample_project.py +++ b/tests/test_sample_project.py @@ -1,5 +1,6 @@ """Test building the sample project produces the expected results""" +import json import os.path import shutil import tempfile @@ -37,7 +38,9 @@ LayerCategories, ExportedEnvironmentPaths, ExportMetadata, + LayerVariants, ) +from venvstacks._injected.postinstall import DEPLOYED_LAYER_CONFIG from venvstacks._util import get_env_python ################################## @@ -344,6 +347,9 @@ def get_exported_python( self.check_deployed_environments(layered_metadata, get_exported_python) + def assertPathExists(self, expected_path: Path) -> None: + self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") + @pytest.mark.slow @pytest.mark.expected_output def test_build_is_reproducible(self) -> None: @@ -363,9 +369,31 @@ def test_build_is_reproducible(self) -> None: expected_tagged_dry_run_result = _get_expected_dry_run_result( build_env, expect_tagged_outputs=True ) - # Test stage 1: ensure lock files can be regenerated without alteration + # Test stage: create and link build environments committed_locked_requirements = _collect_locked_requirements(build_env) build_env.create_environments(lock=True) + subtests_started += 1 + with self.subTest("Check build environments have been linked"): + for env in self.build_env.all_environments(): + config_path = env.env_path / DEPLOYED_LAYER_CONFIG + self.assertPathExists(config_path) + layer_config = json.loads(config_path.read_text(encoding="utf-8")) + python_path = env.env_path / layer_config["python"] + expected_python_path = env.python_path + self.assertEqual(str(python_path), str(expected_python_path)) + base_python_path = env.env_path / layer_config["base_python"] + if env.kind == LayerVariants.RUNTIME: + # base_python should refer to the runtime layer itself + expected_base_python_path = expected_python_path + else: + # base_python should refer to the venv's base Python runtime + self.assertIsNotNone(env.base_python_path) + assert env.base_python_path is not None + base_python_path = Path(os.path.normpath(base_python_path)) + expected_base_python_path = env.base_python_path + self.assertEqual(str(base_python_path), str(expected_base_python_path)) + subtests_passed += 1 + # Test stage: ensure lock files can be regenerated without alteration generated_locked_requirements = _collect_locked_requirements(build_env) export_locked_requirements = True subtests_started += 1 @@ -383,7 +411,7 @@ def test_build_is_reproducible(self) -> None: build_env, list(generated_locked_requirements.keys()), ) - # Test stage 2: ensure environments can be populated without building the artifacts + # Test stage: ensure environments can be populated without building the artifacts build_env.create_environments() # Use committed lock files subtests_started += 1 with self.subTest("Ensure archive publication requests are reproducible"): @@ -408,7 +436,7 @@ def test_build_is_reproducible(self) -> None: post_rebuild_locked_requirements, generated_locked_requirements ) subtests_passed += 1 - # Test stage 3: ensure built artifacts have the expected manifest contents + # Test stage: ensure built artifacts have the expected manifest contents manifest_path, snippet_paths, archive_paths = build_env.publish_artifacts() export_published_archives = True subtests_started += 1