From 185ef12300173bb571bff1338ddec85747f8e077 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Fri, 24 Jan 2025 08:13:05 +0000 Subject: [PATCH] Split config manager --- pyodide_build/build_env.py | 21 ++++- pyodide_build/config.py | 133 ++++++++++++++++------------- pyodide_build/tests/conftest.py | 1 + pyodide_build/tests/test_config.py | 80 ++++++++++++++--- 4 files changed, 164 insertions(+), 71 deletions(-) diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index 1773738..d6d18f8 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -15,7 +15,7 @@ from pyodide_build import __version__ from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname -from pyodide_build.config import ConfigManager +from pyodide_build.config import ConfigManager, CrossBuildEnvConfigManager from pyodide_build.recipe import load_all_recipes RUST_BUILD_PRELUDE = """ @@ -120,7 +120,7 @@ def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]: """ Get common environment variables for the in-tree and out-of-tree build. """ - config_manager = ConfigManager(pyodide_root) + config_manager = CrossBuildEnvConfigManager(pyodide_root) env = config_manager.to_env() env.update( @@ -137,6 +137,12 @@ def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]: return env +@functools.cache +def get_host_build_environment_vars() -> dict[str, str]: + manager = ConfigManager() + return manager.to_env() + + def get_build_flag(name: str) -> str: """ Get a value of a build flag. @@ -149,6 +155,17 @@ def get_build_flag(name: str) -> str: return build_vars[name] +def get_host_build_flag(name: str) -> str: + """ + Get a value of a build flag without accessing the cross-build environment. + """ + build_vars = get_host_build_environment_vars() + if name not in build_vars: + raise ValueError(f"Unknown build flag: {name}") + + return build_vars[name] + + def get_pyversion_major() -> str: return get_build_flag("PYMAJOR") diff --git a/pyodide_build/config.py b/pyodide_build/config.py index 89cdc80..56af0bc 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -1,6 +1,7 @@ import os import subprocess from collections.abc import Mapping +from copy import deepcopy from pathlib import Path from types import MappingProxyType @@ -13,8 +14,81 @@ class ConfigManager: + """ + Configuration manager for pyodide-build. + This class works "before" installing the cross build environment. + So it does not have access to the variables that are retrieved from the cross build environment. + + Most of the times, use CrossBuildEnvConfigManager instead of this class. + But if you need to access the configuration without installing the cross build environment, use this class. + """ + + def __init__(self): + self._config = { + **self._load_default_config(), + **self._load_cross_build_envs(), + **self._load_config_file(Path.cwd(), os.environ), + **self._load_config_from_env(os.environ), + } + + def _load_default_config(self) -> Mapping[str, str]: + return deepcopy(DEFAULT_CONFIG) + + def _load_cross_build_envs(self) -> Mapping[str, str]: + """ + Load environment variables from the cross build environment. + """ + + # This method should be implemented in the subclass. + return {} + + def _load_config_from_env(self, env: Mapping[str, str]) -> Mapping[str, str]: + return { + BUILD_VAR_TO_KEY[key]: env[key] for key in env if key in BUILD_VAR_TO_KEY + } + + def _load_config_file( + self, curdir: Path, env: Mapping[str, str] + ) -> Mapping[str, str]: + pyproject_path, configs = search_pyproject_toml(curdir) + + if pyproject_path is None or configs is None: + return {} + + if ( + "tool" in configs + and "pyodide" in configs["tool"] + and "build" in configs["tool"]["pyodide"] + ): + build_config = {} + for key, v in configs["tool"]["pyodide"]["build"].items(): + if key not in OVERRIDABLE_BUILD_KEYS: + logger.warning( + "WARNING: The provided build key %s is either invalid or not overridable, hence ignored.", + key, + ) + continue + build_config[key] = _environment_substitute_str(v, env) + + return build_config + else: + return {} + + @property + def config(self) -> Mapping[str, str]: + return MappingProxyType(self._config) + + def to_env(self) -> dict[str, str]: + """ + Export the configuration to environment variables. + """ + return {BUILD_KEY_TO_VAR[k]: v for k, v in self.config.items()} + + +class CrossBuildEnvConfigManager(ConfigManager): """ Configuration manager for Package build process. + This class works "after" installing the cross build environment. The configuration manager is responsible for loading configuration from various sources. The configuration can be loaded from the following sources (in order of precedence): @@ -28,22 +102,9 @@ class ConfigManager: def __init__(self, pyodide_root: Path): self.pyodide_root = pyodide_root - self._config = { - **self._load_default_config(), - **self._load_makefile_envs(), - **self._load_config_file(Path.cwd(), os.environ), - **self._load_config_from_env(os.environ), - } + super().__init__() - def _load_default_config(self) -> Mapping[str, str]: - return { - k: _environment_substitute_str( - v, env={"PYODIDE_ROOT": str(self.pyodide_root)} - ) - for k, v in DEFAULT_CONFIG.items() - } - - def _load_makefile_envs(self) -> Mapping[str, str]: + def _load_cross_build_envs(self) -> Mapping[str, str]: makefile_vars = self._get_make_environment_vars() computed_vars = { k: _environment_substitute_str(v, env=makefile_vars) @@ -89,48 +150,6 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: return environment - def _load_config_from_env(self, env: Mapping[str, str]) -> Mapping[str, str]: - return { - BUILD_VAR_TO_KEY[key]: env[key] for key in env if key in BUILD_VAR_TO_KEY - } - - def _load_config_file( - self, curdir: Path, env: Mapping[str, str] - ) -> Mapping[str, str]: - pyproject_path, configs = search_pyproject_toml(curdir) - - if pyproject_path is None or configs is None: - return {} - - if ( - "tool" in configs - and "pyodide" in configs["tool"] - and "build" in configs["tool"]["pyodide"] - ): - build_config = {} - for key, v in configs["tool"]["pyodide"]["build"].items(): - if key not in OVERRIDABLE_BUILD_KEYS: - logger.warning( - "WARNING: The provided build key %s is either invalid or not overridable, hence ignored.", - key, - ) - continue - build_config[key] = _environment_substitute_str(v, env) - - return build_config - else: - return {} - - @property - def config(self) -> Mapping[str, str]: - return MappingProxyType(self._config) - - def to_env(self) -> dict[str, str]: - """ - Export the configuration to environment variables. - """ - return {BUILD_KEY_TO_VAR[k]: v for k, v in self.config.items()} - # Configuration variables and corresponding environment variables. # TODO: distinguish between variables that are overridable by the user and those that are not. diff --git a/pyodide_build/tests/conftest.py b/pyodide_build/tests/conftest.py index 2a50483..0bbd9d4 100644 --- a/pyodide_build/tests/conftest.py +++ b/pyodide_build/tests/conftest.py @@ -61,6 +61,7 @@ def reset_cache(): def _reset(): build_env.get_pyodide_root.cache_clear() + build_env.get_host_build_environment_vars.cache_clear() build_env.get_build_environment_vars.cache_clear() build_env.get_unisolated_packages.cache_clear() diff --git a/pyodide_build/tests/test_config.py b/pyodide_build/tests/test_config.py index 69b4911..d6a1890 100644 --- a/pyodide_build/tests/test_config.py +++ b/pyodide_build/tests/test_config.py @@ -4,27 +4,71 @@ DEFAULT_CONFIG, DEFAULT_CONFIG_COMPUTED, ConfigManager, + CrossBuildEnvConfigManager, ) from pyodide_build.xbuildenv import CrossBuildEnvManager -class TestConfigManager_OutOfTree: +class TestConfigManager: + def test_default_config(self, reset_env_vars, reset_cache): + config_manager = ConfigManager() + default_config = config_manager._load_default_config() + assert default_config.keys() == DEFAULT_CONFIG.keys() + + def test_load_config_from_env(self, reset_env_vars, reset_cache): + config_manager = ConfigManager() + env = { + "CMAKE_TOOLCHAIN_FILE": "/path/to/toolchain", + "MESON_CROSS_FILE": "/path/to/crossfile", + } + + config = config_manager._load_config_from_env(env) + assert config["cmake_toolchain_file"] == "/path/to/toolchain" + assert config["meson_cross_file"] == "/path/to/crossfile" + + def test_load_config_from_file(self, tmp_path, reset_env_vars, reset_cache): + pyproject_file = tmp_path / "pyproject.toml" + + env = { + "MESON_CROSS_FILE": "/path/to/crossfile", + } + + pyproject_file.write_text("""[tool.pyodide.build] + invalid_flags = "this_should_not_be_parsed" + skip_emscripten_version_check = "1" + """) + + config_manager = ConfigManager() + config = config_manager._load_config_file(pyproject_file, env) + + assert "invalid_flags" not in config + assert ( + config["skip_emscripten_version_check"] + == "1" + ) + + +class TestCrossBuildEnvConfigManager_OutOfTree: def test_default_config(self, dummy_xbuildenv, reset_env_vars, reset_cache): xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) default_config = config_manager._load_default_config() assert default_config.keys() == DEFAULT_CONFIG.keys() - def test_makefile_envs(self, dummy_xbuildenv, reset_env_vars, reset_cache): + def test_cross_build_envs(self, dummy_xbuildenv, reset_env_vars, reset_cache): xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) - makefile_vars = config_manager._load_makefile_envs() + makefile_vars = config_manager._load_cross_build_envs() # It should contain information about the cpython and emscripten versions assert "pyversion" in makefile_vars @@ -41,7 +85,9 @@ def test_get_make_environment_vars( xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) make_vars = config_manager._get_make_environment_vars() assert make_vars["PYODIDE_ROOT"] == str(xbuildenv_manager.pyodide_root) @@ -49,9 +95,11 @@ def test_computed_vars(self, dummy_xbuildenv, reset_env_vars, reset_cache): xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) - makefile_vars = config_manager._load_makefile_envs() + makefile_vars = config_manager._load_cross_build_envs() for k, v in DEFAULT_CONFIG_COMPUTED.items(): assert k in makefile_vars @@ -62,7 +110,9 @@ def test_load_config_from_env(self, dummy_xbuildenv, reset_env_vars, reset_cache xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) env = { "CMAKE_TOOLCHAIN_FILE": "/path/to/toolchain", @@ -95,7 +145,9 @@ def test_load_config_from_file( xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) config = config_manager._load_config_file(pyproject_file, env) @@ -110,7 +162,9 @@ def test_config_all(self, dummy_xbuildenv, reset_env_vars, reset_cache): xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) config = config_manager.config for key in BUILD_KEY_TO_VAR.keys(): @@ -120,7 +174,9 @@ def test_to_env(self, dummy_xbuildenv, reset_env_vars, reset_cache): xbuildenv_manager = CrossBuildEnvManager( dummy_xbuildenv / common.xbuildenv_dirname() ) - config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root) + config_manager = CrossBuildEnvConfigManager( + pyodide_root=xbuildenv_manager.pyodide_root + ) env = config_manager.to_env() for env_var in BUILD_KEY_TO_VAR.values(): assert env_var in env