Split config manager

pyodide_build/build_env.py

@@ -15,7 +15,7 @@
 
 from pyodide_build import __version__
 from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname
-from pyodide_build.config import ConfigManager
+from pyodide_build.config import ConfigManager, CrossBuildEnvConfigManager
 from pyodide_build.recipe import load_all_recipes
 
 RUST_BUILD_PRELUDE = """
@@ -120,7 +120,7 @@ def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]:
     """
     Get common environment variables for the in-tree and out-of-tree build.
     """
-    config_manager = ConfigManager(pyodide_root)
+    config_manager = CrossBuildEnvConfigManager(pyodide_root)
     env = config_manager.to_env()
 
     env.update(
@@ -137,6 +137,12 @@ def get_build_environment_vars(pyodide_root: Path) -> dict[str, str]:
     return env
 
 
+@functools.cache
+def get_host_build_environment_vars() -> dict[str, str]:
+    manager = ConfigManager()
+    return manager.to_env()
+
+
 def get_build_flag(name: str) -> str:
     """
     Get a value of a build flag.
@@ -149,6 +155,17 @@ def get_build_flag(name: str) -> str:
     return build_vars[name]
 
 
+def get_host_build_flag(name: str) -> str:
+    """
+    Get a value of a build flag without accessing the cross-build environment.
+    """
+    build_vars = get_host_build_environment_vars()
+    if name not in build_vars:
+        raise ValueError(f"Unknown build flag: {name}")
+
+    return build_vars[name]
+
+
 def get_pyversion_major() -> str:
     return get_build_flag("PYMAJOR")
 

pyodide_build/config.py

@@ -1,6 +1,7 @@
 import os
 import subprocess
 from collections.abc import Mapping
+from copy import deepcopy
 from pathlib import Path
 from types import MappingProxyType
 
@@ -13,8 +14,81 @@
 
 
 class ConfigManager:
+    """
+    Configuration manager for pyodide-build.
+    This class works "before" installing the cross build environment.
+    So it does not have access to the variables that are retrieved from the cross build environment.
+
+    Most of the times, use CrossBuildEnvConfigManager instead of this class.
+    But if you need to access the configuration without installing the cross build environment, use this class.
+    """
+
+    def __init__(self):
+        self._config = {
+            **self._load_default_config(),
+            **self._load_cross_build_envs(),
+            **self._load_config_file(Path.cwd(), os.environ),
+            **self._load_config_from_env(os.environ),
+        }
+
+    def _load_default_config(self) -> Mapping[str, str]:
+        return deepcopy(DEFAULT_CONFIG)
+
+    def _load_cross_build_envs(self) -> Mapping[str, str]:
+        """
+        Load environment variables from the cross build environment.
+        """
+
+        # This method should be implemented in the subclass.
+        return {}
+
+    def _load_config_from_env(self, env: Mapping[str, str]) -> Mapping[str, str]:
+        return {
+            BUILD_VAR_TO_KEY[key]: env[key] for key in env if key in BUILD_VAR_TO_KEY
+        }
+
+    def _load_config_file(
+        self, curdir: Path, env: Mapping[str, str]
+    ) -> Mapping[str, str]:
+        pyproject_path, configs = search_pyproject_toml(curdir)
+
+        if pyproject_path is None or configs is None:
+            return {}
+
+        if (
+            "tool" in configs
+            and "pyodide" in configs["tool"]
+            and "build" in configs["tool"]["pyodide"]
+        ):
+            build_config = {}
+            for key, v in configs["tool"]["pyodide"]["build"].items():
+                if key not in OVERRIDABLE_BUILD_KEYS:
+                    logger.warning(
+                        "WARNING: The provided build key %s is either invalid or not overridable, hence ignored.",
+                        key,
+                    )
+                    continue
+                build_config[key] = _environment_substitute_str(v, env)
+
+            return build_config
+        else:
+            return {}
+
+    @property
+    def config(self) -> Mapping[str, str]:
+        return MappingProxyType(self._config)
+
+    def to_env(self) -> dict[str, str]:
+        """
+        Export the configuration to environment variables.
+        """
+        return {BUILD_KEY_TO_VAR[k]: v for k, v in self.config.items()}
+
+
+class CrossBuildEnvConfigManager(ConfigManager):
     """
     Configuration manager for Package build process.
+    This class works "after" installing the cross build environment.
 
     The configuration manager is responsible for loading configuration from various sources.
     The configuration can be loaded from the following sources (in order of precedence):
@@ -28,22 +102,9 @@ class ConfigManager:
 
     def __init__(self, pyodide_root: Path):
         self.pyodide_root = pyodide_root
-        self._config = {
-            **self._load_default_config(),
-            **self._load_makefile_envs(),
-            **self._load_config_file(Path.cwd(), os.environ),
-            **self._load_config_from_env(os.environ),
-        }
+        super().__init__()
 
-    def _load_default_config(self) -> Mapping[str, str]:
-        return {
-            k: _environment_substitute_str(
-                v, env={"PYODIDE_ROOT": str(self.pyodide_root)}
-            )
-            for k, v in DEFAULT_CONFIG.items()
-        }
-
-    def _load_makefile_envs(self) -> Mapping[str, str]:
+    def _load_cross_build_envs(self) -> Mapping[str, str]:
         makefile_vars = self._get_make_environment_vars()
         computed_vars = {
             k: _environment_substitute_str(v, env=makefile_vars)
@@ -89,48 +150,6 @@ def _get_make_environment_vars(self) -> Mapping[str, str]:
 
         return environment
 
-    def _load_config_from_env(self, env: Mapping[str, str]) -> Mapping[str, str]:
-        return {
-            BUILD_VAR_TO_KEY[key]: env[key] for key in env if key in BUILD_VAR_TO_KEY
-        }
-
-    def _load_config_file(
-        self, curdir: Path, env: Mapping[str, str]
-    ) -> Mapping[str, str]:
-        pyproject_path, configs = search_pyproject_toml(curdir)
-
-        if pyproject_path is None or configs is None:
-            return {}
-
-        if (
-            "tool" in configs
-            and "pyodide" in configs["tool"]
-            and "build" in configs["tool"]["pyodide"]
-        ):
-            build_config = {}
-            for key, v in configs["tool"]["pyodide"]["build"].items():
-                if key not in OVERRIDABLE_BUILD_KEYS:
-                    logger.warning(
-                        "WARNING: The provided build key %s is either invalid or not overridable, hence ignored.",
-                        key,
-                    )
-                    continue
-                build_config[key] = _environment_substitute_str(v, env)
-
-            return build_config
-        else:
-            return {}
-
-    @property
-    def config(self) -> Mapping[str, str]:
-        return MappingProxyType(self._config)
-
-    def to_env(self) -> dict[str, str]:
-        """
-        Export the configuration to environment variables.
-        """
-        return {BUILD_KEY_TO_VAR[k]: v for k, v in self.config.items()}
-
 
 # Configuration variables and corresponding environment variables.
 # TODO: distinguish between variables that are overridable by the user and those that are not.

pyodide_build/tests/conftest.py

@@ -61,6 +61,7 @@ def reset_cache():
 
     def _reset():
         build_env.get_pyodide_root.cache_clear()
+        build_env.get_host_build_environment_vars.cache_clear()
         build_env.get_build_environment_vars.cache_clear()
         build_env.get_unisolated_packages.cache_clear()
 

pyodide_build/tests/test_config.py

@@ -4,27 +4,71 @@
     DEFAULT_CONFIG,
     DEFAULT_CONFIG_COMPUTED,
     ConfigManager,
+    CrossBuildEnvConfigManager,
 )
 from pyodide_build.xbuildenv import CrossBuildEnvManager
 
 
-class TestConfigManager_OutOfTree:
+class TestConfigManager:
+    def test_default_config(self, reset_env_vars, reset_cache):
+        config_manager = ConfigManager()
+        default_config = config_manager._load_default_config()
+        assert default_config.keys() == DEFAULT_CONFIG.keys()
+
+    def test_load_config_from_env(self, reset_env_vars, reset_cache):
+        config_manager = ConfigManager()
+        env = {
+            "CMAKE_TOOLCHAIN_FILE": "/path/to/toolchain",
+            "MESON_CROSS_FILE": "/path/to/crossfile",
+        }
+
+        config = config_manager._load_config_from_env(env)
+        assert config["cmake_toolchain_file"] == "/path/to/toolchain"
+        assert config["meson_cross_file"] == "/path/to/crossfile"
+
+    def test_load_config_from_file(self, tmp_path, reset_env_vars, reset_cache):
+        pyproject_file = tmp_path / "pyproject.toml"
+
+        env = {
+            "MESON_CROSS_FILE": "/path/to/crossfile",
+        }
+
+        pyproject_file.write_text("""[tool.pyodide.build]
+                                  invalid_flags = "this_should_not_be_parsed"
+                                  skip_emscripten_version_check = "1"
+                                  """)
+
+        config_manager = ConfigManager()
+        config = config_manager._load_config_file(pyproject_file, env)
+
+        assert "invalid_flags" not in config
+        assert (
+            config["skip_emscripten_version_check"]
+            == "1"
+        )
+
+
+class TestCrossBuildEnvConfigManager_OutOfTree:
     def test_default_config(self, dummy_xbuildenv, reset_env_vars, reset_cache):
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
 
         default_config = config_manager._load_default_config()
         assert default_config.keys() == DEFAULT_CONFIG.keys()
 
-    def test_makefile_envs(self, dummy_xbuildenv, reset_env_vars, reset_cache):
+    def test_cross_build_envs(self, dummy_xbuildenv, reset_env_vars, reset_cache):
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
 
-        makefile_vars = config_manager._load_makefile_envs()
+        makefile_vars = config_manager._load_cross_build_envs()
 
         # It should contain information about the cpython and emscripten versions
         assert "pyversion" in makefile_vars
@@ -41,17 +85,21 @@ def test_get_make_environment_vars(
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
         make_vars = config_manager._get_make_environment_vars()
         assert make_vars["PYODIDE_ROOT"] == str(xbuildenv_manager.pyodide_root)
 
     def test_computed_vars(self, dummy_xbuildenv, reset_env_vars, reset_cache):
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
 
-        makefile_vars = config_manager._load_makefile_envs()
+        makefile_vars = config_manager._load_cross_build_envs()
 
         for k, v in DEFAULT_CONFIG_COMPUTED.items():
             assert k in makefile_vars
@@ -62,7 +110,9 @@ def test_load_config_from_env(self, dummy_xbuildenv, reset_env_vars, reset_cache
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
 
         env = {
             "CMAKE_TOOLCHAIN_FILE": "/path/to/toolchain",
@@ -95,7 +145,9 @@ def test_load_config_from_file(
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
 
         config = config_manager._load_config_file(pyproject_file, env)
 
@@ -110,7 +162,9 @@ def test_config_all(self, dummy_xbuildenv, reset_env_vars, reset_cache):
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
         config = config_manager.config
 
         for key in BUILD_KEY_TO_VAR.keys():
@@ -120,7 +174,9 @@ def test_to_env(self, dummy_xbuildenv, reset_env_vars, reset_cache):
         xbuildenv_manager = CrossBuildEnvManager(
             dummy_xbuildenv / common.xbuildenv_dirname()
         )
-        config_manager = ConfigManager(pyodide_root=xbuildenv_manager.pyodide_root)
+        config_manager = CrossBuildEnvConfigManager(
+            pyodide_root=xbuildenv_manager.pyodide_root
+        )
         env = config_manager.to_env()
         for env_var in BUILD_KEY_TO_VAR.values():
             assert env_var in env