MAINT Rename recipe-related scripts (#92)

This PR makes a subdirectory `pyodide-build/recipe` and puts all the
recipe-related codes into that directory. No functional changes are
included. This PR is to separate code that is related to the recipes,
making it easier for new contributors to understand the code.

Also renamed the files as follows:

- `buildall.py` -> `graph_builder.py`
-  `buildpkg.py` -> `builder.py`
-  `io.py` -> `spec.py`
-  `mkpkg.py` -> `skeleton.py`
-  `recipe.py` -> `loader.py`

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

pyodide_build/build_env.py

@@ -16,7 +16,6 @@
 from pyodide_build import __version__
 from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname
 from pyodide_build.config import ConfigManager, CrossBuildEnvConfigManager
-from pyodide_build.recipe import load_all_recipes
 
 RUST_BUILD_PRELUDE = """
 rustup default ${RUST_TOOLCHAIN}
@@ -188,13 +187,17 @@ def get_hostsitepackages() -> str:
 
 @functools.cache
 def get_unisolated_packages() -> list[str]:
+    # TODO: Remove this function (and use remote package index)
+    # https://github.com/pyodide/pyodide-build/issues/43
     PYODIDE_ROOT = get_pyodide_root()
 
     unisolated_file = PYODIDE_ROOT / "unisolated.txt"
     if unisolated_file.exists():
         # in xbuild env, read from file
         unisolated_packages = unisolated_file.read_text().splitlines()
     else:
+        from pyodide_build.recipe.loader import load_all_recipes
+
         unisolated_packages = []
         recipe_dir = PYODIDE_ROOT / "packages"
         recipes = load_all_recipes(recipe_dir)

pyodide_build/cli/build.py

@@ -15,7 +15,6 @@
     get_pyodide_root,
     init_environment,
 )
-from pyodide_build.io import _BuildSpecExports, _ExportTypes
 from pyodide_build.logger import logger
 from pyodide_build.out_of_tree import build
 from pyodide_build.out_of_tree.pypi import (
@@ -24,6 +23,7 @@
     fetch_pypi_package,
 )
 from pyodide_build.pypabuild import parse_backend_flags
+from pyodide_build.spec import _BuildSpecExports, _ExportTypes
 
 
 def convert_exports(exports: str) -> _BuildSpecExports:

pyodide_build/cli/build_recipes.py

@@ -4,11 +4,12 @@
 
 import typer
 
-from pyodide_build import build_env, buildall, recipe
+from pyodide_build import build_env
 from pyodide_build.build_env import BuildArgs, init_environment
-from pyodide_build.buildpkg import RecipeBuilder
 from pyodide_build.common import get_num_cores
 from pyodide_build.logger import logger
+from pyodide_build.recipe import graph_builder, loader
+from pyodide_build.recipe.builder import RecipeBuilder
 
 
 @dataclasses.dataclass(eq=False, order=False, kw_only=True)
@@ -115,7 +116,7 @@ def build_recipes_no_deps(
         target_install_dir=target_install_dir,
         host_install_dir=host_install_dir,
     )
-    build_args = buildall.set_default_build_args(build_args)
+    build_args = graph_builder.set_default_build_args(build_args)
     args = Args(
         build_args=build_args,
         build_dir=build_dir,
@@ -127,10 +128,10 @@ def build_recipes_no_deps(
     return build_recipes_no_deps_impl(packages, args, continue_)
 
 
-def _rust_setup(recipe_dir: Path, packages: list[str]):
-    recipes = recipe.load_recipes(recipe_dir, packages, False)
+def _rust_setup(recipe_dir: Path, packages: list[str]) -> None:
+    recipes = loader.load_recipes(recipe_dir, packages, False)
     if any(recipe.is_rust_package() for recipe in recipes.values()):
-        buildall._ensure_rust_toolchain()
+        graph_builder._ensure_rust_toolchain()
 
 
 def build_recipes_no_deps_impl(
@@ -247,7 +248,7 @@ def build_recipes(
         target_install_dir=target_install_dir,
         host_install_dir=host_install_dir,
     )
-    build_args = buildall.set_default_build_args(build_args)
+    build_args = graph_builder.set_default_build_args(build_args)
     args = Args(
         build_args=build_args,
         build_dir=build_dir,
@@ -273,7 +274,7 @@ def build_recipes_impl(
     else:
         targets = ",".join(packages)
 
-    pkg_map = buildall.build_packages(
+    pkg_map = graph_builder.build_packages(
         args.recipe_dir,
         targets=targets,
         build_args=args.build_args,
@@ -283,10 +284,10 @@ def build_recipes_impl(
     )
 
     if log_dir:
-        buildall.copy_logs(pkg_map, log_dir)
+        graph_builder.copy_logs(pkg_map, log_dir)
 
     if install_options:
-        buildall.install_packages(
+        graph_builder.install_packages(
             pkg_map,
             args.install_dir,
             compression_level=install_options.compression_level,

pyodide_build/cli/skeleton.py

@@ -6,8 +6,9 @@
 
 import typer
 
-from pyodide_build import build_env, mkpkg
+from pyodide_build import build_env
 from pyodide_build.logger import logger
+from pyodide_build.recipe import skeleton
 
 app = typer.Typer()
 
@@ -72,20 +73,20 @@ def new_recipe_pypi(
 
     if update or update_patched:
         try:
-            mkpkg.update_package(
+            skeleton.update_package(
                 recipe_dir_,
                 name,
                 version,
                 source_fmt=source_format,  # type: ignore[arg-type]
                 update_patched=update_patched,
             )
-        except mkpkg.MkpkgFailedException as e:
+        except skeleton.MkpkgFailedException as e:
             logger.error("%s update failed: %s", name, e)
             sys.exit(1)
-        except mkpkg.MkpkgSkipped as e:
+        except skeleton.MkpkgSkipped as e:
             logger.warning("%s update skipped: %s", name, e)
         except Exception:
             print(name)
             raise
     else:
-        mkpkg.make_package(recipe_dir_, name, version, source_fmt=source_format)  # type: ignore[arg-type]
+        skeleton.make_package(recipe_dir_, name, version, source_fmt=source_format)  # type: ignore[arg-type]

pyodide_build/out_of_tree/build.py

@@ -5,7 +5,7 @@
 
 from pyodide_build import build_env, common, pypabuild
 from pyodide_build.build_env import get_pyodide_root, wheel_platform
-from pyodide_build.io import _BuildSpecExports
+from pyodide_build.spec import _BuildSpecExports
 
 
 def run(

pyodide_build/out_of_tree/pypi.py

@@ -29,9 +29,9 @@
 
 from pyodide_build import build_env
 from pyodide_build.common import repack_zip_archive
-from pyodide_build.io import _BuildSpecExports
 from pyodide_build.logger import logger
 from pyodide_build.out_of_tree import build
+from pyodide_build.spec import _BuildSpecExports
 
 _PYPI_INDEX = ["https://pypi.org/simple/"]
 _PYPI_TRUSTED_HOSTS = ["pypi.org"]

pyodide_build/pypabuild.py

@@ -23,7 +23,7 @@
     get_unisolated_packages,
     platform,
 )
-from pyodide_build.io import _BuildSpecExports
+from pyodide_build.spec import _BuildSpecExports
 from pyodide_build.vendor._pypabuild import (
     _STYLES,
     _DefaultIsolatedEnv,

pyodide_build/buildpkg.py → pyodide_build/recipe/builder.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 """
 Builds a Pyodide package.
 """
@@ -20,7 +18,6 @@
 import requests
 
 from pyodide_build import common, pypabuild
-from pyodide_build.bash_runner import BashRunnerWithSharedEnvironment, get_bash_runner
 from pyodide_build.build_env import (
     RUST_BUILD_PRELUDE,
     BuildArgs,
@@ -40,8 +37,12 @@
     modify_wheel,
     retag_wheel,
 )
-from pyodide_build.io import MetaConfig, _SourceSpec
 from pyodide_build.logger import logger
+from pyodide_build.recipe.bash_runner import (
+    BashRunnerWithSharedEnvironment,
+    get_bash_runner,
+)
+from pyodide_build.recipe.spec import MetaConfig, _SourceSpec
 
 
 def _make_whlfile(

pyodide_build/buildall.py → pyodide_build/recipe/graph_builder.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 """
 Build all of the packages in a given directory.
 """
@@ -29,18 +27,19 @@
 from rich.spinner import Spinner
 from rich.table import Table
 
-from pyodide_build import build_env, recipe
+from pyodide_build import build_env
 from pyodide_build.build_env import BuildArgs
-from pyodide_build.buildpkg import needs_rebuild
 from pyodide_build.common import (
     exit_with_stdio,
     extract_wheel_metadata_file,
     find_matching_wheels,
     find_missing_executables,
     repack_zip_archive,
 )
-from pyodide_build.io import MetaConfig, _BuildSpecTypes
 from pyodide_build.logger import console_stdout, logger
+from pyodide_build.recipe import loader
+from pyodide_build.recipe.builder import needs_rebuild
+from pyodide_build.recipe.spec import MetaConfig, _BuildSpecTypes
 
 
 class BuildError(Exception):
@@ -375,7 +374,7 @@ def generate_dependency_graph(
     # On first pass add all dependencies regardless of whether
     # disabled since it might happen because of a transitive dependency
     graph = {}
-    all_recipes = recipe.load_all_recipes(packages_dir)
+    all_recipes = loader.load_all_recipes(packages_dir)
     no_numpy_dependents = "no-numpy-dependents" in requested
     requested.discard("no-numpy-dependents")
     packages = requested.copy()
@@ -851,7 +850,7 @@ def build_packages(
     force_rebuild: bool = False,
 ) -> dict[str, BasePackage]:
     requested, disabled = _parse_package_query(targets)
-    requested_packages = recipe.load_recipes(packages_dir, requested)
+    requested_packages = loader.load_recipes(packages_dir, requested)
     pkg_map = generate_dependency_graph(
         packages_dir, set(requested_packages.keys()), disabled
     )

pyodide_build/recipe.py → pyodide_build/recipe/loader.py

@@ -2,8 +2,8 @@
 from collections.abc import Iterable
 from pathlib import Path
 
-from pyodide_build.io import MetaConfig
 from pyodide_build.logger import logger
+from pyodide_build.recipe.spec import MetaConfig
 
 
 @functools.lru_cache(maxsize=1)

pyodide_build/mkpkg.py → pyodide_build/recipe/skeleton.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 import contextlib
 import json
 import shutil

pyodide_build/io.py → pyodide_build/recipe/spec.py

@@ -4,6 +4,8 @@
 import pydantic
 from pydantic import BaseModel, ConfigDict, Field
 
+from pyodide_build.spec import _BuildSpecExports
+
 
 class _PackageSpec(BaseModel):
     name: str
@@ -69,8 +71,6 @@ def _check_patches_extra(self) -> Self:
         return self
 
 
-_ExportTypes = Literal["pyinit", "requested", "whole_archive"]
-_BuildSpecExports = _ExportTypes | list[str]
 _BuildSpecTypes = Literal[
     "package", "static_library", "shared_library", "cpython_module"
 ]

pyodide_build/spec.py

@@ -0,0 +1,4 @@
+from typing import Literal
+
+_ExportTypes = Literal["pyinit", "requested", "whole_archive"]
+_BuildSpecExports = _ExportTypes | list[str]

pyodide_build/tests/test_bash_runner.py → pyodide_build/tests/recipe/test_bash_runner.py

@@ -1,7 +1,7 @@
 import subprocess
 from pathlib import Path
 
-from pyodide_build import bash_runner
+from pyodide_build.recipe import bash_runner
 
 
 def test_subprocess_with_shared_env_1():