From fb114ab01e445bf808359213d8f4110a894fab72 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Thu, 3 Oct 2024 10:16:52 -0700 Subject: [PATCH] chore(docs): improved navigation and structure for API docs (#400) Follows pattern from our other rulesets, makes it easier to link users to docs. --- ### Changes are visible to end-users: no ### Test plan None --------- Co-authored-by: Matt Mackay --- .gitattributes | 1 + CONTRIBUTING.md | 4 +- README.md | 44 +++++-- docs/BUILD.bazel | 31 ++++- docs/migrating.md | 15 +++ docs/pex.md | 44 +++++++ docs/py_binary.md | 92 +++++++++++++ docs/py_library.md | 149 +++++++++++++++++++++ docs/py_test.md | 111 ++++++++++++++++ docs/rules.md | 249 ----------------------------------- docs/venv.md | 62 +++++++++ py/defs.bzl | 47 ++++++- py/private/BUILD.bazel | 16 +-- py/private/py_binary.bzl | 34 +---- py/private/py_library.bzl | 6 +- py/private/py_pex_binary.bzl | 33 +++-- py/private/py_venv.bzl | 28 +++- 17 files changed, 633 insertions(+), 333 deletions(-) create mode 100644 docs/migrating.md create mode 100644 docs/pex.md create mode 100644 docs/py_binary.md create mode 100644 docs/py_library.md create mode 100644 docs/py_test.md delete mode 100644 docs/rules.md create mode 100644 docs/venv.md diff --git a/.gitattributes b/.gitattributes index 151144a8..5e3c1501 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,5 @@ docs/*.md linguist-generated=true +docs/migrating.md linguist-generated=false # Configuration for 'git archive' # see https://git-scm.com/docs/git-archive/2.40.0#ATTRIBUTES diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a00925b..de2e794f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,11 +30,11 @@ artifact or a version fetched from the internet, run this from this directory: ```sh -OVERRIDE="--override_repository=rules_py=$(pwd)/rules_py" +OVERRIDE="--override_repository=aspect_rules_py=$(pwd)/rules_py" echo "common $OVERRIDE" >> ~/.bazelrc ``` -This means that any usage of `@rules_py` on your system will point to this folder. +This means that any usage of `@aspect_rules_py` on your system will point to this folder. ## Releasing diff --git a/README.md b/README.md index 2b51fb4d..809e6739 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,25 @@ The lower layer of `rules_python` is currently reused, dealing with the toolchai However, this ruleset introduces a new implementation of `py_library`, `py_binary`, and `py_test`. Our philosophy is to behave more like idiomatic python ecosystem tools, where rules_python is closely tied to the way Google does Python development in their internal monorepo, google3. +However we try to maintain compatibility with rules_python's rules for most use cases. -| Layer | Legacy | Recommended | -| ------------------------------------------- | ------------ | ---------------- | -| rules: BUILD file UI | rules_python | **rules_py** | -| gazelle: generate BUILD files | rules_python | rules_python [1] | -| pip_parse: fetch and install deps from pypi | rules_python | rules_python | -| toolchain: fetch hermetic interpreter | rules_python | rules_python | +| Layer | Legacy | Recommended | +| ------------------------------------------- | ------------ | -------------------- | +| toolchain: fetch hermetic interpreter | rules_python | rules_python | +| pip.parse: fetch and install deps from pypi | rules_python | rules_python | +| gazelle: generate BUILD files | rules_python | [`aspect configure`] | +| rules: user-facing implementations | rules_python | **rules_py** | + +Watch our video series for a quick tutorial on how rules_py makes it easy to do Python with Bazel: +[![youtube playlist](https://i.ytimg.com/vi/Ms9qX0Yyn0s/hqdefault.jpg)](https://www.youtube.com/playlist?list=PLLU28e_DRwdu46fldnYzyFYvSJLjVFICd) _Need help?_ This ruleset has support provided by https://aspect.dev. -[1] we will likely fork the extension for performance, using TreeSitter to parse Python code rather than a Python program. +[`aspect configure`]: https://docs.aspect.build/cli/commands/aspect_configure ## Differences -We think you'll love rules_py because: +We think you'll love rules_py because it fixes many issues with rules_python's rule implementations: - The launcher uses the Bash toolchain rather than Python, so we have no dependency on a system interpreter. Fixes: - [py_binary with hermetic toolchain requires a system interpreter](https://github.com/bazelbuild/rules_python/issues/691) @@ -32,8 +36,8 @@ We think you'll love rules_py because: - [sys.path[0] breaks out of runfile tree.](https://github.com/bazelbuild/rules_python/issues/382) - [User site-packages directory should be ignored](https://github.com/bazelbuild/rules_python/issues/1059) - We create a python-idiomatic virtualenv to run actions, which means better compatibility with userland implementations of [importlib](https://docs.python.org/3/library/importlib.html). -- Thanks to the virtualenv, you can open the project in an editor like PyCharm and have working auto-complete, jump-to-definition, etc. Fixes: - - [Smooth IDE support for python_rules](https://github.com/bazelbuild/rules_python/issues/1401) +- Thanks to the virtualenv, you can open the project in an editor like PyCharm or VSCode and have working auto-complete, jump-to-definition, etc. + - Fixes [Smooth IDE support for python_rules](https://github.com/bazelbuild/rules_python/issues/1401) > [!NOTE] > What about the "starlarkification" effort in rules_python? @@ -50,7 +54,7 @@ Follow instructions from the release you wish to use: ### Using with Gazelle -In any ancestor `BUILD` file of the Python code, add these lines to instruct [Gazelle] to create rules_py variants of the `py_*` rules: +In any ancestor `BUILD` file of the Python code, add these lines to instruct [Gazelle] to create `rules_py` variants of the `py_*` rules: ``` # gazelle:map_kind py_library py_library @aspect_rules_py//py:defs.bzl @@ -58,4 +62,20 @@ In any ancestor `BUILD` file of the Python code, add these lines to instruct [Ga # gazelle:map_kind py_test py_test @aspect_rules_py//py:defs.bzl ``` -[Gazelle]: https://github.com/bazelbuild/rules_python/blob/main/gazelle/README.md +[gazelle]: https://github.com/bazelbuild/rules_python/blob/main/gazelle/README.md + +# Public API + +## Executables + +- [py_binary](docs/py_binary.md) an executable Python program, used with `bazel run` or as a tool. +- [py_test](docs/py_test.md) a Python program that executes a test runner such as `unittest` or `pytest`, to be used with `bazel test`. +- [py_venv](docs/venv.md) create a virtualenv for a `py_binary` or `py_test` target for use outside Bazel, such as in an editor/IDE. + +## Packaging + +- [py_pex_binary](docs/pex.md) Create a zip file containing a full Python application. + +## Packages + +- [py_library](docs/py_library.md) a unit of Python code, used as a dependency of other rules. diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 763a7279..59785a2d 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -3,8 +3,37 @@ load("@aspect_bazel_lib//lib:docs.bzl", "stardoc_with_diff_test", "update_docs") stardoc_with_diff_test( - name = "rules", + name = "py_library", + bzl_library_target = "//py/private:py_library", +) + +stardoc_with_diff_test( + name = "py_binary", + bzl_library_target = "//py:defs", + symbol_names = [ + "py_binary", + "py_binary_rule", + ], +) + +stardoc_with_diff_test( + name = "py_test", bzl_library_target = "//py:defs", + symbol_names = [ + "py_test", + "py_test_rule", + "py_pytest_main", + ], +) + +stardoc_with_diff_test( + name = "pex", + bzl_library_target = "//py/private:py_pex_binary", +) + +stardoc_with_diff_test( + name = "venv", + bzl_library_target = "//py/private:py_venv", ) update_docs(name = "update") diff --git a/docs/migrating.md b/docs/migrating.md new file mode 100644 index 00000000..c1a13145 --- /dev/null +++ b/docs/migrating.md @@ -0,0 +1,15 @@ +# Migrating rules from rules_python to rules_py + +rules_py tries to closely mirror the API of rules_python. +Migration is a "drop-in replacement" for the majority of use cases. + +## Replace load statements + +Instead of loading from `@rules_python//python:defs.bzl`, load from `@aspect_rules_py//py:defs.bzl`. +The rest of the BUILD file can remain the same. + +If using Gazelle, see the note on [using with Gazelle](/README.md#using-with-gazelle) + +## Remaining notes + +Users are encouraged to send a Pull Request to add more documentation as they uncover issues during migrations. diff --git a/docs/pex.md b/docs/pex.md new file mode 100644 index 00000000..92504a3e --- /dev/null +++ b/docs/pex.md @@ -0,0 +1,44 @@ + + +Create a zip file containing a full Python application. + +Follows [PEP-441 (PEX)](https://peps.python.org/pep-0441/) + +## Ensuring a compatible interpreter is used + +The resulting zip file does *not* contain a Python interpreter. +Users are expected to execute the PEX with a compatible interpreter on the runtime system. + +Use the `python_interpreter_constraints` to provide an error if a wrong interpreter tries to execute the PEX, for example: + +```starlark +py_pex_binary( + python_interpreter_constraints = [ + "CPython=={major}.{minor}.{patch}", + ] +) +``` + + + + +## py_pex_binary + +
+py_pex_binary(name, binary, inject_env, python_interpreter_constraints, python_shebang)
+
+ +Build a pex executable from a py_binary + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| binary | A py_binary target | Label | required | | +| inject_env | Environment variables to set when running the pex binary. | Dictionary: String -> String | optional | {} | +| python_interpreter_constraints | Python interpreter versions this PEX binary is compatible with. A list of semver strings. The placeholder strings {major}, {minor}, {patch} can be used for gathering version information from the hermetic python toolchain. | List of strings | optional | ["CPython=={major}.{minor}.*"] | +| python_shebang | - | String | optional | "#!/usr/bin/env python3" | + + diff --git a/docs/py_binary.md b/docs/py_binary.md new file mode 100644 index 00000000..a384c2f0 --- /dev/null +++ b/docs/py_binary.md @@ -0,0 +1,92 @@ + + +Re-implementations of [py_binary](https://bazel.build/reference/be/python#py_binary) +and [py_test](https://bazel.build/reference/be/python#py_test) + +## Choosing the Python version + +The `python_version` attribute must refer to a python toolchain version +which has been registered in the WORKSPACE or MODULE.bazel file. + +When using WORKSPACE, this may look like this: + +```starlark +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +python_register_toolchains( + name = "python_toolchain_3_8", + python_version = "3.8.12", + # setting set_python_version_constraint makes it so that only matches py_* rule + # which has this exact version set in the `python_version` attribute. + set_python_version_constraint = True, +) + +# It's important to register the default toolchain last it will match any py_* target. +python_register_toolchains( + name = "python_toolchain", + python_version = "3.9", +) +``` + +Configuring for MODULE.bazel may look like this: + +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.8.12", is_default = False) +python.toolchain(python_version = "3.9", is_default = True) +``` + + + + +## py_binary_rule + +
+py_binary_rule(name, data, deps, env, imports, main, package_collisions, python_version,
+               resolutions, srcs)
+
+ +Run a Python program under Bazel. Most users should use the [py_binary macro](#py_binary) instead of loading this directly. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| data | Runtime dependencies of the program.

The transitive closure of the data dependencies will be available in the .runfiles folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. | List of labels | optional | [] | +| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | +| env | Environment variables to set when running the binary. | Dictionary: String -> String | optional | {} | +| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | +| main | Script to execute with the Python interpreter. | Label | required | | +| package_collisions | The action that should be taken when a symlink collision is encountered when creating the venv. A collision can occour when multiple packages providing the same file are installed into the venv. The possible values are:

* "error": When conflicting symlinks are found, an error is reported and venv creation halts. * "warning": When conflicting symlinks are found, an warning is reported, however venv creation continues. * "ignore": When conflicting symlinks are found, no message is reported and venv creation continues. | String | optional | "error" | +| python_version | Whether to build this target and its transitive deps for a specific python version. | String | optional | "" | +| resolutions | FIXME | Dictionary: Label -> String | optional | {} | +| srcs | Python source files. | List of labels | optional | [] | + + + + +## py_binary + +
+py_binary(name, srcs, main, kwargs)
+
+ +Wrapper macro for [`py_binary_rule`](#py_binary_rule). + +Creates a [py_venv](./venv.md) target to constrain the interpreter and packages used at runtime. +Users can `bazel run [name].venv` to create this virtualenv, then use it in the editor or other tools. + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | Name of the rule. | none | +| srcs | Python source files. | [] | +| main | Entry point. Like rules_python, this is treated as a suffix of a file that should appear among the srcs. If absent, then [name].py is tried. As a final fallback, if the srcs has a single file, that is used as the main. | None | +| kwargs | additional named parameters to py_binary_rule. | none | + + diff --git a/docs/py_library.md b/docs/py_library.md new file mode 100644 index 00000000..2f8b2721 --- /dev/null +++ b/docs/py_library.md @@ -0,0 +1,149 @@ + + +A re-implementation of [py_library](https://bazel.build/reference/be/python#py_library). + +Supports "virtual" dependencies with a `virtual_deps` attribute, which lists packages which are required +without binding them to a particular version of that package. + + + + +## py_library + +
+py_library(name, data, deps, imports, resolutions, srcs, virtual_deps)
+
+ + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| data | Runtime dependencies of the program.

The transitive closure of the data dependencies will be available in the .runfiles folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. | List of labels | optional | [] | +| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | +| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | +| resolutions | FIXME | Dictionary: Label -> String | optional | {} | +| srcs | Python source files. | List of labels | optional | [] | +| virtual_deps | - | List of strings | optional | [] | + + + + +## py_library_utils.implementation + +
+py_library_utils.implementation(ctx)
+
+ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |

-

| none | + + + + +## py_library_utils.make_imports_depset + +
+py_library_utils.make_imports_depset(ctx, imports, extra_imports_depsets)
+
+ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |

-

| none | +| imports |

-

| [] | +| extra_imports_depsets |

-

| [] | + + + + +## py_library_utils.make_instrumented_files_info + +
+py_library_utils.make_instrumented_files_info(ctx, extra_source_attributes,
+                                              extra_dependency_attributes)
+
+ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |

-

| none | +| extra_source_attributes |

-

| [] | +| extra_dependency_attributes |

-

| [] | + + + + +## py_library_utils.make_merged_runfiles + +
+py_library_utils.make_merged_runfiles(ctx, extra_depsets, extra_runfiles, extra_runfiles_depsets)
+
+ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |

-

| none | +| extra_depsets |

-

| [] | +| extra_runfiles |

-

| [] | +| extra_runfiles_depsets |

-

| [] | + + + + +## py_library_utils.make_srcs_depset + +
+py_library_utils.make_srcs_depset(ctx)
+
+ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |

-

| none | + + + + +## py_library_utils.resolve_virtuals + +
+py_library_utils.resolve_virtuals(ctx, ignore_missing)
+
+ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |

-

| none | +| ignore_missing |

-

| False | + + diff --git a/docs/py_test.md b/docs/py_test.md new file mode 100644 index 00000000..d194fd5e --- /dev/null +++ b/docs/py_test.md @@ -0,0 +1,111 @@ + + +Re-implementations of [py_binary](https://bazel.build/reference/be/python#py_binary) +and [py_test](https://bazel.build/reference/be/python#py_test) + +## Choosing the Python version + +The `python_version` attribute must refer to a python toolchain version +which has been registered in the WORKSPACE or MODULE.bazel file. + +When using WORKSPACE, this may look like this: + +```starlark +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +python_register_toolchains( + name = "python_toolchain_3_8", + python_version = "3.8.12", + # setting set_python_version_constraint makes it so that only matches py_* rule + # which has this exact version set in the `python_version` attribute. + set_python_version_constraint = True, +) + +# It's important to register the default toolchain last it will match any py_* target. +python_register_toolchains( + name = "python_toolchain", + python_version = "3.9", +) +``` + +Configuring for MODULE.bazel may look like this: + +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.8.12", is_default = False) +python.toolchain(python_version = "3.9", is_default = True) +``` + + + + +## py_test_rule + +
+py_test_rule(name, data, deps, env, imports, main, package_collisions, python_version, resolutions,
+             srcs)
+
+ +Run a Python program under Bazel. Most users should use the [py_test macro](#py_test) instead of loading this directly. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| data | Runtime dependencies of the program.

The transitive closure of the data dependencies will be available in the .runfiles folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. | List of labels | optional | [] | +| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | +| env | Environment variables to set when running the binary. | Dictionary: String -> String | optional | {} | +| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | +| main | Script to execute with the Python interpreter. | Label | required | | +| package_collisions | The action that should be taken when a symlink collision is encountered when creating the venv. A collision can occour when multiple packages providing the same file are installed into the venv. The possible values are:

* "error": When conflicting symlinks are found, an error is reported and venv creation halts. * "warning": When conflicting symlinks are found, an warning is reported, however venv creation continues. * "ignore": When conflicting symlinks are found, no message is reported and venv creation continues. | String | optional | "error" | +| python_version | Whether to build this target and its transitive deps for a specific python version. | String | optional | "" | +| resolutions | FIXME | Dictionary: Label -> String | optional | {} | +| srcs | Python source files. | List of labels | optional | [] | + + + + +## py_pytest_main + +
+py_pytest_main(name, py_library, deps, data, testonly, kwargs)
+
+ +py_pytest_main wraps the template rendering target and the final py_library. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | The name of the runable target that updates the test entry file. | none | +| py_library | Use this attribute to override the default py_library rule. | <unknown object com.google.devtools.build.skydoc.fakebuildapi.FakeStarlarkRuleFunctionsApi$RuleDefinitionIdentifier> | +| deps | A list containing the pytest library target, e.g., @pypi_pytest//:pkg. | [] | +| data | A list of data dependencies to pass to the py_library target. | [] | +| testonly | A boolean indicating if the py_library target is testonly. | True | +| kwargs | The extra arguments passed to the template rendering target. | none | + + + + +## py_test + +
+py_test(name, main, srcs, kwargs)
+
+ +Identical to [py_binary](./py_binary.md), but produces a target that can be used with `bazel test`. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name |

-

| none | +| main |

-

| None | +| srcs |

-

| [] | +| kwargs |

-

| none | + + diff --git a/docs/rules.md b/docs/rules.md deleted file mode 100644 index 9078fd76..00000000 --- a/docs/rules.md +++ /dev/null @@ -1,249 +0,0 @@ - - -Public API re-exports - - - -## py_binary_rule - -
-py_binary_rule(name, data, deps, env, imports, main, package_collisions, python_version,
-               resolutions, srcs)
-
- -Run a Python program under Bazel. Most users should use the [py_binary macro](#py_binary) instead of loading this directly. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this target. | Name | required | | -| data | Runtime dependencies of the program.

The transitive closure of the data dependencies will be available in the .runfiles folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. | List of labels | optional | [] | -| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | -| env | Environment variables to set when running the binary. | Dictionary: String -> String | optional | {} | -| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | -| main | Script to execute with the Python interpreter. | Label | required | | -| package_collisions | The action that should be taken when a symlink collision is encountered when creating the venv. A collision can occour when multiple packages providing the same file are installed into the venv. The possible values are:

* "error": When conflicting symlinks are found, an error is reported and venv creation halts. * "warning": When conflicting symlinks are found, an warning is reported, however venv creation continues. * "ignore": When conflicting symlinks are found, no message is reported and venv creation continues. | String | optional | "error" | -| python_version | Whether to build this target and its transitive deps for a specific python version.

Note that setting this attribute alone will not be enough as the python toolchain for the desired version also needs to be registered in the WORKSPACE or MODULE.bazel file.

When using WORKSPACE, this may look like this,

 load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")

python_register_toolchains(     name = "python_toolchain_3_8",     python_version = "3.8.12",     # setting set_python_version_constraint makes it so that only matches py_* rule       # which has this exact version set in the python_version attribute.     set_python_version_constraint = True, )

# It's important to register the default toolchain last it will match any py_* target.  python_register_toolchains(     name = "python_toolchain",     python_version = "3.9", )


Configuring for MODULE.bazel may look like this:

 python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.8.12", is_default = False) python.toolchain(python_version = "3.9", is_default = True)
| String | optional | "" | -| resolutions | FIXME | Dictionary: Label -> String | optional | {} | -| srcs | Python source files. | List of labels | optional | [] | - - - - -## py_library - -
-py_library(name, data, deps, imports, resolutions, srcs, virtual_deps)
-
- - - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this target. | Name | required | | -| data | Runtime dependencies of the program.

The transitive closure of the data dependencies will be available in the .runfiles folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. | List of labels | optional | [] | -| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | -| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | -| resolutions | FIXME | Dictionary: Label -> String | optional | {} | -| srcs | Python source files. | List of labels | optional | [] | -| virtual_deps | - | List of strings | optional | [] | - - - - -## py_pex_binary - -
-py_pex_binary(name, binary, inject_env, python_interpreter_constraints, python_shebang)
-
- -Build a pex executable from a py_binary - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this target. | Name | required | | -| binary | A py_binary target | Label | required | | -| inject_env | Environment variables to set when running the pex binary. | Dictionary: String -> String | optional | {} | -| python_interpreter_constraints | Python interpreter versions this PEX binary is compatible with. A list of semver strings. The placeholder strings {major}, {minor}, {patch} can be used for gathering version information from the hermetic python toolchain.

For example, to enforce same interpreter version that Bazel uses, following can be used.

starlark py_pex_binary     python_interpreter_constraints = [       "CPython=={major}.{minor}.{patch}"     ] )
| List of strings | optional | ["CPython=={major}.{minor}.*"] | -| python_shebang | - | String | optional | "#!/usr/bin/env python3" | - - - - -## py_test_rule - -
-py_test_rule(name, data, deps, env, imports, main, package_collisions, python_version, resolutions,
-             srcs)
-
- -Run a Python program under Bazel. Most users should use the [py_test macro](#py_test) instead of loading this directly. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this target. | Name | required | | -| data | Runtime dependencies of the program.

The transitive closure of the data dependencies will be available in the .runfiles folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. | List of labels | optional | [] | -| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | -| env | Environment variables to set when running the binary. | Dictionary: String -> String | optional | {} | -| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | -| main | Script to execute with the Python interpreter. | Label | required | | -| package_collisions | The action that should be taken when a symlink collision is encountered when creating the venv. A collision can occour when multiple packages providing the same file are installed into the venv. The possible values are:

* "error": When conflicting symlinks are found, an error is reported and venv creation halts. * "warning": When conflicting symlinks are found, an warning is reported, however venv creation continues. * "ignore": When conflicting symlinks are found, no message is reported and venv creation continues. | String | optional | "error" | -| python_version | Whether to build this target and its transitive deps for a specific python version.

Note that setting this attribute alone will not be enough as the python toolchain for the desired version also needs to be registered in the WORKSPACE or MODULE.bazel file.

When using WORKSPACE, this may look like this,

 load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")

python_register_toolchains(     name = "python_toolchain_3_8",     python_version = "3.8.12",     # setting set_python_version_constraint makes it so that only matches py_* rule       # which has this exact version set in the python_version attribute.     set_python_version_constraint = True, )

# It's important to register the default toolchain last it will match any py_* target.  python_register_toolchains(     name = "python_toolchain",     python_version = "3.9", )


Configuring for MODULE.bazel may look like this:

 python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.8.12", is_default = False) python.toolchain(python_version = "3.9", is_default = True)
| String | optional | "" | -| resolutions | FIXME | Dictionary: Label -> String | optional | {} | -| srcs | Python source files. | List of labels | optional | [] | - - - - -## py_unpacked_wheel - -
-py_unpacked_wheel(name, py_package_name, src)
-
- - - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this target. | Name | required | | -| py_package_name | - | String | required | | -| src | The Wheel file, as defined by https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format | Label | required | | - - - - -## py_binary - -
-py_binary(name, srcs, main, kwargs)
-
- -Wrapper macro for [`py_binary_rule`](#py_binary_rule). - -Creates a virtualenv to constrain the interpreter and packages used at runtime. -Users can `bazel run [name].venv` to produce this, then use it in the editor. - - -**PARAMETERS** - - -| Name | Description | Default Value | -| :------------- | :------------- | :------------- | -| name | Name of the rule. | none | -| srcs | Python source files. | [] | -| main | Entry point. Like rules_python, this is treated as a suffix of a file that should appear among the srcs. If absent, then "[name].py" is tried. As a final fallback, if the srcs has a single file, that is used as the main. | None | -| kwargs | additional named parameters to the py_binary_rule. | none | - - - - -## py_pytest_main - -
-py_pytest_main(name, py_library, deps, data, testonly, kwargs)
-
- -py_pytest_main wraps the template rendering target and the final py_library. - -**PARAMETERS** - - -| Name | Description | Default Value | -| :------------- | :------------- | :------------- | -| name | The name of the runable target that updates the test entry file. | none | -| py_library | Use this attribute to override the default py_library rule. | <unknown object com.google.devtools.build.skydoc.fakebuildapi.FakeStarlarkRuleFunctionsApi$RuleDefinitionIdentifier> | -| deps | A list containing the pytest library target, e.g., @pypi_pytest//:pkg. | [] | -| data | A list of data dependencies to pass to the py_library target. | [] | -| testonly | A boolean indicating if the py_library target is testonly. | True | -| kwargs | The extra arguments passed to the template rendering target. | none | - - - - -## py_test - -
-py_test(name, main, srcs, kwargs)
-
- -Identical to py_binary, but produces a target that can be used with `bazel test`. - -**PARAMETERS** - - -| Name | Description | Default Value | -| :------------- | :------------- | :------------- | -| name |

-

| none | -| main |

-

| None | -| srcs |

-

| [] | -| kwargs |

-

| none | - - - - -## py_venv - -
-py_venv(name, kwargs)
-
- - - -**PARAMETERS** - - -| Name | Description | Default Value | -| :------------- | :------------- | :------------- | -| name |

-

| none | -| kwargs |

-

| none | - - - - -## resolutions.from_requirements - -
-resolutions.from_requirements(base, requirement_fn)
-
- -Returns data representing the resolution for a given set of dependencies - -**PARAMETERS** - - -| Name | Description | Default Value | -| :------------- | :------------- | :------------- | -| base | Base set of requirements to turn into resolutions. | none | -| requirement_fn | Optional function to transform the Python package name into a requirement label. | <function lambda> | - -**RETURNS** - -A resolution struct for use with virtual deps. - - - - -## resolutions.empty - -
-resolutions.empty()
-
- - - - - diff --git a/docs/venv.md b/docs/venv.md new file mode 100644 index 00000000..ab6595a2 --- /dev/null +++ b/docs/venv.md @@ -0,0 +1,62 @@ + + +Create a Python virtualenv directory structure. + +Note that [py_binary](./py_binary.md#py_binary) and [py_test](./py_test.md#py_test) macros automatically provide `[name].venv` targets. +Using `py_venv` directly is only required for cases where those defaults do not apply. + +> [!NOTE] +> As an implementation detail, this currently uses <https://github.com/prefix-dev/rip> which is a very fast Rust-based tool. + + + + +## py_venv_rule + +
+py_venv_rule(name, deps, imports, location, package_collisions, resolutions, venv_name)
+
+ +Create a Python virtual environment with the dependencies listed. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| deps | Targets that produce Python code, commonly py_library rules. | List of labels | optional | [] | +| imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | [] | +| location | Path from the workspace root for where to root the virtial environment | String | optional | "" | +| package_collisions | The action that should be taken when a symlink collision is encountered when creating the venv. A collision can occour when multiple packages providing the same file are installed into the venv. The possible values are:

* "error": When conflicting symlinks are found, an error is reported and venv creation halts. * "warning": When conflicting symlinks are found, an warning is reported, however venv creation continues. * "ignore": When conflicting symlinks are found, no message is reported and venv creation continues. | String | optional | "error" | +| resolutions | FIXME | Dictionary: Label -> String | optional | {} | +| venv_name | Outer folder name for the generated virtual environment | String | optional | "" | + + + + +## py_venv + +
+py_venv(name, kwargs)
+
+ +Wrapper macro for [`py_venv_rule`](#py_venv_rule). + +Chooses a suitable default location for the resulting directory. + +By default, VSCode (and likely other tools) expect to find virtualenv's in the root of the project opened in the editor. +They also provide a nice name to see "which one is open" when discovered this way. +See https://github.com/aspect-build/rules_py/issues/395 + +Use py_venv_rule directly to have more control over the location. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name |

-

| none | +| kwargs |

-

| none | + + diff --git a/py/defs.bzl b/py/defs.bzl index 6fabf75a..abd76926 100644 --- a/py/defs.bzl +++ b/py/defs.bzl @@ -1,4 +1,39 @@ -"Public API re-exports" +"""Re-implementations of [py_binary](https://bazel.build/reference/be/python#py_binary) +and [py_test](https://bazel.build/reference/be/python#py_test) + +## Choosing the Python version + +The `python_version` attribute must refer to a python toolchain version +which has been registered in the WORKSPACE or MODULE.bazel file. + +When using WORKSPACE, this may look like this: + +```starlark +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +python_register_toolchains( + name = "python_toolchain_3_8", + python_version = "3.8.12", + # setting set_python_version_constraint makes it so that only matches py_* rule + # which has this exact version set in the `python_version` attribute. + set_python_version_constraint = True, +) + +# It's important to register the default toolchain last it will match any py_* target. +python_register_toolchains( + name = "python_toolchain", + python_version = "3.9", +) +``` + +Configuring for MODULE.bazel may look like this: + +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.8.12", is_default = False) +python.toolchain(python_version = "3.9", is_default = True) +``` +""" load("@aspect_bazel_lib//lib:utils.bzl", "propagate_common_rule_attributes") load("//py/private:py_binary.bzl", _py_binary = "py_binary", _py_test = "py_test") @@ -57,17 +92,17 @@ def _py_binary_or_test(name, rule, srcs, main, deps = [], resolutions = {}, **kw def py_binary(name, srcs = [], main = None, **kwargs): """Wrapper macro for [`py_binary_rule`](#py_binary_rule). - Creates a virtualenv to constrain the interpreter and packages used at runtime. - Users can `bazel run [name].venv` to produce this, then use it in the editor. + Creates a [py_venv](./venv.md) target to constrain the interpreter and packages used at runtime. + Users can `bazel run [name].venv` to create this virtualenv, then use it in the editor or other tools. Args: name: Name of the rule. srcs: Python source files. main: Entry point. Like rules_python, this is treated as a suffix of a file that should appear among the srcs. - If absent, then "[name].py" is tried. As a final fallback, if the srcs has a single file, + If absent, then `[name].py` is tried. As a final fallback, if the srcs has a single file, that is used as the main. - **kwargs: additional named parameters to the py_binary_rule. + **kwargs: additional named parameters to `py_binary_rule`. """ # For a clearer DX when updating resolutions, the resolutions dict is "string" -> "label", @@ -79,7 +114,7 @@ def py_binary(name, srcs = [], main = None, **kwargs): _py_binary_or_test(name = name, rule = _py_binary, srcs = srcs, main = main, resolutions = resolutions, **kwargs) def py_test(name, main = None, srcs = [], **kwargs): - "Identical to py_binary, but produces a target that can be used with `bazel test`." + "Identical to [py_binary](./py_binary.md), but produces a target that can be used with `bazel test`." # Ensure that any other targets we write will be testonly like the py_test target kwargs["testonly"] = True diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 7cd39349..943d0fb8 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -1,5 +1,10 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +package(default_visibility = [ + "//docs:__pkg__", + "//py:__subpackages__", +]) + exports_files( [ "run.tmpl.sh", @@ -20,7 +25,6 @@ exports_files( bzl_library( name = "py_binary", srcs = ["py_binary.bzl"], - visibility = ["//:__subpackages__"], deps = [ ":py_library", "@aspect_bazel_lib//lib:expand_make_vars", @@ -31,7 +35,6 @@ bzl_library( bzl_library( name = "py_library", srcs = ["py_library.bzl"], - visibility = ["//:__subpackages__"], deps = [ ":providers", ":py_semantics", @@ -39,19 +42,18 @@ bzl_library( "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", "@bazel_skylib//lib:types", + "@rules_python//python:defs_bzl", ], ) bzl_library( name = "py_wheel", - visibility = ["//py:__subpackages__"], deps = [":providers"], ) bzl_library( name = "py_pytest_main", srcs = ["py_pytest_main.bzl"], - visibility = ["//py:__subpackages__"], deps = [ "@rules_python//docs:bazel_repo_tools", "@rules_python//python:defs_bzl", @@ -61,7 +63,6 @@ bzl_library( bzl_library( name = "py_semantics", srcs = ["py_semantics.bzl"], - visibility = ["//py:__subpackages__"], deps = [ "//py/private/toolchain:types", "@bazel_skylib//rules:common_settings", @@ -71,7 +72,6 @@ bzl_library( bzl_library( name = "py_unpacked_wheel", srcs = ["py_unpacked_wheel.bzl"], - visibility = ["//py:__subpackages__"], deps = [ ":py_library", ":py_semantics", @@ -83,7 +83,6 @@ bzl_library( bzl_library( name = "py_venv", srcs = ["py_venv.bzl"], - visibility = ["//py:__subpackages__"], deps = [ ":providers", ":py_library", @@ -109,15 +108,14 @@ bzl_library( bzl_library( name = "py_pex_binary", srcs = ["py_pex_binary.bzl"], - visibility = ["//py:__subpackages__"], deps = [ ":py_semantics", "//py/private/toolchain:types", + "@rules_python//python:defs_bzl", ], ) bzl_library( name = "virtual", srcs = ["virtual.bzl"], - visibility = ["//py:__subpackages__"], ) diff --git a/py/private/py_binary.bzl b/py/private/py_binary.bzl index 8fa8e74c..645f227e 100644 --- a/py/private/py_binary.bzl +++ b/py/private/py_binary.bzl @@ -141,39 +141,7 @@ _attrs = dict({ mandatory = True, ), "python_version": attr.string( - doc = """Whether to build this target and its transitive deps for a specific python version. - -Note that setting this attribute alone will not be enough as the python toolchain for the desired version -also needs to be registered in the WORKSPACE or MODULE.bazel file. - -When using WORKSPACE, this may look like this, - -``` -load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") - -python_register_toolchains( - name = "python_toolchain_3_8", - python_version = "3.8.12", - # setting set_python_version_constraint makes it so that only matches py_* rule - # which has this exact version set in the `python_version` attribute. - set_python_version_constraint = True, -) - -# It's important to register the default toolchain last it will match any py_* target. -python_register_toolchains( - name = "python_toolchain", - python_version = "3.9", -) -``` - -Configuring for MODULE.bazel may look like this: - -``` -python = use_extension("@rules_python//python/extensions:python.bzl", "python") -python.toolchain(python_version = "3.8.12", is_default = False) -python.toolchain(python_version = "3.9", is_default = True) -``` -""", + doc = """Whether to build this target and its transitive deps for a specific python version.""", ), "package_collisions": attr.string( doc = """The action that should be taken when a symlink collision is encountered when creating the venv. diff --git a/py/private/py_library.bzl b/py/private/py_library.bzl index 2b52bc8f..ca570a58 100644 --- a/py/private/py_library.bzl +++ b/py/private/py_library.bzl @@ -1,4 +1,8 @@ -"""Implementation for the py_library rule""" +"""A re-implementation of [py_library](https://bazel.build/reference/be/python#py_library). + +Supports "virtual" dependencies with a `virtual_deps` attribute, which lists packages which are required +without binding them to a particular version of that package. +""" load("@rules_python//python:defs.bzl", "PyInfo") load("@bazel_skylib//lib:paths.bzl", "paths") diff --git a/py/private/py_pex_binary.bzl b/py/private/py_pex_binary.bzl index 92af316f..90fcf114 100644 --- a/py/private/py_pex_binary.bzl +++ b/py/private/py_pex_binary.bzl @@ -1,4 +1,22 @@ -"Create python zip file https://peps.python.org/pep-0441/ (PEX)" +"""Create a zip file containing a full Python application. + +Follows [PEP-441 (PEX)](https://peps.python.org/pep-0441/) + +## Ensuring a compatible interpreter is used + +The resulting zip file does *not* contain a Python interpreter. +Users are expected to execute the PEX with a compatible interpreter on the runtime system. + +Use the `python_interpreter_constraints` to provide an error if a wrong interpreter tries to execute the PEX, for example: + +```starlark +py_pex_binary( + python_interpreter_constraints = [ + "CPython=={major}.{minor}.{patch}", + ] +) +``` +""" load("@rules_python//python:defs.bzl", "PyInfo") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") @@ -52,10 +70,9 @@ def _map_srcs(f, workspace): return ["--distinfo={}".format(f.dirname)] return ["--dep={}".format(f.dirname)] + elif site_packages_i == -1: # If the path does not have a `site-packages` in it, then put it into # the standard runfiles tree. - - elif site_packages_i == -1: return ["--source={}={}".format(f.path, dest_path)] return [] @@ -133,16 +150,6 @@ _attrs = dict({ Python interpreter versions this PEX binary is compatible with. A list of semver strings. The placeholder strings `{major}`, `{minor}`, `{patch}` can be used for gathering version information from the hermetic python toolchain. - -For example, to enforce same interpreter version that Bazel uses, following can be used. - -```starlark -py_pex_binary - python_interpreter_constraints = [ - "CPython=={major}.{minor}.{patch}" - ] -) -``` """, ), # NB: this is read by _resolve_toolchain in py_semantics. diff --git a/py/private/py_venv.bzl b/py/private/py_venv.bzl index 5ccd7c63..234ccd57 100644 --- a/py/private/py_venv.bzl +++ b/py/private/py_venv.bzl @@ -1,4 +1,11 @@ -"""Implementation for the py_binary and py_test rules.""" +"""Create a Python virtualenv directory structure. + +Note that [py_binary](./py_binary.md#py_binary) and [py_test](./py_test.md#py_test) macros automatically provide `[name].venv` targets. +Using `py_venv` directly is only required for cases where those defaults do not apply. + +> [!NOTE] +> As an implementation detail, this currently uses which is a very fast Rust-based tool. +""" load("@rules_python//python:defs.bzl", "PyInfo") load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_rlocation_path") @@ -85,8 +92,8 @@ def _py_venv_rule_imp(ctx): ), ] -_py_venv = rule( - doc = "Create a Python virtual environment with the dependencies listed.", +py_venv_rule = rule( + doc = """Create a Python virtual environment with the dependencies listed.""", implementation = _py_venv_rule_imp, attrs = { "deps": attr.label_list( @@ -140,11 +147,18 @@ A collision can occour when multiple packages providing the same file are instal ) def py_venv(name, **kwargs): - # By default, VSCode (and likely other tools) expect to find virtualenv's in the root of the project opened in the editor. - # They also provide a nice name to see "which one is open" when discovered this way. - # See https://github.com/aspect-build/rules_py/issues/395 + """Wrapper macro for [`py_venv_rule`](#py_venv_rule). + + Chooses a suitable default location for the resulting directory. + + By default, VSCode (and likely other tools) expect to find virtualenv's in the root of the project opened in the editor. + They also provide a nice name to see "which one is open" when discovered this way. + See https://github.com/aspect-build/rules_py/issues/395 + + Use py_venv_rule directly to have more control over the location. + """ default_venv_name = ".{}".format(paths.join(native.package_name(), name).replace("/", "+")) - _py_venv( + py_venv_rule( name = name, venv_name = kwargs.pop("venv_name", default_venv_name), **kwargs