From 36199b01631ddab46f17fd56305ad9fae0ac3565 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sun, 16 Jul 2023 23:42:38 +0900 Subject: [PATCH 1/9] feat: support alias rendering for python aware toolchain targets --- python/private/render_pkg_aliases.bzl | 189 ++++++++++++++++++ .../render_pkg_aliases/BUILD.bazel | 3 + .../render_pkg_aliases_test.bzl | 174 ++++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 python/private/render_pkg_aliases.bzl create mode 100644 tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel create mode 100644 tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl new file mode 100644 index 0000000000..28042e0507 --- /dev/null +++ b/python/private/render_pkg_aliases.bzl @@ -0,0 +1,189 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""render_pkg_aliases is a function to generate BUILD.bazel contents used to create user-friendly aliases. + +This is used in bzlmod and non-bzlmod setups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":version_label.bzl", "version_label") + +_DEFAULT = """\ +alias( + name = "{name}", + actual = "@{repo_name}_{dep}//:{target}", +)""" + +_SELECT = """\ +alias( + name = "{name}", + actual = select({{{selects}}}), +)""" + +def _render_alias( + *, + name, + repo_name, + dep, + target, + default_version, + versions, + rules_python): + """Render an alias for common targets + + If the versions is passed, then the `rules_python` must be passed as well and + an alias with a select statement based on the python version is going to be + generated. + """ + if versions == None: + return _DEFAULT.format( + name = name, + repo_name = repo_name, + dep = dep, + target = target, + ) + + # Create the alias repositories which contains different select + # statements These select statements point to the different pip + # whls that are based on a specific version of Python. + selects = {} + for full_version in versions: + condition = "@@{rules_python}//python/config_settings:is_python_{full_python_version}".format( + rules_python = rules_python, + full_python_version = full_version, + ) + actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(full_version), + dep = dep, + target = target, + ) + selects[condition] = actual + + default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(default_version), + dep = dep, + target = target, + ) + selects["//conditions:default"] = default_actual + + return _SELECT.format( + name = name, + selects = "\n{} ".format( + "".join([ + " {}: {},\n".format(repr(k), repr(v)) + for k, v in selects.items() + ]), + ), + ) + +def _render_entry_points(repo_name, dep): + return """\ +load("@{repo_name}_{dep}//:entry_points.bzl", "entry_points") + +[ + alias( + name = script, + actual = "@{repo_name}_{dep}//:" + target, + visibility = ["//visibility:public"], + ) + for script, target in entry_points.items() +] +""".format( + repo_name = repo_name, + dep = dep, + ) + +def _render_common_aliases(repo_name, name, versions = None, default_version = None, rules_python = None): + return "\n\n".join([ + """package(default_visibility = ["//visibility:public"])""", + _render_alias( + name = name, + repo_name = repo_name, + dep = name, + target = "pkg", + versions = versions, + default_version = default_version, + rules_python = rules_python, + ), + ] + [ + _render_alias( + name = target, + repo_name = repo_name, + dep = name, + target = target, + versions = versions, + default_version = default_version, + rules_python = rules_python, + ) + for target in ["pkg", "whl", "data", "dist_info"] + ]) + +def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_python = None, default_version = None): + """Create alias declarations for each PyPI package. + + The aliases should be appended to the pip_repository BUILD.bazel file. These aliases + allow users to use requirement() without needed a corresponding `use_repo()` for each dep + when using bzlmod. + + Args: + repo_name: the repository name of the hub repository that is visible to the users that is + also used as the prefix for the spoke repo names (e.g. "pip", "pypi"). + bzl_packages: the list of packages to setup, if not specified, whl_map.keys() will be used instead. + whl_map: the whl_map for generating Python version aware aliases. + default_version: the default version to be used for the aliases. + rules_python: the name of the rules_python workspace. + + Returns: + A dict of file paths and their contents. + """ + if not bzl_packages and whl_map: + bzl_packages = list(whl_map.keys()) + + contents = {} + for name in bzl_packages: + versions = None + if whl_map != None: + versions = whl_map[name] + name = normalize_name(name) + + filename = "{}/BUILD.bazel".format(name) + contents[filename] = _render_common_aliases( + repo_name = repo_name, + name = name, + versions = versions, + rules_python = rules_python, + default_version = default_version, + ).strip() + + if versions == None: + # NOTE: this code would be normally executed in the non-bzlmod + # scenario, where we are requesting friendly aliases to be + # generated. In that case, we will not be creating aliases for + # entry_points to leave the behaviour unchanged from previous + # rules_python versions. + continue + + # NOTE @aignas 2023-07-07: we are not creating aliases using a select + # and the version specific aliases because we would need to fetch the + # package for all versions in order to construct the said select. + for version in versions: + filename = "{}/bin_py{}/BUILD.bazel".format(name, version_label(version)) + contents[filename] = _render_entry_points( + repo_name = "{}_{}".format(repo_name, version_label(version)), + dep = name, + ).strip() + + return contents diff --git a/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel b/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel new file mode 100644 index 0000000000..f2e0126666 --- /dev/null +++ b/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel @@ -0,0 +1,3 @@ +load(":render_pkg_aliases_test.bzl", "render_pkg_aliases_test_suite") + +render_pkg_aliases_test_suite(name = "render_pkg_aliases_tests") diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl new file mode 100644 index 0000000000..eed8653705 --- /dev/null +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -0,0 +1,174 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"render_pkg_aliases tests" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_legacy_aliases(env): + actual = render_pkg_aliases( + bzl_packages = ["foo"], + repo_name = "pypi", + ) + + want = { + "foo/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "foo", + actual = "@pypi_foo//:pkg", +) + +alias( + name = "pkg", + actual = "@pypi_foo//:pkg", +) + +alias( + name = "whl", + actual = "@pypi_foo//:whl", +) + +alias( + name = "data", + actual = "@pypi_foo//:data", +) + +alias( + name = "dist_info", + actual = "@pypi_foo//:dist_info", +)""", + } + + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_legacy_aliases) + +def _test_all_legacy_aliases_are_created(env): + actual = render_pkg_aliases( + bzl_packages = ["foo", "bar"], + repo_name = "pypi", + ) + + want_files = ["bar/BUILD.bazel", "foo/BUILD.bazel"] + + env.expect.that_dict(actual).keys().contains_exactly(want_files) + +_tests.append(_test_all_legacy_aliases_are_created) + +def _test_bzlmod_aliases(env): + actual = render_pkg_aliases( + default_version = "3.2.3", + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar-baz": ["3.2.3"], + }, + ) + + want = { + "bar_baz/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "bar_baz", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:pkg", + "//conditions:default": "@pypi_32_bar_baz//:pkg", + }), +) + +alias( + name = "pkg", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:pkg", + "//conditions:default": "@pypi_32_bar_baz//:pkg", + }), +) + +alias( + name = "whl", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:whl", + "//conditions:default": "@pypi_32_bar_baz//:whl", + }), +) + +alias( + name = "data", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:data", + "//conditions:default": "@pypi_32_bar_baz//:data", + }), +) + +alias( + name = "dist_info", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:dist_info", + "//conditions:default": "@pypi_32_bar_baz//:dist_info", + }), +)""", + "bar_baz/bin_py32/BUILD.bazel": """\ +load("@pypi_32_bar_baz//:entry_points.bzl", "entry_points") + +[ + alias( + name = script, + actual = "@pypi_32_bar_baz//:" + target, + visibility = ["//visibility:public"], + ) + for script, target in entry_points.items() +]""", + } + + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_bzlmod_aliases) + +def _test_bzlmod_aliases_are_created_for_all_wheels(env): + actual = render_pkg_aliases( + default_version = "3.2.3", + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar": ["3.1.2", "3.2.3"], + "foo": ["3.1.2", "3.2.3"], + }, + ) + + want_files = [ + "bar/BUILD.bazel", + "bar/bin_py31/BUILD.bazel", + "bar/bin_py32/BUILD.bazel", + "foo/BUILD.bazel", + "foo/bin_py31/BUILD.bazel", + "foo/bin_py32/BUILD.bazel", + ] + + env.expect.that_dict(actual).keys().contains_exactly(want_files) + +_tests.append(_test_bzlmod_aliases_are_created_for_all_wheels) + +def render_pkg_aliases_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 631382259c5e86a7895bbb1252d92dde7320f0c7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:04:46 +0900 Subject: [PATCH 2/9] refactor: use render_pkg_aliases --- python/pip_install/pip_repository.bzl | 55 ++++----------------------- 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 99d1fb05b1..0e80542747 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -21,6 +21,7 @@ load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") CPPFLAGS = "CPPFLAGS" @@ -268,56 +269,12 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _pkg_aliases(rctx, repo_name, bzl_packages): - """Create alias declarations for each python dependency. - - The aliases should be appended to the pip_repository BUILD.bazel file. These aliases - allow users to use requirement() without needed a corresponding `use_repo()` for each dep - when using bzlmod. - - Args: - rctx: the repository context. - repo_name: the repository name of the parent that is visible to the users. - bzl_packages: the list of packages to setup. - """ - for name in bzl_packages: - build_content = """package(default_visibility = ["//visibility:public"]) - -alias( - name = "{name}", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "pkg", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "whl", - actual = "@{repo_name}_{dep}//:whl", -) - -alias( - name = "data", - actual = "@{repo_name}_{dep}//:data", -) - -alias( - name = "dist_info", - actual = "@{repo_name}_{dep}//:dist_info", -) -""".format( - name = name, - repo_name = repo_name, - dep = name, - ) - rctx.file("{}/BUILD.bazel".format(name), build_content) - def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): repo_name = rctx.attr.repo_name build_contents = _BUILD_FILE_CONTENTS - _pkg_aliases(rctx, repo_name, bzl_packages) + aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) # NOTE: we are using the canonical name with the double '@' in order to # always uniquely identify a repository, as the labels are being passed as @@ -458,7 +415,9 @@ def _pip_repository_impl(rctx): config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) if rctx.attr.incompatible_generate_aliases: - _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { From 5eab849ee24f05152da7dda82514092da9e3d8b3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sun, 16 Jul 2023 23:42:58 +0900 Subject: [PATCH 3/9] test: add a helper to distinguish between bazel 6 --- tests/test_env.bzl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/test_env.bzl diff --git a/tests/test_env.bzl b/tests/test_env.bzl new file mode 100644 index 0000000000..ee9a32437f --- /dev/null +++ b/tests/test_env.bzl @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Functions for inspecting the test environment. + +Currently contains: +* A check to see if we are on Bazel 6.0+ +""" + +def _is_bazel_6_or_higher(): + return testing.ExecutionInfo == testing.ExecutionInfo + +test_env = struct( + is_bazel_6_or_higher = _is_bazel_6_or_higher, +) From 79fc0586d6f0c19d25688efd32c79a6d8b4fbd95 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sun, 16 Jul 2023 23:43:27 +0900 Subject: [PATCH 4/9] feat: add a full_version helper --- python/private/full_version.bzl | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 python/private/full_version.bzl diff --git a/python/private/full_version.bzl b/python/private/full_version.bzl new file mode 100644 index 0000000000..db4411cf79 --- /dev/null +++ b/python/private/full_version.bzl @@ -0,0 +1,35 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"A small helper to ensure that we are working with full versions." + +load("//python:versions.bzl", "MINOR_MAPPING") + +def full_version(version): + """Return a full version. + + Args: + version: the version in `X.Y` or `X.Y.Z` format. + + Returns: + a full version given the version string. If the string is already a + major version then we return it as is. + """ + parts = version.split(".") + if len(parts) == 2: + return MINOR_MAPPING[version] + elif len(parts) == 3: + return version + else: + fail("Unknown version format: {}".format(version)) From 590830bcbdf93a5731b6ee8f93d199e0df9fd50f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:10:11 +0900 Subject: [PATCH 5/9] refactor: use full_version --- python/extensions/private/pythons_hub.bzl | 11 +++-------- python/pip.bzl | 4 ++-- python/repositories.bzl | 5 ++--- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/python/extensions/private/pythons_hub.bzl b/python/extensions/private/pythons_hub.bzl index a64f203bd6..f36ce45521 100644 --- a/python/extensions/private/pythons_hub.bzl +++ b/python/extensions/private/pythons_hub.bzl @@ -14,7 +14,8 @@ "Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" -load("//python:versions.bzl", "MINOR_MAPPING", "WINDOWS_NAME") +load("//python:versions.bzl", "WINDOWS_NAME") +load("//python/private:full_version.bzl", "full_version") load( "//python/private:toolchains_repo.bzl", "get_host_os_arch", @@ -28,12 +29,6 @@ def _have_same_length(*lists): fail("expected at least one list") return len({len(length): None for length in lists}) == 1 -def _get_version(python_version): - # we need to get the MINOR_MAPPING or use the full version - if python_version in MINOR_MAPPING: - python_version = MINOR_MAPPING[python_version] - return python_version - def _python_toolchain_build_file_content( prefixes, python_versions, @@ -55,7 +50,7 @@ def _python_toolchain_build_file_content( # build the toolchain content by calling python_toolchain_build_file_content return "\n".join([python_toolchain_build_file_content( prefix = prefixes[i], - python_version = _get_version(python_versions[i]), + python_version = full_version(python_versions[i]), set_python_version_constraint = set_python_version_constraints[i], user_repository_name = user_repository_names[i], rules_python = rules_python, diff --git a/python/pip.bzl b/python/pip.bzl index cae15919b0..352694f97f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -17,7 +17,7 @@ load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annot load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") -load(":versions.bzl", "MINOR_MAPPING") +load("//python/private:full_version.bzl", "full_version") compile_pip_requirements = _compile_pip_requirements package_annotation = _package_annotation @@ -296,7 +296,7 @@ alias( for [python_version, repo_prefix] in version_map: alias.append("""\ "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( - full_python_version = MINOR_MAPPING[python_version] if python_version in MINOR_MAPPING else python_version, + full_python_version = full_version(python_version), actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( repo_prefix = repo_prefix, wheel_name = wheel_name, diff --git a/python/repositories.bzl b/python/repositories.bzl index 62d94210e0..8ac750dda6 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -21,6 +21,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archi load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") +load("//python/private:full_version.bzl", "full_version") load( "//python/private:toolchains_repo.bzl", "multi_toolchain_aliases", @@ -30,7 +31,6 @@ load( load( ":versions.bzl", "DEFAULT_RELEASE_BASE_URL", - "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS", "get_release_info", @@ -505,8 +505,7 @@ def python_register_toolchains( base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) - if python_version in MINOR_MAPPING: - python_version = MINOR_MAPPING[python_version] + python_version = full_version(python_version) toolchain_repo_name = "{name}_toolchains".format(name = name) From d1f62cb27255dca26e35ed631917e76c77a154d8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:18:26 +0900 Subject: [PATCH 6/9] feat: generate entry_points.bzl for each whl_library --- .../tools/wheel_installer/wheel_installer.py | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 9b363c3068..7d75cb31ca 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -18,7 +18,6 @@ import json import os import re -import shutil import subprocess import sys import textwrap @@ -226,7 +225,7 @@ def _generate_build_file_contents( "**/* *", "**/*.py", "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created # RECORD is known to contain sha256 checksums of files which might include the checksums # of generated files produced when wheels are installed. The file is ignored to avoid # Bazel caching issues. @@ -329,7 +328,7 @@ def _extract_wheel( bazel.sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps ] - entry_points = [] + entry_points = {} for name, (module, attribute) in sorted(whl.entry_points().items()): # There is an extreme edge-case with entry_points that end with `.py` # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 @@ -341,16 +340,14 @@ def _extract_wheel( (installation_dir / entry_point_script_name).write_text( _generate_entry_point_contents(module, attribute) ) - entry_points.append( - _generate_entry_point_rule( - entry_point_target_name, - entry_point_script_name, - bazel.PY_LIBRARY_LABEL, - ) + entry_points[entry_point_without_py] = _generate_entry_point_rule( + entry_point_target_name, + entry_point_script_name, + bazel.PY_LIBRARY_LABEL, ) with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file: - additional_content = entry_points + additional_content = list(entry_points.values()) data = [] data_exclude = pip_data_exclude srcs_exclude = [] @@ -381,6 +378,31 @@ def _extract_wheel( ) build_file.write(contents) + with open(installation_dir / "entry_points.bzl", "w") as entry_point_file: + entry_points_str = "" + if entry_points: + entry_points_str = "\n" + "".join( + [ + f' "{script}": "{bazel.WHEEL_ENTRY_POINT_PREFIX}_{script}",\n' + for script in sorted(entry_points.keys()) + ] + ) + + contents = textwrap.dedent( + """\ + \"\"\" + This file contains the entry_point script names as a dict, where the keys + are the script names and the values are the target names. + + generated by @rules_python//python/pip_install/tools/wheel_installer/wheel_installer.py + \"\"\" + + entry_points = {{{}}} + """ + ).format(entry_points_str) + + entry_point_file.write(contents) + def main() -> None: parser = argparse.ArgumentParser( From b9a2f5f66ad58b96d815b8c2aa010f82df1ce2f9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:24:31 +0900 Subject: [PATCH 7/9] feat: consolidate bzlmod pip hub repos and support entry_points --- examples/bzlmod/MODULE.bazel | 5 +- examples/bzlmod/entry_point/BUILD.bazel | 2 +- python/extensions/pip.bzl | 45 ++------ .../extensions/private/pip_hub_repository.bzl | 95 ++++++++++++++++ .../extensions/private/requirements.bzl.tmpl | 52 +++++++++ python/pip_install/entry_point.bzl | 69 ++++++++++++ ...ub_repository_requirements_bzlmod.bzl.tmpl | 35 ------ python/pip_install/pip_repository.bzl | 102 ------------------ ...ip_repository_requirements_bzlmod.bzl.tmpl | 33 ------ .../entry_point/BUILD.bazel | 17 +++ .../entry_point/entry_point_test.bzl | 94 ++++++++++++++++ 11 files changed, 336 insertions(+), 213 deletions(-) create mode 100644 python/extensions/private/pip_hub_repository.bzl create mode 100644 python/extensions/private/requirements.bzl.tmpl create mode 100644 python/pip_install/entry_point.bzl delete mode 100644 python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl delete mode 100644 python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl create mode 100644 tests/pip_hub_repository/entry_point/BUILD.bazel create mode 100644 tests/pip_hub_repository/entry_point/entry_point_test.bzl diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index df88ae8490..7c5083c82a 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -118,10 +118,7 @@ pip.parse( "@whl_mods_hub//:wheel.json": "wheel", }, ) - -# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't -# yet support entry points; see https://github.com/bazelbuild/rules_python/issues/1262 -use_repo(pip, "pip", "pip_39") +use_repo(pip, "pip") bazel_dep(name = "other_module", version = "", repo_name = "our_other_module") local_path_override( diff --git a/examples/bzlmod/entry_point/BUILD.bazel b/examples/bzlmod/entry_point/BUILD.bazel index f68552c3ef..dfc02b00a0 100644 --- a/examples/bzlmod/entry_point/BUILD.bazel +++ b/examples/bzlmod/entry_point/BUILD.bazel @@ -1,4 +1,4 @@ -load("@pip_39//:requirements.bzl", "entry_point") +load("@pip//:requirements.bzl", "entry_point") load("@rules_python//python:defs.bzl", "py_test") alias( diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index add69a4c64..699fb751f6 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -15,17 +15,16 @@ "pip module extension for use with bzlmod" load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load("@rules_python//python:pip.bzl", "whl_library_alias") load( "@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", - "pip_hub_repository_bzlmod", "pip_repository_attrs", - "pip_repository_bzlmod", "use_isolated", "whl_library", ) load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/extensions/private:pip_hub_repository.bzl", "pip_hub_repository") +load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version_label.bzl", "version_label") @@ -111,16 +110,6 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): requirements = parse_result.requirements extra_pip_args = pip_attr.extra_pip_args + parse_result.options - # Create the repository where users load the `requirement` macro. Under bzlmod - # this does not create the install_deps() macro. - # TODO: we may not need this repository once we have entry points - # supported. For now a user can access this repository and use - # the entrypoint functionality. - pip_repository_bzlmod( - name = pip_name, - repo_name = pip_name, - requirements_lock = pip_attr.requirements_lock, - ) if hub_name not in whl_map: whl_map[hub_name] = {} @@ -155,9 +144,9 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): ) if whl_name not in whl_map[hub_name]: - whl_map[hub_name][whl_name] = {} + whl_map[hub_name][whl_name] = [] - whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_" + whl_map[hub_name][whl_name].append(full_version(pip_attr.python_version)) def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. @@ -295,32 +284,12 @@ def _pip_impl(module_ctx): _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map) for hub_name, whl_map in hub_whl_map.items(): - for whl_name, version_map in whl_map.items(): - if DEFAULT_PYTHON_VERSION not in version_map: - fail(( - "Default python version '{version}' missing in pip " + - "hub '{hub}': update your pip.parse() calls so that " + - 'includes `python_version = "{version}"`' - ).format( - version = DEFAULT_PYTHON_VERSION, - hub = hub_name, - )) - - # Create the alias repositories which contains different select - # statements These select statements point to the different pip - # whls that are based on a specific version of Python. - whl_library_alias( - name = hub_name + "_" + whl_name, - wheel_name = whl_name, - default_version = DEFAULT_PYTHON_VERSION, - version_map = version_map, - ) - # Create the hub repository for pip. - pip_hub_repository_bzlmod( + pip_hub_repository( name = hub_name, repo_name = hub_name, - whl_library_alias_names = whl_map.keys(), + whl_map = whl_map, + default_version = full_version(DEFAULT_PYTHON_VERSION), ) def _pip_parse_ext_attrs(): diff --git a/python/extensions/private/pip_hub_repository.bzl b/python/extensions/private/pip_hub_repository.bzl new file mode 100644 index 0000000000..b73c003500 --- /dev/null +++ b/python/extensions/private/pip_hub_repository.bzl @@ -0,0 +1,95 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A pip_hub_repository rule used to create bzlmod hub repos for PyPI packages. + +It assumes that version aware toolchain is used and is responsible for setting up +aliases for entry points and the actual package targets. +""" + +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") + +_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +def _impl(rctx): + bzl_packages = rctx.attr.whl_map.keys() + repo_name = rctx.attr.repo_name + + aliases = render_pkg_aliases( + repo_name = repo_name, + whl_map = rctx.attr.whl_map, + default_version = rctx.attr.default_version, + rules_python = rctx.attr._template.workspace_name, + ) + for path, contents in aliases.items(): + rctx.file(path, contents) + + # NOTE: we are using the canonical name with the double '@' in order to + # always uniquely identify a repository, as the labels are being passed as + # a string and the resolution of the label happens at the call-site of the + # `requirement`, et al. macros. + macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) + + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + "%%ALL_DATA_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, p) + for p in bzl_packages + ]), + "%%ALL_WHL_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, "whl") + for p in bzl_packages + ]), + "%%DEFAULT_PY_VERSION%%": repr(rctx.attr.default_version), + "%%MACRO_TMPL%%": macro_tmpl, + "%%NAME%%": rctx.attr.name, + "%%PACKAGE_AVAILABILITY%%": repr({ + k: [v for v in versions] + for k, versions in rctx.attr.whl_map.items() + }), + "%%RULES_PYTHON%%": rctx.attr._template.workspace_name, + }) + +pip_hub_repository = repository_rule( + attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ +This is the default python version in the format of X.Y.Z. This should match +what is setup by the 'python' extension using the 'is_default = True' +setting.""", + ), + "repo_name": attr.string( + mandatory = True, + doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", + ), + "whl_map": attr.string_list_dict( + mandatory = True, + doc = "The wheel map where values are python versions", + ), + "_template": attr.label(default = ":requirements.bzl.tmpl"), + }, + doc = """A rule for creating bzlmod hub repo for PyPI packages. PRIVATE USE ONLY.""", + implementation = _impl, +) diff --git a/python/extensions/private/requirements.bzl.tmpl b/python/extensions/private/requirements.bzl.tmpl new file mode 100644 index 0000000000..4243990b2b --- /dev/null +++ b/python/extensions/private/requirements.bzl.tmpl @@ -0,0 +1,52 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip.parse extension. + +This file is different from the other bzlmod template +because we do not support entry_point yet. +""" + +load("@@%%RULES_PYTHON%%//python/pip_install:entry_point.bzl", _entry_point = "entry_point") + +all_requirements = %%ALL_REQUIREMENTS%% + +all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% + +all_data_requirements = %%ALL_DATA_REQUIREMENTS%% + +_default_py_version = %%DEFAULT_PY_VERSION%% +_packages = %%PACKAGE_AVAILABILITY%% + +def _clean_name(name): + return name.replace("-", "_").replace(".", "_").lower() + +def requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") + +def whl_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") + +def data_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "data") + +def dist_info_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") + +def entry_point(pkg, script = None): + """Returns a select() expression to locate the version-specific entry point. + """ + # TODO: not implemented + # selects = _entry_point( + # tmpl = "@@%%NAME%%//{pkg}/bin_py{version_label}:{script}", + # pkg = _clean_name(pkg), + # script = script, + # packages = _packages, + # default_version = _default_py_version, + # ) + # if selects == None: + # fail("Package '{}' does not exist, select one from: {}".format(pkg, _packages.keys())) + + # # NOTE: We return a select() expression instead of an alias to such an expression + # # to avoid having to eagerly load all versions of the wheel. See + # # https://github.com/bazelbuild/rules_python/issues/1262 for discussion. + # return select(selects) diff --git a/python/pip_install/entry_point.bzl b/python/pip_install/entry_point.bzl new file mode 100644 index 0000000000..71bc96e68b --- /dev/null +++ b/python/pip_install/entry_point.bzl @@ -0,0 +1,69 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""entry_point macro implementation for bzlmod. PRIVATE USE ONLY. + +NOTE(2023-07-11): We cannot set the visibility of this utility function, because the hub +repo needs to be able to access this. +""" + +load("//python/private:version_label.bzl", "version_label") + +def entry_point(*, pkg, packages, default_version, tmpl, script = None): + """Return an entry_point script dictionary for a select statement. + + PRIVATE USE ONLY. + + Args: + pkg: the PyPI package name (e.g. "pylint"). + script: the script name to use (e.g. "epylint"), defaults to the `pkg` arg. + packages: the mapping of PyPI packages to python versions that are supported. + default_version: the default Python version. + tmpl: the template that will be interpolated by this function. The + following keys are going to be replaced: 'version_label', 'pkg' and + 'script'. + + Returns: + A dict that can be used in select statement or None if the pkg is not + in the supplied packages dictionary. + """ + if not script: + script = pkg + + if pkg not in packages: + # This is an error case, the caller should execute 'fail' and we are not doing it because + # we want easier testability. + return None + + selects = {} + default = "" + for full_version in packages[pkg]: + # Label() is called to evaluate this in the context of rules_python, not the pip repo + condition = str(Label("//python/config_settings:is_python_{}".format(full_version))) + + entry_point = tmpl.format( + version_label = version_label(full_version), + pkg = pkg, + script = script, + ) + + if full_version == default_version: + default = entry_point + else: + selects[condition] = entry_point + + if default: + selects["//conditions:default"] = default + + return selects diff --git a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 4a3d512ae7..0000000000 --- a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. - -This file is different from the other bzlmod template -because we do not support entry_point yet. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - # TODO: https://github.com/bazelbuild/rules_python/issues/1262 - print("not implemented") diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 0e80542747..c0a15581c8 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -269,108 +269,6 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): - repo_name = rctx.attr.repo_name - build_contents = _BUILD_FILE_CONTENTS - aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) - for path, contents in aliases.items(): - rctx.file(path, contents) - - # NOTE: we are using the canonical name with the double '@' in order to - # always uniquely identify a repository, as the labels are being passed as - # a string and the resolution of the label happens at the call-site of the - # `requirement`, et al. macros. - macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - - rctx.file("BUILD.bazel", build_contents) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, p) - for p in bzl_packages - ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "whl") - for p in bzl_packages - ]), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%REQUIREMENTS_LOCK%%": requirements, - }) - -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_library_alias_names - _create_pip_repository_bzlmod(rctx, bzl_packages, "") - -pip_hub_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_library_alias_names": attr.string_list( - mandatory = True, - doc = "The list of whl alias that we use to build aliases and the whl names", - ), - "_template": attr.label( - default = ":pip_hub_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_hub_repository_bzlmod = repository_rule( - attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_hub_repository_bzlmod_impl, -) - -def _pip_repository_bzlmod_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) - - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] - - bzl_packages = sorted([name for name, _ in packages]) - _create_pip_repository_bzlmod(rctx, bzl_packages, str(requirements_txt)) - -pip_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_repository_bzlmod = repository_rule( - attrs = pip_repository_bzlmod_attrs, - doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", - implementation = _pip_repository_bzlmod_impl, -) - def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 2df60b0b52..0000000000 --- a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,33 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - if not script: - script = pkg - return "@@%%NAME%%_{}//:rules_python_wheel_entry_point_{}".format(_clean_name(pkg), script) diff --git a/tests/pip_hub_repository/entry_point/BUILD.bazel b/tests/pip_hub_repository/entry_point/BUILD.bazel new file mode 100644 index 0000000000..a8a441c428 --- /dev/null +++ b/tests/pip_hub_repository/entry_point/BUILD.bazel @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":entry_point_test.bzl", "entry_point_test_suite") + +entry_point_test_suite(name = "entry_point_tests") diff --git a/tests/pip_hub_repository/entry_point/entry_point_test.bzl b/tests/pip_hub_repository/entry_point/entry_point_test.bzl new file mode 100644 index 0000000000..87f540dec6 --- /dev/null +++ b/tests/pip_hub_repository/entry_point/entry_point_test.bzl @@ -0,0 +1,94 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/pip_install:entry_point.bzl", "entry_point") +load("//tests:test_env.bzl", "test_env") + +def _label(label_str): + # Bazel 5.4 is stringifying the labels differently. + # + # This function can be removed when the minimum supported version is 6+ + if test_env.is_bazel_6_or_higher(): + return label_str + else: + return label_str.lstrip("@") + +_tests = [] + +def _test_unknown_entry_point_returns_none(env): + actual = entry_point( + pkg = "foo", + packages = {}, + tmpl = "dummy", + default_version = "dummy", + ) + + # None is returned if the package is not found, we will fail in the place + # where this is called. + want = None + + # FIXME @aignas 2023-07-11: currently the rules_testing does not accept a + # None to the dict subject. + env.expect.that_int(actual).equals(want) + +_tests.append(_test_unknown_entry_point_returns_none) + +def _test_constraint_values_are_set_correctly(env): + actual = entry_point( + pkg = "foo", + packages = {"foo": ["1.2.0", "1.2.3", "1.2.5"]}, + tmpl = "dummy", + default_version = "1.2.3", + ) + + # Python constraints are set correctly + want = { + # NOTE @aignas 2023-07-07: label will contain the rules_python + # when the macro is used outside rules_python + _label("@//python/config_settings:is_python_1.2.0"): "dummy", + _label("@//python/config_settings:is_python_1.2.5"): "dummy", + "//conditions:default": "dummy", + } + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_constraint_values_are_set_correctly) + +def _test_template_is_interpolated_correctly(env): + actual = entry_point( + pkg = "foo", + script = "bar", + packages = {"foo": ["1.3.3", "1.2.5"]}, + tmpl = "pkg={pkg} script={script} version={version_label}", + default_version = "1.2.5", + ) + + # Template is interpolated correctly + want = { + _label("@//python/config_settings:is_python_1.3.3"): "pkg=foo script=bar version=13", + "//conditions:default": "pkg=foo script=bar version=12", + } + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_template_is_interpolated_correctly) + +def entry_point_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 9d28b0f6340ed3fc8978c072961ea4ae17861047 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:24:11 +0900 Subject: [PATCH 8/9] docs: regenerate docs --- docs/pip_repository.md | 46 ------------------------------------------ 1 file changed, 46 deletions(-) diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 853605276f..f58d90c396 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -2,27 +2,6 @@ - - -## pip_hub_repository_bzlmod - -
-pip_hub_repository_bzlmod(name, repo_mapping, repo_name, whl_library_alias_names)
-
- -A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this repository. | Name | required | | -| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | -| repo_name | The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name. | String | required | | -| whl_library_alias_names | The list of whl alias that we use to build aliases and the whl names | List of strings | required | | - - ## pip_repository @@ -101,31 +80,6 @@ py_binary( | timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | - - -## pip_repository_bzlmod - -
-pip_repository_bzlmod(name, repo_mapping, repo_name, requirements_darwin, requirements_linux,
-                      requirements_lock, requirements_windows)
-
- -A rule for bzlmod pip_repository creation. Intended for private use only. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this repository. | Name | required | | -| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | -| repo_name | The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name | String | required | | -| requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | -| requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | -| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | Label | optional | None | -| requirements_windows | Override the requirements_lock attribute when the host platform is Windows | Label | optional | None | - - ## whl_library From 1b88ae3fc4a802002c5c3c8fce04c54f552dbb7a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Tue, 25 Jul 2023 18:19:13 +0900 Subject: [PATCH 9/9] A different external API for entrypoints without a macro --- examples/bzlmod/MODULE.bazel | 6 ++ examples/bzlmod/entry_point/BUILD.bazel | 3 +- python/extensions/pip.bzl | 30 +++++-- .../extensions/private/pip_hub_repository.bzl | 8 ++ python/private/render_pkg_aliases.bzl | 86 ++++++++++--------- 5 files changed, 86 insertions(+), 47 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 7c5083c82a..5d6e98f2f9 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -94,6 +94,9 @@ use_repo(pip, "whl_mods_hub") # Because we do not have a python_version defined here # pip.parse uses the python toolchain that is set as default. pip.parse( + entry_points = { + "yamllint": ["yamllint"], + }, hub_name = "pip", requirements_lock = "//:requirements_lock_3_9.txt", requirements_windows = "//:requirements_windows_3_9.txt", @@ -106,6 +109,9 @@ pip.parse( }, ) pip.parse( + entry_points = { + "yamllint": ["yamllint"], + }, hub_name = "pip", python_version = "3.10", requirements_lock = "//:requirements_lock_3_10.txt", diff --git a/examples/bzlmod/entry_point/BUILD.bazel b/examples/bzlmod/entry_point/BUILD.bazel index dfc02b00a0..8e3f3338e4 100644 --- a/examples/bzlmod/entry_point/BUILD.bazel +++ b/examples/bzlmod/entry_point/BUILD.bazel @@ -1,9 +1,8 @@ -load("@pip//:requirements.bzl", "entry_point") load("@rules_python//python:defs.bzl", "py_test") alias( name = "yamllint", - actual = entry_point("yamllint"), + actual = "@pip//yamllint/bin:yamllint", ) py_test( diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index 699fb751f6..2dc42420ac 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -111,7 +111,10 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): extra_pip_args = pip_attr.extra_pip_args + parse_result.options if hub_name not in whl_map: - whl_map[hub_name] = {} + whl_map[hub_name] = struct( + wheels = {}, + entry_points = {}, + ) whl_modifications = {} if pip_attr.whl_modifications != None: @@ -143,10 +146,20 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): environment = pip_attr.environment, ) - if whl_name not in whl_map[hub_name]: - whl_map[hub_name][whl_name] = [] + if whl_name not in whl_map[hub_name].wheels: + whl_map[hub_name].wheels[whl_name] = [] + + whl_map[hub_name].wheels[whl_name].append(full_version(pip_attr.python_version)) + + for whl_name, scripts in pip_attr.entry_points.items(): + if whl_name not in whl_map[hub_name].entry_points: + whl_map[hub_name].entry_points[whl_name] = {} - whl_map[hub_name][whl_name].append(full_version(pip_attr.python_version)) + for script in scripts: + if script not in whl_map[hub_name].entry_points[whl_name]: + whl_map[hub_name].entry_points[whl_name][script] = [] + + whl_map[hub_name].entry_points[whl_name][script].append(full_version(pip_attr.python_version)) def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. @@ -288,12 +301,19 @@ def _pip_impl(module_ctx): pip_hub_repository( name = hub_name, repo_name = hub_name, - whl_map = whl_map, + whl_map = whl_map.wheels, + whl_entry_points = { + whl_name: json.encode(values) + for whl_name, values in whl_map.entry_points.items() + }, default_version = full_version(DEFAULT_PYTHON_VERSION), ) def _pip_parse_ext_attrs(): attrs = dict({ + "entry_points": attr.string_list_dict( + doc = "TODO", + ), "hub_name": attr.string( mandatory = True, doc = """ diff --git a/python/extensions/private/pip_hub_repository.bzl b/python/extensions/private/pip_hub_repository.bzl index b73c003500..9e4f7551bd 100644 --- a/python/extensions/private/pip_hub_repository.bzl +++ b/python/extensions/private/pip_hub_repository.bzl @@ -35,6 +35,10 @@ def _impl(rctx): aliases = render_pkg_aliases( repo_name = repo_name, whl_map = rctx.attr.whl_map, + whl_entry_points = { + whl_name: json.decode(values) + for whl_name, values in rctx.attr.whl_entry_points.items() + }, default_version = rctx.attr.default_version, rules_python = rctx.attr._template.workspace_name, ) @@ -84,6 +88,10 @@ setting.""", mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "whl_entry_points": attr.string_dict( + mandatory = False, + doc = "The entry points that we will create aliases for.", + ), "whl_map": attr.string_list_dict( mandatory = True, doc = "The wheel map where values are python versions", diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl index 28042e0507..ca66b06719 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/render_pkg_aliases.bzl @@ -37,9 +37,9 @@ def _render_alias( repo_name, dep, target, - default_version, versions, - rules_python): + rules_python, + default_version = None): """Render an alias for common targets If the versions is passed, then the `rules_python` must be passed as well and @@ -71,13 +71,14 @@ def _render_alias( ) selects[condition] = actual - default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( - repo_name = repo_name, - version = version_label(default_version), - dep = dep, - target = target, - ) - selects["//conditions:default"] = default_actual + if default_version: + default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(default_version), + dep = dep, + target = target, + ) + selects["//conditions:default"] = default_actual return _SELECT.format( name = name, @@ -89,22 +90,21 @@ def _render_alias( ), ) -def _render_entry_points(repo_name, dep): - return """\ -load("@{repo_name}_{dep}//:entry_points.bzl", "entry_points") - -[ - alias( - name = script, - actual = "@{repo_name}_{dep}//:" + target, - visibility = ["//visibility:public"], - ) - for script, target in entry_points.items() -] -""".format( - repo_name = repo_name, - dep = dep, - ) +def _render_entry_points(repo_name, dep, entry_points, default_version = None, rules_python = None, prefix = "rules_python_wheel_entry_point_"): + return "\n\n".join([ + """package(default_visibility = ["//visibility:public"])""", + ] + [ + _render_alias( + name = normalize_name(script), + repo_name = repo_name, + dep = dep, + target = prefix + normalize_name(script), + versions = versions, + default_version = default_version, + rules_python = rules_python, + ) + for script, versions in entry_points.items() + ]) def _render_common_aliases(repo_name, name, versions = None, default_version = None, rules_python = None): return "\n\n".join([ @@ -131,7 +131,14 @@ def _render_common_aliases(repo_name, name, versions = None, default_version = N for target in ["pkg", "whl", "data", "dist_info"] ]) -def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_python = None, default_version = None): +def render_pkg_aliases( + *, + repo_name, + bzl_packages = None, + whl_map = None, + whl_entry_points = None, + rules_python = None, + default_version = None): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -155,8 +162,14 @@ def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_ contents = {} for name in bzl_packages: versions = None + entry_points = None + if whl_map != None: versions = whl_map[name] + + if whl_entry_points != None: + entry_points = whl_entry_points.get(name, {}) + name = normalize_name(name) filename = "{}/BUILD.bazel".format(name) @@ -168,22 +181,15 @@ def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_ default_version = default_version, ).strip() - if versions == None: - # NOTE: this code would be normally executed in the non-bzlmod - # scenario, where we are requesting friendly aliases to be - # generated. In that case, we will not be creating aliases for - # entry_points to leave the behaviour unchanged from previous - # rules_python versions. - continue - - # NOTE @aignas 2023-07-07: we are not creating aliases using a select - # and the version specific aliases because we would need to fetch the - # package for all versions in order to construct the said select. - for version in versions: - filename = "{}/bin_py{}/BUILD.bazel".format(name, version_label(version)) + if entry_points: + # Generate aliases where we have the select statement + filename = "{}/bin/BUILD.bazel".format(name) contents[filename] = _render_entry_points( - repo_name = "{}_{}".format(repo_name, version_label(version)), + repo_name = repo_name, dep = name, + rules_python = rules_python, + default_version = default_version, + entry_points = entry_points, ).strip() return contents