diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index df88ae8490..7c5083c82a 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -118,10 +118,7 @@ pip.parse( "@whl_mods_hub//:wheel.json": "wheel", }, ) - -# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't -# yet support entry points; see https://github.com/bazelbuild/rules_python/issues/1262 -use_repo(pip, "pip", "pip_39") +use_repo(pip, "pip") bazel_dep(name = "other_module", version = "", repo_name = "our_other_module") local_path_override( diff --git a/examples/bzlmod/entry_point/BUILD.bazel b/examples/bzlmod/entry_point/BUILD.bazel index f68552c3ef..dfc02b00a0 100644 --- a/examples/bzlmod/entry_point/BUILD.bazel +++ b/examples/bzlmod/entry_point/BUILD.bazel @@ -1,4 +1,4 @@ -load("@pip_39//:requirements.bzl", "entry_point") +load("@pip//:requirements.bzl", "entry_point") load("@rules_python//python:defs.bzl", "py_test") alias( diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index add69a4c64..699fb751f6 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -15,17 +15,16 @@ "pip module extension for use with bzlmod" load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load("@rules_python//python:pip.bzl", "whl_library_alias") load( "@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", - "pip_hub_repository_bzlmod", "pip_repository_attrs", - "pip_repository_bzlmod", "use_isolated", "whl_library", ) load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/extensions/private:pip_hub_repository.bzl", "pip_hub_repository") +load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version_label.bzl", "version_label") @@ -111,16 +110,6 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): requirements = parse_result.requirements extra_pip_args = pip_attr.extra_pip_args + parse_result.options - # Create the repository where users load the `requirement` macro. Under bzlmod - # this does not create the install_deps() macro. - # TODO: we may not need this repository once we have entry points - # supported. For now a user can access this repository and use - # the entrypoint functionality. - pip_repository_bzlmod( - name = pip_name, - repo_name = pip_name, - requirements_lock = pip_attr.requirements_lock, - ) if hub_name not in whl_map: whl_map[hub_name] = {} @@ -155,9 +144,9 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): ) if whl_name not in whl_map[hub_name]: - whl_map[hub_name][whl_name] = {} + whl_map[hub_name][whl_name] = [] - whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_" + whl_map[hub_name][whl_name].append(full_version(pip_attr.python_version)) def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. @@ -295,32 +284,12 @@ def _pip_impl(module_ctx): _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map) for hub_name, whl_map in hub_whl_map.items(): - for whl_name, version_map in whl_map.items(): - if DEFAULT_PYTHON_VERSION not in version_map: - fail(( - "Default python version '{version}' missing in pip " + - "hub '{hub}': update your pip.parse() calls so that " + - 'includes `python_version = "{version}"`' - ).format( - version = DEFAULT_PYTHON_VERSION, - hub = hub_name, - )) - - # Create the alias repositories which contains different select - # statements These select statements point to the different pip - # whls that are based on a specific version of Python. - whl_library_alias( - name = hub_name + "_" + whl_name, - wheel_name = whl_name, - default_version = DEFAULT_PYTHON_VERSION, - version_map = version_map, - ) - # Create the hub repository for pip. - pip_hub_repository_bzlmod( + pip_hub_repository( name = hub_name, repo_name = hub_name, - whl_library_alias_names = whl_map.keys(), + whl_map = whl_map, + default_version = full_version(DEFAULT_PYTHON_VERSION), ) def _pip_parse_ext_attrs(): diff --git a/python/extensions/private/pip_hub_repository.bzl b/python/extensions/private/pip_hub_repository.bzl new file mode 100644 index 0000000000..b73c003500 --- /dev/null +++ b/python/extensions/private/pip_hub_repository.bzl @@ -0,0 +1,95 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A pip_hub_repository rule used to create bzlmod hub repos for PyPI packages. + +It assumes that version aware toolchain is used and is responsible for setting up +aliases for entry points and the actual package targets. +""" + +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") + +_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +def _impl(rctx): + bzl_packages = rctx.attr.whl_map.keys() + repo_name = rctx.attr.repo_name + + aliases = render_pkg_aliases( + repo_name = repo_name, + whl_map = rctx.attr.whl_map, + default_version = rctx.attr.default_version, + rules_python = rctx.attr._template.workspace_name, + ) + for path, contents in aliases.items(): + rctx.file(path, contents) + + # NOTE: we are using the canonical name with the double '@' in order to + # always uniquely identify a repository, as the labels are being passed as + # a string and the resolution of the label happens at the call-site of the + # `requirement`, et al. macros. + macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) + + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + "%%ALL_DATA_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, p) + for p in bzl_packages + ]), + "%%ALL_WHL_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, "whl") + for p in bzl_packages + ]), + "%%DEFAULT_PY_VERSION%%": repr(rctx.attr.default_version), + "%%MACRO_TMPL%%": macro_tmpl, + "%%NAME%%": rctx.attr.name, + "%%PACKAGE_AVAILABILITY%%": repr({ + k: [v for v in versions] + for k, versions in rctx.attr.whl_map.items() + }), + "%%RULES_PYTHON%%": rctx.attr._template.workspace_name, + }) + +pip_hub_repository = repository_rule( + attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ +This is the default python version in the format of X.Y.Z. This should match +what is setup by the 'python' extension using the 'is_default = True' +setting.""", + ), + "repo_name": attr.string( + mandatory = True, + doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", + ), + "whl_map": attr.string_list_dict( + mandatory = True, + doc = "The wheel map where values are python versions", + ), + "_template": attr.label(default = ":requirements.bzl.tmpl"), + }, + doc = """A rule for creating bzlmod hub repo for PyPI packages. PRIVATE USE ONLY.""", + implementation = _impl, +) diff --git a/python/extensions/private/requirements.bzl.tmpl b/python/extensions/private/requirements.bzl.tmpl new file mode 100644 index 0000000000..4243990b2b --- /dev/null +++ b/python/extensions/private/requirements.bzl.tmpl @@ -0,0 +1,52 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip.parse extension. + +This file is different from the other bzlmod template +because we do not support entry_point yet. +""" + +load("@@%%RULES_PYTHON%%//python/pip_install:entry_point.bzl", _entry_point = "entry_point") + +all_requirements = %%ALL_REQUIREMENTS%% + +all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% + +all_data_requirements = %%ALL_DATA_REQUIREMENTS%% + +_default_py_version = %%DEFAULT_PY_VERSION%% +_packages = %%PACKAGE_AVAILABILITY%% + +def _clean_name(name): + return name.replace("-", "_").replace(".", "_").lower() + +def requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") + +def whl_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") + +def data_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "data") + +def dist_info_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") + +def entry_point(pkg, script = None): + """Returns a select() expression to locate the version-specific entry point. + """ + # TODO: not implemented + # selects = _entry_point( + # tmpl = "@@%%NAME%%//{pkg}/bin_py{version_label}:{script}", + # pkg = _clean_name(pkg), + # script = script, + # packages = _packages, + # default_version = _default_py_version, + # ) + # if selects == None: + # fail("Package '{}' does not exist, select one from: {}".format(pkg, _packages.keys())) + + # # NOTE: We return a select() expression instead of an alias to such an expression + # # to avoid having to eagerly load all versions of the wheel. See + # # https://github.com/bazelbuild/rules_python/issues/1262 for discussion. + # return select(selects) diff --git a/python/pip_install/entry_point.bzl b/python/pip_install/entry_point.bzl new file mode 100644 index 0000000000..71bc96e68b --- /dev/null +++ b/python/pip_install/entry_point.bzl @@ -0,0 +1,69 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""entry_point macro implementation for bzlmod. PRIVATE USE ONLY. + +NOTE(2023-07-11): We cannot set the visibility of this utility function, because the hub +repo needs to be able to access this. +""" + +load("//python/private:version_label.bzl", "version_label") + +def entry_point(*, pkg, packages, default_version, tmpl, script = None): + """Return an entry_point script dictionary for a select statement. + + PRIVATE USE ONLY. + + Args: + pkg: the PyPI package name (e.g. "pylint"). + script: the script name to use (e.g. "epylint"), defaults to the `pkg` arg. + packages: the mapping of PyPI packages to python versions that are supported. + default_version: the default Python version. + tmpl: the template that will be interpolated by this function. The + following keys are going to be replaced: 'version_label', 'pkg' and + 'script'. + + Returns: + A dict that can be used in select statement or None if the pkg is not + in the supplied packages dictionary. + """ + if not script: + script = pkg + + if pkg not in packages: + # This is an error case, the caller should execute 'fail' and we are not doing it because + # we want easier testability. + return None + + selects = {} + default = "" + for full_version in packages[pkg]: + # Label() is called to evaluate this in the context of rules_python, not the pip repo + condition = str(Label("//python/config_settings:is_python_{}".format(full_version))) + + entry_point = tmpl.format( + version_label = version_label(full_version), + pkg = pkg, + script = script, + ) + + if full_version == default_version: + default = entry_point + else: + selects[condition] = entry_point + + if default: + selects["//conditions:default"] = default + + return selects diff --git a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 4a3d512ae7..0000000000 --- a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. - -This file is different from the other bzlmod template -because we do not support entry_point yet. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - # TODO: https://github.com/bazelbuild/rules_python/issues/1262 - print("not implemented") diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 0e80542747..c0a15581c8 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -269,108 +269,6 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): - repo_name = rctx.attr.repo_name - build_contents = _BUILD_FILE_CONTENTS - aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) - for path, contents in aliases.items(): - rctx.file(path, contents) - - # NOTE: we are using the canonical name with the double '@' in order to - # always uniquely identify a repository, as the labels are being passed as - # a string and the resolution of the label happens at the call-site of the - # `requirement`, et al. macros. - macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - - rctx.file("BUILD.bazel", build_contents) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, p) - for p in bzl_packages - ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "whl") - for p in bzl_packages - ]), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%REQUIREMENTS_LOCK%%": requirements, - }) - -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_library_alias_names - _create_pip_repository_bzlmod(rctx, bzl_packages, "") - -pip_hub_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_library_alias_names": attr.string_list( - mandatory = True, - doc = "The list of whl alias that we use to build aliases and the whl names", - ), - "_template": attr.label( - default = ":pip_hub_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_hub_repository_bzlmod = repository_rule( - attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_hub_repository_bzlmod_impl, -) - -def _pip_repository_bzlmod_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) - - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] - - bzl_packages = sorted([name for name, _ in packages]) - _create_pip_repository_bzlmod(rctx, bzl_packages, str(requirements_txt)) - -pip_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_repository_bzlmod = repository_rule( - attrs = pip_repository_bzlmod_attrs, - doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", - implementation = _pip_repository_bzlmod_impl, -) - def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 2df60b0b52..0000000000 --- a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,33 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - if not script: - script = pkg - return "@@%%NAME%%_{}//:rules_python_wheel_entry_point_{}".format(_clean_name(pkg), script) diff --git a/tests/pip_hub_repository/entry_point/BUILD.bazel b/tests/pip_hub_repository/entry_point/BUILD.bazel new file mode 100644 index 0000000000..a8a441c428 --- /dev/null +++ b/tests/pip_hub_repository/entry_point/BUILD.bazel @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":entry_point_test.bzl", "entry_point_test_suite") + +entry_point_test_suite(name = "entry_point_tests") diff --git a/tests/pip_hub_repository/entry_point/entry_point_test.bzl b/tests/pip_hub_repository/entry_point/entry_point_test.bzl new file mode 100644 index 0000000000..87f540dec6 --- /dev/null +++ b/tests/pip_hub_repository/entry_point/entry_point_test.bzl @@ -0,0 +1,94 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/pip_install:entry_point.bzl", "entry_point") +load("//tests:test_env.bzl", "test_env") + +def _label(label_str): + # Bazel 5.4 is stringifying the labels differently. + # + # This function can be removed when the minimum supported version is 6+ + if test_env.is_bazel_6_or_higher(): + return label_str + else: + return label_str.lstrip("@") + +_tests = [] + +def _test_unknown_entry_point_returns_none(env): + actual = entry_point( + pkg = "foo", + packages = {}, + tmpl = "dummy", + default_version = "dummy", + ) + + # None is returned if the package is not found, we will fail in the place + # where this is called. + want = None + + # FIXME @aignas 2023-07-11: currently the rules_testing does not accept a + # None to the dict subject. + env.expect.that_int(actual).equals(want) + +_tests.append(_test_unknown_entry_point_returns_none) + +def _test_constraint_values_are_set_correctly(env): + actual = entry_point( + pkg = "foo", + packages = {"foo": ["1.2.0", "1.2.3", "1.2.5"]}, + tmpl = "dummy", + default_version = "1.2.3", + ) + + # Python constraints are set correctly + want = { + # NOTE @aignas 2023-07-07: label will contain the rules_python + # when the macro is used outside rules_python + _label("@//python/config_settings:is_python_1.2.0"): "dummy", + _label("@//python/config_settings:is_python_1.2.5"): "dummy", + "//conditions:default": "dummy", + } + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_constraint_values_are_set_correctly) + +def _test_template_is_interpolated_correctly(env): + actual = entry_point( + pkg = "foo", + script = "bar", + packages = {"foo": ["1.3.3", "1.2.5"]}, + tmpl = "pkg={pkg} script={script} version={version_label}", + default_version = "1.2.5", + ) + + # Template is interpolated correctly + want = { + _label("@//python/config_settings:is_python_1.3.3"): "pkg=foo script=bar version=13", + "//conditions:default": "pkg=foo script=bar version=12", + } + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_template_is_interpolated_correctly) + +def entry_point_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests)