From d83ef8bc5d2ff26f91223fac0b3e526049d5a1d4 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Fri, 1 Mar 2024 16:15:42 -0800 Subject: [PATCH] feat: don't require users to build tools from source (#287) Wire up the toolchains along two different codepaths, based on whether our version was stamped as a release, or otherwise a pre-release. When a release: - fetch tools from GitHub releases using `rctx.download` - each fetched tool goes into a separate external repo with its own `toolchain` declaration, so Bazel fetches only the needed ones - the resulting repos are returned to repositories.bzl which registers them When a pre-release: - continue to build the tools from source, using existing toolchains --- ### Type of change - New feature or functionality (change which adds functionality) ### Test plan - Manual testing; please provide instructions so we can reproduce: Use my personal fork to test against a release I performed there. --- .github/workflows/ci.yaml | 6 + e2e/smoke/WORKSPACE.bazel | 1 + py/private/toolchain/tools.bzl | 144 +++++++++++++++++------- py/private/toolchain/unpack/BUILD.bazel | 13 +-- py/private/toolchain/venv/BUILD.bazel | 18 ++- py/repositories.bzl | 29 +++-- tools/integrity.bzl | 12 +- tools/version.bzl | 2 +- 8 files changed, 153 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ecb14a9f..af6aa3c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,3 +29,9 @@ jobs: {"folder": ".", "bazelversion": "6.4.0"}, {"folder": ".", "bzlmodEnabled": true} ] + + verify-bcr-patches: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: patch --dry-run -p1 < .bcr/patches/*.patch diff --git a/e2e/smoke/WORKSPACE.bazel b/e2e/smoke/WORKSPACE.bazel index 0152cb43..70e3a8f7 100644 --- a/e2e/smoke/WORKSPACE.bazel +++ b/e2e/smoke/WORKSPACE.bazel @@ -53,6 +53,7 @@ crate_repositories() # Alternatively, you can skip calling this function, so long as you've # already fetched all the dependencies. load("@aspect_rules_py//py:repositories.bzl", "rules_py_dependencies") + rules_py_dependencies() # "Installation" for rules_python diff --git a/py/private/toolchain/tools.bzl b/py/private/toolchain/tools.bzl index 7d2cd006..69ba45cc 100644 --- a/py/private/toolchain/tools.bzl +++ b/py/private/toolchain/tools.bzl @@ -1,31 +1,41 @@ -"""Toolchain for making toolchains""" +"""Utilities for making toolchains""" +load("//tools:integrity.bzl", "RELEASED_BINARY_INTEGRITY") +load("//tools:version.bzl", "VERSION") -load("//tools:version.bzl", "IS_PRERELEASE") +# The expected config for each tool, whether it runs in an action or at runtime +RUST_BIN_CFG = { + "unpack": "exec", + "venv": "target", +} TOOLCHAIN_PLATFORMS = { "darwin_amd64": struct( - release_platform = "macos-amd64", + arch = "x86_64", + vendor_os_abi = "apple-darwin", compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", ], ), "darwin_arm64": struct( - release_platform = "macos-arm64", + arch = "aarch64", + vendor_os_abi = "apple-darwin", compatible_with = [ "@platforms//os:macos", "@platforms//cpu:aarch64", ], ), "linux_amd64": struct( - release_platform = "linux-amd64", + arch = "x86_64", + vendor_os_abi = "unknown-linux-gnu", compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", ], ), "linux_arm64": struct( - release_platform = "linux-arm64", + arch = "aarch64", + vendor_os_abi = "unknown-linux-gnu", compatible_with = [ "@platforms//os:linux", "@platforms//cpu:aarch64", @@ -56,7 +66,7 @@ def _toolchain_impl(ctx): return [toolchain_info, default_info, template_variables] -_toolchain = rule( +py_tool_toolchain = rule( implementation = _toolchain_impl, attrs = { "bin": attr.label( @@ -69,49 +79,97 @@ _toolchain = rule( }, ) -def _make_toolchain_name(name, platform): - return "{}_{}_toolchain".format(name, platform) - -def make_toolchain(name, toolchain_type, tools, cfg = "exec"): +def source_toolchain(name, toolchain_type, bin): """Makes vtool toolchain and repositories Args: name: Override the prefix for the generated toolchain repositories. toolchain_type: Toolchain type reference. - tools: Mapping of tool binary to platform. - cfg: Generate a toolchain for the target or exec config. + bin: the rust_binary target """ - if IS_PRERELEASE: - toolchain_rule = "{}_toolchain_source".format(name) - _toolchain( - name = toolchain_rule, - bin = tools["from-source"], - template_var = "{}_BIN".format(name.upper()), - ) - native.toolchain( - name = _make_toolchain_name(name, "source"), - toolchain = toolchain_rule, - toolchain_type = toolchain_type, + toolchain_rule = "{}_toolchain_source".format(name) + py_tool_toolchain( + name = toolchain_rule, + bin = bin, + template_var = "{}_BIN".format(name.upper()), + ) + native.toolchain( + name = "{}_source_toolchain".format(name), + toolchain = toolchain_rule, + toolchain_type = toolchain_type, + ) + +def _tool_repo_impl(rctx): + build_content = """\ +# Generated by @aspect_rules_py//py/private/toolchain:tools.bzl +load("@aspect_rules_py//py/private/toolchain:tools.bzl", "py_tool_toolchain") + +package(default_visibility = ["//visibility:public"]) +""" + # For manual testing, override these environment variables + # TODO: use rctx.getenv when available, see https://github.com/bazelbuild/bazel/pull/20944 + release_fork = "aspect-build" + release_version = VERSION + if "RULES_PY_RELEASE_FORK" in rctx.os.environ: + release_fork = rctx.os.environ["RULES_PY_RELEASE_FORK"] + if "RULES_PY_RELEASE_VERSION" in rctx.os.environ: + release_version = rctx.os.environ["RULES_PY_RELEASE_VERSION"] + + for tool, cfg in RUST_BIN_CFG.items(): + filename = "-".join([ + tool, + TOOLCHAIN_PLATFORMS[rctx.attr.platform].arch, + TOOLCHAIN_PLATFORMS[rctx.attr.platform].vendor_os_abi, + ]) + url = "https://github.com/{}/rules_py/releases/download/v{}/{}".format( + release_fork, + release_version, + filename ) - return - - for [platform, meta] in TOOLCHAIN_PLATFORMS.items(): - toolchain_rule = "{}_toolchain_{}".format(name, platform) - _toolchain( - name = toolchain_rule, - bin = tools[meta.release_platform], - template_var = "{}_BIN".format(name.upper()), + rctx.download( + url = url, + sha256 = RELEASED_BINARY_INTEGRITY[filename], + executable = True, + output = tool, ) + build_content += """\ +py_tool_toolchain(name = "concrete_{tool}_toolchain", bin = "{tool}", template_var = "{tool_upper}_BIN") + +toolchain( + name = "{tool}_toolchain", + {cfg}_compatible_with = {compatible_with}, + toolchain = "concrete_{tool}_toolchain", + toolchain_type = "@aspect_rules_py//py/private/toolchain/{tool}:toolchain_type", +) +""".format( + cfg = cfg, + compatible_with = TOOLCHAIN_PLATFORMS[rctx.attr.platform].compatible_with, + tool = tool, + tool_upper = tool.upper(), +) + + rctx.file("BUILD.bazel", build_content) - args = dict({ - "name": _make_toolchain_name(name, platform), - "toolchain": toolchain_rule, - "toolchain_type": toolchain_type, - }) - if cfg == "exec": - args.update({"exec_compatible_with": meta.compatible_with}) - else: - args.update({"target_compatible_with": meta.compatible_with}) - - native.toolchain(**args) +_tool_repo = repository_rule( + doc = "Download pre-built binary tools and create toolchains for them", + implementation = _tool_repo_impl, + attrs = { + "platform": attr.string(mandatory = True, values = TOOLCHAIN_PLATFORMS.keys()), + } +) + +def binary_tool_repos(name): + """Create a downloaded toolchain for every tool under every supported platform. + + Args: + name: prefix used in created repositories + Returns: + list of toolchain targets to register + """ + result = [] + for platform in TOOLCHAIN_PLATFORMS.keys(): + plat_repo_name = ".".join([name, platform]) + result.append("@{}//:all".format(plat_repo_name)) + _tool_repo(name = plat_repo_name, platform = platform) + return result diff --git a/py/private/toolchain/unpack/BUILD.bazel b/py/private/toolchain/unpack/BUILD.bazel index d670d7d7..3efb2688 100644 --- a/py/private/toolchain/unpack/BUILD.bazel +++ b/py/private/toolchain/unpack/BUILD.bazel @@ -1,17 +1,12 @@ -load("//py/private/toolchain:tools.bzl", "make_toolchain") +load("//py/private/toolchain:tools.bzl", "source_toolchain") toolchain_type( name = "toolchain_type", + visibility = ["//visibility:public"], ) -make_toolchain( +source_toolchain( name = "unpack", toolchain_type = ":toolchain_type", - tools = { - "from-source": "//py/tools/unpack_bin", - "macos-amd64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-x86_64-apple-darwin", - "macos-arm64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-aarch64-apple-darwin", - "linux-amd64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-x86_64-unknown-linux-musl", - "linux-arm64": "@aspect_rules_py//py/tools/unpack_bin/bins:unpack-aarch64-unknown-linux-musl", - }, + bin = "//py/tools/unpack_bin", ) diff --git a/py/private/toolchain/venv/BUILD.bazel b/py/private/toolchain/venv/BUILD.bazel index 66083b84..a407433c 100644 --- a/py/private/toolchain/venv/BUILD.bazel +++ b/py/private/toolchain/venv/BUILD.bazel @@ -1,18 +1,16 @@ -load("//py/private/toolchain:tools.bzl", "make_toolchain") +load("//py/private/toolchain:tools.bzl", "source_toolchain") toolchain_type( name = "toolchain_type", + visibility = ["//visibility:public"], ) -make_toolchain( +source_toolchain( name = "venv", - cfg = "target", + # When running from source, we assume the target is the exec platform. + # TODO: when we have cross-compile Mac -> Linux, pre-release users should be able to + # build a py_image on their Mac and have the linux venv rust_binary end up in the container. + # cfg = "target", toolchain_type = ":toolchain_type", - tools = { - "from-source": "//py/tools/venv_bin", - "macos-amd64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-x86_64-apple-darwin", - "macos-arm64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-aarch64-apple-darwin", - "linux-amd64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-x86_64-unknown-linux-musl", - "linux-arm64": "@aspect_rules_py//py/tools/venv_bin/bins:venv-aarch64-unknown-linux-musl", - }, + bin = "//py/tools/venv_bin", ) diff --git a/py/repositories.bzl b/py/repositories.bzl index dc973405..297f6fa4 100644 --- a/py/repositories.bzl +++ b/py/repositories.bzl @@ -7,6 +7,8 @@ See https://docs.bazel.build/versions/main/skylark/deploying.html#dependencies load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//py/private/toolchain:autodetecting.bzl", _register_autodetecting_python_toolchain = "register_autodetecting_python_toolchain") +load("//py/private/toolchain:tools.bzl", "binary_tool_repos") +load("//tools:version.bzl", "IS_PRERELEASE") def http_archive(name, **kwargs): maybe(_http_archive, name = name, **kwargs) @@ -21,11 +23,13 @@ register_autodetecting_python_toolchain = _register_autodetecting_python_toolcha # and released only in semver majors. # buildifier: disable=unnamed-macro -def rules_py_dependencies(register_py_toolchains = True): - """Fetch rules_py's dependencies, and optionally register toolchains. +def rules_py_dependencies(name = "rules_py_tools", register = True, prerelease = IS_PRERELEASE): + """Fetch rules_py's dependencies Args: - register_py_toolchains: When true, rules_py's toolchains are automatically registered. + name: prefix for generated repositories + register: whether to register the toolchains created, should be false under bzlmod + prerelease: whether to build rust tools from source, rather than download pre-built binaries """ # The minimal version of bazel_skylib we require @@ -49,9 +53,18 @@ def rules_py_dependencies(register_py_toolchains = True): strip_prefix = "rules_python-0.31.0", url = "https://github.com/bazelbuild/rules_python/releases/download/0.31.0/rules_python-0.31.0.tar.gz", ) + + # When running from a release version: + # Fetch remote tools from the release and create toolchain for them + if prerelease: + if register: + native.register_toolchains( + "@aspect_rules_py//py/private/toolchain/venv/...", + "@aspect_rules_py//py/private/toolchain/unpack/...", + ) + return - if register_py_toolchains: - native.register_toolchains( - "@aspect_rules_py//py/private/toolchain/venv/...", - "@aspect_rules_py//py/private/toolchain/unpack/...", - ) + toolchains = binary_tool_repos(name) + if register: + native.register_toolchains(*toolchains) + \ No newline at end of file diff --git a/tools/integrity.bzl b/tools/integrity.bzl index 69a24787..54b6b9a6 100644 --- a/tools/integrity.bzl +++ b/tools/integrity.bzl @@ -4,4 +4,14 @@ This file contents are entirely replaced during release publishing. The checked in content is only here to allow load() statements in the sources to resolve. """ -RELEASED_BINARY_INTEGRITY = {} +# FROM https://github.com/alexeagle/rules_py/releases/tag/v0.101.0 +RELEASED_BINARY_INTEGRITY = { + "unpack-aarch64-apple-darwin": "5199cb98d59535ba8609c4a538be99d8759e971ab9d1e680d932091529860868", + "unpack-x86_64-apple-darwin": "23d0eb9ec09609c00d65779234f1210506eba1bf64090a01cef71d5175e3e6b6", + "venv-aarch64-apple-darwin": "e02b478358641abf72ca5192948101e18a3dabccb48e7ee704759ad39c2f718a", + "venv-x86_64-apple-darwin": "13d0c08a23b1064489df522d2327cfbaf0058b9364ca2ffee51b5b6f06a6930e", + "unpack-aarch64-unknown-linux-gnu": "48552e399a1f2ab97e62ca7fce5783b6214e284330c7555383f43acf82446636", + "unpack-x86_64-unknown-linux-gnu": "fd265552bfd236efef519f81ce783322a50d8d7ab5af5d08a713e519cedff87f", + "venv-aarch64-unknown-linux-gnu": "b05d6cbe485e8ed1872f330012929126778744208e47485d7e2be285a027c7ca", + "venv-x86_64-unknown-linux-gnu": "61f53fe45da5f2f160907c87c1a2d663bc57eee667520a46490a837825be61c9" +} diff --git a/tools/version.bzl b/tools/version.bzl index 36434ac2..82f7e302 100644 --- a/tools/version.bzl +++ b/tools/version.bzl @@ -1,6 +1,6 @@ "version information. replaced with stamped info with each release" -# This is automagically replace by git during git archive using `git export-subst` +# Automagically "stamped" by git during `git archive` thanks to `export-subst` line in .gitattributes. # See https://git-scm.com/docs/git-archive#Documentation/git-archive.txt-export-subst _VERSION_PRIVATE = "$Format:%(describe:tags=true)$"