From 41ff2a05c0beff439bad7acfd564e6827e34b13b Mon Sep 17 00:00:00 2001 From: Marcus Boerger <6420169+helly25@users.noreply.github.com> Date: Mon, 15 May 2023 18:48:29 +0200 Subject: [PATCH] Auto-detect toolchain from llvm_versions attr (#196) The `llvm_versions` attr is currently not being used to auto-detect toolchains and is only used for the `urls` and `toolchain_roots` attributes. Users who want to simply specify different LLVM versions for different host OS/arch currently also need to specify the `urls` attribute even when the auto-detection logic would otherwise be sufficient (see default toolchain in tests/WORKSPACE). With this change, the `urls` and associated `strip_prefix` and `sha256` attributes will be unnecessary. The default toolchain in `tests/WORKSPACE` has also been changed to reflect this. --------- Co-authored-by: Siddhartha Bagaria --- README.md | 37 +++++++++++++++++++---- tests/WORKSPACE | 13 +++++++- toolchain/internal/llvm_distributions.bzl | 14 +++++++-- toolchain/rules.bzl | 17 +++++++---- 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 5a100657..81c97da2 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") llvm_toolchain( name = "llvm_toolchain", - llvm_version = "15.0.6", + llvm_version = "16.0.0", ) load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains") @@ -70,6 +70,30 @@ attributes to `llvm_toolchain`. ## Advanced Usage +#### Per host architecture LLVM version + +LLVM does not come with distributions for all host architectures in each +version. In particular patch versions often come with few prebuilt packages. +This means that a single version probably is not enough to address all hosts +one wants to support. + +This can be solved by providing a target/version map with a default version. +The example below selects `15.0.6` as the default version for all targets not +specified explicitly. This is like providing `llvm_version = "15.0.6"`, just +like in the example on the top. However, here we provide two more entries that +map their respective target to a distinct version: + +``` +llvm_toolchain( + name = "llvm_toolchain", + llvm_versions = { + "": "15.0.6", + "darwin-aarch64": "15.0.7", + "darwin-x86_64": "15.0.7", + }, +) +``` + #### Customizations We currently offer limited customizability through attributes of the @@ -132,11 +156,12 @@ and instead rely on the `--incompatible_enable_cc_toolchain_resolution` flag. The following mechanisms are available for using an LLVM toolchain: 1. Host OS information is used to find the right pre-built binary distribution - from llvm.org, given the `llvm_version` attribute. The LLVM toolchain - archive is downloaded and extracted as a separate repository with the suffix - `_llvm`. The detection is not perfect, so you may have to use other options - for some host OS type and versions. We expect the detection logic to grow - through community contributions. We welcome PRs. + from llvm.org, given the `llvm_version` or `llvm_versions` attribute. The + LLVM toolchain archive is downloaded and extracted as a separate repository + with the suffix `_llvm`. The detection logic for `llvm_version` is not + perfect, so you may have to use `llvm_versions` for some host OS type and + versions. We expect the detection logic to grow through community + contributions. We welcome PRs. 2. You can use the `urls` attribute to specify your own URLs for each OS type, version and architecture. For example, you can specify a different URL for Arch Linux and a different one for Ubuntu. Just as with the option above, diff --git a/tests/WORKSPACE b/tests/WORKSPACE index bc0a71c4..eb298161 100644 --- a/tests/WORKSPACE +++ b/tests/WORKSPACE @@ -27,7 +27,7 @@ bazel_toolchain_dependencies() load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") # When updating this version, also update the versions associated with -# llvm_toolchain below, sys_paths_test in the worflows file, and xcompile_test +# llvm_toolchain below, sys_paths_test in the workflows file, and xcompile_test # through the `llvm_toolchain_with_sysroot` toolchain. LLVM_VERSION = "15.0.6" @@ -38,6 +38,17 @@ llvm_toolchain( "darwin-aarch64": "15.0.7", "darwin-x86_64": "15.0.7", }, +) + +# Example toolchain with user provided URLs. +# TODO(siddharthab): Add test. +llvm_toolchain( + name = "llvm_toolchain_with_urls", + llvm_versions = { + "": "15.0.6", + "darwin-aarch64": "15.0.7", + "darwin-x86_64": "15.0.7", + }, sha256 = { "": "38bc7f5563642e73e69ac5626724e206d6d539fbef653541b34cae0ba9c3f036", "darwin-aarch64": "867c6afd41158c132ef05a8f1ddaecf476a26b91c85def8e124414f9a9ba188d", diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 8174062b..63d6b74c 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -13,7 +13,7 @@ # limitations under the License. load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "use_netrc") -load("//toolchain/internal:common.bzl", _arch = "arch", _attr_dict = "attr_dict", _host_os_arch_dict_value = "host_os_arch_dict_value") +load("//toolchain/internal:common.bzl", _arch = "arch", _attr_dict = "attr_dict", _host_os_arch_dict_value = "host_os_arch_dict_value", _os = "os") load("//toolchain/internal:release_name.bzl", _llvm_release_name = "llvm_release_name") # If a new LLVM version is missing from this list, please add the shasums here @@ -334,8 +334,18 @@ def _urls(rctx): return urls, sha256, strip_prefix, key +def _get_llvm_version(rctx): + if rctx.attr.llvm_version: + return rctx.attr.llvm_version + if not rctx.attr.llvm_versions: + fail("Neither 'llvm_version' nor 'llvm_versions' given.") + (key, llvm_version) = _host_os_arch_dict_value(rctx, "llvm_versions") + if not llvm_version: + fail("LLVM version string missing for ({os}, {arch})", os=_os(rctx), arch=_arch(rctx)) + return llvm_version + def _distribution_urls(rctx): - llvm_version = rctx.attr.llvm_version + llvm_version = _get_llvm_version(rctx) if rctx.attr.distribution == "auto": basename = _llvm_release_name(rctx, llvm_version) diff --git a/toolchain/rules.bzl b/toolchain/rules.bzl index 23e28560..bf6dbdfc 100644 --- a/toolchain/rules.bzl +++ b/toolchain/rules.bzl @@ -29,6 +29,16 @@ _target_pairs = ", ".join(_supported_os_arch_keys()) # Atributes common to both `llvm` and `toolchain` repository rules. _common_attrs = { + "llvm_versions": attr.string_dict( + mandatory = False, + doc = ("LLVM version strings, keyed by host OS release name and architecture, " + + "e.g. darwin-x86_64, darwin-aarch64, ubuntu-20.04-x86_64, etc., or a " + + "less specific OS and arch pair ({}). ".format(_target_pairs) + + "An empty key is used to specify a fallback default for all hosts. " + + "If no `toolchain_roots` is given, then the toolchain will be looked up " + + "in the list of known llvm_distributions using the provided version. " + + "If unset, a default value is set from the `llvm_version` attribute."), + ), } _llvm_repo_attrs = dict(_common_attrs) @@ -42,7 +52,7 @@ _llvm_repo_attrs.update({ mandatory = False, doc = ("URLs to LLVM pre-built binary distribution archives, keyed by host OS " + "release name and architecture, e.g. darwin-x86_64, darwin-aarch64, " + - "ubuntu-20.04-x86_64, etc, or a less specific OS and arch pair " + + "ubuntu-20.04-x86_64, etc., or a less specific OS and arch pair " + "({}). ".format(_target_pairs) + "May also need the `strip_prefix` attribute. " + "Consider also setting the `sha256` attribute. An empty key is " + @@ -239,11 +249,6 @@ _llvm_config_attrs.update({ "paths. Else, the value will be assumed to be a bazel package containing the " + "filegroup targets as in BUILD.llvm_repo."), ), - "llvm_versions": attr.string_dict( - mandatory = True, - doc = ("LLVM version strings for each entry in the `toolchain_roots` attribute; " + - "if unset, a default value is set from the `llvm_version` attribute."), - ), "absolute_paths": attr.bool( default = False, doc = "Use absolute paths in the toolchain. Avoids sandbox overhead.",