Auto-detect toolchain from llvm_versions attr (#196)

The `llvm_versions` attr is currently not being used to auto-detect toolchains
and is only used for the `urls` and `toolchain_roots` attributes.

Users who want to simply specify different LLVM versions for different host
OS/arch currently also need to specify the `urls` attribute even when the
auto-detection logic would otherwise be sufficient (see default toolchain in
tests/WORKSPACE).

With this change, the `urls` and associated `strip_prefix` and `sha256`
attributes will be unnecessary. The default toolchain in `tests/WORKSPACE`
has also been changed to reflect this.

---------

Co-authored-by: Siddhartha Bagaria <[email protected]>

README.md

@@ -45,7 +45,7 @@ load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
 
 llvm_toolchain(
     name = "llvm_toolchain",
-    llvm_version = "15.0.6",
+    llvm_version = "16.0.0",
 )
 
 load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
@@ -70,6 +70,30 @@ attributes to `llvm_toolchain`.
 
 ## Advanced Usage
 
+#### Per host architecture LLVM version
+
+LLVM does not come with distributions for all host architectures in each
+version. In particular patch versions often come with few prebuilt packages.
+This means that a single version probably is not enough to address all hosts
+one wants to support.
+
+This can be solved by providing a target/version map with a default version.
+The example below selects `15.0.6` as the default version for all targets not
+specified explicitly. This is like providing `llvm_version = "15.0.6"`, just
+like in the example on the top. However, here we provide two more entries that
+map their respective target to a distinct version:
+
+```
+llvm_toolchain(
+    name = "llvm_toolchain",
+    llvm_versions = {
+        "": "15.0.6",
+        "darwin-aarch64": "15.0.7",
+        "darwin-x86_64": "15.0.7",
+    },
+)
+```
+
 #### Customizations
 
 We currently offer limited customizability through attributes of the
@@ -132,11 +156,12 @@ and instead rely on the `--incompatible_enable_cc_toolchain_resolution` flag.
 The following mechanisms are available for using an LLVM toolchain:
 
 1. Host OS information is used to find the right pre-built binary distribution
-   from llvm.org, given the `llvm_version` attribute. The LLVM toolchain
-   archive is downloaded and extracted as a separate repository with the suffix
-   `_llvm`. The detection is not perfect, so you may have to use other options
-   for some host OS type and versions. We expect the detection logic to grow
-   through community contributions. We welcome PRs.
+   from llvm.org, given the `llvm_version` or `llvm_versions` attribute. The
+   LLVM toolchain archive is downloaded and extracted as a separate repository
+   with the suffix `_llvm`. The detection logic for `llvm_version` is not
+   perfect, so you may have to use `llvm_versions` for some host OS type and
+   versions. We expect the detection logic to grow through community
+   contributions. We welcome PRs.
 2. You can use the `urls` attribute to specify your own URLs for each OS type,
    version and architecture. For example, you can specify a different URL for
    Arch Linux and a different one for Ubuntu. Just as with the option above,

tests/WORKSPACE

@@ -27,7 +27,7 @@ bazel_toolchain_dependencies()
 load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
 
 # When updating this version, also update the versions associated with
-# llvm_toolchain below, sys_paths_test in the worflows file, and xcompile_test
+# llvm_toolchain below, sys_paths_test in the workflows file, and xcompile_test
 # through the `llvm_toolchain_with_sysroot` toolchain.
 LLVM_VERSION = "15.0.6"
 
@@ -38,6 +38,17 @@ llvm_toolchain(
         "darwin-aarch64": "15.0.7",
         "darwin-x86_64": "15.0.7",
     },
+)
+
+# Example toolchain with user provided URLs.
+# TODO(siddharthab): Add test.
+llvm_toolchain(
+    name = "llvm_toolchain_with_urls",
+    llvm_versions = {
+        "": "15.0.6",
+        "darwin-aarch64": "15.0.7",
+        "darwin-x86_64": "15.0.7",
+    },
     sha256 = {
         "": "38bc7f5563642e73e69ac5626724e206d6d539fbef653541b34cae0ba9c3f036",
         "darwin-aarch64": "867c6afd41158c132ef05a8f1ddaecf476a26b91c85def8e124414f9a9ba188d",

toolchain/internal/llvm_distributions.bzl

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "use_netrc")
-load("//toolchain/internal:common.bzl", _arch = "arch", _attr_dict = "attr_dict", _host_os_arch_dict_value = "host_os_arch_dict_value")
+load("//toolchain/internal:common.bzl", _arch = "arch", _attr_dict = "attr_dict", _host_os_arch_dict_value = "host_os_arch_dict_value", _os = "os")
 load("//toolchain/internal:release_name.bzl", _llvm_release_name = "llvm_release_name")
 
 # If a new LLVM version is missing from this list, please add the shasums here
@@ -334,8 +334,18 @@ def _urls(rctx):
 
     return urls, sha256, strip_prefix, key
 
+def _get_llvm_version(rctx):
+    if rctx.attr.llvm_version:
+        return rctx.attr.llvm_version
+    if not rctx.attr.llvm_versions:
+        fail("Neither 'llvm_version' nor 'llvm_versions' given.")
+    (key, llvm_version) = _host_os_arch_dict_value(rctx, "llvm_versions")
+    if not llvm_version:
+        fail("LLVM version string missing for ({os}, {arch})", os=_os(rctx), arch=_arch(rctx))
+    return llvm_version
+
 def _distribution_urls(rctx):
-    llvm_version = rctx.attr.llvm_version
+    llvm_version = _get_llvm_version(rctx)
 
     if rctx.attr.distribution == "auto":
         basename = _llvm_release_name(rctx, llvm_version)

toolchain/rules.bzl

@@ -29,6 +29,16 @@ _target_pairs = ", ".join(_supported_os_arch_keys())
 
 # Atributes common to both `llvm` and `toolchain` repository rules.
 _common_attrs = {
+    "llvm_versions": attr.string_dict(
+        mandatory = False,
+        doc = ("LLVM version strings, keyed by host OS release name and architecture, " +
+               "e.g. darwin-x86_64, darwin-aarch64, ubuntu-20.04-x86_64, etc., or a " +
+               "less specific OS and arch pair ({}). ".format(_target_pairs) +
+               "An empty key is used to specify a fallback default for all hosts. " +
+               "If no `toolchain_roots` is given, then the toolchain will be looked up " +
+               "in the list of known llvm_distributions using the provided version. " +
+               "If unset, a default value is set from the `llvm_version` attribute."),
+    ),
 }
 
 _llvm_repo_attrs = dict(_common_attrs)
@@ -42,7 +52,7 @@ _llvm_repo_attrs.update({
         mandatory = False,
         doc = ("URLs to LLVM pre-built binary distribution archives, keyed by host OS " +
                "release name and architecture, e.g. darwin-x86_64, darwin-aarch64, " +
-               "ubuntu-20.04-x86_64, etc, or a less specific OS and arch pair " +
+               "ubuntu-20.04-x86_64, etc., or a less specific OS and arch pair " +
                "({}). ".format(_target_pairs) +
                "May also need the `strip_prefix` attribute. " +
                "Consider also setting the `sha256` attribute. An empty key is " +
@@ -239,11 +249,6 @@ _llvm_config_attrs.update({
                "paths. Else, the value will be assumed to be a bazel package containing the " +
                "filegroup targets as in BUILD.llvm_repo."),
     ),
-    "llvm_versions": attr.string_dict(
-        mandatory = True,
-        doc = ("LLVM version strings for each entry in the `toolchain_roots` attribute; " +
-               "if unset, a default value is set from the `llvm_version` attribute."),
-    ),
     "absolute_paths": attr.bool(
         default = False,
         doc = "Use absolute paths in the toolchain. Avoids sandbox overhead.",