[build] Use py_cc_toolchain for configuring pybind11 (#22346)

The py_cc_toolchain is a rules_python concept that bridges the Python
and C++ toolchains by providing cc_library targets for the headers and
libraries for compiling C code against the Python interpreter selected
by the Python toolchain.

This commit changes our python repository to define that new kind of
toolchain and our pydrake bindings to use it. The toolchains are now
encapsulated as `@python//:all`, moved from `//tools/py_toolchain`.

Important note for future bzlmod users: because our python repository
rule can fail when run on a system without our default interpreter,
our MODULE.bazel will no longer register our default python toolchain
automatically. Downstream Bazel projects using Drake as a module will
need to opt-in to the local toolchain.

We provide new labels in //tools/workspace/python as a single point of
control for depending on Python. The comments in python/repository.bzl
provide an overview of how the pieces all fit together.

The legacy libraries `@python` and `@python//:python_direct_link` are
deprecated.

CMakeLists.txt

@@ -358,6 +358,9 @@ endfunction()
 set(BAZEL_WORKSPACE_EXTRA)
 set(BAZEL_WORKSPACE_EXCLUDES)
 
+# Our cmake/WORKSPACE.bzlmod always provides @python.
+list(APPEND BAZEL_WORKSPACE_EXCLUDES "python")
+
 macro(override_repository NAME)
   set(repo "${CMAKE_CURRENT_BINARY_DIR}/external/workspace/${NAME}")
   string(APPEND BAZEL_WORKSPACE_EXTRA

MODULE.bazel

@@ -23,11 +23,6 @@ cc_configure = use_extension(
 )
 use_repo(cc_configure, "local_config_cc")
 
-register_toolchains(
-    "//tools/py_toolchain:toolchain",
-    "//tools/py_toolchain:exec_tools_toolchain",
-)
-
 # TODO(#20731) Move all of our dependencies from WORKSPACE.bzlmod into this
 # file, so that downstream projects can consume Drake exclusively via bzlmod
 # (and so that we can delete our WORKSPACE files prior to Bazel 9 which drops

cmake/WORKSPACE.bzlmod.in

@@ -16,11 +16,8 @@ python_repository(
 # Custom repository rules injected by CMake.
 @BAZEL_WORKSPACE_EXTRA@
 
-# The list of repositories already provided via BAZEL_WORKSPACE_EXTRA.
-_BAZEL_WORKSPACE_EXCLUDES = split_cmake_list("@BAZEL_WORKSPACE_EXCLUDES@")
-
 # For anything not already overridden, use Drake's default externals.
 add_default_workspace(
-    repository_excludes = ["python"] + _BAZEL_WORKSPACE_EXCLUDES,
+    repository_excludes = split_cmake_list("@BAZEL_WORKSPACE_EXCLUDES@"),
     bzlmod = True,
 )

cmake/bazel.rc.in

@@ -10,6 +10,9 @@ startup --output_base="@BAZEL_OUTPUT_BASE@"
 # Environment variables to be used in repository rules (if any).
 common @BAZEL_REPO_ENV@
 
+# Use the Python interpreter from our cmake/WORKSPACE.bzlmod.in.
+build --extra_toolchains=@python//:all
+
 # Disable the "convenience symlinks" intended for Bazel users; they only add
 # confusion for the CMake use case.
 build --symlink_prefix=/

tools/bazel.rc

@@ -12,6 +12,9 @@ build -c opt
 build --strip=never
 build --strict_system_includes
 
+# Use the host Python interpreter by default.
+build --extra_toolchains=@python//:all
+
 # Use C++20 by default.
 build --cxxopt=-std=c++20
 build --host_cxxopt=-std=c++20

tools/install/install.bzl

@@ -1,4 +1,3 @@
-load("@python//:version.bzl", "PYTHON_VERSION")
 load("@rules_license//rules:providers.bzl", "LicenseInfo")
 load("//tools/skylark:cc.bzl", "CcInfo")
 load("//tools/skylark:drake_java.bzl", "MainClassInfo")
@@ -48,6 +47,16 @@ def _depset_to_list(x):
     iter_list = x.to_list() if type(x) == "depset" else x
     return iter_list
 
+#------------------------------------------------------------------------------
+
+_PY_CC_TOOLCHAIN_TYPE = "@rules_python//python/cc:toolchain_type"
+
+def _python_version(ctx):
+    """Returns a string a containing the major.minor version number of the
+    current Python toolchain."""
+    py_cc_toolchain = ctx.toolchains[_PY_CC_TOOLCHAIN_TYPE].py_cc_toolchain
+    return py_cc_toolchain.python_version
+
 #------------------------------------------------------------------------------
 def _output_path(ctx, input_file, strip_prefix = [], ignore_errors = False):
     """Compute output path (without destination prefix) for install action.
@@ -123,7 +132,7 @@ def _install_action(
     if "@WORKSPACE@" in dest:
         dest = dest.replace("@WORKSPACE@", _workspace(ctx))
     if "@PYTHON_VERSION@" in dest:
-        dest = dest.replace("@PYTHON_VERSION@", PYTHON_VERSION)
+        dest = dest.replace("@PYTHON_VERSION@", _python_version(ctx))
 
     if type(strip_prefixes) == "dict":
         strip_prefix = strip_prefixes.get(
@@ -608,6 +617,10 @@ _install_rule = rule(
     },
     executable = True,
     implementation = _install_impl,
+    toolchains = [
+        # Used to discern the major.minor site-packages path to install into.
+        _PY_CC_TOOLCHAIN_TYPE,
+    ],
 )
 
 def install(tags = [], **kwargs):

tools/install/libdrake/BUILD.bazel

@@ -4,7 +4,6 @@ load(
     "@drake//tools/workspace:cmake_configure_file.bzl",
     "cmake_configure_file",
 )
-load("@python//:version.bzl", "PYTHON_VERSION")
 load(
     "//third_party:com_github_bazelbuild_rules_cc/whole_archive.bzl",
     "cc_whole_archive_library",
@@ -29,6 +28,13 @@ load(":header_lint.bzl", "cc_check_allowed_headers")
 
 package(default_visibility = ["//visibility:private"])
 
+genrule(
+    name = "python_version_cmake",
+    srcs = ["//tools/workspace/python:python_version.txt"],
+    outs = ["python_version.cmake"],
+    cmd = """echo "set(PYTHON_VERSION \\"$$(cat $<)\\")" > $@""",
+)
+
 genrule(
     name = "stamp_version",
     outs = ["stamp_version.txt"],
@@ -57,9 +63,9 @@ cmake_configure_file(
     src = "drake-config.cmake.in",
     out = "drake-config.cmake",
     atonly = True,
-    cmakelists = ["stamp_version.cmake"],
-    defines = [
-        "PYTHON_VERSION=" + PYTHON_VERSION,
+    cmakelists = [
+        ":python_version.cmake",
+        ":stamp_version.cmake",
     ],
 )
 

tools/skylark/pybind.bzl

@@ -289,8 +289,8 @@ def drake_pybind_cc_googletest(
         deps = cc_deps + [
             "//:drake_shared_library",
             "//bindings/pydrake:pydrake_pybind",
+            "//tools/workspace/python:cc_libpython",
             "@pybind11",
-            "@python//:python_direct_link",
         ],
         copts = cc_copts,
         use_default_main = False,

tools/workspace/default.bzl

@@ -391,16 +391,12 @@ def add_default_toolchains(
           set this to True if you are using bzlmod.
     """
     if bzlmod:
-        # All toolchains are in MODULE.bazel already.
+        # The cc toolchain is in MODULE.bazel already.
+        # The py toolchain is in tools/bazel.rc already.
         return
 
     if "py" not in excludes:
-        native.register_toolchains(
-            "//tools/py_toolchain:toolchain",
-        )
-        native.register_toolchains(
-            "//tools/py_toolchain:exec_tools_toolchain",
-        )
+        native.register_toolchains("@python//:all")
     if "rust" not in excludes:
         register_rust_toolchains()
 

tools/workspace/lcm/package.BUILD.bazel

@@ -195,7 +195,8 @@ cc_binary(
     visibility = ["//visibility:private"],
     deps = [
         ":lcm",
-        "@python",
+        "@drake//tools/workspace/python:cc_headers",
+        "@drake//tools/workspace/python:cc_libs",
     ],
 )
 

tools/workspace/pybind11/package.BUILD.bazel

@@ -61,7 +61,8 @@ cc_library(
     includes = ["include"],
     deps = [
         "@eigen",
-        "@python",
+        "@drake//tools/workspace/python:cc_headers",
+        "@drake//tools/workspace/python:cc_libs",
     ],
 )
 

tools/workspace/python/BUILD.bazel

@@ -1,6 +1,42 @@
-# This file exists to make our directory into a Bazel package, so that our
-# neighboring *.bzl file can be loaded elsewhere.
-
 load("//tools/lint:lint.bzl", "add_lint_tests")
+load(":defs.bzl", "current_py_cc_libpython", "python_version_txt")
+
+package(default_visibility = ["//visibility:public"])
+
+# Provides a text file containing the major.minor version number of the current
+# Python toolchain, without any newlines. This file may be used as srcs or data
+# for any other rule whose action needs to know the current python version.
+python_version_txt(
+    name = "python_version.txt",
+)
+
+# Provides a single point of control within Drake for how to compile a native
+# C/C++ Python module (e.g., for pybind11) for the current Python toolchain.
+# This may be used like it was a cc_library target that listed hdrs= and
+# includes= for the current Python's header files.
+alias(
+    name = "cc_headers",
+    actual = "@rules_python//python/cc:current_py_cc_headers",
+)
+
+# Provides a single point of control within Drake for how to link a native
+# C/C++ Python module (e.g., for pybind11) for the current Python toolchain.
+# This may be used like it was a cc_library target that listed linkopts= for
+# any libraries use by a Python module. Note that this is intended for linking
+# native modules, and does NOT link the libpython embedded runtime; for that,
+# use cc_libpython below.
+alias(
+    name = "cc_libs",
+    actual = "@rules_python//python/cc:current_py_cc_libs",
+)
+
+# Provides a single point of control within Drake for how to link a C/C++
+# executable that embeds the current Python toolchain's interpreter (e.g., for
+# Python unit tests which are implemented as C++ programs). This alias may be
+# used like it was a cc_library target that listed linkopts= for the libpython
+# embedded runtime.
+current_py_cc_libpython(
+    name = "cc_libpython",
+)
 
 add_lint_tests()

tools/workspace/python/defs.bzl

@@ -0,0 +1,55 @@
+load("@rules_cc//cc/common:cc_common.bzl", "cc_common")
+load("//tools/skylark:cc.bzl", "CcInfo")
+
+_PY_CC_TOOLCHAIN_TYPE = "@rules_python//python/cc:toolchain_type"
+
+# These rules are intended for use only by our neighboring BUILD.bazel file.
+# The comments in that file explain how and why to use these rules' output.
+
+def _current_py_cc_libpython(ctx):
+    py_cc_toolchain = ctx.toolchains[_PY_CC_TOOLCHAIN_TYPE].py_cc_toolchain
+    linkopts = ["-lpython{}".format(py_cc_toolchain.python_version)]
+    return [
+        CcInfo(
+            compilation_context = None,
+            linking_context = cc_common.create_linking_context(
+                linker_inputs = depset(
+                    direct = [
+                        cc_common.create_linker_input(
+                            owner = ctx.label,
+                            user_link_flags = depset(linkopts),
+                        ),
+                    ],
+                ),
+            ),
+        ),
+    ]
+
+current_py_cc_libpython = rule(
+    implementation = _current_py_cc_libpython,
+    toolchains = [_PY_CC_TOOLCHAIN_TYPE],
+    provides = [CcInfo],
+    doc = """Provides the linker flags for how to link a C/C++ executable
+    that embeds a Python interpreter (e.g., for unit testing), based on the
+    current Python toolchain. Use this rule like a cc_library.""",
+)
+
+def _python_version_txt(ctx):
+    py_cc_toolchain = ctx.toolchains[_PY_CC_TOOLCHAIN_TYPE].py_cc_toolchain
+    output = ctx.actions.declare_file(ctx.label.name)
+    ctx.actions.write(
+        output = output,
+        content = py_cc_toolchain.python_version,
+        is_executable = False,
+    )
+    return [DefaultInfo(
+        files = depset([output]),
+        data_runfiles = ctx.runfiles(files = [output]),
+    )]
+
+python_version_txt = rule(
+    implementation = _python_version_txt,
+    toolchains = [_PY_CC_TOOLCHAIN_TYPE],
+    doc = """Generates a text file containing the major.minor version number of
+    the current Python toolchain, without any newlines.""",
+)