[Python] Support musl-linux binary wheels on arm64 (grpc#38223)

This is a rebase of grpc#32941 to fix conflicts and align to what changed in the meantime.

I'm very confused on how the tooling works here:
- no idea how to generate `tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile` so I copied the x64 one and update accordingly
- `tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version` and `tools/dockerfile/distribtest/python_alpine_aarch64.current_version` need to be refreshed

<!--

If you know who should review your pull request, please assign it to that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the appropriate
lang label.

-->

Closes grpc#38223

PiperOrigin-RevId: 720993782

src/python/grpcio_observability/setup.py

@@ -177,6 +177,8 @@ def get_ext_filename(self, ext_name):
         EXTRA_ENV_LINK_ARGS += " -lpthread"
         if check_linker_need_libatomic():
             EXTRA_ENV_LINK_ARGS += " -latomic"
+    if "linux" in sys.platform:
+        EXTRA_ENV_LINK_ARGS += " -static-libgcc"
 
 # This enables the standard link-time optimizer, which help us prevent some undefined symbol errors by
 # remove some unused symbols from .so file.

templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template

@@ -0,0 +1,27 @@
+%YAML 1.2
+--- |
+  # Copyright 2024 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!!
+
+  FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546
+
+  <%include file="../python_pip_builds.include"/>
+
+  <%include file="../ccache.include"/>
+
+  <%include file="../git_config.include"/>
+
+  RUN apk add openssl openssl-dev

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -36,6 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd",
     "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0",
     "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0",
+    "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:7f727b0896c5e86ff7e214255e2e05dee45689c1793d6e698905b3465a2752e9",
     "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d",
     "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1",
     "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb",
@@ -68,6 +69,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:263ea79d940c905233624b02751194474408338b0f5ae2fb822966b2f20f47ce",
     "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:0e102df67f31aeb6afe68250603288c2f1c98ccf360d1c42d751b8451da94b48",
     "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:527e2e9ec4db0c52a53b50abfd59907a1b7e221168dc401686f6a48d33bddc5c",
+    "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27",
     "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:94b57e5ea31ebc29af734474bcaff3074770778e5d27557cdc06d755ee8bc7ed",
     "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:edf4a0c8333c9309e52f323aa7315bbc0e5643216613cab4ecd2bce3d1ec26c0",
     "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878",

tools/distrib/python/grpcio_tools/setup.py

@@ -227,6 +227,8 @@ def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
         EXTRA_ENV_LINK_ARGS += " -lpthread"
         if check_linker_need_libatomic():
             EXTRA_ENV_LINK_ARGS += " -latomic"
+    if "linux" in sys.platform:
+        EXTRA_ENV_LINK_ARGS += " -static-libgcc"
 
 # Explicitly link Core Foundation framework for MacOS to ensure no symbol is
 # missing when compiled using package managers like Conda.

tools/dockerfile/distribtest/python_alpine_aarch64.current_version

@@ -0,0 +1 @@
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:e78a4b229d02acb518073cddbeb1d359dce6e789@sha256:7f727b0896c5e86ff7e214255e2e05dee45689c1793d6e698905b3465a2752e9

tools/internal_ci/linux/grpc_distribtests_python_arm64.sh → tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright 2025 The gRPC Authors
+# Copyright 2023 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+# this an aarch64 image
+FROM arm64v8/python:3.13-alpine
 
-# change to grpc repo root
-cd $(dirname $0)/../../..
+# Our test infrastructure demands bash
+RUN apk update && apk add bash
 
-echo "TODO: Building arm64 python artifacts"
+RUN pip3 install virtualenv

tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version

@@ -0,0 +1 @@
+us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:b171ae144fc76a473fdfad8e25c36a02b886f6a9@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27

tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile

@@ -0,0 +1,66 @@
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!!
+
+FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546
+
+#===================================
+# Install Python build requirements
+RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp313-cp313/bin/pip install --upgrade 'cython<4.0.0rc1'
+
+#=================
+# Install ccache
+
+# Install ccache from source since ccache 3.x packaged with most linux distributions
+# does not support Redis backend for caching.
+RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.7.5/ccache-4.7.5.tar.gz \
+    && tar -zxf ccache.tar.gz \
+    && cd ccache-4.7.5 \
+    && mkdir build && cd build \
+    && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \
+    && make -j4 && make install \
+    && cd ../.. \
+    && rm -rf ccache-4.7.5 ccache.tar.gz
+
+
+# TODO: simplify the list of third_party modules list
+# NOTE: git>=2.46 allows leading paths like third_party/* to include all subdirectories
+# current docker base images use git versions lower than 2.46 and hence require separate configs for each submodule  
+RUN git config --global --add safe.directory /var/local/jenkins/grpc
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/bloaty
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/xds
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googleapis
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googletest
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opencensus-proto
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/boringssl-with-bazel
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/envoy-api
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protobuf
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/zlib
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/benchmark
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/re2
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/abseil-cpp
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry-cpp
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protoc-gen-validate
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/cares/cares
+RUN git config --global protocol.file.allow always
+
+RUN apk add openssl openssl-dev

tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg

@@ -15,7 +15,7 @@
 # Config file for the internal CI (in protobuf text format)
 
 # Location of the continuous shell script in repository.
-build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python_arm64.sh"
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
 timeout_mins: 240
 action {
   define_artifacts {
@@ -24,3 +24,8 @@ action {
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 musllinux_1_1"
+}

tools/internal_ci/linux/grpc_distribtests_python.cfg

@@ -24,3 +24,8 @@ action {
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64 musllinux_1_1"
+}

tools/internal_ci/linux/grpc_distribtests_python.sh

@@ -23,10 +23,19 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-# some distribtests use a pre-registered binfmt_misc hook
-# to automatically execute foreign binaries (such as aarch64)
-# under qemu emulator.
-source tools/internal_ci/helper_scripts/prepare_qemu_rc
+IS_AARCH64_MUSL=""
+if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" || "${TASK_RUNNER_EXTRA_FILTERS}" == "presubmit aarch64 musllinux_1_1" ]]; then
+  IS_AARCH64_MUSL="True"
+fi
+
+if [[ "${IS_AARCH64_MUSL}" == "True" ]]; then
+  echo "Skipping prepare_qemu_rc'"
+else
+  # some distribtests use a pre-registered binfmt_misc hook
+  # to automatically execute foreign binaries (such as aarch64)
+  # under qemu emulator.
+  source tools/internal_ci/helper_scripts/prepare_qemu_rc
+fi
 
 # configure ccache
 source tools/internal_ci/helper_scripts/prepare_ccache_rc
@@ -40,7 +49,12 @@ mkdir -p input_artifacts
 cp -r artifacts/* input_artifacts/ || true
 
 # This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/
-tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+if [[ "${IS_AARCH64_MUSL}" == "True" ]]; then
+  # Not using TASK_RUNNER_EXTRA_FILTERS since we don't have a target with presubmit tag.
+  tools/run_tests/task_runner.py -f package linux python musllinux_1_1 aarch64 -x build_packages/sponge_log.xml || FAILED="true"
+else
+  tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+fi
 
 # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
 # in addition to that, preserve the contents of "artifacts" directory since we want kokoro
@@ -52,7 +66,12 @@ cp -r artifacts/* input_artifacts/ || true
 # Run all python linux distribtests
 # We run the distribtests even if some of the artifacts have failed to build, since that gives
 # a better signal about which distribtest are affected by the currently broken artifact builds.
-tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+if [[ "${IS_AARCH64_MUSL}" == "True" ]]; then
+  # We're using alpine as tag in distribtest targets.
+  tools/run_tests/task_runner.py -f distribtest linux python aarch64 alpine -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+else
+  tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+fi
 
 # This step checks if any of the artifacts exceeds a per-file size limit.
 tools/internal_ci/helper_scripts/check_python_artifacts_size.sh

tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg

@@ -27,5 +27,5 @@ action {
 
 env_vars {
   key: "TASK_RUNNER_EXTRA_FILTERS"
-  value: "presubmit"
+  value: "presubmit -e aarch64 musllinux_1_1"
 }

tools/internal_ci/linux/pull_request/grpc_distribtests_python_arm64.cfg

@@ -15,7 +15,7 @@
 # Config file for the internal CI (in protobuf text format)
 
 # Location of the continuous shell script in repository.
-build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python_arm64.sh"
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
 timeout_mins: 240
 action {
   define_artifacts {
@@ -27,5 +27,5 @@ action {
 
 env_vars {
   key: "TASK_RUNNER_EXTRA_FILTERS"
-  value: "presubmit"
+  value: "presubmit aarch64 musllinux_1_1"
 }

tools/internal_ci/linux/release/grpc_distribtests_python.cfg

@@ -24,3 +24,8 @@ action {
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64 musllinux_1_1"
+}

tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg

@@ -26,5 +26,5 @@ action {
 
 env_vars {
   key: "RUN_TESTS_FLAGS"
-  value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=5400"
+  value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=7200"
 }

tools/run_tests/artifacts/artifact_targets.py

@@ -200,19 +200,27 @@ def build_jobspec(self, inner_jobs=None):
             )
             environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version)
             environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE"
-            environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
             environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE"
 
-            if self.arch == "x86":
+            if self.arch in ("x86", "aarch64"):
                 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE"
 
+            if self.arch == "aarch64":
+                # As we won't strip the binary with auditwheel (see below), strip
+                # it at link time.
+                environ["LDFLAGS"] = "-s"
+                # We're using musllinux aarch64 image to build this artifact so no crosscompiling required.
+                environ["GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS"] = "TRUE"
+            else:
+                environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
+
             return create_docker_jobspec(
                 self.name,
                 "tools/dockerfile/grpc_artifact_python_%s_%s"
                 % (self.platform, self.arch),
                 "tools/run_tests/artifacts/build_artifact_python.sh",
                 environ=environ,
-                timeout_seconds=60 * 60 * 2,
+                timeout_seconds=60 * 60 * 4,
             )
         elif self.platform == "windows":
             environ["EXT_COMPILER"] = "msvc"
@@ -452,6 +460,16 @@ def targets():
             PythonArtifact(
                 "musllinux_1_1", "x86", "cp313-cp313", presubmit=True
             ),
+            PythonArtifact(
+                "musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True
+            ),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp311-cp311"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp312-cp312"),
+            PythonArtifact(
+                "musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True
+            ),
             PythonArtifact("macos", "x64", "python3.8", presubmit=True),
             PythonArtifact("macos", "x64", "python3.9"),
             PythonArtifact("macos", "x64", "python3.10"),

tools/run_tests/artifacts/distribtest_targets.py

@@ -461,6 +461,7 @@ def targets():
         PythonDistribTest(
             "linux", "aarch64", "python38_buster", presubmit=True
         ),
+        PythonDistribTest("linux", "aarch64", "alpine"),
         PythonDistribTest(
             "linux", "x64", "alpine3.18", source=True, presubmit=True
         ),

tools/run_tests/artifacts/package_targets.py

@@ -142,9 +142,17 @@ def build_jobspec(self, inner_jobs=None):
 class PythonPackage:
     """Collects python eggs and wheels created in the artifact phase"""
 
-    def __init__(self):
+    def __init__(self, platform="", arch=""):
         self.name = "python_package"
         self.labels = ["package", "python", "linux"]
+        self.platform = platform
+        self.arch = arch
+        if self.platform:
+            self.labels.append(platform)
+            self.name += "_" + platform
+        if self.arch:
+            self.labels.append(arch)
+            self.name += "_" + arch
 
     def pre_build_jobspecs(self):
         return []
@@ -154,9 +162,16 @@ def build_jobspec(self, inner_jobs=None):
         # since the python package build does very little, we can use virtually
         # any image that has new-enough python, so reusing one of the images used
         # for artifact building seems natural.
+        dockerfile_dir = (
+            "tools/dockerfile/grpc_artifact_python_manylinux2014_x64"
+        )
+        if "musllinux_1_1" in self.platform and "aarch64" in self.arch:
+            dockerfile_dir = (
+                "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64"
+            )
         return create_docker_jobspec(
             self.name,
-            "tools/dockerfile/grpc_artifact_python_manylinux2014_x64",
+            dockerfile_dir,
             "tools/run_tests/artifacts/build_package_python.sh",
             environ={"PYTHON": "/opt/python/cp39-cp39/bin/python"},
         )
@@ -189,5 +204,6 @@ def targets():
         CSharpPackage("windows"),
         RubyPackage(),
         PythonPackage(),
+        PythonPackage("musllinux_1_1", "aarch64"),
         PHPPackage(),
     ]

tools/run_tests/task_runner.py

@@ -71,6 +71,14 @@ def _create_build_map():
     default=[],
     help="Filter targets to build with AND semantics.",
 )
+argp.add_argument(
+    "-e",
+    "--exclude",
+    choices=sorted(_BUILD_MAP.keys()),
+    nargs="+",
+    default=[],
+    help="Target labels to exclude from building.",
+)
 argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int)
 argp.add_argument(
     "-x",
@@ -106,6 +114,9 @@ def _create_build_map():
 # Among targets selected by -b, filter out those that don't match the filter
 targets = [t for t in targets if all(f in t.labels for f in args.filter)]
 
+# Exclude target if it has ALL of the specified exclude labels.
+targets = [t for t in targets if not all(l in args.exclude for l in t.labels)]
+
 print("Will build %d targets:" % len(targets))
 for target in targets:
     print("  %s, labels %s" % (target.name, target.labels))