diff --git a/src/python/grpcio_observability/setup.py b/src/python/grpcio_observability/setup.py index 201efe27bd5b2..c0be8a2758b27 100644 --- a/src/python/grpcio_observability/setup.py +++ b/src/python/grpcio_observability/setup.py @@ -177,6 +177,8 @@ def get_ext_filename(self, ext_name): EXTRA_ENV_LINK_ARGS += " -lpthread" if check_linker_need_libatomic(): EXTRA_ENV_LINK_ARGS += " -latomic" + if "linux" in sys.platform: + EXTRA_ENV_LINK_ARGS += " -static-libgcc" # This enables the standard link-time optimizer, which help us prevent some undefined symbol errors by # remove some unused symbols from .so file. diff --git a/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template b/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template new file mode 100644 index 0000000000000..dc34c01c34b60 --- /dev/null +++ b/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template @@ -0,0 +1,27 @@ +%YAML 1.2 +--- | + # Copyright 2024 The gRPC Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!! + + FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546 + + <%include file="../python_pip_builds.include"/> + + <%include file="../ccache.include"/> + + <%include file="../git_config.include"/> + + RUN apk add openssl openssl-dev diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index f2dde165429c0..13145eb3ba35c 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -36,6 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd", "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0", "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0", + "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:7f727b0896c5e86ff7e214255e2e05dee45689c1793d6e698905b3465a2752e9", "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d", "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1", "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb", @@ -68,6 +69,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:263ea79d940c905233624b02751194474408338b0f5ae2fb822966b2f20f47ce", "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:0e102df67f31aeb6afe68250603288c2f1c98ccf360d1c42d751b8451da94b48", "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:527e2e9ec4db0c52a53b50abfd59907a1b7e221168dc401686f6a48d33bddc5c", + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27", "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:94b57e5ea31ebc29af734474bcaff3074770778e5d27557cdc06d755ee8bc7ed", "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:edf4a0c8333c9309e52f323aa7315bbc0e5643216613cab4ecd2bce3d1ec26c0", "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878", diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 1a997df8de114..a30222be92432 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -227,6 +227,8 @@ def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): EXTRA_ENV_LINK_ARGS += " -lpthread" if check_linker_need_libatomic(): EXTRA_ENV_LINK_ARGS += " -latomic" + if "linux" in sys.platform: + EXTRA_ENV_LINK_ARGS += " -static-libgcc" # Explicitly link Core Foundation framework for MacOS to ensure no symbol is # missing when compiled using package managers like Conda. diff --git a/tools/dockerfile/distribtest/python_alpine_aarch64.current_version b/tools/dockerfile/distribtest/python_alpine_aarch64.current_version new file mode 100644 index 0000000000000..5f4f5771719be --- /dev/null +++ b/tools/dockerfile/distribtest/python_alpine_aarch64.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:e78a4b229d02acb518073cddbeb1d359dce6e789@sha256:7f727b0896c5e86ff7e214255e2e05dee45689c1793d6e698905b3465a2752e9 \ No newline at end of file diff --git a/tools/internal_ci/linux/grpc_distribtests_python_arm64.sh b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile similarity index 74% rename from tools/internal_ci/linux/grpc_distribtests_python_arm64.sh rename to tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile index 9a6c7060ebd6c..686989ad9e704 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python_arm64.sh +++ b/tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile @@ -1,5 +1,4 @@ -#!/usr/bin/env bash -# Copyright 2025 The gRPC Authors +# Copyright 2023 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex +# this an aarch64 image +FROM arm64v8/python:3.13-alpine -# change to grpc repo root -cd $(dirname $0)/../../.. +# Our test infrastructure demands bash +RUN apk update && apk add bash -echo "TODO: Building arm64 python artifacts" +RUN pip3 install virtualenv diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version new file mode 100644 index 0000000000000..39e0c47d2a033 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:b171ae144fc76a473fdfad8e25c36a02b886f6a9@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27 \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile new file mode 100644 index 0000000000000..bb392b6b39bfe --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile @@ -0,0 +1,66 @@ +# Copyright 2024 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!! + +FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546 + +#=================================== +# Install Python build requirements +RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<4.0.0rc1' +RUN /opt/python/cp313-cp313/bin/pip install --upgrade 'cython<4.0.0rc1' + +#================= +# Install ccache + +# Install ccache from source since ccache 3.x packaged with most linux distributions +# does not support Redis backend for caching. +RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.7.5/ccache-4.7.5.tar.gz \ + && tar -zxf ccache.tar.gz \ + && cd ccache-4.7.5 \ + && mkdir build && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \ + && make -j4 && make install \ + && cd ../.. \ + && rm -rf ccache-4.7.5 ccache.tar.gz + + +# TODO: simplify the list of third_party modules list +# NOTE: git>=2.46 allows leading paths like third_party/* to include all subdirectories +# current docker base images use git versions lower than 2.46 and hence require separate configs for each submodule +RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/bloaty +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/xds +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googleapis +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googletest +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opencensus-proto +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/boringssl-with-bazel +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/envoy-api +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protobuf +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/zlib +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/benchmark +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/re2 +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/abseil-cpp +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry-cpp +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protoc-gen-validate +RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/cares/cares +RUN git config --global protocol.file.allow always + +RUN apk add openssl openssl-dev diff --git a/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg b/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg index ffc02e606df42..f166d92fd5f16 100644 --- a/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg +++ b/tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg @@ -15,7 +15,7 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python_arm64.sh" +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" timeout_mins: 240 action { define_artifacts { @@ -24,3 +24,8 @@ action { regex: "github/grpc/artifacts/**" } } + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "aarch64 musllinux_1_1" +} diff --git a/tools/internal_ci/linux/grpc_distribtests_python.cfg b/tools/internal_ci/linux/grpc_distribtests_python.cfg index 307b3b61f0abb..987183791d649 100644 --- a/tools/internal_ci/linux/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_python.cfg @@ -24,3 +24,8 @@ action { regex: "github/grpc/artifacts/**" } } + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "-e aarch64 musllinux_1_1" +} diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index dd5dba63dcf95..ce3541485d3ae 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -23,10 +23,19 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -# some distribtests use a pre-registered binfmt_misc hook -# to automatically execute foreign binaries (such as aarch64) -# under qemu emulator. -source tools/internal_ci/helper_scripts/prepare_qemu_rc +IS_AARCH64_MUSL="" +if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" || "${TASK_RUNNER_EXTRA_FILTERS}" == "presubmit aarch64 musllinux_1_1" ]]; then + IS_AARCH64_MUSL="True" +fi + +if [[ "${IS_AARCH64_MUSL}" == "True" ]]; then + echo "Skipping prepare_qemu_rc'" +else + # some distribtests use a pre-registered binfmt_misc hook + # to automatically execute foreign binaries (such as aarch64) + # under qemu emulator. + source tools/internal_ci/helper_scripts/prepare_qemu_rc +fi # configure ccache source tools/internal_ci/helper_scripts/prepare_ccache_rc @@ -40,7 +49,12 @@ mkdir -p input_artifacts cp -r artifacts/* input_artifacts/ || true # This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/ -tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" +if [[ "${IS_AARCH64_MUSL}" == "True" ]]; then + # Not using TASK_RUNNER_EXTRA_FILTERS since we don't have a target with presubmit tag. + tools/run_tests/task_runner.py -f package linux python musllinux_1_1 aarch64 -x build_packages/sponge_log.xml || FAILED="true" +else + tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" +fi # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. # in addition to that, preserve the contents of "artifacts" directory since we want kokoro @@ -52,7 +66,12 @@ cp -r artifacts/* input_artifacts/ || true # Run all python linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" +if [[ "${IS_AARCH64_MUSL}" == "True" ]]; then + # We're using alpine as tag in distribtest targets. + tools/run_tests/task_runner.py -f distribtest linux python aarch64 alpine -j 12 -x distribtests/sponge_log.xml || FAILED="true" +else + tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" +fi # This step checks if any of the artifacts exceeds a per-file size limit. tools/internal_ci/helper_scripts/check_python_artifacts_size.sh diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg index de62ae1c80b99..86741bc1f32e8 100644 --- a/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "TASK_RUNNER_EXTRA_FILTERS" - value: "presubmit" + value: "presubmit -e aarch64 musllinux_1_1" } diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_python_arm64.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_python_arm64.cfg index 80f51498c0795..0eb0a74388553 100644 --- a/tools/internal_ci/linux/pull_request/grpc_distribtests_python_arm64.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_python_arm64.cfg @@ -15,7 +15,7 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python_arm64.sh" +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" timeout_mins: 240 action { define_artifacts { @@ -27,5 +27,5 @@ action { env_vars { key: "TASK_RUNNER_EXTRA_FILTERS" - value: "presubmit" + value: "presubmit aarch64 musllinux_1_1" } diff --git a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg index 6a916e938710e..fc4da2ed1e5de 100644 --- a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg +++ b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg @@ -24,3 +24,8 @@ action { regex: "github/grpc/artifacts/**" } } + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "-e aarch64 musllinux_1_1" +} diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg index ab541436b407b..fa2fc655ee7e9 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=5400" + value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=7200" } diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 67508bafcb1f7..4d54621a96593 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -200,19 +200,27 @@ def build_jobspec(self, inner_jobs=None): ) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" - environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE" - if self.arch == "x86": + if self.arch in ("x86", "aarch64"): environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" + if self.arch == "aarch64": + # As we won't strip the binary with auditwheel (see below), strip + # it at link time. + environ["LDFLAGS"] = "-s" + # We're using musllinux aarch64 image to build this artifact so no crosscompiling required. + environ["GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS"] = "TRUE" + else: + environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" + return create_docker_jobspec( self.name, "tools/dockerfile/grpc_artifact_python_%s_%s" % (self.platform, self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, - timeout_seconds=60 * 60 * 2, + timeout_seconds=60 * 60 * 4, ) elif self.platform == "windows": environ["EXT_COMPILER"] = "msvc" @@ -452,6 +460,16 @@ def targets(): PythonArtifact( "musllinux_1_1", "x86", "cp313-cp313", presubmit=True ), + PythonArtifact( + "musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True + ), + PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"), + PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"), + PythonArtifact("musllinux_1_1", "aarch64", "cp311-cp311"), + PythonArtifact("musllinux_1_1", "aarch64", "cp312-cp312"), + PythonArtifact( + "musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True + ), PythonArtifact("macos", "x64", "python3.8", presubmit=True), PythonArtifact("macos", "x64", "python3.9"), PythonArtifact("macos", "x64", "python3.10"), diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 2a01514aca771..9741c186d71c4 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -461,6 +461,7 @@ def targets(): PythonDistribTest( "linux", "aarch64", "python38_buster", presubmit=True ), + PythonDistribTest("linux", "aarch64", "alpine"), PythonDistribTest( "linux", "x64", "alpine3.18", source=True, presubmit=True ), diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index dabfb452880d9..8cffe7b98106b 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -142,9 +142,17 @@ def build_jobspec(self, inner_jobs=None): class PythonPackage: """Collects python eggs and wheels created in the artifact phase""" - def __init__(self): + def __init__(self, platform="", arch=""): self.name = "python_package" self.labels = ["package", "python", "linux"] + self.platform = platform + self.arch = arch + if self.platform: + self.labels.append(platform) + self.name += "_" + platform + if self.arch: + self.labels.append(arch) + self.name += "_" + arch def pre_build_jobspecs(self): return [] @@ -154,9 +162,16 @@ def build_jobspec(self, inner_jobs=None): # since the python package build does very little, we can use virtually # any image that has new-enough python, so reusing one of the images used # for artifact building seems natural. + dockerfile_dir = ( + "tools/dockerfile/grpc_artifact_python_manylinux2014_x64" + ) + if "musllinux_1_1" in self.platform and "aarch64" in self.arch: + dockerfile_dir = ( + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64" + ) return create_docker_jobspec( self.name, - "tools/dockerfile/grpc_artifact_python_manylinux2014_x64", + dockerfile_dir, "tools/run_tests/artifacts/build_package_python.sh", environ={"PYTHON": "/opt/python/cp39-cp39/bin/python"}, ) @@ -189,5 +204,6 @@ def targets(): CSharpPackage("windows"), RubyPackage(), PythonPackage(), + PythonPackage("musllinux_1_1", "aarch64"), PHPPackage(), ] diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index fb2502d2a282e..1ecaaa63a69c8 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -71,6 +71,14 @@ def _create_build_map(): default=[], help="Filter targets to build with AND semantics.", ) +argp.add_argument( + "-e", + "--exclude", + choices=sorted(_BUILD_MAP.keys()), + nargs="+", + default=[], + help="Target labels to exclude from building.", +) argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) argp.add_argument( "-x", @@ -106,6 +114,9 @@ def _create_build_map(): # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] +# Exclude target if it has ALL of the specified exclude labels. +targets = [t for t in targets if not all(l in args.exclude for l in t.labels)] + print("Will build %d targets:" % len(targets)) for target in targets: print(" %s, labels %s" % (target.name, target.labels))