Test mac tweaks to fix flaky failures and lag

PiperOrigin-RevId: 508555693
protocolbuffers · Feb 15, 2023 · 009665c · 009665c
1 parent 4b1cd0d
commit 009665c
Show file tree

Hide file tree

Showing 9 changed files with 30 additions and 293 deletions.
diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml
@@ -93,16 +93,29 @@ runs:
       shell: bash
       run: bazelisk version
 
+    - name: Warm up Xcode
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        mkdir -p mac_bazel_workaround
+        bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS
+        XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation"
+        SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3"
+        COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3"
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc
+
     - name: Run Bash
       if: ${{ inputs.bash }}
       run: ${{ inputs.bash }}
       shell: bash
 
     - name: Run Bazel
       if: ${{ !inputs.bash }}
-      run: >-
-        bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
-        ${{ inputs.bazel }} $BAZEL_FLAGS
+      run: bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} ${{ inputs.bazel }} $BAZEL_FLAGS
       shell: bash
 
     - name: Save Bazel repository cache

diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml
@@ -42,7 +42,10 @@ runs:
     - name: Initialize MacOS-specific Bazel flags
       if: runner.os == 'macOS'
       shell: bash
-      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV
+      run: |
+        echo "BAZEL_FLAGS=$BAZEL_FLAGS --toolchain_resolution_debug --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV
+        echo "BAZEL_USE_XCODE_TOOLCHAIN=1" >> $GITHUB_ENV
+        echo "DEVELOPER_DIR=/Applications/Xcode_14.1.app/Contents/Developer" >> $GITHUB_ENV
 
     - name: Configure Bazel caching
       # Skip bazel cache for local act runs due to issue with credential files

diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml
@@ -9,132 +9,12 @@ on:
         type: string
 
 jobs:
-  linux:
-    strategy:
-      fail-fast: false   # Don't cancel all jobs if one fails.
-      matrix:
-        include:
-          - name: 7.3 Debug
-            version: 7.3.28-dbg
-            command: composer test \&\& composer test_c
-          - name: 7.4 Debug
-            version: 7.4.18-dbg
-            command: composer test \&\& composer test_c
-          - name: 8.0 Optimized
-            version: 8.0.5
-            command: composer test \&\& composer test_c
-          - name: 8.0 Debug
-            version: 8.0.5-dbg
-            command: composer test \&\& composer test_c
-          - name: 8.0 Memory Leak
-            version: 8.0.5-dbg
-            # Run specialized memory leak & multirequest tests.
-            command: composer test_c \&\& tests/multirequest.sh \&\& tests/memory_leak_test.sh
-          - name: 8.0 Valgrind
-            version: 8.0.5-dbg
-            command: composer test_valgrind
-
-    name: Linux ${{ matrix.name}}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          submodules: recursive
-          ref: ${{ inputs.safe-checkout }}
-      - name: Run tests
-        uses: ./.github/actions/docker
-        with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          command: ${{ matrix.command }}
-
-  linux-32bit:
-    strategy:
-      fail-fast: false   # Don't cancel all jobs if one fails.
-      matrix:
-        version: ['7.4', '8.0']
-        suffix: [ '', '-zts']
-        test: ['test', 'test_c']
-        exclude:
-          - suffix: '-zts'
-            test: 'test'
-        include:
-          - suffix: '-zts'
-            suffix_name: ' Thread Safe'
-          - test: 'test_c'
-            test_name: ' Extension'
-
-    name: Linux 32-bit ${{ matrix.version}}${{ matrix.suffix_name }}${{ matrix.test_name }}
-    runs-on: ubuntu-latest
-    env:
-      image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:97f50ab24582380012d7ddef5f82f08e19b9dff55d09a4a8d90a87421ae66a45
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          submodules: recursive
-          ref: ${{ inputs.safe-checkout }}
-
-      - name: Cross compile protoc for i386
-        id: cross-compile
-        uses: ./.github/actions/cross-compile-protoc
-        with:
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          architecture: linux-i386
-
-      - name: Run tests
-        uses: ./.github/actions/docker
-        with:
-          image: ${{ env.image }}
-          skip-staleness-check: true
-          platform: linux/386
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          command: >-
-            /bin/bash -c '
-            cd php && php -v && php -m;
-            composer update --ignore-platform-reqs;
-            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
-            PATH="/usr/local/php-${{ matrix.version }}${{matrix.suffix}}/bin:$PATH"
-            composer ${{ matrix.test }}'
-
-  linux-aarch64:
-    name: Linux aarch64
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          ref: ${{ inputs.safe-checkout }}
-
-      - name: Cross compile protoc for aarch64
-        id: cross-compile
-        uses: ./.github/actions/cross-compile-protoc
-        with:
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          architecture: linux-aarch64
-
-      - name: Run tests
-        uses: ./.github/actions/docker
-        with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310
-          platform: linux/arm64
-          skip-staleness-check: true
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          command: >-
-            -c '
-            cd php;
-            composer update --ignore-platform-reqs;
-            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
-            composer test;
-            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
-            composer test_c'
-
   macos:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         version: ['7.4', '8.0']
+        noop: [0,1,2,3,4,5]
 
     name: MacOS PHP ${{ matrix.version }}
     runs-on: macos-12

diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml
@@ -9,49 +9,14 @@ on:
         type: string
 
 jobs:
-  linux:
-    strategy:
-      fail-fast: false   # Don't cancel all jobs if one fails.
-      matrix:
-        type: [ Pure, C++]
-        version: [ "3.7", "3.8", "3.9", "3.10" ]
-        include:
-          - type: Pure
-            targets: //python/... @upb//python/... //python:python_version
-            flags: --define=use_fast_cpp_protos=false
-          - type: C++
-            targets: //python/... //python:python_version
-            flags: --define=use_fast_cpp_protos=true
-          - type: C++
-            version: aarch64
-            targets: //python/... //python:aarch64_test
-            # TODO(b/262628111) Enable this once conformance tests are fixed.
-            flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
-            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b
-
-    name: Linux ${{ matrix.type }} ${{ matrix.version }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          ref: ${{ inputs.safe-checkout }}
-      - name: Run tests
-        uses: ./.github/actions/bazel-docker
-        with:
-          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }}
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
-          bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION
-
-
   macos:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         type: [ Pure, C++]
         # TODO(b/267550843) Consider expanding this set of versions.
-        version: [ "3.10" ]
+        version: [ "3.7", "3.8", "3.9", "3.10" ]
+        noop: [0,1,2,3,4,5,6,7,8,9]
         include:
           - type: Pure
             targets: //python/... @upb//python/... //python:python_version

diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml
@@ -9,73 +9,13 @@ on:
         type: string
 
 jobs:
-  linux:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - { name: Ruby 2.6, ruby: ruby-2.6.0, bazel: 5.1.1}
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1}
-          - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1}
-          - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1}
-          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1}
-          - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1}
-          - { name: JRuby 9.3, ruby: jruby-9.3.4.0, bazel: 5.1.1}
-          - { name: Ruby 2.6 (Bazel6), ruby: ruby-2.6.0, bazel: 6.0.0}
-          - { name: JRuby 9.2 (Bazel6), ruby: jruby-9.2.20.1, bazel: 6.0.0}
-
-    name: Linux ${{ matrix.name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          ref: ${{ inputs.safe-checkout }}
-      - name: Run tests
-        uses: ./.github/actions/bazel-docker
-        with:
-          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }}
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
-          bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
-
-  linux-aarch64:
-    name: Linux aarch64
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          ref: ${{ inputs.safe-checkout }}
-
-      - name: Cross compile protoc for aarch64
-        id: cross-compile
-        uses: ./.github/actions/cross-compile-protoc
-        with:
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          architecture: linux-aarch64
-
-      - name: Run tests
-        uses: ./.github/actions/docker
-        with:
-          image: arm64v8/ruby:2.7.3-buster
-          skip-staleness-check: true
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          command: >-
-            /bin/bash -c '
-            gem install bundler;
-            cd /workspace/ruby;
-            bundle;
-            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake;
-            rake clobber_package gem;
-            PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake test'
-
   macos:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         # This is the full set of versions we support on MacOS.
         version: [ "2.7", "3.0", "3.1", "3.2" ]
+        noop: [0,1,2,3,4,5]
 
     name: MacOS Ruby ${{ matrix.version }}
     runs-on: macos-12
@@ -100,40 +40,3 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: ruby_macos/${{ matrix.version }}
           bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }}
-
-  test_ruby_gems:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - { name: Ruby 2.6, ruby: ruby-2.6.0, bazel: 5.1.1}
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1}
-          - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1}
-          - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1}
-          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1}
-          - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1}
-          - { name: JRuby 9.3, ruby: jruby-9.3.4.0, bazel: 5.1.1}
-          - { name: Ruby 2.6 (Bazel6), ruby: ruby-2.6.0, bazel: 6.0.0}
-          - { name: JRuby 9.2 (Bazel6), ruby: jruby-9.2.20.1, bazel: 6.0.0}
-    name: Install ${{ matrix.name }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          ref: ${{ inputs.safe-checkout }}
-      - name: Run tests
-        uses: ./.github/actions/bazel-docker
-        with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }}
-          bash: >
-            bazel --version;
-            ruby --version;
-            bazel build //ruby:release //:protoc $BAZEL_FLAGS;
-            gem install bazel-bin/ruby/google-protobuf-*;
-            bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto;
-            bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto;
-            ruby ruby/tests/basic.rb
-
diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml
@@ -98,21 +98,6 @@ jobs:
   # Note: this pattern of passing the head sha is vulnerable to PWN requests for
   # pull_request_target events. We carefully limit those workflows to require a
   # human stamp before continuing.
-  cpp:
-    name: C++
-    needs: [check-tag]
-    uses: ./.github/workflows/test_cpp.yml
-    with:
-      safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
-    secrets: inherit
-
-  java:
-    name: Java
-    needs: [check-tag]
-    uses: ./.github/workflows/test_java.yml
-    with:
-      safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
-    secrets: inherit
 
   python:
     name: Python
@@ -138,22 +123,6 @@ jobs:
       safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
     secrets: inherit
 
-  php-ext:
-    name: PHP Extension
-    needs: [check-tag]
-    uses: ./.github/workflows/test_php_ext.yml
-    with:
-      safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
-    secrets: inherit
-
-  csharp:
-    name: C#
-    needs: [check-tag]
-    uses: ./.github/workflows/test_csharp.yml
-    with:
-      safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
-    secrets: inherit
-
   objectivec:
     name: Objective-C
     needs: [check-tag]

diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py
@@ -407,6 +407,7 @@ def __setstate__(self, state):
     self.ParseFromString(serialized)
 
   def __reduce__(self):
+    message_descriptor = None
     message_descriptor = self.DESCRIPTOR
     if message_descriptor.containing_type is None:
       return type(self), (), self.__getstate__()