From 0e01308f4822b3da67429959719543bbd1e3cbe1 Mon Sep 17 00:00:00 2001
From: Mike Kruskal <mkruskal@google.com>
Date: Wed, 15 Feb 2023 10:22:37 -0800
Subject: [PATCH] Implement a workaround for macOS/Bazel flakes.

Bazel has a 2 minute timeout for their internal `xcrun` call, which can be exceeded on our github runners about 5% of the time.  This leads to flakes and opaque errors, but is a one-time cost.  Subsequent xcruns finish in seconds, so we can just do an initial call w/o a timeout before running Bazel.

With this change our total flake rate drops from ~30% to nearly 0% for our full suite of tests

See https://github.com/bazelbuild/bazel/issues/17437 for background.

PiperOrigin-RevId: 509869860
---
 .github/actions/bazel/action.yml              | 23 +++++++++++++++++++
 .../actions/internal/bazel-setup/action.yml   |  4 +++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml
index 141527db0267..5fd3e9a90171 100644
--- a/.github/actions/bazel/action.yml
+++ b/.github/actions/bazel/action.yml
@@ -93,6 +93,29 @@ runs:
       shell: bash
       run: bazelisk version
 
+    # Bazel has multiple Xcode calls with hardcoded timeouts.  Many of these
+    # end up timing out on our github runners, causing flakes on every mac
+    # build that invoked Bazel.  To work around this, we manually inoke these
+    # calls before running Bazel to make sure they end up in Xcode's cache for
+    # quicker runs later.  All of these calls are obtained from xcrun calls in
+    # https://github.com/bazelbuild/bazel/blob/e8a69f5d5acaeb6af760631490ecbf73e8a04eeb/tools/cpp/osx_cc_configure.bzl.
+    # See https://github.com/bazelbuild/bazel/issues/17437 for more details.
+    # TODO(b/269503614) Remove this once Bazel provides an official solution.
+    - name: Warm up Xcode
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        mkdir -p mac_bazel_workaround
+        bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS
+        XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation"
+        SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3"
+        COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3"
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc
+        time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc
+
     - name: Run Bash
       if: ${{ inputs.bash }}
       run: ${{ inputs.bash }}
diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml
index 20e724199fde..0ef5d5a2266a 100644
--- a/.github/actions/internal/bazel-setup/action.yml
+++ b/.github/actions/internal/bazel-setup/action.yml
@@ -42,7 +42,9 @@ runs:
     - name: Initialize MacOS-specific Bazel flags
       if: runner.os == 'macOS'
       shell: bash
-      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV
+      run: |
+        echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV
+        echo "DEVELOPER_DIR=${{ env.DEVELOPER_DIR || '/Applications/Xcode_14.1.app/Contents/Developer' }}" >> $GITHUB_ENV
 
     - name: Configure Bazel caching
       # Skip bazel cache for local act runs due to issue with credential files