Move ASan and TSan jobs to their own workflows. (iree-org#18003)

Progress on iree-org#17957. This moves the ASan (Address Sanitizer) and TSan (Thread Sanitizer) jobs into their own workflows, so they can run based on independent triggers and can be individually enabled/disabled as needed. For now these are the triggers: * ASan runs on `pull_request` and `push` events, similar to how it ran as part of `ci.yml` * TSan runs on a nightly `schedule` and on-demand using `workflow_dispatch`. We can add other ways to trigger it like via pull request labels or git trailers as needed. Both of these jobs need more disk space than GitHub's standard runners have, so they are running on larger self-hosted runners. If we have enough runner capacity then both jobs could run on every commit (`pull_request` and `push`). I think ASan is valuable enough for that but TSan is more situational. --- For more information about these sanitizers, see * https://iree.dev/developers/debugging/sanitizers/ * https://github.com/google/sanitizers * https://clang.llvm.org/docs/AddressSanitizer.html * https://clang.llvm.org/docs/ThreadSanitizer.html * https://clang.llvm.org/docs/MemorySanitizer.html
vinayakdsci · Jul 24, 2024 · 52eff2d · 52eff2d
1 parent ae00c4f
commit 52eff2d
Show file tree

Hide file tree

Showing 3 changed files with 135 additions and 40 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -432,45 +432,6 @@ jobs:
             ./build_tools/cmake/ctest_all.sh \
             "${BUILD_DIR}"
 
-  sanitizers:
-    needs: setup
-    name: "sanitizers :: ${{ matrix.name }}"
-    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'sanitizers')
-    runs-on:
-      - self-hosted # must come first
-      - runner-group=${{ needs.setup.outputs.runner-group }}
-      - environment=${{ needs.setup.outputs.runner-env }}
-      - cpu
-      - os-family=Linux
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - name: asan
-            script: ./build_tools/cmake/build_and_test_asan.sh
-          # Disabled to reduce self-hosted runners needed. See #17957
-          # - name: tsan
-          #   script: ./build_tools/cmake/build_and_test_tsan.sh
-    steps:
-      - name: "Checking out repository"
-        uses: actions/[email protected]
-        with:
-          submodules: true
-      - name: "Building and testing"
-        env:
-          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
-          SCRIPT: ${{ matrix.script }}
-        run: |
-          # Note that this uses the latest version of the clang compiler, etc.
-          # This gives us access to the latest features and validates that IREE
-          # builds using the latest versions.
-          ./build_tools/github_actions/docker_run.sh \
-            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-            --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \
-            gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \
-            ${SCRIPT}
-
   small_runtime:
     needs: setup
     if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'small_runtime')
@@ -744,7 +705,6 @@ jobs:
 
       # Configurations
       - build_test_runtime
-      - sanitizers
       - small_runtime
       # - gcc
       - tracing

diff --git a/.github/workflows/ci_linux_x64_clang_asan.yml b/.github/workflows/ci_linux_x64_clang_asan.yml
@@ -0,0 +1,78 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: CI - Linux x64 clang ASan
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/ci_linux_x64_clang_asan.yml"
+      - "CMakeLists.txt"
+      - "build_tools/*"
+      - "compiler/*"
+      - "llvm-external-projects/*"
+      - "runtime/*"
+      - "samples/*"
+      - "tests/*"
+      - "third_party/*"
+      - "tools/*"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/ci_linux_x64_clang_asan.yml"
+      - "CMakeLists.txt"
+      - "build_tools/*"
+      - "compiler/*"
+      - "llvm-external-projects/*"
+      - "runtime/*"
+      - "samples/*"
+      - "tests/*"
+      - "third_party/*"
+      - "tools/*"
+  workflow_dispatch:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    uses: ./.github/workflows/setup.yml
+
+  linux_x64_clang_asan:
+    needs: setup
+    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'linux_x64_clang_asan')
+    runs-on:
+      - self-hosted # must come first
+      - runner-group=${{ needs.setup.outputs.runner-group }}
+      - environment=${{ needs.setup.outputs.runner-env }}
+      - cpu
+      - os-family=Linux
+    steps:
+      - name: "Checking out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+      - name: "Building and testing with ASan"
+        env:
+          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
+        run: |
+          # Note that this uses the latest version of the clang compiler, etc.
+          # This gives us access to the latest features and validates that IREE
+          # builds using the latest versions.
+          ./build_tools/github_actions/docker_run.sh \
+            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
+            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
+            --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \
+            gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \
+            ./build_tools/cmake/build_and_test_asan.sh
diff --git a/.github/workflows/ci_linux_x64_clang_tsan.yml b/.github/workflows/ci_linux_x64_clang_tsan.yml
@@ -0,0 +1,57 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: CI - Linux x64 clang TSan
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/ci_linux_x64_clang_tsan.yml"
+  schedule:
+    # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8).
+    - cron: "15 9 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    uses: ./.github/workflows/setup.yml
+
+  linux_x64_clang_tsan:
+    needs: setup
+    runs-on:
+      - self-hosted # must come first
+      - runner-group=${{ needs.setup.outputs.runner-group }}
+      - environment=${{ needs.setup.outputs.runner-env }}
+      - cpu
+      - os-family=Linux
+    steps:
+      - name: "Checking out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+      - name: "Building and testing with TSan"
+        env:
+          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
+        run: |
+          # Note that this uses the latest version of the clang compiler, etc.
+          # This gives us access to the latest features and validates that IREE
+          # builds using the latest versions.
+          ./build_tools/github_actions/docker_run.sh \
+            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
+            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
+            --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \
+            gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \
+            ./build_tools/cmake/build_and_test_tsan.sh