Fix GHA caches (#719)

nod-ai · Aug 30, 2024 · 3a8f993 · 3a8f993
1 parent 1f32c6c
commit 3a8f993
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 30 deletions.
diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml
@@ -9,6 +9,11 @@ on:
     branches:
       - main
 
+  schedule:
+    # At minute 0 past every 6th hour. (see https://crontab.guru)
+    # this job is to keep the ccache cache warm
+    - cron: '0 */6 * * *'
+
 concurrency:
   # A PR number if a pull request and otherwise the commit hash. This cancels
   # queued and in-progress runs for the same PR (presubmit) or commit
@@ -24,6 +29,8 @@ jobs:
       fail-fast: true
     env:
       CACHE_DIR: ${{ github.workspace }}/.container-cache
+      # either the PR number or `branch-N` where N always increments
+      CACHE_KEY: linux-build-test-cpp-asserts-manylinux-v2-${{ github.event.number || format('{0}-{1}', github.ref_name, github.run_number) }}
     steps:
       - name: Set unified TZ
         uses: szenius/[email protected]
@@ -42,11 +49,11 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Install static libs
+      - name: Install deps
         run: |
-          dnf install -y almalinux-release-devel
           yum remove -y openssl-devel zlib-devel || true
           yum install -y protobuf-devel protobuf-compiler
+
       - name: Sync source deps
         run: |
           python ./sync_deps.py
@@ -59,9 +66,18 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: ${{ env.CACHE_DIR }}
-          key: linux-build-test-cpp-asserts-manylinux-v2-${{ github.sha }}
+          key:  ${{ env.CACHE_KEY }}
           restore-keys: linux-build-test-cpp-
 
+      - name: Configure ccache debug logs
+        run: |
+          # https://interrupt.memfault.com/blog/ccache-debugging
+          echo CCACHE_DEBUG=1 >> $GITHUB_ENV
+          echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV
+          echo CCACHE_DEBUGDIR="$HOME/ccache-debug" >> $GITHUB_ENV
+          echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV
+          echo CCACHE_COMPILERCHECK=content >> $GITHUB_ENV
+
       - name: Build packages
         run: |
           export cache_dir="${{ env.CACHE_DIR }}"
@@ -72,6 +88,20 @@ jobs:
         run: |
           tar cf iree-dist-linux.tar -C iree-install .
 
+      - name: Tar ccache logs
+        if: ${{ !cancelled() }}
+        run: |
+          ccache --show-stats -v
+          tar cf linux-ccache-logs.tar $CCACHE_DEBUGDIR
+
+      - name: Upload ccache logs
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ccache_logs
+          path: linux-ccache-logs.tar
+          if-no-files-found: warn
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -82,10 +112,10 @@ jobs:
 
       - name: Save cache
         uses: actions/cache/save@v3
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && github.event_name != 'pull_request' }}
         with:
           path: ${{ env.CACHE_DIR }}
-          key: linux-build-test-cpp-asserts-manylinux-v2-${{ github.sha }}
+          key: ${{ env.CACHE_KEY }}
 
   test_linux:
     name: E2E Test linux

diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
@@ -9,10 +9,10 @@ on:
     branches:
       - main
 
+  schedule:
+    - cron: '0 */6 * * *'
+
 concurrency:
-  # A PR number if a pull request and otherwise the commit hash. This cancels
-  # queued and in-progress runs for the same PR (presubmit) or commit
-  # (postsubmit).
   group: ci-build-test-cpp-macos-${{ github.event.number || github.sha }}
   cancel-in-progress: true
 
@@ -26,6 +26,7 @@ jobs:
         runs-on: [macos-12, macos-14]
     env:
       CACHE_DIR: ${{ github.workspace }}/.container-cache
+      CACHE_KEY: ${{ matrix.runs-on }}-build-test-cpp-asserts-v1-${{ github.event.number || format('{0}-{1}', github.ref_name, github.run_number) }}
     steps:
       - name: Set unified TZ
         uses: szenius/[email protected]
@@ -60,11 +61,16 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: ${{ env.CACHE_DIR }}
-          # without datetime stamps you'll get collisions for the cache warming runs
-          # ("Failed to save: Unable to reserve cache with key ..., another job may be creating this cache.")
-          key: ${{ matrix.runs-on }}-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }}
+          key: ${{ env.CACHE_KEY }}
           restore-keys: ${{ matrix.runs-on }}-build-test-cpp-
 
+      - name: Configure ccache debug logs
+        run: |
+          echo CCACHE_DEBUG=1 >> $GITHUB_ENV
+          echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV
+          echo CCACHE_DEBUGDIR="$HOME/ccache-debug" >> $GITHUB_ENV
+          echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV
+
       - name: Build packages
         run: |
           export cache_dir="${{ env.CACHE_DIR }}"
@@ -73,9 +79,21 @@ jobs:
       - name: Create artifacts
         if: ${{ !cancelled() }}
         run: |
-          rm -f iree-install/bin/clang*
-          rm -f iree-install/bin/llvm-link*
-          tar cf iree-dist-${{ matrix.runs-on }}.tar -C iree-install . -C ../iree-build tools/testing/e2e/iree-e2e-matmul-test
+          tar cf iree-dist-${{ matrix.runs-on }}.tar -C iree-install .
+
+      - name: Tar ccache logs
+        if: ${{ !cancelled() }}
+        run: |
+          ccache --show-stats -v
+          tar cf macos-ccache-logs.tar $CCACHE_DEBUGDIR
+
+      - name: Upload ccache logs
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ccache_logs
+          path: macos-ccache-logs.tar
+          if-no-files-found: warn
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -87,7 +105,7 @@ jobs:
 
       - name: Save cache
         uses: actions/cache/save@v3
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && github.event_name != 'pull_request' }}
         with:
           path: ${{ env.CACHE_DIR }}
-          key: ${{ matrix.runs-on }}-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }}
+          key: ${{ env.CACHE_KEY }}
diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml
@@ -10,14 +10,9 @@ on:
       - main
 
   schedule:
-      # At minute 0 past every 12th hour. (see https://crontab.guru)
-      # this job is to keep the ccache cache warm
-    - cron: '0 */12 * * *'
+    - cron: '0 */6 * * *'
 
 concurrency:
-  # A PR number if a pull request and otherwise the commit hash. This cancels
-  # queued and in-progress runs for the same PR (presubmit) or commit
-  # (postsubmit).
   group: ci-build-test-cpp-windows-${{ github.event.number || github.sha }}
   cancel-in-progress: true
 
@@ -34,6 +29,7 @@ jobs:
       fail-fast: true
     env:
       CACHE_DIR: ${{ github.workspace }}/.container-cache
+      CACHE_KEY: windows-build-test-cpp-asserts-v1-${{ github.event.number || format('{0}-{1}', github.ref_name, github.run_number) }}
     steps:
       - name: Set unified TZ
         uses: szenius/[email protected]
@@ -73,11 +69,17 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: ${{ env.CACHE_DIR }}
-          # without datetime stamps you'll get collisions for the cache warming runs
-          # ("Failed to save: Unable to reserve cache with key ..., another job may be creating this cache.")
-          key: windows-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }}
+          key: ${{ env.CACHE_KEY }}
           restore-keys: windows-build-test-cpp-
 
+      - name: Configure ccache debug logs
+        run: |
+          echo CCACHE_DEBUG=1 >> $GITHUB_ENV
+          echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV
+          echo CCACHE_DEBUGDIR="/c/ccache-debug" >> $GITHUB_ENV
+          echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV
+          echo CCACHE_COMPILERCHECK=content >> $GITHUB_ENV
+
       - name: Build packages
         run: |
           export cache_dir="${{ env.CACHE_DIR }}"
@@ -88,6 +90,20 @@ jobs:
         run: |
           tar cf iree-dist-windows.tar -C iree-install .
 
+      - name: Tar ccache logs
+        if: ${{ !cancelled() }}
+        run: |
+          ccache --show-stats -v
+          tar cf windows-ccache-logs.tar $CCACHE_DEBUGDIR
+
+      - name: Upload ccache logs
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: ccache_logs
+          path: windows-ccache-logs.tar
+          if-no-files-found: warn
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -98,10 +114,10 @@ jobs:
 
       - name: Save cache
         uses: actions/cache/save@v3
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && github.event_name != 'pull_request' }}
         with:
           path: ${{ env.CACHE_DIR }}
-          key: windows-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }}
+          key: ${{ env.CACHE_KEY }}
 
   test_windows:
     name: E2E Test windows

diff --git a/build_tools/ci/build_test_cpp.sh b/build_tools/ci/build_test_cpp.sh
@@ -115,9 +115,6 @@ elif [[ "$OSTYPE" == "msys"* ]]; then
   ctest --test-dir "$build_dir" -R amd-aie --output-on-failure -j --repeat until-pass:5
 fi
 
-# Show ccache stats.
-ccache --show-stats
-
 rm -f "$install_dir"/bin/clang*
 rm -f "$install_dir"/bin/llvm-link*
 cp "$build_dir"/tools/testing/e2e/iree-e2e-matmul-test "$install_dir"/bin