From 3a8f9938c837989902db1c21b4d3f4b7178a8356 Mon Sep 17 00:00:00 2001 From: Maksim Levental Date: Thu, 29 Aug 2024 17:02:45 -0700 Subject: [PATCH] Fix GHA caches (#719) --- .github/workflows/ci-linux.yml | 40 ++++++++++++++++++++++++++++---- .github/workflows/ci-macos.yml | 40 +++++++++++++++++++++++--------- .github/workflows/ci-windows.yml | 38 +++++++++++++++++++++--------- build_tools/ci/build_test_cpp.sh | 3 --- 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index 100ea0eca..d0529f584 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -9,6 +9,11 @@ on: branches: - main + schedule: + # At minute 0 past every 6th hour. (see https://crontab.guru) + # this job is to keep the ccache cache warm + - cron: '0 */6 * * *' + concurrency: # A PR number if a pull request and otherwise the commit hash. This cancels # queued and in-progress runs for the same PR (presubmit) or commit @@ -24,6 +29,8 @@ jobs: fail-fast: true env: CACHE_DIR: ${{ github.workspace }}/.container-cache + # either the PR number or `branch-N` where N always increments + CACHE_KEY: linux-build-test-cpp-asserts-manylinux-v2-${{ github.event.number || format('{0}-{1}', github.ref_name, github.run_number) }} steps: - name: Set unified TZ uses: szenius/set-timezone@v2.0 @@ -42,11 +49,11 @@ jobs: with: submodules: recursive - - name: Install static libs + - name: Install deps run: | - dnf install -y almalinux-release-devel yum remove -y openssl-devel zlib-devel || true yum install -y protobuf-devel protobuf-compiler + - name: Sync source deps run: | python ./sync_deps.py @@ -59,9 +66,18 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CACHE_DIR }} - key: linux-build-test-cpp-asserts-manylinux-v2-${{ github.sha }} + key: ${{ env.CACHE_KEY }} restore-keys: linux-build-test-cpp- + - name: Configure ccache debug logs + run: | + # https://interrupt.memfault.com/blog/ccache-debugging + echo CCACHE_DEBUG=1 >> $GITHUB_ENV + echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV + echo CCACHE_DEBUGDIR="$HOME/ccache-debug" >> $GITHUB_ENV + echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV + echo CCACHE_COMPILERCHECK=content >> $GITHUB_ENV + - name: Build packages run: | export cache_dir="${{ env.CACHE_DIR }}" @@ -72,6 +88,20 @@ jobs: run: | tar cf iree-dist-linux.tar -C iree-install . + - name: Tar ccache logs + if: ${{ !cancelled() }} + run: | + ccache --show-stats -v + tar cf linux-ccache-logs.tar $CCACHE_DEBUGDIR + + - name: Upload ccache logs + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ccache_logs + path: linux-ccache-logs.tar + if-no-files-found: warn + - name: Upload artifacts uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} @@ -82,10 +112,10 @@ jobs: - name: Save cache uses: actions/cache/save@v3 - if: ${{ !cancelled() }} + if: ${{ !cancelled() && github.event_name != 'pull_request' }} with: path: ${{ env.CACHE_DIR }} - key: linux-build-test-cpp-asserts-manylinux-v2-${{ github.sha }} + key: ${{ env.CACHE_KEY }} test_linux: name: E2E Test linux diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index eda8ee471..069831e34 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -9,10 +9,10 @@ on: branches: - main + schedule: + - cron: '0 */6 * * *' + concurrency: - # A PR number if a pull request and otherwise the commit hash. This cancels - # queued and in-progress runs for the same PR (presubmit) or commit - # (postsubmit). group: ci-build-test-cpp-macos-${{ github.event.number || github.sha }} cancel-in-progress: true @@ -26,6 +26,7 @@ jobs: runs-on: [macos-12, macos-14] env: CACHE_DIR: ${{ github.workspace }}/.container-cache + CACHE_KEY: ${{ matrix.runs-on }}-build-test-cpp-asserts-v1-${{ github.event.number || format('{0}-{1}', github.ref_name, github.run_number) }} steps: - name: Set unified TZ uses: szenius/set-timezone@v2.0 @@ -60,11 +61,16 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CACHE_DIR }} - # without datetime stamps you'll get collisions for the cache warming runs - # ("Failed to save: Unable to reserve cache with key ..., another job may be creating this cache.") - key: ${{ matrix.runs-on }}-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }} + key: ${{ env.CACHE_KEY }} restore-keys: ${{ matrix.runs-on }}-build-test-cpp- + - name: Configure ccache debug logs + run: | + echo CCACHE_DEBUG=1 >> $GITHUB_ENV + echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV + echo CCACHE_DEBUGDIR="$HOME/ccache-debug" >> $GITHUB_ENV + echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV + - name: Build packages run: | export cache_dir="${{ env.CACHE_DIR }}" @@ -73,9 +79,21 @@ jobs: - name: Create artifacts if: ${{ !cancelled() }} run: | - rm -f iree-install/bin/clang* - rm -f iree-install/bin/llvm-link* - tar cf iree-dist-${{ matrix.runs-on }}.tar -C iree-install . -C ../iree-build tools/testing/e2e/iree-e2e-matmul-test + tar cf iree-dist-${{ matrix.runs-on }}.tar -C iree-install . + + - name: Tar ccache logs + if: ${{ !cancelled() }} + run: | + ccache --show-stats -v + tar cf macos-ccache-logs.tar $CCACHE_DEBUGDIR + + - name: Upload ccache logs + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ccache_logs + path: macos-ccache-logs.tar + if-no-files-found: warn - name: Upload artifacts uses: actions/upload-artifact@v4 @@ -87,7 +105,7 @@ jobs: - name: Save cache uses: actions/cache/save@v3 - if: ${{ !cancelled() }} + if: ${{ !cancelled() && github.event_name != 'pull_request' }} with: path: ${{ env.CACHE_DIR }} - key: ${{ matrix.runs-on }}-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }} + key: ${{ env.CACHE_KEY }} diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 4804d14eb..d7cb8a929 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -10,14 +10,9 @@ on: - main schedule: - # At minute 0 past every 12th hour. (see https://crontab.guru) - # this job is to keep the ccache cache warm - - cron: '0 */12 * * *' + - cron: '0 */6 * * *' concurrency: - # A PR number if a pull request and otherwise the commit hash. This cancels - # queued and in-progress runs for the same PR (presubmit) or commit - # (postsubmit). group: ci-build-test-cpp-windows-${{ github.event.number || github.sha }} cancel-in-progress: true @@ -34,6 +29,7 @@ jobs: fail-fast: true env: CACHE_DIR: ${{ github.workspace }}/.container-cache + CACHE_KEY: windows-build-test-cpp-asserts-v1-${{ github.event.number || format('{0}-{1}', github.ref_name, github.run_number) }} steps: - name: Set unified TZ uses: szenius/set-timezone@v2.0 @@ -73,11 +69,17 @@ jobs: uses: actions/cache/restore@v3 with: path: ${{ env.CACHE_DIR }} - # without datetime stamps you'll get collisions for the cache warming runs - # ("Failed to save: Unable to reserve cache with key ..., another job may be creating this cache.") - key: windows-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }} + key: ${{ env.CACHE_KEY }} restore-keys: windows-build-test-cpp- + - name: Configure ccache debug logs + run: | + echo CCACHE_DEBUG=1 >> $GITHUB_ENV + echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV + echo CCACHE_DEBUGDIR="/c/ccache-debug" >> $GITHUB_ENV + echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV + echo CCACHE_COMPILERCHECK=content >> $GITHUB_ENV + - name: Build packages run: | export cache_dir="${{ env.CACHE_DIR }}" @@ -88,6 +90,20 @@ jobs: run: | tar cf iree-dist-windows.tar -C iree-install . + - name: Tar ccache logs + if: ${{ !cancelled() }} + run: | + ccache --show-stats -v + tar cf windows-ccache-logs.tar $CCACHE_DEBUGDIR + + - name: Upload ccache logs + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ccache_logs + path: windows-ccache-logs.tar + if-no-files-found: warn + - name: Upload artifacts uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} @@ -98,10 +114,10 @@ jobs: - name: Save cache uses: actions/cache/save@v3 - if: ${{ !cancelled() }} + if: ${{ !cancelled() && github.event_name != 'pull_request' }} with: path: ${{ env.CACHE_DIR }} - key: windows-build-test-cpp-asserts-v1-${{ github.sha }}-${{ github.event.repository.updated_at }} + key: ${{ env.CACHE_KEY }} test_windows: name: E2E Test windows diff --git a/build_tools/ci/build_test_cpp.sh b/build_tools/ci/build_test_cpp.sh index 65c61332a..296dfc353 100644 --- a/build_tools/ci/build_test_cpp.sh +++ b/build_tools/ci/build_test_cpp.sh @@ -115,9 +115,6 @@ elif [[ "$OSTYPE" == "msys"* ]]; then ctest --test-dir "$build_dir" -R amd-aie --output-on-failure -j --repeat until-pass:5 fi -# Show ccache stats. -ccache --show-stats - rm -f "$install_dir"/bin/clang* rm -f "$install_dir"/bin/llvm-link* cp "$build_dir"/tools/testing/e2e/iree-e2e-matmul-test "$install_dir"/bin