From 7a97fb38dd86a0bde736e8e95ad4e33ca33b21bd Mon Sep 17 00:00:00 2001 From: Maksim Levental Date: Thu, 29 Aug 2024 20:50:26 -0700 Subject: [PATCH] [Hot fix] Linux CI (#727) --- .github/workflows/ci-linux.yml | 24 +--- .github/workflows/ci-macos.yml | 22 +--- .github/workflows/ci-windows.yml | 23 +--- build_tools/ci/build_test_cpp.sh | 3 + sync_deps.py | 189 +++++++++++++++---------------- 5 files changed, 100 insertions(+), 161 deletions(-) diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index d0529f584..968cf4630 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -69,18 +69,10 @@ jobs: key: ${{ env.CACHE_KEY }} restore-keys: linux-build-test-cpp- - - name: Configure ccache debug logs - run: | - # https://interrupt.memfault.com/blog/ccache-debugging - echo CCACHE_DEBUG=1 >> $GITHUB_ENV - echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV - echo CCACHE_DEBUGDIR="$HOME/ccache-debug" >> $GITHUB_ENV - echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV - echo CCACHE_COMPILERCHECK=content >> $GITHUB_ENV - - name: Build packages run: | export cache_dir="${{ env.CACHE_DIR }}" + export CCACHE_COMPILERCHECK="string:$(clang --version)" bash build_tools/ci/build_test_cpp.sh - name: Create artifacts @@ -88,20 +80,6 @@ jobs: run: | tar cf iree-dist-linux.tar -C iree-install . - - name: Tar ccache logs - if: ${{ !cancelled() }} - run: | - ccache --show-stats -v - tar cf linux-ccache-logs.tar $CCACHE_DEBUGDIR - - - name: Upload ccache logs - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ccache_logs - path: linux-ccache-logs.tar - if-no-files-found: warn - - name: Upload artifacts uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 069831e34..cf05c9376 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -64,16 +64,10 @@ jobs: key: ${{ env.CACHE_KEY }} restore-keys: ${{ matrix.runs-on }}-build-test-cpp- - - name: Configure ccache debug logs - run: | - echo CCACHE_DEBUG=1 >> $GITHUB_ENV - echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV - echo CCACHE_DEBUGDIR="$HOME/ccache-debug" >> $GITHUB_ENV - echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV - - name: Build packages run: | export cache_dir="${{ env.CACHE_DIR }}" + export CCACHE_COMPILERCHECK="string:$(clang --version)" bash build_tools/ci/build_test_cpp.sh - name: Create artifacts @@ -81,20 +75,6 @@ jobs: run: | tar cf iree-dist-${{ matrix.runs-on }}.tar -C iree-install . - - name: Tar ccache logs - if: ${{ !cancelled() }} - run: | - ccache --show-stats -v - tar cf macos-ccache-logs.tar $CCACHE_DEBUGDIR - - - name: Upload ccache logs - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ccache_logs - path: macos-ccache-logs.tar - if-no-files-found: warn - - name: Upload artifacts uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index d7cb8a929..de71a4f9a 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -72,17 +72,10 @@ jobs: key: ${{ env.CACHE_KEY }} restore-keys: windows-build-test-cpp- - - name: Configure ccache debug logs - run: | - echo CCACHE_DEBUG=1 >> $GITHUB_ENV - echo CCACHE_DEBUGLEVEL=2 >> $GITHUB_ENV - echo CCACHE_DEBUGDIR="/c/ccache-debug" >> $GITHUB_ENV - echo CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros >> $GITHUB_ENV - echo CCACHE_COMPILERCHECK=content >> $GITHUB_ENV - - name: Build packages run: | export cache_dir="${{ env.CACHE_DIR }}" + export CCACHE_COMPILERCHECK="string:$(clang-cl.exe --version)" bash build_tools/ci/build_test_cpp.sh - name: Create artifacts @@ -90,20 +83,6 @@ jobs: run: | tar cf iree-dist-windows.tar -C iree-install . - - name: Tar ccache logs - if: ${{ !cancelled() }} - run: | - ccache --show-stats -v - tar cf windows-ccache-logs.tar $CCACHE_DEBUGDIR - - - name: Upload ccache logs - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ccache_logs - path: windows-ccache-logs.tar - if-no-files-found: warn - - name: Upload artifacts uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} diff --git a/build_tools/ci/build_test_cpp.sh b/build_tools/ci/build_test_cpp.sh index 296dfc353..38c98ca9d 100644 --- a/build_tools/ci/build_test_cpp.sh +++ b/build_tools/ci/build_test_cpp.sh @@ -36,10 +36,12 @@ elif [[ "$OSTYPE" == "msys"* ]]; then export CC=clang-cl.exe export CXX=clang-cl.exe fi + export CCACHE_DIR="${cache_dir}/ccache" export CCACHE_MAXSIZE="700M" export CMAKE_C_COMPILER_LAUNCHER=ccache export CMAKE_CXX_COMPILER_LAUNCHER=ccache +export CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros # Clear ccache stats. ccache -z @@ -64,6 +66,7 @@ CMAKE_ARGS="\ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$install_dir \ -DCMAKE_INSTALL_LIBDIR=lib \ + -DIREE_ERROR_ON_MISSING_SUBMODULES=OFF \ -DIREE_ENABLE_ASSERTIONS=ON \ -DIREE_BUILD_SAMPLES=OFF \ -DIREE_BUILD_PYTHON_BINDINGS=ON \ diff --git a/sync_deps.py b/sync_deps.py index 0d0e470c7..149b11533 100644 --- a/sync_deps.py +++ b/sync_deps.py @@ -2,122 +2,121 @@ ### AUTO-GENERATED: DO NOT EDIT ### Casual developers and CI bots invoke this to do the most ### efficient checkout of dependencies. -### Cross-repo project development should use the +### Cross-repo project development should use the ### 'shark-workspace' dev tool for more full featured setup. ### Update with: shark-workspace pin PINNED_VERSIONS = { - "iree": "60b65f30c932eaf967922785253a85a1aa14cebb", + "iree": "60b65f30c932eaf967922785253a85a1aa14cebb", } ORIGINS = { - "iree": "https://github.com/iree-org/iree.git", + "iree": "https://github.com/iree-org/iree.git", } -SUBMODULES = { - "iree": 1 -} +SUBMODULES = {"iree"} +EXCLUDED_SUBMODULES = {"third_party/torch-mlir"} ### Update support: import argparse -from pathlib import Path import re import shlex import subprocess +from pathlib import Path def main(): - parser = argparse.ArgumentParser(description="Source deps sync") - parser.add_argument( - "--exclude-submodule", - nargs="*", - help="Exclude submodules by regex (relative to '{project}:{path})") - parser.add_argument("--exclude-dep", - nargs="*", - help="Excludes dependencies by regex") - parser.add_argument("--depth", - type=int, - default=0, - help="Fetch revisions with --depth") - parser.add_argument("--submodules-depth", - type=int, - default=0, - help="Update submodules with --depth") - args = parser.parse_args() - - workspace_dir = Path(__file__).resolve().parent.parent - for repo_name, revision in PINNED_VERSIONS.items(): - # Exclude this dep? - exclude_repo = False - for exclude_pattern in (args.exclude_dep or ()): - if re.search(exclude_pattern, repo_name): - exclude_repo = True - if exclude_repo: - print(f"Excluding {repo_name} based on --exclude-dep") - continue - - print(f"Syncing {repo_name}") - repo_dir = workspace_dir / repo_name - if not repo_dir.exists(): - # Shallow clone - print(f" Cloning {repo_name}...") - repo_dir.mkdir() - run(["init"], repo_dir) - run(["remote", "add", "origin", ORIGINS[repo_name]], repo_dir) - # Checkout detached head. - fetch_args = ["fetch"] - if args.depth > 0: - fetch_args.extend(["--depth=1"]) - fetch_args.extend(["origin", revision]) - run(fetch_args, repo_dir) - run(["-c", "advice.detachedHead=false", "checkout", revision], repo_dir) - if SUBMODULES.get(repo_name): - print(f" Initializing submodules for {repo_name}") - cp = run(["submodule", "status"], - repo_dir, - silent=True, - capture_output=True) - submodules = [] - for submodule_status_line in cp.stdout.decode().splitlines(): - submodule_status_parts = submodule_status_line.split() - submodule_path = submodule_status_parts[1] - exclude_submodule = False - for exclude_pattern in (args.exclude_submodule or ()): - if re.search(exclude_pattern, f"{repo_name}:{submodule_path}"): - exclude_submodule = True - if exclude_submodule: - print(f" Excluding {submodule_path} based on --exclude-submodule") - continue - submodules.append(submodule_path) - - update_args = ["submodule", "update", "--init"] - if args.submodules_depth > 0: - update_args.extend(["--depth", "1"]) - update_args.extend(["--"]) - update_args.extend(submodules) - run(update_args, repo_dir) - - -def run(args, - cwd, - *, - capture_output: bool = False, - check: bool = True, - silent: bool = False): - args = ["git"] + args - args_text = ' '.join([shlex.quote(arg) for arg in args]) - if not silent: - print(f" [{cwd}]$ {args_text}") - cp = subprocess.run(args, cwd=str(cwd), capture_output=capture_output) - if check and cp.returncode != 0: - addl_info = f":\n({cp.stderr.decode()})" if capture_output else "" - raise RuntimeError(f"Git command failed: {args_text} (from {cwd})" - f"{addl_info}") - return cp + parser = argparse.ArgumentParser(description="Source deps sync") + parser.add_argument( + "--exclude-submodule", + nargs="*", + help="Exclude submodules by regex (relative to '{project}:{path})", + default=(), + ) + parser.add_argument( + "--exclude-dep", nargs="*", help="Excludes dependencies by regex" + ) + parser.add_argument( + "--depth", type=int, default=0, help="Fetch revisions with --depth" + ) + parser.add_argument( + "--submodules-depth", type=int, default=0, help="Update submodules with --depth" + ) + args = parser.parse_args() + + workspace_dir = Path(__file__).resolve().parent.parent + for repo_name, revision in PINNED_VERSIONS.items(): + # Exclude this dep? + exclude_repo = False + for exclude_pattern in args.exclude_dep or (): + if re.search(exclude_pattern, repo_name): + exclude_repo = True + if exclude_repo: + print(f"Excluding {repo_name} based on --exclude-dep") + continue + + print(f"Syncing {repo_name}") + repo_dir = workspace_dir / repo_name + if not repo_dir.exists(): + # Shallow clone + print(f" Cloning {repo_name}...") + repo_dir.mkdir() + run(["init"], repo_dir) + run(["remote", "add", "origin", ORIGINS[repo_name]], repo_dir) + # Checkout detached head. + fetch_args = ["fetch"] + if args.depth > 0: + fetch_args.extend(["--depth=1"]) + fetch_args.extend(["origin", revision]) + run(fetch_args, repo_dir) + run(["-c", "advice.detachedHead=false", "checkout", revision], repo_dir) + if repo_name in SUBMODULES: + print(f" Initializing submodules for {repo_name}") + cp = run( + ["submodule", "status"], repo_dir, silent=True, capture_output=True + ) + submodules = [] + for submodule_status_line in cp.stdout.decode().splitlines(): + submodule_status_parts = submodule_status_line.split() + submodule_path = submodule_status_parts[1] + exclude_submodule = False + for exclude_pattern in args.exclude_submodule + tuple( + EXCLUDED_SUBMODULES + ): + if re.search(exclude_pattern, f"{repo_name}:{submodule_path}"): + exclude_submodule = True + if exclude_submodule: + print(f" Excluding {submodule_path} based on --exclude-submodule") + continue + submodules.append(submodule_path) + + update_args = ["submodule", "update", "--init"] + if args.submodules_depth > 0: + update_args.extend(["--depth", "1"]) + update_args.extend(["--"]) + update_args.extend(submodules) + print(update_args) + run(update_args, repo_dir) + + +def run( + args, cwd, *, capture_output: bool = False, check: bool = True, silent: bool = False +): + args = ["git"] + args + args_text = " ".join([shlex.quote(arg) for arg in args]) + if not silent: + print(f" [{cwd}]$ {args_text}") + cp = subprocess.run(args, cwd=str(cwd), capture_output=capture_output) + if check and cp.returncode != 0: + addl_info = f":\n({cp.stderr.decode()})" if capture_output else "" + raise RuntimeError( + f"Git command failed: {args_text} (from {cwd})" f"{addl_info}" + ) + return cp if __name__ == "__main__": - main() + main()