From b8f64ac94bffdcf0b5c69ad77ef33a91a8d32a5e Mon Sep 17 00:00:00 2001 From: Maksim Levental Date: Wed, 4 Sep 2024 16:46:35 -0500 Subject: [PATCH] write powershell --- .github/workflows/ci-windows.yml | 48 ++++---- README.md | 1 - build_tools/build_llvm.ps1 | 91 ++++++++++++++ build_tools/build_llvm.sh | 19 +-- build_tools/build_test_cpp.ps1 | 114 ++++++++++++++++++ build_tools/build_test_cpp.sh | 24 +--- .../Target/tests/aie_cdo_gen_test.cxx | 8 +- 7 files changed, 241 insertions(+), 64 deletions(-) create mode 100644 build_tools/build_llvm.ps1 create mode 100644 build_tools/build_test_cpp.ps1 diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 9406bdc80..92323a676 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -24,11 +24,6 @@ concurrency: group: ci-build-test-cpp-windows-${{ github.event.number || github.sha }} cancel-in-progress: true -defaults: - run: - # force bash for windows - shell: bash - jobs: build_and_ctest: name: Build and Test (windows, ASSERTIONS) @@ -38,6 +33,9 @@ jobs: env: CACHE_DIR: ${{ github.workspace }}/.container-cache CACHE_KEY: windows-build-test-cpp-asserts-v1-${{ format('{0}-{1}', github.ref_name, github.run_number) }} + defaults: + run: + shell: powershell steps: - name: Set unified TZ uses: szenius/set-timezone@v2.0 @@ -49,18 +47,15 @@ jobs: - name: "Checking out repository" env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + BRANCH_NAME: ${{ github.ref }} REPO_ADDRESS: ${{ github.server_url }}/${{ github.repository }} + shell: bash run: | - git \ - -c submodule."third_party/torch-mlir".update=none \ - -c submodule."third_party/stablehlo".update=none \ - -c submodule."src/runtime_src/core/common/aiebu".update=none \ - clone \ - --recursive \ - --shallow-submodules \ - --depth 1 \ - -b $BRANCH_NAME $REPO_ADDRESS . + git init + git remote add origin $REPO_ADDRESS + git -c protocol.version=2 fetch --depth 1 origin $BRANCH_NAME + git reset --hard FETCH_HEAD + git -c submodule."third_party/torch-mlir".update=none -c submodule."third_party/stablehlo".update=none -c submodule."src/runtime_src/core/common/aiebu".update=none submodule update --init --recursive --depth 1 --single-branch -j 10 - name: Setup Cpp uses: aminya/setup-cpp@v1 @@ -86,22 +81,26 @@ jobs: key: ${{ env.CACHE_KEY }} restore-keys: windows-build-test-cpp- - - name: Build packages + - name: Hack IREE + shell: bash run: | - export cache_dir="${{ env.CACHE_DIR }}" - export CCACHE_COMPILERCHECK="string:$(clang-cl.exe --version)" - bash build_tools/build_llvm.sh - rm -rf llvm-build - export llvm_install_dir=$PWD/llvm-install # TODO(max): send IREE a fix for this # target_compile_definitions may only set INTERFACE properties on IMPORTED sed -i '114s/PRIVATE/INTERFACE/g' third_party/iree/compiler/src/iree/compiler/API/CMakeLists.txt - bash build_tools/build_test_cpp.sh + + - name: Build packages + run: | + $env:cache_dir = "${{ env.CACHE_DIR }}" + $env:CCACHE_COMPILERCHECK = "string:$(clang-cl.exe --version)" + powershell build_tools/build_llvm.ps1 + Remove-Item llvm-build -Force + powershell build_tools/build_test_cpp.ps1 - name: Create artifacts if: ${{ !cancelled() }} run: | - pushd third_party/iree/third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd + pushd third_party/iree/third_party/llvm-project + $llvm_sha_short = "$(git rev-parse --short HEAD)" tar cf llvm-dist-windows-$llvm_sha_short.tar llvm-install tar cf iree-dist-windows.tar iree-install @@ -138,6 +137,9 @@ jobs: name: E2E Test windows runs-on: windows-phoenix needs: build_and_ctest + defaults: + run: + shell: bash strategy: fail-fast: true steps: diff --git a/README.md b/README.md index f5c972052..687c3c539 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,6 @@ git \ -c submodule."src/runtime_src/core/common/aiebu".update=none \ clone \ --recursive \ - --depth 1 \ --shallow-submodules \ https://github.com/nod-ai/iree-amd-aie.git ``` diff --git a/build_tools/build_llvm.ps1 b/build_tools/build_llvm.ps1 new file mode 100644 index 000000000..9648206fe --- /dev/null +++ b/build_tools/build_llvm.ps1 @@ -0,0 +1,91 @@ +# Enable strict mode +$ErrorActionPreference = 'Stop' +Set-PSDebug -Trace 2 + +$this_dir = Split-Path -Path $MyInvocation.MyCommand.Path -Parent +$repo_root = Resolve-Path -Path "$this_dir/.." +$llvm_dir = Resolve-Path -Path "$repo_root/third_party/iree/third_party/llvm-project/llvm" +echo "llvm_dir $llvm_dir" +$build_dir = "$repo_root/llvm-build" +$install_dir = "$repo_root/llvm-install" + +if (-not (Test-Path $build_dir)) +{ + New-Item -Path $build_dir -ItemType Directory | Out-Null +} +$build_dir = Resolve-Path -Path $build_dir +$cache_dir = $env:cache_dir + +if (-not $cache_dir) +{ + $cache_dir = "$repo_root/.build-cache" + if (-not (Test-Path $cache_dir)) + { + New-Item -Path $cache_dir -ItemType Directory | Out-Null + } + $cache_dir = Resolve-Path -Path $cache_dir +} +echo "Caching to $cache_dir" + +if (-not (Test-Path "$cache_dir/ccache")) +{ + New-Item -Path "$cache_dir/ccache" -ItemType Directory | Out-Null +} +if (-not (Test-Path "$cache_dir/pip")) +{ + New-Item -Path "$cache_dir/pip" -ItemType Directory | Out-Null +} + +$python = (Get-Command python -ErrorAction SilentlyContinue).Source +echo "Using python: $python" + +$env:CC = 'clang-cl.exe' +$env:CXX = 'clang-cl.exe' +$env:CCACHE_DIR = "$cache_dir/ccache" +$env:CCACHE_MAXSIZE = '700M' +$env:CMAKE_C_COMPILER_LAUNCHER = 'ccache' +$env:CMAKE_CXX_COMPILER_LAUNCHER = 'ccache' +$env:CCACHE_SLOPPINESS = 'include_file_ctime,include_file_mtime,time_macros' + +& ccache -z + +& cmake ` + -GNinja ` + -DCMAKE_BUILD_TYPE=Release ` + -DCMAKE_INSTALL_PREFIX=$install_dir ` + -DCMAKE_OBJECT_PATH_MAX=4096 ` + -DCMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld ` + -DCMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld ` + -DCMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld ` + -DCMAKE_C_COMPILER="$env:CC" ` + -DCMAKE_CXX_COMPILER="$env:CXX" ` + -DCMAKE_C_FLAGS="-DMLIR_CAPI_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DMLIR_CAPI_BUILDING_LIBRARY=1" ` + -DCMAKE_CXX_FLAGS="-DMLIR_CAPI_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DMLIR_CAPI_BUILDING_LIBRARY=1" ` + -DLLVM_INCLUDE_EXAMPLES=OFF ` + -DLLVM_INCLUDE_TESTS=OFF ` + -DLLVM_INCLUDE_BENCHMARKS=OFF ` + -DLLVM_APPEND_VC_REV=OFF ` + -DLLVM_ENABLE_ASSERTIONS=ON ` + -DLLVM_ENABLE_IDE=ON ` + -DLLVM_ENABLE_BINDINGS=OFF ` + -DLLVM_ENABLE_LIBEDIT=OFF ` + -DLLVM_ENABLE_LIBXML2=OFF ` + -DLLVM_ENABLE_TERMINFO=OFF ` + -DLLVM_ENABLE_ZLIB=OFF ` + -DLLVM_ENABLE_ZSTD=OFF ` + -DLLVM_FORCE_ENABLE_STATS=ON ` + -DLLVM_INSTALL_UTILS=ON ` + -DMLIR_ENABLE_BINDINGS_PYTHON=ON ` + -DLLVM_ENABLE_PROJECTS="mlir;clang;lld" ` + -DLLVM_TARGET_ARCH=X86 ` + -DLLVM_TARGETS_TO_BUILD=X86 ` + -S "$llvm_dir" -B "$build_dir" + +echo "Building all" +echo "------------" +cmake --build $build_dir -- -k 0 + +echo "Installing" +echo "----------" +echo "Install to: $install_dir" +cmake --build $build_dir --target install diff --git a/build_tools/build_llvm.sh b/build_tools/build_llvm.sh index 82e27ea99..5f3f2383f 100644 --- a/build_tools/build_llvm.sh +++ b/build_tools/build_llvm.sh @@ -32,9 +32,6 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then export CMAKE_TOOLCHAIN_FILE="$this_dir/linux_default_toolchain.cmake" export CC=clang export CXX=clang++ -elif [[ "$OSTYPE" == "msys"* ]]; then - export CC=clang-cl.exe - export CXX=clang-cl.exe fi export CCACHE_DIR="${cache_dir}/ccache" @@ -68,19 +65,7 @@ CMAKE_ARGS="\ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DLLVM_ENABLE_PROJECTS=mlir;clang;lld" -if [[ "$OSTYPE" == "msys"* ]]; then - cmake $CMAKE_ARGS \ - -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_C_FLAGS="-DMLIR_CAPI_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DMLIR_CAPI_BUILDING_LIBRARY=1" \ - -DLLVM_ENABLE_EH=ON \ - -DCMAKE_C_COMPILER="${CC}" \ - -DCMAKE_CXX_COMPILER="${CXX}" \ - -DLLVM_TARGET_ARCH=X86 \ - -DLLVM_TARGETS_TO_BUILD=X86 \ - -S "$llvm_dir" -B "$build_dir" -elif [[ "$OSTYPE" == "linux-gnu"* ]]; then +if [[ "$OSTYPE" == "linux-gnu"* ]]; then cmake $CMAKE_ARGS \ -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ @@ -90,7 +75,7 @@ elif [[ "$OSTYPE" == "linux-gnu"* ]]; then -DLLVM_TARGET_ARCH=X86 \ -DLLVM_TARGETS_TO_BUILD=X86 \ -S "$llvm_dir" -B "$build_dir" -else +elif [[ "$OSTYPE" == "darwin"* ]]; then cmake $CMAKE_ARGS \ -DLLVM_TARGET_ARCH="X86;AArch64" \ -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ diff --git a/build_tools/build_test_cpp.ps1 b/build_tools/build_test_cpp.ps1 new file mode 100644 index 000000000..0a34e9614 --- /dev/null +++ b/build_tools/build_test_cpp.ps1 @@ -0,0 +1,114 @@ +# Enable strict mode +$ErrorActionPreference = 'Stop' + +$this_dir = Split-Path -Path $MyInvocation.MyCommand.Path -Parent +$repo_root = Resolve-Path -Path "$this_dir/.." +$iree_dir = Resolve-Path -Path "$repo_root/third_party/iree" +$build_dir = "$repo_root/iree-build" +$install_dir = "$repo_root/iree-install" + +if (-not (Test-Path $build_dir)) +{ + New-Item -Path $build_dir -ItemType Directory | Out-Null +} +$build_dir = Resolve-Path -Path $build_dir +$cache_dir = $env:cache_dir +$llvm_install_dir = $env:llvm_install_dir + +if (-not $cache_dir) +{ + $cache_dir = "$repo_root/.build-cache" + if (-not (Test-Path $cache_dir)) + { + New-Item -Path $cache_dir -ItemType Directory | Out-Null + } + $cache_dir = Resolve-Path -Path $cache_dir +} +echo "Caching to $cache_dir" + +if (-not (Test-Path "$cache_dir/ccache")) +{ + New-Item -Path "$cache_dir/ccache" -ItemType Directory | Out-Null +} +if (-not (Test-Path "$cache_dir/pip")) +{ + New-Item -Path "$cache_dir/pip" -ItemType Directory | Out-Null +} + +$python = (Get-Command python -ErrorAction SilentlyContinue).Source +echo "Using python: $python" + +$env:CC = 'clang-cl.exe' +$env:CXX = 'clang-cl.exe' +$env:CCACHE_DIR = "$cache_dir/ccache" +$env:CCACHE_MAXSIZE = '700M' +$env:CMAKE_C_COMPILER_LAUNCHER = 'ccache' +$env:CMAKE_CXX_COMPILER_LAUNCHER = 'ccache' +$env:CCACHE_SLOPPINESS = 'include_file_ctime,include_file_mtime,time_macros' + +& ccache -z + +echo "Building IREE" + +$CMAKE_ARGS = @( + "-GNinja" + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX=$install_dir" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld" + "-DCMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld" + "-DCMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld" + "-DCMAKE_C_COMPILER=$env:CC" + "-DCMAKE_CXX_COMPILER=$env:CXX" + "-DLLVM_TARGET_ARCH=X86" + "-DLLVM_TARGETS_TO_BUILD=X86" + "-DIREE_LINK_COMPILER_SHARED_LIBRARY=OFF" + "-DIREE_EXTERNAL_HAL_DRIVERS=xrt" + "-DIREE_ERROR_ON_MISSING_SUBMODULES=OFF" + "-DIREE_ENABLE_ASSERTIONS=ON" + "-DIREE_BUILD_SAMPLES=OFF" + "-DIREE_BUILD_PYTHON_BINDINGS=ON" + "-DIREE_BUILD_BINDINGS_TFLITE=OFF" + "-DIREE_HAL_DRIVER_DEFAULTS=OFF" + "-DIREE_HAL_DRIVER_LOCAL_SYNC=ON" + "-DIREE_HAL_DRIVER_LOCAL_TASK=ON" + "-DIREE_TARGET_BACKEND_DEFAULTS=OFF" + "-DIREE_TARGET_BACKEND_LLVM_CPU=ON" + "-DIREE_INPUT_TOSA=OFF" + "-DIREE_INPUT_STABLEHLO=OFF" + "-DIREE_INPUT_TORCH=OFF" + "-DCMAKE_OBJECT_PATH_MAX=4096" + "-DIREE_CMAKE_PLUGIN_PATHS=$repo_root" +) + +if (Test-Path $llvm_install_dir) +{ + $CMAKE_ARGS += @( + "-DIREE_BUILD_BUNDLED_LLVM=OFF" + "-DClang_DIR=$llvm_install_dir/lib/cmake/clang" + "-DLLD_DIR=$llvm_install_dir/lib/cmake/lld" + "-DMLIR_DIR=$llvm_install_dir/lib/cmake/mlir" + "-DLLVM_DIR=$llvm_install_dir/lib/cmake/llvm" + ) +} + +& cmake $CMAKE_ARGS "-S $iree_dir" -B $build_dir + +echo "Building all" +echo "------------" +& cmake --build $build_dir -- -k 0 + +echo "Installing" +echo "----------" +echo "Install to: $install_dir" +& cmake --build $build_dir --target iree-install-dist + +echo "CTest" +echo "-----" + +# Hack while Windows is flaky to get past failing tests +& ctest --test-dir $build_dir -R amd-aie --output-on-failure -j --repeat until-pass: 5 + +Remove-Item -Path "$install_dir/bin/clang*" -Force +Remove-Item -Path "$install_dir/bin/llvm-link*" -Force +Copy-Item -Path "$build_dir/tools/testing/e2e/iree-e2e-matmul-test" -Destination "$install_dir/bin -Force" diff --git a/build_tools/build_test_cpp.sh b/build_tools/build_test_cpp.sh index 82e9e96b6..6cf1f82c3 100644 --- a/build_tools/build_test_cpp.sh +++ b/build_tools/build_test_cpp.sh @@ -33,9 +33,6 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then export CMAKE_TOOLCHAIN_FILE="$this_dir/linux_default_toolchain.cmake" export CC=clang export CXX=clang++ -elif [[ "$OSTYPE" == "msys"* ]]; then - export CC=clang-cl.exe - export CXX=clang-cl.exe fi export CCACHE_DIR="${cache_dir}/ccache" @@ -68,6 +65,7 @@ CMAKE_ARGS="\ -DCMAKE_INSTALL_PREFIX=$install_dir \ -DCMAKE_INSTALL_LIBDIR=lib \ -DIREE_ERROR_ON_MISSING_SUBMODULES=OFF \ + -DIREE_LINK_COMPILER_SHARED_LIBRARY=OFF \ -DIREE_ENABLE_ASSERTIONS=ON \ -DIREE_BUILD_SAMPLES=OFF \ -DIREE_BUILD_PYTHON_BINDINGS=ON \ @@ -92,20 +90,7 @@ if [ -d "${llvm_install_dir}" ]; then -DLLVM_DIR=$llvm_install_dir/lib/cmake/llvm" fi -if [[ "$OSTYPE" == "msys"* ]]; then - cmake $CMAKE_ARGS \ - -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_C_FLAGS="-DIREE_EMBED_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DIREE_EMBED_BUILDING_LIBRARY=1" \ - -DCMAKE_CXX_FLAGS="-DIREE_EMBED_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DIREE_EMBED_BUILDING_LIBRARY=1" \ - -DCMAKE_C_COMPILER="${CC}" \ - -DCMAKE_CXX_COMPILER="${CXX}" \ - -DLLVM_TARGET_ARCH=X86 \ - -DLLVM_TARGETS_TO_BUILD=X86 \ - -DIREE_EXTERNAL_HAL_DRIVERS=xrt \ - -S $iree_dir -B $build_dir -elif [[ "$OSTYPE" == "linux-gnu"* ]]; then +if [[ "$OSTYPE" == "linux-gnu"* ]]; then cmake $CMAKE_ARGS \ -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ @@ -116,7 +101,7 @@ elif [[ "$OSTYPE" == "linux-gnu"* ]]; then -DLLVM_TARGETS_TO_BUILD=X86 \ -DIREE_EXTERNAL_HAL_DRIVERS=xrt \ -S $iree_dir -B $build_dir -else +elif [[ "$OSTYPE" == "darwin"* ]]; then cmake $CMAKE_ARGS \ -DLLVM_TARGET_ARCH="X86;ARM" \ -DLLVM_TARGETS_TO_BUILD="X86;ARM" \ @@ -138,9 +123,6 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then ctest --test-dir "$build_dir" -R amd-aie --output-on-failure -j elif [[ "$OSTYPE" == "darwin"* ]]; then ctest --test-dir "$build_dir" -R amd-aie -E "matmul_pack_peel_air_e2e|matmul_elementwise_pack_peel_air_e2e|conv_fill_spec_pad" --output-on-failure -j --repeat until-pass:5 -elif [[ "$OSTYPE" == "msys"* ]]; then - # hack while windows is flaky to get past failing tests - ctest --test-dir "$build_dir" -R amd-aie --output-on-failure -j --repeat until-pass:5 fi rm -f "$install_dir"/bin/clang* diff --git a/compiler/plugins/target/AMD-AIE/iree-amd-aie/Target/tests/aie_cdo_gen_test.cxx b/compiler/plugins/target/AMD-AIE/iree-amd-aie/Target/tests/aie_cdo_gen_test.cxx index 7ea4b8269..a4b457712 100644 --- a/compiler/plugins/target/AMD-AIE/iree-amd-aie/Target/tests/aie_cdo_gen_test.cxx +++ b/compiler/plugins/target/AMD-AIE/iree-amd-aie/Target/tests/aie_cdo_gen_test.cxx @@ -43,15 +43,17 @@ int main(int argc, char **argv) { auto deviceOps = moduleOp.getOps(); auto nDeviceOps = std::distance(deviceOps.begin(), deviceOps.end()); - if (nDeviceOps != 1){ + if (nDeviceOps != 1) { std::cerr << "Error: Expected exactly one xilinx.aie.device op\n"; return 1; } - auto deviceOp = *deviceOps.begin(); + auto deviceOp = *deviceOps.begin(); llvm::DebugFlag = true; const char *debugTypes[3] = {"aie-generate-cdo", "iree-aie-runtime", "iree-aie-cdo-emitter"}; +#ifndef NDEBUG llvm::setCurrentDebugTypes(debugTypes, 3); +#endif auto status = AIETranslateToCDODirect(deviceOp, workDir, false, false, false); std::vector diagnostics; ScopedDiagnosticHandler handler(moduleOp.getContext(), [&](Diagnostic &d) { @@ -63,7 +65,9 @@ int main(int argc, char **argv) { for (const auto &diagnostic : diagnostics) std::cerr << diagnostic << "\n"; llvm::DebugFlag = false; +#ifndef NDEBUG llvm::setCurrentDebugType("aie-cdo-driver-debug"); +#endif status = AIETranslateToCDODirect(deviceOp, workDir, false, false, true); if (failed(status)) for (const auto &diagnostic : diagnostics) std::cerr << diagnostic << "\n";