write powershell

nod-ai · Sep 4, 2024 · b8f64ac · b8f64ac
1 parent 230d645
commit b8f64ac
Show file tree

Hide file tree

Showing 7 changed files with 241 additions and 64 deletions.
diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml
@@ -24,11 +24,6 @@ concurrency:
   group: ci-build-test-cpp-windows-${{ github.event.number || github.sha }}
   cancel-in-progress: true
 
-defaults:
-  run:
-    # force bash for windows
-    shell: bash
-
 jobs:
   build_and_ctest:
     name: Build and Test (windows, ASSERTIONS)
@@ -38,6 +33,9 @@ jobs:
     env:
       CACHE_DIR: ${{ github.workspace }}/.container-cache
       CACHE_KEY: windows-build-test-cpp-asserts-v1-${{ format('{0}-{1}', github.ref_name, github.run_number) }}
+    defaults:
+      run:
+        shell: powershell
     steps:
       - name: Set unified TZ
         uses: szenius/[email protected]
@@ -49,18 +47,15 @@ jobs:
 
       - name: "Checking out repository"
         env:
-          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+          BRANCH_NAME: ${{ github.ref }}
           REPO_ADDRESS: ${{ github.server_url }}/${{ github.repository }}
+        shell: bash
         run: |
-          git \
-            -c submodule."third_party/torch-mlir".update=none \
-            -c submodule."third_party/stablehlo".update=none \
-            -c submodule."src/runtime_src/core/common/aiebu".update=none \
-            clone \
-            --recursive \
-            --shallow-submodules \
-            --depth 1 \
-            -b $BRANCH_NAME $REPO_ADDRESS .
+          git init
+          git remote add origin $REPO_ADDRESS
+          git -c protocol.version=2 fetch --depth 1 origin $BRANCH_NAME
+          git reset --hard FETCH_HEAD
+          git -c submodule."third_party/torch-mlir".update=none -c submodule."third_party/stablehlo".update=none -c submodule."src/runtime_src/core/common/aiebu".update=none submodule update --init --recursive --depth 1 --single-branch -j 10
 
       - name: Setup Cpp
         uses: aminya/setup-cpp@v1
@@ -86,22 +81,26 @@ jobs:
           key: ${{ env.CACHE_KEY }}
           restore-keys: windows-build-test-cpp-
 
-      - name: Build packages
+      - name: Hack IREE
+        shell: bash
         run: |
-          export cache_dir="${{ env.CACHE_DIR }}"
-          export CCACHE_COMPILERCHECK="string:$(clang-cl.exe --version)"
-          bash build_tools/build_llvm.sh
-          rm -rf llvm-build
-          export llvm_install_dir=$PWD/llvm-install
           # TODO(max): send IREE a fix for this
           # target_compile_definitions may only set INTERFACE properties on IMPORTED
           sed -i '114s/PRIVATE/INTERFACE/g' third_party/iree/compiler/src/iree/compiler/API/CMakeLists.txt
-          bash build_tools/build_test_cpp.sh
+
+      - name: Build packages
+        run: |
+          $env:cache_dir = "${{ env.CACHE_DIR }}"
+          $env:CCACHE_COMPILERCHECK = "string:$(clang-cl.exe --version)"
+          powershell build_tools/build_llvm.ps1
+          Remove-Item llvm-build -Force
+          powershell build_tools/build_test_cpp.ps1
 
       - name: Create artifacts
         if: ${{ !cancelled() }}
         run: |
-          pushd third_party/iree/third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd
+          pushd third_party/iree/third_party/llvm-project 
+          $llvm_sha_short = "$(git rev-parse --short HEAD)"
           tar cf llvm-dist-windows-$llvm_sha_short.tar llvm-install
           tar cf iree-dist-windows.tar iree-install
 
@@ -138,6 +137,9 @@ jobs:
     name: E2E Test windows
     runs-on: windows-phoenix
     needs: build_and_ctest
+    defaults:
+      run:
+        shell: bash
     strategy:
       fail-fast: true
     steps:

diff --git a/README.md b/README.md
@@ -38,7 +38,6 @@ git \
   -c submodule."src/runtime_src/core/common/aiebu".update=none \
   clone \
   --recursive \
-  --depth 1 \
   --shallow-submodules \
   https://github.com/nod-ai/iree-amd-aie.git
 ```

diff --git a/build_tools/build_llvm.ps1 b/build_tools/build_llvm.ps1
@@ -0,0 +1,91 @@
+# Enable strict mode
+$ErrorActionPreference = 'Stop'
+Set-PSDebug -Trace 2
+
+$this_dir = Split-Path -Path $MyInvocation.MyCommand.Path -Parent
+$repo_root = Resolve-Path -Path "$this_dir/.."
+$llvm_dir = Resolve-Path -Path "$repo_root/third_party/iree/third_party/llvm-project/llvm"
+echo "llvm_dir $llvm_dir"
+$build_dir = "$repo_root/llvm-build"
+$install_dir = "$repo_root/llvm-install"
+
+if (-not (Test-Path $build_dir))
+{
+    New-Item -Path $build_dir -ItemType Directory | Out-Null
+}
+$build_dir = Resolve-Path -Path $build_dir
+$cache_dir = $env:cache_dir
+
+if (-not $cache_dir)
+{
+    $cache_dir = "$repo_root/.build-cache"
+    if (-not (Test-Path $cache_dir))
+    {
+        New-Item -Path $cache_dir -ItemType Directory | Out-Null
+    }
+    $cache_dir = Resolve-Path -Path $cache_dir
+}
+echo "Caching to $cache_dir"
+
+if (-not (Test-Path "$cache_dir/ccache"))
+{
+    New-Item -Path "$cache_dir/ccache" -ItemType Directory | Out-Null
+}
+if (-not (Test-Path "$cache_dir/pip"))
+{
+    New-Item -Path "$cache_dir/pip" -ItemType Directory | Out-Null
+}
+
+$python = (Get-Command python -ErrorAction SilentlyContinue).Source
+echo "Using python: $python"
+
+$env:CC = 'clang-cl.exe'
+$env:CXX = 'clang-cl.exe'
+$env:CCACHE_DIR = "$cache_dir/ccache"
+$env:CCACHE_MAXSIZE = '700M'
+$env:CMAKE_C_COMPILER_LAUNCHER = 'ccache'
+$env:CMAKE_CXX_COMPILER_LAUNCHER = 'ccache'
+$env:CCACHE_SLOPPINESS = 'include_file_ctime,include_file_mtime,time_macros'
+
+& ccache -z
+
+& cmake `
+  -GNinja `
+  -DCMAKE_BUILD_TYPE=Release `
+  -DCMAKE_INSTALL_PREFIX=$install_dir `
+  -DCMAKE_OBJECT_PATH_MAX=4096 `
+  -DCMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld `
+  -DCMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld `
+  -DCMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld `
+  -DCMAKE_C_COMPILER="$env:CC" `
+  -DCMAKE_CXX_COMPILER="$env:CXX" `
+  -DCMAKE_C_FLAGS="-DMLIR_CAPI_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DMLIR_CAPI_BUILDING_LIBRARY=1" `
+  -DCMAKE_CXX_FLAGS="-DMLIR_CAPI_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DMLIR_CAPI_BUILDING_LIBRARY=1" `
+  -DLLVM_INCLUDE_EXAMPLES=OFF `
+  -DLLVM_INCLUDE_TESTS=OFF `
+  -DLLVM_INCLUDE_BENCHMARKS=OFF `
+  -DLLVM_APPEND_VC_REV=OFF `
+  -DLLVM_ENABLE_ASSERTIONS=ON `
+  -DLLVM_ENABLE_IDE=ON `
+  -DLLVM_ENABLE_BINDINGS=OFF `
+  -DLLVM_ENABLE_LIBEDIT=OFF `
+  -DLLVM_ENABLE_LIBXML2=OFF `
+  -DLLVM_ENABLE_TERMINFO=OFF `
+  -DLLVM_ENABLE_ZLIB=OFF `
+  -DLLVM_ENABLE_ZSTD=OFF `
+  -DLLVM_FORCE_ENABLE_STATS=ON `
+  -DLLVM_INSTALL_UTILS=ON `
+  -DMLIR_ENABLE_BINDINGS_PYTHON=ON `
+  -DLLVM_ENABLE_PROJECTS="mlir;clang;lld" `
+  -DLLVM_TARGET_ARCH=X86 `
+  -DLLVM_TARGETS_TO_BUILD=X86 `
+  -S "$llvm_dir" -B "$build_dir"
+
+echo "Building all"
+echo "------------"
+cmake --build $build_dir -- -k 0
+
+echo "Installing"
+echo "----------"
+echo "Install to: $install_dir"
+cmake --build $build_dir --target install
diff --git a/build_tools/build_llvm.sh b/build_tools/build_llvm.sh
@@ -32,9 +32,6 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
   export CMAKE_TOOLCHAIN_FILE="$this_dir/linux_default_toolchain.cmake"
   export CC=clang
   export CXX=clang++
-elif [[ "$OSTYPE" == "msys"* ]]; then
-  export CC=clang-cl.exe
-  export CXX=clang-cl.exe
 fi
 
 export CCACHE_DIR="${cache_dir}/ccache"
@@ -68,19 +65,7 @@ CMAKE_ARGS="\
   -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
   -DLLVM_ENABLE_PROJECTS=mlir;clang;lld"
 
-if [[ "$OSTYPE" == "msys"* ]]; then
-  cmake $CMAKE_ARGS \
-    -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \
-    -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \
-    -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \
-    -DCMAKE_C_FLAGS="-DMLIR_CAPI_ENABLE_WINDOWS_DLL_DECLSPEC=1 -DMLIR_CAPI_BUILDING_LIBRARY=1" \
-    -DLLVM_ENABLE_EH=ON \
-    -DCMAKE_C_COMPILER="${CC}" \
-    -DCMAKE_CXX_COMPILER="${CXX}" \
-    -DLLVM_TARGET_ARCH=X86 \
-    -DLLVM_TARGETS_TO_BUILD=X86 \
-    -S "$llvm_dir" -B "$build_dir"
-elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
   cmake $CMAKE_ARGS \
     -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \
     -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \
@@ -90,7 +75,7 @@ elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
     -DLLVM_TARGET_ARCH=X86 \
     -DLLVM_TARGETS_TO_BUILD=X86 \
     -S "$llvm_dir" -B "$build_dir"
-else
+elif [[ "$OSTYPE" == "darwin"* ]]; then
   cmake $CMAKE_ARGS \
     -DLLVM_TARGET_ARCH="X86;AArch64" \
     -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \

diff --git a/build_tools/build_test_cpp.ps1 b/build_tools/build_test_cpp.ps1
@@ -0,0 +1,114 @@
+# Enable strict mode
+$ErrorActionPreference = 'Stop'
+
+$this_dir = Split-Path -Path $MyInvocation.MyCommand.Path -Parent
+$repo_root = Resolve-Path -Path "$this_dir/.."
+$iree_dir = Resolve-Path -Path "$repo_root/third_party/iree"
+$build_dir = "$repo_root/iree-build"
+$install_dir = "$repo_root/iree-install"
+
+if (-not (Test-Path $build_dir))
+{
+    New-Item -Path $build_dir -ItemType Directory | Out-Null
+}
+$build_dir = Resolve-Path -Path $build_dir
+$cache_dir = $env:cache_dir
+$llvm_install_dir = $env:llvm_install_dir
+
+if (-not $cache_dir)
+{
+    $cache_dir = "$repo_root/.build-cache"
+    if (-not (Test-Path $cache_dir))
+    {
+        New-Item -Path $cache_dir -ItemType Directory | Out-Null
+    }
+    $cache_dir = Resolve-Path -Path $cache_dir
+}
+echo "Caching to $cache_dir"
+
+if (-not (Test-Path "$cache_dir/ccache"))
+{
+    New-Item -Path "$cache_dir/ccache" -ItemType Directory | Out-Null
+}
+if (-not (Test-Path "$cache_dir/pip"))
+{
+    New-Item -Path "$cache_dir/pip" -ItemType Directory | Out-Null
+}
+
+$python = (Get-Command python -ErrorAction SilentlyContinue).Source
+echo "Using python: $python"
+
+$env:CC = 'clang-cl.exe'
+$env:CXX = 'clang-cl.exe'
+$env:CCACHE_DIR = "$cache_dir/ccache"
+$env:CCACHE_MAXSIZE = '700M'
+$env:CMAKE_C_COMPILER_LAUNCHER = 'ccache'
+$env:CMAKE_CXX_COMPILER_LAUNCHER = 'ccache'
+$env:CCACHE_SLOPPINESS = 'include_file_ctime,include_file_mtime,time_macros'
+
+& ccache -z
+
+echo "Building IREE"
+
+$CMAKE_ARGS = @(
+    "-GNinja"
+    "-DCMAKE_BUILD_TYPE=Release"
+    "-DCMAKE_INSTALL_PREFIX=$install_dir"
+    "-DCMAKE_INSTALL_LIBDIR=lib"
+    "-DCMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld"
+    "-DCMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld"
+    "-DCMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld"
+    "-DCMAKE_C_COMPILER=$env:CC"
+    "-DCMAKE_CXX_COMPILER=$env:CXX"
+    "-DLLVM_TARGET_ARCH=X86"
+    "-DLLVM_TARGETS_TO_BUILD=X86"
+    "-DIREE_LINK_COMPILER_SHARED_LIBRARY=OFF"
+    "-DIREE_EXTERNAL_HAL_DRIVERS=xrt"
+    "-DIREE_ERROR_ON_MISSING_SUBMODULES=OFF"
+    "-DIREE_ENABLE_ASSERTIONS=ON"
+    "-DIREE_BUILD_SAMPLES=OFF"
+    "-DIREE_BUILD_PYTHON_BINDINGS=ON"
+    "-DIREE_BUILD_BINDINGS_TFLITE=OFF"
+    "-DIREE_HAL_DRIVER_DEFAULTS=OFF"
+    "-DIREE_HAL_DRIVER_LOCAL_SYNC=ON"
+    "-DIREE_HAL_DRIVER_LOCAL_TASK=ON"
+    "-DIREE_TARGET_BACKEND_DEFAULTS=OFF"
+    "-DIREE_TARGET_BACKEND_LLVM_CPU=ON"
+    "-DIREE_INPUT_TOSA=OFF"
+    "-DIREE_INPUT_STABLEHLO=OFF"
+    "-DIREE_INPUT_TORCH=OFF"
+    "-DCMAKE_OBJECT_PATH_MAX=4096"
+    "-DIREE_CMAKE_PLUGIN_PATHS=$repo_root"
+)
+
+if (Test-Path $llvm_install_dir)
+{
+    $CMAKE_ARGS += @(
+        "-DIREE_BUILD_BUNDLED_LLVM=OFF"
+        "-DClang_DIR=$llvm_install_dir/lib/cmake/clang"
+        "-DLLD_DIR=$llvm_install_dir/lib/cmake/lld"
+        "-DMLIR_DIR=$llvm_install_dir/lib/cmake/mlir"
+        "-DLLVM_DIR=$llvm_install_dir/lib/cmake/llvm"
+    )
+}
+
+& cmake $CMAKE_ARGS "-S $iree_dir" -B $build_dir
+
+echo "Building all"
+echo "------------"
+& cmake --build $build_dir -- -k 0
+
+echo "Installing"
+echo "----------"
+echo "Install to: $install_dir"
+& cmake --build $build_dir --target iree-install-dist
+
+echo "CTest"
+echo "-----"
+
+# Hack while Windows is flaky to get past failing tests
+& ctest --test-dir $build_dir -R amd-aie --output-on-failure -j --repeat until-pass: 5
+
+Remove-Item -Path "$install_dir/bin/clang*" -Force
+Remove-Item -Path "$install_dir/bin/llvm-link*" -Force
+Copy-Item -Path "$build_dir/tools/testing/e2e/iree-e2e-matmul-test" -Destination "$install_dir/bin -Force"